[PATCH v3 12/26] read-cache: basic hand shaking to the file watcher

["Nguyễn Thái Ngọc Duy" <pclouds@gmail.com>]

read_cache() connects to the file watcher, specified by
filewatcher.path config, and performs basic hand shaking. CE_WATCHED
is cleared if git and file watcher have different views on the index
state.

All send/receive calls must be complete within a limited time to avoid
a buggy file-watcher hang "git status" forever. And the whole point of
doing this is speed. If file watcher can't respond fast enough, for
whatever reason, then it should not be used.

Signed-off-by: Nguyễn Thái Ngọc Duy <pclouds@gmail.com>
---
 Documentation/config.txt           |  10 +++
 Documentation/git-file-watcher.txt |   4 +-
 Makefile                           |   1 +
 cache.h                            |   1 +
 file-watcher-lib.c (new)           |  91 ++++++++++++++++++++++
 file-watcher-lib.h (new)           |   6 ++
 file-watcher.c                     | 152 ++++++++++++++++++++++++++++++++++++-
 read-cache.c                       |   6 ++
 8 files changed, 269 insertions(+), 2 deletions(-)
 create mode 100644 file-watcher-lib.c
 create mode 100644 file-watcher-lib.h

diff --git a/Documentation/config.txt b/Documentation/config.txt
index 5f4d793..6ad653a 100644
--- a/Documentation/config.txt
+++ b/Documentation/config.txt
@@ -1042,6 +1042,16 @@ difftool.<tool>.cmd::
 difftool.prompt::
 	Prompt before each invocation of the diff tool.
 
+filewatcher.path::
+	The directory that contains the socket of `git	file-watcher`.
+	If it's not an absolute path, it's relative to $GIT_DIR. An
+	empty path means no connection to file watcher.
+
+filewatcher.timeout::
+	This is the maximum time in milliseconds that Git waits for
+	the file watcher to respond before giving up. Default value is
+	50. Setting to -1 makes Git wait forever.
+
 fetch.recurseSubmodules::
 	This option can be either set to a boolean value or to 'on-demand'.
 	Setting it to a boolean changes the behavior of fetch and pull to
diff --git a/Documentation/git-file-watcher.txt b/Documentation/git-file-watcher.txt
index ec81f18..d91caf3 100644
--- a/Documentation/git-file-watcher.txt
+++ b/Documentation/git-file-watcher.txt
@@ -14,7 +14,9 @@ DESCRIPTION
 -----------
 This program watches file changes in a git working directory and let
 Git now what files have been changed so that Git does not have to call
-lstat(2) to detect that itself.
+lstat(2) to detect that itself. Config key filewatcher.path needs to
+be set to `<socket directory>` so Git knows how to contact to the file
+watcher.
 
 OPTIONS
 -------
diff --git a/Makefile b/Makefile
index 8eef0d6..1c4d659 100644
--- a/Makefile
+++ b/Makefile
@@ -798,6 +798,7 @@ LIB_OBJS += entry.o
 LIB_OBJS += environment.o
 LIB_OBJS += exec_cmd.o
 LIB_OBJS += fetch-pack.o
+LIB_OBJS += file-watcher-lib.o
 LIB_OBJS += fsck.o
 LIB_OBJS += gettext.o
 LIB_OBJS += gpg-interface.o
diff --git a/cache.h b/cache.h
index a0af2a5..b3ea574 100644
--- a/cache.h
+++ b/cache.h
@@ -283,6 +283,7 @@ struct index_state {
 	struct hash_table name_hash;
 	struct hash_table dir_hash;
 	unsigned char sha1[20];
+	int watcher;
 };
 
 extern struct index_state the_index;
diff --git a/file-watcher-lib.c b/file-watcher-lib.c
new file mode 100644
index 0000000..d0636cc
--- /dev/null
+++ b/file-watcher-lib.c
@@ -0,0 +1,91 @@
+#include "cache.h"
+#include "file-watcher-lib.h"
+#include "pkt-line.h"
+#include "unix-socket.h"
+
+static char *watcher_path;
+static int WAIT_TIME = 50;	/* in ms */
+
+static int connect_watcher(const char *path)
+{
+	struct strbuf sb = STRBUF_INIT;
+	int fd;
+
+	if (!path || !*path)
+		return -1;
+
+	strbuf_addf(&sb, "%s/socket", path);
+	fd = unix_stream_connect(sb.buf);
+	strbuf_release(&sb);
+	return fd;
+}
+
+static void reset_watches(struct index_state *istate, int disconnect)
+{
+	int i;
+	for (i = 0; i < istate->cache_nr; i++)
+		if (istate->cache[i]->ce_flags & CE_WATCHED) {
+			istate->cache[i]->ce_flags &= ~(CE_WATCHED | CE_VALID);
+			istate->cache_changed = 1;
+		}
+	if (disconnect && istate->watcher > 0) {
+		close(istate->watcher);
+		istate->watcher = -1;
+	}
+}
+
+static int watcher_config(const char *var, const char *value, void *data)
+{
+	if (!strcmp(var, "filewatcher.path")) {
+		if (is_absolute_path(value))
+			watcher_path = xstrdup(value);
+		else if (*value == '~')
+			watcher_path = expand_user_path(value);
+		else
+			watcher_path = git_pathdup("%s", value);
+		return 0;
+	}
+	if (!strcmp(var, "filewatcher.timeout")) {
+		WAIT_TIME = git_config_int(var, value);
+		return 0;
+	}
+	return 0;
+}
+
+void open_watcher(struct index_state *istate)
+{
+	static int read_config = 0;
+	char *msg;
+
+	if (!get_git_work_tree()) {
+		reset_watches(istate, 1);
+		return;
+	}
+
+	if (!read_config) {
+		/*
+		 * can't hook into git_default_config because
+		 * read_cache() may be called even before git_config()
+		 * call.
+		 */
+		git_config(watcher_config, NULL);
+		read_config = 1;
+	}
+
+	istate->watcher = connect_watcher(watcher_path);
+	if (packet_write_timeout(istate->watcher, WAIT_TIME, "hello") <= 0 ||
+	    (msg = packet_read_line_timeout(istate->watcher, WAIT_TIME, NULL)) == NULL ||
+	    strcmp(msg, "hello")) {
+		reset_watches(istate, 1);
+		return;
+	}
+
+	if (packet_write_timeout(istate->watcher, WAIT_TIME, "index %s %s",
+				 sha1_to_hex(istate->sha1),
+				 get_git_work_tree()) <= 0 ||
+	    (msg = packet_read_line_timeout(istate->watcher, WAIT_TIME, NULL)) == NULL ||
+	    strcmp(msg, "ok")) {
+		reset_watches(istate, 0);
+		return;
+	}
+}
diff --git a/file-watcher-lib.h b/file-watcher-lib.h
new file mode 100644
index 0000000..eb6edf5
--- /dev/null
+++ b/file-watcher-lib.h
@@ -0,0 +1,6 @@
+#ifndef __FILE_WATCHER_LIB__
+#define __FILE_WATCHER_LIB__
+
+void open_watcher(struct index_state *istate);
+
+#endif
diff --git a/file-watcher.c b/file-watcher.c
index 1e1ccad..6df3a48 100644
--- a/file-watcher.c
+++ b/file-watcher.c
@@ -3,20 +3,78 @@
 #include "parse-options.h"
 #include "exec_cmd.h"
 #include "unix-socket.h"
+#include "pkt-line.h"
 
 static const char *const file_watcher_usage[] = {
 	N_("git file-watcher [options] <socket directory>"),
 	NULL
 };
 
+struct repository {
+	char *work_tree;
+	char index_signature[41];
+	/*
+	 * At least with inotify we don't keep track down to "/". So
+	 * if worktree is /abc/def and someone moves /abc to /ghi, and
+	 * /jlk to /abc (and /jlk/def exists before the move), we
+	 * cant' detect that /abc/def is totally new. Checking inode
+	 * is probably enough for this case.
+	 */
+	ino_t inode;
+};
+
+const char *invalid_signature = "0000000000000000000000000000000000000000";
+
+static struct repository **repos;
+static int nr_repos;
+
 struct connection {
-	int sock;
+	int sock, polite;
+	struct repository *repo;
 };
 
 static struct connection **conns;
 static struct pollfd *pfd;
 static int conns_alloc, pfd_nr, pfd_alloc;
 
+static struct repository *get_repo(const char *work_tree)
+{
+	int first, last;
+	struct repository *repo;
+
+	first = 0;
+	last = nr_repos;
+	while (last > first) {
+		int next = (last + first) >> 1;
+		int cmp = strcmp(work_tree, repos[next]->work_tree);
+		if (!cmp)
+			return repos[next];
+		if (cmp < 0) {
+			last = next;
+			continue;
+		}
+		first = next+1;
+	}
+
+	nr_repos++;
+	repos = xrealloc(repos, sizeof(*repos) * nr_repos);
+	if (nr_repos > first + 1)
+		memmove(repos + first + 1, repos + first,
+			(nr_repos - first - 1) * sizeof(*repos));
+	repo = xmalloc(sizeof(*repo));
+	memset(repo, 0, sizeof(*repo));
+	repo->work_tree = xstrdup(work_tree);
+	memset(repo->index_signature, '0', 40);
+	repos[first] = repo;
+	return repo;
+}
+
+static void reset_repo(struct repository *repo, ino_t inode)
+{
+	memcpy(repo->index_signature, invalid_signature, 40);
+	repo->inode = inode;
+}
+
 static int shutdown_connection(int id)
 {
 	struct connection *conn = conns[id];
@@ -31,6 +89,98 @@ static int shutdown_connection(int id)
 
 static int handle_command(int conn_id)
 {
+	int fd = conns[conn_id]->sock;
+	int len;
+	const char *arg;
+	char *msg;
+
+	/*
+	 * ">" denotes an incoming packet, "<" outgoing. The lack of
+	 * "<" means no reply expected.
+	 *
+	 * < "error" SP ERROR-STRING
+	 *
+	 * This can be sent whenever the client violates the protocol.
+	 */
+
+	msg = packet_read_line(fd, &len);
+	if (!msg) {
+		packet_write(fd, "error invalid pkt-line");
+		return shutdown_connection(conn_id);
+	}
+
+	/*
+	 * > "hello" [SP CAP [SP CAP..]]
+	 * < "hello" [SP CAP [SP CAP..]]
+	 *
+	 * Advertise capabilities of both sides. File watcher may
+	 * disconnect if the client does not advertise the required
+	 * capabilities. Capabilities in uppercase MUST be
+	 * supported. If any side does not understand any of the
+	 * advertised uppercase capabilities, it must disconnect.
+	 */
+	if ((arg = skip_prefix(msg, "hello"))) {
+		if (*arg) {	/* no capabilities supported yet */
+			packet_write(fd, "error capabilities not supported");
+			return shutdown_connection(conn_id);
+		}
+		packet_write(fd, "hello");
+		conns[conn_id]->polite = 1;
+	}
+
+	/*
+	 * > "index" SP INDEX-SIGNATURE SP WORK-TREE-PATH
+	 * < "ok" | "inconsistent"
+	 *
+	 * INDEX-SIGNATURE consists of 40 hexadecimal letters
+	 * WORK-TREE-PATH must be absolute and normalized path
+	 *
+	 * Watch file changes in index. The client sends the index and
+	 * work tree info. File watcher validates that it holds the
+	 * same info. If so it sends "ok" back indicating both sides
+	 * are on the same page and CE_WATCHED bits can be ketpt.
+	 *
+	 * Otherwise it sends "inconsistent" and both sides must reset
+	 * back to initial state. File watcher keeps its index
+	 * signature all-zero until the client has updated the index
+	 * ondisk and request to update index signature.
+	 *
+	 * "hello" must be exchanged first. After this command the
+	 * connection is associated with a worktree/index. Many
+	 * commands may require this to proceed.
+	 */
+	else if (starts_with(msg, "index ")) {
+		struct repository *repo;
+		struct stat st;
+		if (!conns[conn_id]->polite) {
+			packet_write(fd, "error why did you not greet me? go away");
+			return shutdown_connection(conn_id);
+		}
+		if (len < 47 || msg[46] != ' ' || !is_absolute_path(msg + 47)) {
+			packet_write(fd, "error invalid index line %s", msg);
+			return shutdown_connection(conn_id);
+		}
+
+		if (lstat(msg + 47, &st) || !S_ISDIR(st.st_mode)) {
+			packet_write(fd, "error work tree does not exist: %s",
+				     strerror(errno));
+			return shutdown_connection(conn_id);
+		}
+		repo = get_repo(msg + 47);
+		conns[conn_id]->repo = repo;
+		if (memcmp(msg + 6, repo->index_signature, 40) ||
+		    !memcmp(msg + 6, invalid_signature, 40) ||
+		    repo->inode != st.st_ino) {
+			packet_write(fd, "inconsistent");
+			reset_repo(repo, st.st_ino);
+			return 0;
+		}
+		packet_write(fd, "ok");
+	}
+	else {
+		packet_write(fd, "error unrecognized command %s", msg);
+		return shutdown_connection(conn_id);
+	}
 	return 0;
 }
 
diff --git a/read-cache.c b/read-cache.c
index 8961864..a7e5735 100644
--- a/read-cache.c
+++ b/read-cache.c
@@ -14,6 +14,7 @@
 #include "resolve-undo.h"
 #include "strbuf.h"
 #include "varint.h"
+#include "file-watcher-lib.h"
 
 static struct cache_entry *refresh_cache_entry(struct cache_entry *ce, int really);
 
@@ -1528,6 +1529,7 @@ int read_index_from(struct index_state *istate, const char *path)
 		src_offset += extsize;
 	}
 	munmap(mmap, mmap_size);
+	open_watcher(istate);
 	return istate->cache_nr;
 
 unmap:
@@ -1553,6 +1555,10 @@ int discard_index(struct index_state *istate)
 	istate->timestamp.nsec = 0;
 	free_name_hash(istate);
 	cache_tree_free(&(istate->cache_tree));
+	if (istate->watcher > 0) {
+		close(istate->watcher);
+		istate->watcher = -1;
+	}
 	istate->initialized = 0;
 	free(istate->cache);
 	istate->cache = NULL;
-- 
1.8.5.2.240.g8478abd