From: David Turner <novalis@novalis.org>
To: git@vger.kernel.org, pclouds@gmail.com, kamggg@gmail.com
Cc: David Turner <dturner@twopensource.com>,
Junio C Hamano <gitster@pobox.com>
Subject: [PATCH v13 10/20] watchman: support watchman to reduce index refresh cost
Date: Sun, 26 Jun 2016 00:14:14 -0400 [thread overview]
Message-ID: <1466914464-10358-10-git-send-email-novalis@novalis.org> (raw)
In-Reply-To: <1466914464-10358-1-git-send-email-novalis@novalis.org>
From: Nguyễn Thái Ngọc Duy <pclouds@gmail.com>
The previous patch has the logic to clear bits in 'WAMA' bitmap. This
patch has logic to set bits as told by watchman. The missing bit,
_using_ these bits, are not here yet.
A lot of this code is written by David Turner originally, mostly from
[1]. I'm just copying and polishing it a bit.
[1] http://article.gmane.org/gmane.comp.version-control.git/248006
Signed-off-by: Nguyễn Thái Ngọc Duy <pclouds@gmail.com>
Signed-off-by: David Turner <dturner@twopensource.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
---
Makefile | 12 +++++
cache.h | 1 +
config.c | 5 ++
configure.ac | 8 ++++
environment.c | 2 +
watchman-support.c | 135 +++++++++++++++++++++++++++++++++++++++++++++++++++++
watchman-support.h | 7 +++
7 files changed, 170 insertions(+)
create mode 100644 watchman-support.c
create mode 100644 watchman-support.h
diff --git a/Makefile b/Makefile
index 2b2ac05..7f4ab41 100644
--- a/Makefile
+++ b/Makefile
@@ -451,6 +451,7 @@ MSGFMT = msgfmt
CURL_CONFIG = curl-config
PTHREAD_LIBS = -lpthread
PTHREAD_CFLAGS =
+WATCHMAN_LIBS =
GCOV = gcov
export TCL_PATH TCLTK_PATH
@@ -1416,6 +1417,13 @@ else
LIB_OBJS += thread-utils.o
endif
+ifdef USE_WATCHMAN
+ LIB_H += watchman-support.h
+ LIB_OBJS += watchman-support.o
+ WATCHMAN_LIBS = -lwatchman
+ BASIC_CFLAGS += -DUSE_WATCHMAN
+endif
+
ifdef HAVE_PATHS_H
BASIC_CFLAGS += -DHAVE_PATHS_H
endif
@@ -2027,6 +2035,9 @@ git-remote-testsvn$X: remote-testsvn.o GIT-LDFLAGS $(GITLIBS) $(VCSSVN_LIB)
$(QUIET_LINK)$(CC) $(ALL_CFLAGS) -o $@ $(ALL_LDFLAGS) $(filter %.o,$^) $(LIBS) \
$(VCSSVN_LIB)
+git-index-helper$X: index-helper.o GIT-LDFLAGS $(GITLIBS)
+ $(QUIET_LINK)$(CC) $(ALL_CFLAGS) -o $@ $(ALL_LDFLAGS) $(filter %.o,$^) $(LIBS) $(WATCHMAN_LIBS)
+
$(REMOTE_CURL_ALIASES): $(REMOTE_CURL_PRIMARY)
$(QUIET_LNCP)$(RM) $@ && \
ln $< $@ 2>/dev/null || \
@@ -2166,6 +2177,7 @@ GIT-BUILD-OPTIONS: FORCE
@echo NO_PYTHON=\''$(subst ','\'',$(subst ','\'',$(NO_PYTHON)))'\' >>$@+
@echo NO_UNIX_SOCKETS=\''$(subst ','\'',$(subst ','\'',$(NO_UNIX_SOCKETS)))'\' >>$@+
@echo NO_MMAP=\''$(subst ','\'',$(subst ','\'',$(NO_MMAP)))'\' >>$@+
+ @echo USE_WATCHMAN=\''$(subst ','\'',$(subst ','\'',$(USE_WATCHMAN)))'\' >>$@+
ifdef TEST_OUTPUT_DIRECTORY
@echo TEST_OUTPUT_DIRECTORY=\''$(subst ','\'',$(subst ','\'',$(TEST_OUTPUT_DIRECTORY)))'\' >>$@+
endif
diff --git a/cache.h b/cache.h
index f10992d..452aea2 100644
--- a/cache.h
+++ b/cache.h
@@ -696,6 +696,7 @@ extern char *git_replace_ref_base;
extern int fsync_object_files;
extern int core_preload_index;
+extern int core_watchman_sync_timeout;
extern int core_apply_sparse_checkout;
extern int precomposed_unicode;
extern int protect_hfs;
diff --git a/config.c b/config.c
index 9ba40bc..e6dc141 100644
--- a/config.c
+++ b/config.c
@@ -882,6 +882,11 @@ static int git_default_core_config(const char *var, const char *value)
return 0;
}
+ if (!strcmp(var, "core.watchmansynctimeout")) {
+ core_watchman_sync_timeout = git_config_int(var, value);
+ return 0;
+ }
+
if (!strcmp(var, "core.createobject")) {
if (!strcmp(value, "rename"))
object_creation_mode = OBJECT_CREATION_USES_RENAMES;
diff --git a/configure.ac b/configure.ac
index 0cd9f46..334d63b 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1099,6 +1099,14 @@ AC_COMPILE_IFELSE([BSD_SYSCTL_SRC],
HAVE_BSD_SYSCTL=])
GIT_CONF_SUBST([HAVE_BSD_SYSCTL])
+#
+# Check for watchman client library
+
+AC_CHECK_LIB([watchman], [watchman_connect],
+ [USE_WATCHMAN=YesPlease],
+ [USE_WATCHMAN=])
+GIT_CONF_SUBST([USE_WATCHMAN])
+
## Other checks.
# Define USE_PIC if you need the main git objects to be built with -fPIC
# in order to build and link perl/Git.so. x86-64 seems to need this.
diff --git a/environment.c b/environment.c
index 6dec9d0..ad41015 100644
--- a/environment.c
+++ b/environment.c
@@ -94,6 +94,8 @@ int core_preload_index = 1;
*/
int ignore_untracked_cache_config;
+int core_watchman_sync_timeout = 300;
+
/* This is set by setup_git_dir_gently() and/or git_default_config() */
char *git_work_tree_cfg;
static char *work_tree;
diff --git a/watchman-support.c b/watchman-support.c
new file mode 100644
index 0000000..8421fe0
--- /dev/null
+++ b/watchman-support.c
@@ -0,0 +1,135 @@
+#include "cache.h"
+#include "watchman-support.h"
+#include "strbuf.h"
+#include "dir.h"
+#include <watchman.h>
+
+static struct watchman_query *make_query(const char *last_update)
+{
+ struct watchman_query *query = watchman_query();
+ watchman_query_set_fields(query, WATCHMAN_FIELD_NAME |
+ WATCHMAN_FIELD_EXISTS |
+ WATCHMAN_FIELD_NEWER);
+ watchman_query_set_empty_on_fresh(query, 1);
+ query->sync_timeout = core_watchman_sync_timeout;
+ if (*last_update)
+ watchman_query_set_since_oclock(query, last_update);
+ return query;
+}
+
+static struct watchman_query_result *query_watchman(
+ struct index_state *istate, struct watchman_connection *connection,
+ const char *fs_path, const char *last_update)
+{
+ struct watchman_error wm_error;
+ struct watchman_query *query;
+ struct watchman_expression *expr;
+ struct watchman_query_result *result;
+
+ query = make_query(last_update);
+ expr = watchman_true_expression();
+ result = watchman_do_query(connection, fs_path, query, expr, &wm_error);
+ watchman_free_query(query);
+ watchman_free_expression(expr);
+
+ if (!result)
+ warning("Watchman query error: %s (at %s)",
+ wm_error.message,
+ *last_update ? last_update : "the beginning");
+
+ return result;
+}
+
+static void update_index(struct index_state *istate,
+ struct watchman_query_result *result)
+{
+ int i;
+
+ if (result->is_fresh_instance) {
+ /* let refresh clear them later */
+ for (i = 0; i < istate->cache_nr; i++)
+ istate->cache[i]->ce_flags |= CE_WATCHMAN_DIRTY;
+ goto done;
+ }
+
+ for (i = 0; i < result->nr; i++) {
+ struct watchman_stat *wm = result->stats + i;
+ int pos;
+
+ if (S_ISDIR(wm->mode) ||
+ !strncmp(wm->name, ".git/", 5) ||
+ strstr(wm->name, "/.git/"))
+ continue;
+
+ pos = index_name_pos(istate, wm->name, strlen(wm->name));
+ if (pos < 0) {
+ if (istate->untracked) {
+ char *name = xstrdup(wm->name);
+ char *dname = dirname(name);
+
+ /*
+ * dirname() returns '.' for the root,
+ * but we call it ''.
+ */
+ if (dname[0] == '.' && dname[1] == 0)
+ string_list_append(&istate->untracked->invalid_untracked, "");
+ else
+ string_list_append(&istate->untracked->invalid_untracked,
+ dname);
+ free(name);
+ }
+ continue;
+ }
+ /* FIXME: ignore staged entries and gitlinks too? */
+
+ istate->cache[pos]->ce_flags |= CE_WATCHMAN_DIRTY;
+ }
+
+done:
+ free(istate->last_update);
+ istate->last_update = xstrdup(result->clock);
+ istate->cache_changed |= WATCHMAN_CHANGED;
+ if (istate->untracked)
+ string_list_remove_duplicates(&istate->untracked->invalid_untracked, 0);
+}
+
+int check_watchman(struct index_state *istate)
+{
+ struct watchman_error wm_error;
+ struct watchman_connection *connection;
+ struct watchman_query_result *result;
+ const char *fs_path;
+ struct timeval timeout;
+ /*
+ * Convert core_watchman_sync_timeout, in milliseconds, to
+ * struct timeval, in seconds and microseconds.
+ */
+
+ fs_path = get_git_work_tree();
+ if (!fs_path)
+ return -1;
+
+ timeout.tv_sec = core_watchman_sync_timeout / 1000;
+ timeout.tv_usec = (core_watchman_sync_timeout % 1000) * 1000;
+ connection = watchman_connect(timeout, &wm_error);
+
+ if (!connection) {
+ warning("Watchman watch error: %s", wm_error.message);
+ return -1;
+ }
+
+ if (watchman_watch(connection, fs_path, &wm_error)) {
+ warning("Watchman watch error: %s", wm_error.message);
+ watchman_connection_close(connection);
+ return -1;
+ }
+
+ result = query_watchman(istate, connection, fs_path,
+ istate->last_update ? istate->last_update : "");
+ watchman_connection_close(connection);
+ if (!result)
+ return -1;
+ update_index(istate, result);
+ watchman_free_query_result(result);
+ return 0;
+}
diff --git a/watchman-support.h b/watchman-support.h
new file mode 100644
index 0000000..ee1ef2c
--- /dev/null
+++ b/watchman-support.h
@@ -0,0 +1,7 @@
+#ifndef WATCHMAN_SUPPORT_H
+#define WATCHMAN_SUPPORT_H
+
+struct index_state;
+int check_watchman(struct index_state *index);
+
+#endif /* WATCHMAN_SUPPORT_H */
--
1.9.1
next prev parent reply other threads:[~2016-06-26 4:15 UTC|newest]
Thread overview: 35+ messages / expand[flat|nested] mbox.gz Atom feed top
2016-06-26 4:14 [PATCH v13 01/20] read-cache.c: fix constness of verify_hdr() David Turner
2016-06-26 4:14 ` [PATCH v13 02/20] read-cache: allow to keep mmap'd memory after reading David Turner
2016-06-26 4:14 ` [PATCH v13 03/20] pkt-line: add gentle version of packet_write David Turner
2016-06-26 4:14 ` [PATCH v13 04/20] index-helper: new daemon for caching index and related stuff David Turner
[not found] ` <CAFUO74nENapwVsM3CUst9AHqy5LcKTFBCnJxGXPk8E952t+X5Q@mail.gmail.com>
2016-06-27 16:53 ` David Turner
2016-06-30 13:06 ` Johannes Schindelin
2016-06-30 15:04 ` Duy Nguyen
2016-07-02 11:20 ` Johannes Schindelin
2016-07-02 12:43 ` Duy Nguyen
2016-06-26 4:14 ` [PATCH v13 05/20] index-helper: add --strict David Turner
2016-06-26 4:14 ` [PATCH v13 06/20] daemonize(): set a flag before exiting the main process David Turner
2016-06-26 4:14 ` [PATCH v13 07/20] index-helper: add --detach David Turner
2016-06-26 4:14 ` [PATCH v13 08/20] index-helper: log warnings David Turner
2016-06-26 4:14 ` [PATCH v13 09/20] read-cache: add watchman 'WAMA' extension David Turner
2016-06-26 4:14 ` David Turner [this message]
2016-06-26 4:14 ` [PATCH v13 11/20] index-helper: use watchman to avoid refreshing index with lstat() David Turner
2016-06-30 17:55 ` Ben Peart
2016-06-30 19:14 ` Duy Nguyen
2016-06-30 23:54 ` Ben Peart
2016-07-03 12:28 ` Duy Nguyen
2016-07-06 16:54 ` Ben Peart
2016-07-08 16:32 ` Duy Nguyen
2016-07-08 16:39 ` Duy Nguyen
2016-06-26 4:14 ` [PATCH v13 12/20] update-index: enable/disable watchman support David Turner
2016-06-26 4:14 ` [PATCH v13 13/20] unpack-trees: preserve index extensions David Turner
2016-06-26 4:14 ` [PATCH v13 14/20] watchman: add a config option to enable the extension David Turner
2016-06-26 4:14 ` [PATCH v13 15/20] index-helper: kill mode David Turner
2016-06-26 4:14 ` [PATCH v13 16/20] index-helper: don't run if already running David Turner
2016-06-26 4:14 ` [PATCH v13 17/20] index-helper: autorun mode David Turner
2016-06-26 4:14 ` [PATCH v13 18/20] index-helper: optionally automatically run David Turner
2016-06-26 4:14 ` [PATCH v13 19/20] trace: measure where the time is spent in the index-heavy operations David Turner
2016-06-26 12:09 ` [PATCH v13 21/20] unix-socket.c: add stub implementation when unix sockets are not supported Nguyễn Thái Ngọc Duy
2016-06-27 12:14 ` Johannes Schindelin
2016-06-27 16:16 ` Duy Nguyen
2016-06-28 10:06 ` Johannes Schindelin
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
List information: http://vger.kernel.org/majordomo-info.html
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=1466914464-10358-10-git-send-email-novalis@novalis.org \
--to=novalis@novalis.org \
--cc=dturner@twopensource.com \
--cc=git@vger.kernel.org \
--cc=gitster@pobox.com \
--cc=kamggg@gmail.com \
--cc=pclouds@gmail.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
Code repositories for project(s) associated with this public inbox
https://80x24.org/mirrors/git.git
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).