[PATCHv7 0/8] fetch submodules in parallel

[Stefan Beller <sbeller@google.com>]

fixing two smaller nits noticed by Ramsay
 * make struct parallel_processes static
 * omit the forward declaration of get_next_submodule by reordering.

Jonathan Nieder (1):
  submodule.c: write "Fetching submodule <foo>" to stderr

Stefan Beller (7):
  xread: poll on non blocking fds
  xread_nonblock: add functionality to read from fds without blocking
  strbuf: add strbuf_read_once to read without blocking
  sigchain: add command to pop all common signals
  run-command: add an asynchronous parallel child processor
  fetch_populated_submodules: use new parallel job processing
  submodules: allow parallel fetching, add tests and documentation

 Documentation/fetch-options.txt |   7 +
 builtin/fetch.c                 |   6 +-
 builtin/pull.c                  |   6 +
 git-compat-util.h               |   1 +
 run-command.c                   | 350 ++++++++++++++++++++++++++++++++++++++++
 run-command.h                   |  78 +++++++++
 sigchain.c                      |   9 ++
 sigchain.h                      |   1 +
 strbuf.c                        |  11 ++
 strbuf.h                        |   9 ++
 submodule.c                     | 144 ++++++++++++-----
 submodule.h                     |   2 +-
 t/t0061-run-command.sh          |  20 +++
 t/t5526-fetch-submodules.sh     |  70 +++++---
 test-run-command.c              |  25 +++
 wrapper.c                       |  35 +++-
 16 files changed, 701 insertions(+), 73 deletions(-)

 $ git diff origin/sb/submodule-parallel-fetch

diff --git a/run-command.c b/run-command.c
index 28048a7..ef3da27 100644
--- a/run-command.c
+++ b/run-command.c
@@ -855,7 +855,7 @@ int capture_command(struct child_process *cmd, struct strbuf *buf, size_t hint)
 	return finish_command(cmd);
 }
 
-struct parallel_processes {
+static struct parallel_processes {
 	void *data;
 
 	int max_processes;
diff --git a/submodule.c b/submodule.c
index cf8bf5d..c21b265 100644
--- a/submodule.c
+++ b/submodule.c
@@ -628,68 +628,6 @@ struct submodule_parallel_fetch {
 #define SPF_INIT {0, ARGV_ARRAY_INIT, NULL, NULL, 0, 0, 0}
 
 static int get_next_submodule(void **task_cb, struct child_process *cp,
-			      struct strbuf *err, void *data);
-
-static int fetch_start_failure(struct child_process *cp,
-			       struct strbuf *err,
-			       void *cb, void *task_cb)
-{
-	struct submodule_parallel_fetch *spf = cb;
-
-	spf->result = 1;
-
-	return 0;
-}
-
-static int fetch_finish(int retvalue, struct child_process *cp,
-			struct strbuf *err, void *cb, void *task_cb)
-{
-	struct submodule_parallel_fetch *spf = cb;
-
-	if (retvalue)
-		spf->result = 1;
-
-	return 0;
-}
-
-int fetch_populated_submodules(const struct argv_array *options,
-			       const char *prefix, int command_line_option,
-			       int quiet, int max_parallel_jobs)
-{
-	int i;
-	struct submodule_parallel_fetch spf = SPF_INIT;
-
-	spf.work_tree = get_git_work_tree();
-	spf.command_line_option = command_line_option;
-	spf.quiet = quiet;
-	spf.prefix = prefix;
-
-	if (!spf.work_tree)
-		goto out;
-
-	if (read_cache() < 0)
-		die("index file corrupt");
-
-	argv_array_push(&spf.args, "fetch");
-	for (i = 0; i < options->argc; i++)
-		argv_array_push(&spf.args, options->argv[i]);
-	argv_array_push(&spf.args, "--recurse-submodules-default");
-	/* default value, "--submodule-prefix" and its value are added later */
-
-	calculate_changed_submodule_paths();
-	run_processes_parallel(max_parallel_jobs,
-			       get_next_submodule,
-			       fetch_start_failure,
-			       fetch_finish,
-			       &spf);
-
-	argv_array_clear(&spf.args);
-out:
-	string_list_clear(&changed_submodule_paths, 1);
-	return spf.result;
-}
-
-static int get_next_submodule(void **task_cb, struct child_process *cp,
 			      struct strbuf *err, void *data)
 {
 	int ret = 0;
@@ -775,6 +713,65 @@ static int get_next_submodule(void **task_cb, struct child_process *cp,
 	return 0;
 }
 
+static int fetch_start_failure(struct child_process *cp,
+			       struct strbuf *err,
+			       void *cb, void *task_cb)
+{
+	struct submodule_parallel_fetch *spf = cb;
+
+	spf->result = 1;
+
+	return 0;
+}
+
+static int fetch_finish(int retvalue, struct child_process *cp,
+			struct strbuf *err, void *cb, void *task_cb)
+{
+	struct submodule_parallel_fetch *spf = cb;
+
+	if (retvalue)
+		spf->result = 1;
+
+	return 0;
+}
+
+int fetch_populated_submodules(const struct argv_array *options,
+			       const char *prefix, int command_line_option,
+			       int quiet, int max_parallel_jobs)
+{
+	int i;
+	struct submodule_parallel_fetch spf = SPF_INIT;
+
+	spf.work_tree = get_git_work_tree();
+	spf.command_line_option = command_line_option;
+	spf.quiet = quiet;
+	spf.prefix = prefix;
+
+	if (!spf.work_tree)
+		goto out;
+
+	if (read_cache() < 0)
+		die("index file corrupt");
+
+	argv_array_push(&spf.args, "fetch");
+	for (i = 0; i < options->argc; i++)
+		argv_array_push(&spf.args, options->argv[i]);
+	argv_array_push(&spf.args, "--recurse-submodules-default");
+	/* default value, "--submodule-prefix" and its value are added later */
+
+	calculate_changed_submodule_paths();
+	run_processes_parallel(max_parallel_jobs,
+			       get_next_submodule,
+			       fetch_start_failure,
+			       fetch_finish,
+			       &spf);
+
+	argv_array_clear(&spf.args);
+out:
+	string_list_clear(&changed_submodule_paths, 1);
+	return spf.result;
+}
+
 unsigned is_submodule_modified(const char *path, int ignore_untracked)
 {
 	ssize_t len;

-- 
2.5.0.268.g453a26a