From: Stefan Beller <sbeller@google.com>
To: git@vger.kernel.org
Cc: gitster@pobox.com, jrnieder@gmail.com, hvoigt@hvoigt.net,
jens.lehmann@web.de, Stefan Beller <sbeller@google.com>
Subject: [WIP/PATCH 3/3] submodule: helper to run foreach in parallel
Date: Thu, 20 Aug 2015 18:40:37 -0700 [thread overview]
Message-ID: <1440121237-24576-3-git-send-email-sbeller@google.com> (raw)
In-Reply-To: <1440121237-24576-1-git-send-email-sbeller@google.com>
This runs a command on each submodule in parallel and should eventually
replace `git submodule foreach`.
There is a new option -j/--jobs (inspired by make) to specify the number
of parallel threads.
The jobs=1 case needs to be special cases to exactly replicate the current
default behavior of `git submodule foreach` such as working stdin input.
For more than one job there is no input possible and the output of both
stdout/stderr of the command are put into the stderr in an ordered fashion,
i.e. the tasks to not intermingle their output in races.
what currently works:
git submodule--helper foreach "echo \$toplevel-\$name-\$path-\$sha1"
which I took for testing during development from t7407.
Signed-off-by: Stefan Beller <sbeller@google.com>
---
This is still WIP, but comments are welcome.
builtin/submodule--helper.c | 147 +++++++++++++++++++++++++++++++++++++++++++-
git-submodule.sh | 9 +++
2 files changed, 154 insertions(+), 2 deletions(-)
diff --git a/builtin/submodule--helper.c b/builtin/submodule--helper.c
index ae74b80..9823302 100644
--- a/builtin/submodule--helper.c
+++ b/builtin/submodule--helper.c
@@ -9,7 +9,9 @@
#include "submodule-config.h"
#include "string-list.h"
#include "run-command.h"
-
+#ifndef NO_PTHREADS
+#include <semaphore.h>
+#endif
static const struct cache_entry **ce_entries;
static int ce_alloc, ce_used;
static const char *alternative_path;
@@ -275,6 +277,144 @@ static int module_clone(int argc, const char **argv, const char *prefix)
return 0;
}
+struct submodule_args {
+ const char *name;
+ const char *path;
+ const char *sha1;
+ const char *toplevel;
+ const char *prefix;
+ const char **cmd;
+ struct submodule_output *out;
+ sem_t *mutex;
+};
+
+int run_cmd_submodule(struct task_queue *aq, void *task)
+{
+ int i;
+ struct submodule_args *args = task;
+ struct strbuf out = STRBUF_INIT;
+ struct strbuf sb = STRBUF_INIT;
+ struct child_process *cp = xmalloc(sizeof(*cp));
+ char buf[1024];
+
+ strbuf_addf(&out, N_("Entering %s\n"), relative_path(args->path, args->prefix, &sb));
+
+ child_process_init(cp);
+ argv_array_pushv(&cp->args, args->cmd);
+
+ argv_array_pushf(&cp->env_array, "name=%s", args->name);
+ argv_array_pushf(&cp->env_array, "path=%s", args->path);
+ argv_array_pushf(&cp->env_array, "sha1=%s", args->sha1);
+ argv_array_pushf(&cp->env_array, "toplevel=%s", args->toplevel);
+
+ for (i = 0; local_repo_env[i]; i++)
+ argv_array_push(&cp->env_array, local_repo_env[i]);
+
+ cp->no_stdin = 1;
+ cp->out = 0;
+ cp->err = -1;
+ cp->dir = args->path;
+ cp->stdout_to_stderr = 1;
+ cp->use_shell = 1;
+
+ if (start_command(cp)) {
+ die("Could not start command");
+ for (i = 0; cp->args.argv; i++)
+ fprintf(stderr, "%s\n", cp->args.argv[i]);
+ }
+
+ while (1) {
+ ssize_t len = xread(cp->err, buf, sizeof(buf));
+ if (len < 0)
+ die("Read from child failed");
+ else if (len == 0)
+ break;
+ else {
+ strbuf_add(&out, buf, len);
+ }
+ }
+ if (finish_command(cp))
+ die("command died with error");
+
+ sem_wait(args->mutex);
+ fputs(out.buf, stderr);
+ sem_post(args->mutex);
+
+ return 0;
+}
+
+int module_foreach_parallel(int argc, const char **argv, const char *prefix)
+{
+ int i, recursive = 0, number_threads = 1, quiet = 0;
+ static struct pathspec pathspec;
+ struct strbuf sb = STRBUF_INIT;
+ struct task_queue *aq;
+ char **cmd;
+ const char **nullargv = {NULL};
+ sem_t *mutex = xmalloc(sizeof(*mutex));
+
+ struct option module_update_options[] = {
+ OPT_STRING(0, "prefix", &alternative_path,
+ N_("path"),
+ N_("alternative anchor for relative paths")),
+ OPT_STRING(0, "cmd", &cmd,
+ N_("string"),
+ N_("command to run")),
+ OPT_BOOL('r', "--recursive", &recursive,
+ N_("Recurse into nexted submodules")),
+ OPT_INTEGER('j', "jobs", &number_threads,
+ N_("Recurse into nexted submodules")),
+ OPT__QUIET(&quiet, N_("Suppress output")),
+ OPT_END()
+ };
+
+ static const char * const git_submodule_helper_usage[] = {
+ N_("git submodule--helper foreach [--prefix=<path>] [<path>...]"),
+ NULL
+ };
+
+ argc = parse_options(argc, argv, prefix, module_update_options,
+ git_submodule_helper_usage, 0);
+
+ if (module_list_compute(0, nullargv, NULL, &pathspec) < 0)
+ return 1;
+
+ gitmodules_config();
+
+ aq = create_task_queue(number_threads);
+
+ for (i = 0; i < ce_used; i++) {
+ const struct submodule *sub;
+ const struct cache_entry *ce = ce_entries[i];
+ struct submodule_args *args = malloc(sizeof(*args));
+
+ if (ce_stage(ce))
+ args->sha1 = xstrdup(sha1_to_hex(null_sha1));
+ else
+ args->sha1 = xstrdup(sha1_to_hex(ce->sha1));
+
+ strbuf_reset(&sb);
+ strbuf_addf(&sb, "%s/.git", ce->name);
+ if (!file_exists(sb.buf))
+ continue;
+
+ args->path = ce->name;
+ sub = submodule_from_path(null_sha1, args->path);
+ if (!sub)
+ die("No submodule mapping found in .gitmodules for path '%s'", args->path);
+
+ args->name = sub->name;
+ args->toplevel = xstrdup(xgetcwd());
+ args->cmd = argv;
+ args->mutex = mutex;
+ args->prefix = alternative_path;
+ add_task(aq, run_cmd_submodule, args);
+ }
+
+ finish_task_queue(aq);
+ return 0;
+}
+
int cmd_submodule__helper(int argc, const char **argv, const char *prefix)
{
if (argc < 2)
@@ -289,6 +429,9 @@ int cmd_submodule__helper(int argc, const char **argv, const char *prefix)
if (!strcmp(argv[1], "module_clone"))
return module_clone(argc - 1, argv + 1, prefix);
+ if (!strcmp(argv[1], "foreach"))
+ return module_foreach_parallel(argc - 1, argv + 1, prefix);
+
usage:
- usage("git submodule--helper [module_list module_name module_clone]\n");
+ usage("git submodule--helper [module_list module_name module_clone foreach]\n");
}
diff --git a/git-submodule.sh b/git-submodule.sh
index fb5155e..fa18434 100755
--- a/git-submodule.sh
+++ b/git-submodule.sh
@@ -431,6 +431,15 @@ cmd_foreach()
}
#
+# Execute an arbitrary command sequence in each checked out
+# submodule in parallel.
+#
+cmd_foreach_parallel()
+{
+ git submodule--helper module_foreach_parallel --prefix "$wt_prefix" $@
+}
+
+#
# Register submodules in .git/config
#
# $@ = requested paths (default to all)
--
2.5.0.264.g01b5c38.dirty
next prev parent reply other threads:[~2015-08-21 1:40 UTC|newest]
Thread overview: 16+ messages / expand[flat|nested] mbox.gz Atom feed top
2015-08-21 1:40 [PATCH 1/3] submodule: implement `module_clone` as a builtin helper Stefan Beller
2015-08-21 1:40 ` [RFC PATCH 2/3] run-commands: add an async queue processor Stefan Beller
2015-08-21 19:05 ` Junio C Hamano
2015-08-21 19:44 ` Jeff King
2015-08-21 19:48 ` Stefan Beller
2015-08-21 19:51 ` Jeff King
2015-08-21 20:12 ` Stefan Beller
2015-08-21 20:41 ` Junio C Hamano
2015-08-21 23:40 ` Stefan Beller
2015-08-24 21:22 ` Junio C Hamano
2015-08-21 19:45 ` Stefan Beller
2015-08-21 20:47 ` Junio C Hamano
2015-08-21 20:56 ` Stefan Beller
2015-08-21 1:40 ` Stefan Beller [this message]
2015-08-21 19:23 ` [WIP/PATCH 3/3] submodule: helper to run foreach in parallel Junio C Hamano
2015-08-21 20:21 ` Stefan Beller
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
List information: http://vger.kernel.org/majordomo-info.html
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=1440121237-24576-3-git-send-email-sbeller@google.com \
--to=sbeller@google.com \
--cc=git@vger.kernel.org \
--cc=gitster@pobox.com \
--cc=hvoigt@hvoigt.net \
--cc=jens.lehmann@web.de \
--cc=jrnieder@gmail.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
Code repositories for project(s) associated with this public inbox
https://80x24.org/mirrors/git.git
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).