From: Calvin Wan <calvinwan@google.com>
To: git@vger.kernel.org
Cc: Calvin Wan <calvinwan@google.com>,
emilyshaffer@google.com, avarab@gmail.com,
phillip.wood123@gmail.com, myriamanis@google.com
Subject: [PATCH v4 1/5] run-command: add duplicate_output_fn to run_processes_parallel_opts
Date: Tue, 8 Nov 2022 18:41:56 +0000 [thread overview]
Message-ID: <20221108184200.2813458-2-calvinwan@google.com> (raw)
In-Reply-To: <https://lore.kernel.org/git/20221020232532.1128326-1-calvinwan@google.com/>
Add duplicate_output_fn as an optionally set function in
run_process_parallel_opts. If set, output from each child process is
copied and passed to the callback function whenever output from the
child process is buffered to allow for separate parsing.
Signed-off-by: Calvin Wan <calvinwan@google.com>
---
run-command.c | 13 +++++++++++--
run-command.h | 24 +++++++++++++++++++++++
t/helper/test-run-command.c | 21 ++++++++++++++++++++
t/t0061-run-command.sh | 39 +++++++++++++++++++++++++++++++++++++
4 files changed, 95 insertions(+), 2 deletions(-)
diff --git a/run-command.c b/run-command.c
index c772acd743..b8f430eb03 100644
--- a/run-command.c
+++ b/run-command.c
@@ -1560,6 +1560,9 @@ static void pp_init(struct parallel_processes *pp,
if (!opts->get_next_task)
BUG("you need to specify a get_next_task function");
+
+ if (opts->duplicate_output && opts->ungroup)
+ BUG("duplicate_output and ungroup are incompatible with each other");
CALLOC_ARRAY(pp->children, n);
if (!opts->ungroup)
@@ -1680,8 +1683,14 @@ static void pp_buffer_stderr(struct parallel_processes *pp,
for (size_t i = 0; i < opts->processes; i++) {
if (pp->children[i].state == GIT_CP_WORKING &&
pp->pfd[i].revents & (POLLIN | POLLHUP)) {
- int n = strbuf_read_once(&pp->children[i].err,
- pp->children[i].process.err, 0);
+ struct strbuf buf = STRBUF_INIT;
+ int n = strbuf_read_once(&buf, pp->children[i].process.err, 0);
+ strbuf_addbuf(&pp->children[i].err, &buf);
+ if (opts->duplicate_output)
+ opts->duplicate_output(&buf, &pp->children[i].err,
+ opts->data,
+ pp->children[i].data);
+ strbuf_release(&buf);
if (n == 0) {
close(pp->children[i].process.err);
pp->children[i].state = GIT_CP_WAIT_CLEANUP;
diff --git a/run-command.h b/run-command.h
index e3e1ea01ad..dd6d6a86c2 100644
--- a/run-command.h
+++ b/run-command.h
@@ -440,6 +440,24 @@ typedef int (*start_failure_fn)(struct strbuf *out,
void *pp_cb,
void *pp_task_cb);
+/**
+ * This callback is called whenever output from a child process is buffered
+ *
+ * "struct strbuf *process_out" contains the output from the child process
+ *
+ * See run_processes_parallel() below for a discussion of the "struct
+ * strbuf *out" parameter.
+ *
+ * pp_cb is the callback cookie as passed into run_processes_parallel,
+ * pp_task_cb is the callback cookie as passed into get_next_task_fn.
+ *
+ * This function is incompatible with "ungroup"
+ */
+typedef void (*duplicate_output_fn)(struct strbuf *process_out,
+ struct strbuf *out,
+ void *pp_cb,
+ void *pp_task_cb);
+
/**
* This callback is called on every child process that finished processing.
*
@@ -493,6 +511,12 @@ struct run_process_parallel_opts
*/
start_failure_fn start_failure;
+ /**
+ * duplicate_output: See duplicate_output_fn() above. This should be
+ * NULL unless process specific output is needed
+ */
+ duplicate_output_fn duplicate_output;
+
/**
* task_finished: See task_finished_fn() above. This can be
* NULL to omit any special handling.
diff --git a/t/helper/test-run-command.c b/t/helper/test-run-command.c
index 3ecb830f4a..40dd329e02 100644
--- a/t/helper/test-run-command.c
+++ b/t/helper/test-run-command.c
@@ -52,6 +52,21 @@ static int no_job(struct child_process *cp,
return 0;
}
+static void duplicate_output(struct strbuf *process_out,
+ struct strbuf *out,
+ void *pp_cb,
+ void *pp_task_cb)
+{
+ struct string_list list = STRING_LIST_INIT_DUP;
+
+ string_list_split(&list, process_out->buf, '\n', -1);
+ for (size_t i = 0; i < list.nr; i++) {
+ if (strlen(list.items[i].string) > 0)
+ fprintf(stderr, "duplicate_output: %s\n", list.items[i].string);
+ }
+ string_list_clear(&list, 0);
+}
+
static int task_finished(int result,
struct strbuf *err,
void *pp_cb,
@@ -439,6 +454,12 @@ int cmd__run_command(int argc, const char **argv)
opts.ungroup = 1;
}
+ if (!strcmp(argv[1], "--duplicate-output")) {
+ argv += 1;
+ argc -= 1;
+ opts.duplicate_output = duplicate_output;
+ }
+
jobs = atoi(argv[2]);
strvec_clear(&proc.args);
strvec_pushv(&proc.args, (const char **)argv + 3);
diff --git a/t/t0061-run-command.sh b/t/t0061-run-command.sh
index 7b5423eebd..130aec7c68 100755
--- a/t/t0061-run-command.sh
+++ b/t/t0061-run-command.sh
@@ -134,6 +134,15 @@ test_expect_success 'run_command runs in parallel with more jobs available than
test_cmp expect actual
'
+test_expect_success 'run_command runs in parallel with more jobs available than tasks --duplicate-output' '
+ test-tool run-command --duplicate-output run-command-parallel 5 sh -c "printf \"%s\n%s\n\" Hello World" >out 2>err &&
+ test_must_be_empty out &&
+ test 4 = $(grep -c "duplicate_output: Hello" err) &&
+ test 4 = $(grep -c "duplicate_output: World" err) &&
+ sed "/duplicate_output/d" err > err1 &&
+ test_cmp expect err1
+'
+
test_expect_success 'run_command runs ungrouped in parallel with more jobs available than tasks' '
test-tool run-command --ungroup run-command-parallel 5 sh -c "printf \"%s\n%s\n\" Hello World" >out 2>err &&
test_line_count = 8 out &&
@@ -145,6 +154,15 @@ test_expect_success 'run_command runs in parallel with as many jobs as tasks' '
test_cmp expect actual
'
+test_expect_success 'run_command runs in parallel with as many jobs as tasks --duplicate-output' '
+ test-tool run-command --duplicate-output run-command-parallel 4 sh -c "printf \"%s\n%s\n\" Hello World" >out 2>err &&
+ test_must_be_empty out &&
+ test 4 = $(grep -c "duplicate_output: Hello" err) &&
+ test 4 = $(grep -c "duplicate_output: World" err) &&
+ sed "/duplicate_output/d" err > err1 &&
+ test_cmp expect err1
+'
+
test_expect_success 'run_command runs ungrouped in parallel with as many jobs as tasks' '
test-tool run-command --ungroup run-command-parallel 4 sh -c "printf \"%s\n%s\n\" Hello World" >out 2>err &&
test_line_count = 8 out &&
@@ -156,6 +174,15 @@ test_expect_success 'run_command runs in parallel with more tasks than jobs avai
test_cmp expect actual
'
+test_expect_success 'run_command runs in parallel with more tasks than jobs available --duplicate-output' '
+ test-tool run-command --duplicate-output run-command-parallel 3 sh -c "printf \"%s\n%s\n\" Hello World" >out 2>err &&
+ test_must_be_empty out &&
+ test 4 = $(grep -c "duplicate_output: Hello" err) &&
+ test 4 = $(grep -c "duplicate_output: World" err) &&
+ sed "/duplicate_output/d" err > err1 &&
+ test_cmp expect err1
+'
+
test_expect_success 'run_command runs ungrouped in parallel with more tasks than jobs available' '
test-tool run-command --ungroup run-command-parallel 3 sh -c "printf \"%s\n%s\n\" Hello World" >out 2>err &&
test_line_count = 8 out &&
@@ -176,6 +203,12 @@ test_expect_success 'run_command is asked to abort gracefully' '
test_cmp expect actual
'
+test_expect_success 'run_command is asked to abort gracefully --duplicate-output' '
+ test-tool run-command --duplicate-output run-command-abort 3 false >out 2>err &&
+ test_must_be_empty out &&
+ test_cmp expect err
+'
+
test_expect_success 'run_command is asked to abort gracefully (ungroup)' '
test-tool run-command --ungroup run-command-abort 3 false >out 2>err &&
test_must_be_empty out &&
@@ -191,6 +224,12 @@ test_expect_success 'run_command outputs ' '
test_cmp expect actual
'
+test_expect_success 'run_command outputs --duplicate-output' '
+ test-tool run-command --duplicate-output run-command-no-jobs 3 sh -c "printf \"%s\n%s\n\" Hello World" >out 2>err &&
+ test_must_be_empty out &&
+ test_cmp expect err
+'
+
test_expect_success 'run_command outputs (ungroup) ' '
test-tool run-command --ungroup run-command-no-jobs 3 sh -c "printf \"%s\n%s\n\" Hello World" >out 2>err &&
test_must_be_empty out &&
--
2.38.1.431.g37b22c650d-goog
next prev parent reply other threads:[~2022-11-08 18:42 UTC|newest]
Thread overview: 26+ messages / expand[flat|nested] mbox.gz Atom feed top
[not found] <https://lore.kernel.org/git/20221020232532.1128326-1-calvinwan@google.com/>
2022-11-08 18:41 ` [PATCH v4 0/5] submodule: parallelize diff Calvin Wan
2022-11-23 17:49 ` Glen Choo
2023-01-15 9:31 ` Junio C Hamano
2023-01-17 19:31 ` Calvin Wan
2022-11-08 18:41 ` Calvin Wan [this message]
2022-11-28 20:45 ` [PATCH v4 1/5] run-command: add duplicate_output_fn to run_processes_parallel_opts Jonathan Tan
2022-11-30 18:46 ` Calvin Wan
2022-11-29 5:11 ` Elijah Newren
2022-11-30 18:47 ` Calvin Wan
2022-11-29 23:29 ` Glen Choo
2022-11-30 9:53 ` Ævar Arnfjörð Bjarmason
2022-11-30 10:26 ` Phillip Wood
2022-11-30 19:02 ` Calvin Wan
2022-11-30 10:28 ` Phillip Wood
2022-11-30 10:57 ` Ævar Arnfjörð Bjarmason
2022-11-08 18:41 ` [PATCH v4 2/5] submodule: strbuf variable rename Calvin Wan
2022-11-08 18:41 ` [PATCH v4 3/5] submodule: move status parsing into function Calvin Wan
2022-11-08 18:41 ` [PATCH v4 4/5] diff-lib: refactor match_stat_with_submodule Calvin Wan
2022-11-30 14:36 ` Phillip Wood
2022-11-30 19:08 ` Calvin Wan
2022-11-08 18:42 ` [PATCH v4 5/5] diff-lib: parallelize run_diff_files for submodules Calvin Wan
2022-11-28 21:01 ` Jonathan Tan
2022-11-29 22:29 ` Glen Choo
2022-11-30 18:11 ` Calvin Wan
2022-11-29 5:13 ` Elijah Newren
2022-11-30 18:04 ` Calvin Wan
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
List information: http://vger.kernel.org/majordomo-info.html
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20221108184200.2813458-2-calvinwan@google.com \
--to=calvinwan@google.com \
--cc=avarab@gmail.com \
--cc=emilyshaffer@google.com \
--cc=git@vger.kernel.org \
--cc=myriamanis@google.com \
--cc=phillip.wood123@gmail.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
Code repositories for project(s) associated with this public inbox
https://80x24.org/mirrors/git.git
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).