From f0cc8e6795077bc7a04e579655a1c5e90c80a4e8 Mon Sep 17 00:00:00 2001 From: Eric Wong Date: Thu, 23 May 2019 09:36:45 +0000 Subject: xapcmd: support spawn options copydatabase(1) is exceptionally noisy and it's output is confusing when run in parallel. Support redirects at least, and env while we're at it to give us future options. We can also stuff a -jobs parameter into the options to limit parallelism since it can be useful for low-priority upgrade jobs. --- lib/PublicInbox/Xapcmd.pm | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) (limited to 'lib/PublicInbox/Xapcmd.pm') diff --git a/lib/PublicInbox/Xapcmd.pm b/lib/PublicInbox/Xapcmd.pm index 586d7e6a..999ddd1c 100644 --- a/lib/PublicInbox/Xapcmd.pm +++ b/lib/PublicInbox/Xapcmd.pm @@ -25,7 +25,8 @@ sub commit_changes ($$$) { } sub run { - my ($ibx, $cmd) = @_; + my ($ibx, $cmd, $env, $opt) = @_; + $opt ||= {}; my $dir = $ibx->{mainrepo} or die "no mainrepo in inbox\n"; which($cmd->[0]) or die "$cmd->[0] not found in PATH\n"; $ibx->umask_prepare; @@ -50,13 +51,21 @@ sub run { die "No Xapian parts found in $old\n" unless @cmds; } my $im = $ibx->importer(0); + my $max = $opt->{jobs} || scalar(@cmds); $ibx->with_umask(sub { $im->lock_acquire; - my %pids = map {; spawn($_) => join(' ', @$_) } @cmds; - while (scalar keys %pids) { - my $pid = waitpid(-1, 0); - my $desc = delete $pids{$pid}; - die "$desc failed: $?\n" if $?; + my %pids; + while (@cmds) { + while (scalar(keys(%pids)) < $max && scalar(@cmds)) { + my $x = shift @cmds; + $pids{spawn($x, $env, $opt)} = $x; + } + + while (scalar keys %pids) { + my $pid = waitpid(-1, 0); + my $x = delete $pids{$pid}; + die join(' ', @$x)." failed: $?\n" if $?; + } } commit_changes($im, $old, $new); }); -- cgit v1.2.3-24-ge0c7