diff options
author | Eric Wong <e@yhbt.net> | 2020-08-12 09:17:16 +0000 |
---|---|---|
committer | Eric Wong <e@yhbt.net> | 2020-08-13 02:22:57 +0000 |
commit | eace79f62e5cab2ecaf05d4e8773ab99f7662f48 (patch) | |
tree | 2ab81db76f8dfaf9b94e106559169b5568fde412 | |
parent | a3aebf796f9b06a04ebb6156240b74b6cc53fdb8 (diff) | |
download | public-inbox-eace79f62e5cab2ecaf05d4e8773ab99f7662f48.tar.gz |
In case there's unbalanced shards AND we're limiting parallelism while using many shards, spawn the next task in the queue ASAP once a task is done, instead of waiting for all tasks to finish before spawning the next batch. Unbalanced shards probably isn't a big issue for most users; however many smaller shards with few jobs can be useful for HDD users to reduce the effect of random writes.
-rw-r--r-- | lib/PublicInbox/Xapcmd.pm | 7 |
1 files changed, 5 insertions, 2 deletions
diff --git a/lib/PublicInbox/Xapcmd.pm b/lib/PublicInbox/Xapcmd.pm index 6fcc9e90..b6279218 100644 --- a/lib/PublicInbox/Xapcmd.pm +++ b/lib/PublicInbox/Xapcmd.pm @@ -9,7 +9,7 @@ use PublicInbox::SearchIdx; use File::Temp 0.19 (); # ->newdir use File::Path qw(remove_tree); use File::Basename qw(dirname); -use POSIX (); +use POSIX qw(WNOHANG); # support testing with dev versions of Xapian which installs # commands with a version number suffix (e.g. "xapian-compact-1.5") @@ -151,14 +151,17 @@ sub process_queue { $pids{cb_spawn($cb, $args, $opt)} = $args; } + my $flags = 0; while (scalar keys %pids) { - my $pid = waitpid(-1, 0); + my $pid = waitpid(-1, $flags) or last; + last if $pid < 0; my $args = delete $pids{$pid}; if ($args) { die join(' ', @$args)." failed: $?\n" if $?; } else { warn "unknown PID($pid) reaped: $?\n"; } + $flags = WNOHANG if scalar(@$queue); } } } |