From eace79f62e5cab2ecaf05d4e8773ab99f7662f48 Mon Sep 17 00:00:00 2001 From: Eric Wong Date: Wed, 12 Aug 2020 09:17:16 +0000 Subject: xapcmd: reduce CPU idling when shards exceeds job count In case there's unbalanced shards AND we're limiting parallelism while using many shards, spawn the next task in the queue ASAP once a task is done, instead of waiting for all tasks to finish before spawning the next batch. Unbalanced shards probably isn't a big issue for most users; however many smaller shards with few jobs can be useful for HDD users to reduce the effect of random writes. --- lib/PublicInbox/Xapcmd.pm | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/PublicInbox/Xapcmd.pm b/lib/PublicInbox/Xapcmd.pm index 6fcc9e90..b6279218 100644 --- a/lib/PublicInbox/Xapcmd.pm +++ b/lib/PublicInbox/Xapcmd.pm @@ -9,7 +9,7 @@ use PublicInbox::SearchIdx; use File::Temp 0.19 (); # ->newdir use File::Path qw(remove_tree); use File::Basename qw(dirname); -use POSIX (); +use POSIX qw(WNOHANG); # support testing with dev versions of Xapian which installs # commands with a version number suffix (e.g. "xapian-compact-1.5") @@ -151,14 +151,17 @@ sub process_queue { $pids{cb_spawn($cb, $args, $opt)} = $args; } + my $flags = 0; while (scalar keys %pids) { - my $pid = waitpid(-1, 0); + my $pid = waitpid(-1, $flags) or last; + last if $pid < 0; my $args = delete $pids{$pid}; if ($args) { die join(' ', @$args)." failed: $?\n" if $?; } else { warn "unknown PID($pid) reaped: $?\n"; } + $flags = WNOHANG if scalar(@$queue); } } } -- cgit v1.2.3-24-ge0c7