diff options
author | Eric Wong <e@80x24.org> | 2019-06-14 16:25:39 +0000 |
---|---|---|
committer | Eric Wong <e@80x24.org> | 2019-06-14 16:25:39 +0000 |
commit | 4bb34fb8f29530f6bda5f0d563a74f6289ac312d (patch) | |
tree | fb26e006105b9e4b06cd6210412e13a3ef031f79 /lib/PublicInbox/V2Writable.pm | |
parent | 3c30532aed6256a984c535530c6667552c2e6a84 (diff) | |
parent | e9eb3af852778a67533e9579b14695763535d262 (diff) | |
download | public-inbox-4bb34fb8f29530f6bda5f0d563a74f6289ac312d.tar.gz |
* origin/reshard: xcpdb: support resharding v2 repos xcpdb: use destination shard as progress prefix xapcmd: preserve indexlevel based on the destination v2writable: use a smaller default for Xapian partitions
Diffstat (limited to 'lib/PublicInbox/V2Writable.pm')
-rw-r--r-- | lib/PublicInbox/V2Writable.pm | 18 |
1 files changed, 16 insertions, 2 deletions
diff --git a/lib/PublicInbox/V2Writable.pm b/lib/PublicInbox/V2Writable.pm index 09ed4e7b..3329d79f 100644 --- a/lib/PublicInbox/V2Writable.pm +++ b/lib/PublicInbox/V2Writable.pm @@ -23,7 +23,14 @@ use IO::Handle; # an estimate of the post-packed size to the raw uncompressed size my $PACKING_FACTOR = 0.4; -# assume 2 cores if GNU nproc(1) is not available +# SATA storage lags behind what CPUs are capable of, so relying on +# nproc(1) can be misleading and having extra Xapian partions is a +# waste of FDs and space. It can also lead to excessive IO latency +# and slow things down. Users on NVME or other fast storage can +# use the NPROC env or switches in our script/public-inbox-* programs +# to increase Xapian partitions. +our $NPROC_MAX_DEFAULT = 4; + sub nproc_parts ($) { my ($creat_opt) = @_; if (ref($creat_opt) eq 'HASH') { @@ -32,7 +39,14 @@ sub nproc_parts ($) { } } - my $n = int($ENV{NPROC} || `nproc 2>/dev/null` || 2); + my $n = $ENV{NPROC}; + if (!$n) { + chomp($n = `nproc 2>/dev/null`); + # assume 2 cores if GNU nproc(1) is not available + $n = 2 if !$n; + $n = $NPROC_MAX_DEFAULT if $NPROC_MAX_DEFAULT > 4; + } + # subtract for the main process and git-fast-import $n -= 1; $n < 1 ? 1 : $n; |