diff options
| author | Eric Wong <e@80x24.org> | 2023-03-21 23:07:16 +0000 |
|---|---|---|
| committer | Eric Wong <e@80x24.org> | 2023-03-25 09:37:39 +0000 |
| commit | 9225145d3e98d83889065f6d4dc4913226e5f121 (patch) | |
| tree | 94754b906b611b1ce31493574d420a0fc10ae3b3 /lib/PublicInbox/V2Writable.pm | |
| parent | cdeac2a60363b9659d5381503074a016d947925a (diff) | |
| download | public-inbox-9225145d3e98d83889065f6d4dc4913226e5f121.tar.gz | |
We'll be using nproc_shards for indexing non-Inbox stuff.
Diffstat (limited to 'lib/PublicInbox/V2Writable.pm')
| -rw-r--r-- | lib/PublicInbox/V2Writable.pm | 26 |
1 files changed, 1 insertions, 25 deletions
diff --git a/lib/PublicInbox/V2Writable.pm b/lib/PublicInbox/V2Writable.pm index ed5182ae..d3d13941 100644 --- a/lib/PublicInbox/V2Writable.pm +++ b/lib/PublicInbox/V2Writable.pm @@ -8,7 +8,7 @@ use strict; use v5.10.1; use parent qw(PublicInbox::Lock PublicInbox::IPC); use PublicInbox::SearchIdxShard; -use PublicInbox::IPC; +use PublicInbox::IPC qw(nproc_shards); use PublicInbox::Eml; use PublicInbox::Git; use PublicInbox::Import; @@ -29,30 +29,6 @@ my $OID = qr/[a-f0-9]{40,}/; # an estimate of the post-packed size to the raw uncompressed size our $PACKING_FACTOR = 0.4; -# SATA storage lags behind what CPUs are capable of, so relying on -# nproc(1) can be misleading and having extra Xapian shards is a -# waste of FDs and space. It can also lead to excessive IO latency -# and slow things down. Users on NVME or other fast storage can -# use the NPROC env or switches in our script/public-inbox-* programs -# to increase Xapian shards -our $NPROC_MAX_DEFAULT = 4; - -sub nproc_shards ($) { - my ($creat_opt) = @_; - my $n = $creat_opt->{nproc} if ref($creat_opt) eq 'HASH'; - $n //= $ENV{NPROC}; - if (!$n) { - # assume 2 cores if not detectable or zero - state $NPROC_DETECTED = PublicInbox::IPC::detect_nproc() || 2; - $n = $NPROC_DETECTED; - $n = $NPROC_MAX_DEFAULT if $n > $NPROC_MAX_DEFAULT; - } - - # subtract for the main process and git-fast-import - $n -= 1; - $n < 1 ? 1 : $n; -} - sub count_shards ($) { my ($self) = @_; # always load existing shards in case core count changes: |