diff options
author | Eric Wong (Contractor, The Linux Foundation) <e@80x24.org> | 2018-03-06 07:28:56 +0000 |
---|---|---|
committer | Eric Wong (Contractor, The Linux Foundation) <e@80x24.org> | 2018-03-06 07:28:56 +0000 |
commit | cab22861f2a22482b74d20c05afbd6108a4590cb (patch) | |
tree | 893cbc6655a0066878105e2b04c35fe5124ab57c /lib | |
parent | fe99e1e4900b5fbe8da4aef25bb0dd05eacb25a3 (diff) | |
download | public-inbox-cab22861f2a22482b74d20c05afbd6108a4590cb.tar.gz |
We need to detect the number of partitions the repository was created with to ensure Xapian DBs can work across different machines (or even CPU affinity changes) without leaving messages unaffected by search.
Diffstat (limited to 'lib')
-rw-r--r-- | lib/PublicInbox/V2Writable.pm | 22 |
1 files changed, 20 insertions, 2 deletions
diff --git a/lib/PublicInbox/V2Writable.pm b/lib/PublicInbox/V2Writable.pm index 3bcea37d..7728b91a 100644 --- a/lib/PublicInbox/V2Writable.pm +++ b/lib/PublicInbox/V2Writable.pm @@ -19,7 +19,9 @@ use PublicInbox::Inbox; my $PACKING_FACTOR = 0.4; # assume 2 cores if GNU nproc(1) is not available -my $NPROC = int($ENV{NPROC} || `nproc 2>/dev/null` || 2); +sub nproc () { + int($ENV{NPROC} || `nproc 2>/dev/null` || 2); +} sub new { my ($class, $v2ibx, $creat) = @_; @@ -32,12 +34,28 @@ sub new { die "$dir does not exist\n"; } } + + my $nparts = 0; + my $xpfx = "$dir/xap" . PublicInbox::Search::SCHEMA_VERSION; + + # always load existing partitions in case core count changes: + if (-d $xpfx) { + foreach my $part (<$xpfx/*>) { + -d $part && $part =~ m!/\d+\z! or next; + eval { + Search::Xapian::Database->new($part)->close; + $nparts++; + }; + } + } + $nparts = nproc() if ($nparts == 0); + my $self = { -inbox => $v2ibx, im => undef, # PublicInbox::Import xap_rw => undef, # PublicInbox::V2SearchIdx xap_ro => undef, - partitions => $NPROC, + partitions => $nparts, transact_bytes => 0, # limit each repo to 1GB or so rotate_bytes => int((1024 * 1024 * 1024) / $PACKING_FACTOR), |