diff options
| author | Eric Wong <e@yhbt.net> | 2020-08-07 10:52:14 +0000 |
|---|---|---|
| committer | Eric Wong <e@yhbt.net> | 2020-08-07 23:46:06 +0000 |
| commit | 826e0992d956aa60711e779afeb8493884c41cc2 (patch) | |
| tree | 7257c8ace2c772b20198de3e53209e3cf9cf360f /lib | |
| parent | 8b1950055d51d4360e596446e5ac0f41008e357d (diff) | |
| download | public-inbox-826e0992d956aa60711e779afeb8493884c41cc2.tar.gz | |
We need to account for whether shard parallelization is enabled or not, since users of parallelization are expected to have more RAM.
Diffstat (limited to 'lib')
| -rw-r--r-- | lib/PublicInbox/V2Writable.pm | 10 |
1 files changed, 8 insertions, 2 deletions
diff --git a/lib/PublicInbox/V2Writable.pm b/lib/PublicInbox/V2Writable.pm index a029fe4c..f7a318e5 100644 --- a/lib/PublicInbox/V2Writable.pm +++ b/lib/PublicInbox/V2Writable.pm @@ -152,6 +152,12 @@ sub add { $self->{ibx}->with_umask(\&_add, $self, $eml, $check_cb); } +sub batch_bytes ($) { + my ($self) = @_; + ($self->{parallel} ? $self->{shards} : 1) * + $PublicInbox::SearchIdx::BATCH_BYTES; +} + # indexes a message, returns true if checkpointing is needed sub do_idx ($$$$) { my ($self, $msgref, $mime, $smsg) = @_; @@ -160,7 +166,7 @@ sub do_idx ($$$$) { my $idx = idx_shard($self, $smsg->{num} % $self->{shards}); $idx->index_raw($msgref, $mime, $smsg); my $n = $self->{transact_bytes} += $smsg->{raw_bytes}; - $n >= ($PublicInbox::SearchIdx::BATCH_BYTES * $self->{shards}); + $n >= batch_bytes($self); } sub _add { @@ -1195,7 +1201,7 @@ sub index_xap_step ($$$;$) { my $ibx = $self->{ibx}; my $all = $ibx->git; my $over = $ibx->over; - my $batch_bytes = $PublicInbox::SearchIdx::BATCH_BYTES; + my $batch_bytes = batch_bytes($self); $step //= $self->{shards}; my $end = $sync->{art_end}; if (my $pr = $sync->{-opt}->{-progress}) { |