From dda6a67ce8865641edad51f83aab2eb2fc27bda8 Mon Sep 17 00:00:00 2001 From: Eric Wong Date: Sun, 21 Feb 2021 07:41:34 +0000 Subject: lei2mail: parallel augment for lock-free stores This lets us make use of multiple cores on IMAP and Maildir backed by SSD (or better) storage. This benefits IMAP stores with high network latency, but may still penalize IMAP servers with rotational storage. --- lib/PublicInbox/NetReader.pm | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) (limited to 'lib/PublicInbox/NetReader.pm') diff --git a/lib/PublicInbox/NetReader.pm b/lib/PublicInbox/NetReader.pm index 4c412491..0956d5da 100644 --- a/lib/PublicInbox/NetReader.pm +++ b/lib/PublicInbox/NetReader.pm @@ -363,8 +363,11 @@ sub _imap_fetch_all ($$$) { } return if $l_uid >= $r_uid; # nothing to do $l_uid ||= 1; - - warn "# $uri fetching UID $l_uid:$r_uid\n" unless $self->{quiet}; + my ($mod, $shard) = @{$self->{shard_info} // []}; + unless ($self->{quiet}) { + my $m = $mod ? " [(UID % $mod) == $shard]" : ''; + warn "# $uri fetching UID $l_uid:$r_uid$m\n"; + } $mic->Uid(1); # the default, we hope my $bs = $self->{imap_opt}->{$sec}->{batch_size} // 1; my $req = $mic->imap4rev1 ? 'BODY.PEEK[]' : 'RFC822.PEEK'; @@ -391,6 +394,8 @@ sub _imap_fetch_all ($$$) { $l_uid = $uids->[-1] + 1; # for next search my $last_uid; my $n = $self->{max_batch}; + + @$uids = grep { ($_ % $mod) == $shard } @$uids if $mod; while (scalar @$uids) { my @batch = splice(@$uids, 0, $bs); $batch = join(',', @batch); -- cgit v1.2.3-24-ge0c7