lei tag: parallelize Maildir access

Since Maildir isn't guaranteed to have any sort of order, we
can parallelize inputs, here.  On a 4-core system, this reduced
one of my tag invocations from 5.5 to 1.4s.

3 files changed, 11 insertions, 15 deletions

diff --git a/lib/PublicInbox/LeiImport.pm b/lib/PublicInbox/LeiImport.pm
index cddd5619..e3cb69ca 100644
--- a/lib/PublicInbox/LeiImport.pm
+++ b/lib/PublicInbox/LeiImport.pm
@@ -35,13 +35,13 @@ sub pmdir_cb { # called via wq_io_do from LeiPmdir->each_mdir_fn
                 die "BUG: $f was not from a Maildir?\n";
         my $kw = PublicInbox::MdirReader::flags2kw($fl);
         substr($folder, 0, 0) = 'maildir:'; # add prefix
-        my $lms = $self->{-lms_ro};
+        my $lse = $self->{lse} //= $self->{lei}->{sto}->search;
+        my $lms = $self->{-lms_ro} //= $lse->lms; # may be 0 or undef
         my $oidbin = $lms ? $lms->name_oidbin($folder, $bn) : undef;
-        my @docids = defined($oidbin) ?
-                        $self->{over}->oidbin_exists($oidbin) : ();
+        my @docids = defined($oidbin) ? $lse->over->oidbin_exists($oidbin) : ();
         my $vmd = $self->{-import_kw} ? { kw => $kw } : undef;
         if (scalar @docids) {
-                $self->{lse}->kw_changed(undef, $kw, \@docids) or return;
+                $lse->kw_changed(undef, $kw, \@docids) or return;
         }
         if (my $eml = eml_from_path($f)) {
                 $vmd->{sync_info} = [ $folder, \$bn ] if $self->{-mail_sync};
diff --git a/lib/PublicInbox/LeiPmdir.pm b/lib/PublicInbox/LeiPmdir.pm
index aa9ce713..760f276c 100644
--- a/lib/PublicInbox/LeiPmdir.pm
+++ b/lib/PublicInbox/LeiPmdir.pm
@@ -30,15 +30,9 @@ sub new {
 
 sub ipc_atfork_child {
         my ($self) = @_;
-        my $lei = $self->{lei};
-        $lei->_lei_atfork_child;
         my $ipt = $self->{ipt} // die 'BUG: no self->{ipt}';
-        $ipt->{lei} = $lei;
-        $ipt->{sto} = $lei->{sto} // die 'BUG: no lei->{sto}';
-        $ipt->{lse} = $ipt->{sto}->search;
-        $ipt->{over} = $ipt->{lse}->over;
-        $ipt->{-lms_ro} //= $ipt->{lse}->lms; # may be undef or '0'
-        $self->SUPER::ipc_atfork_child;
+        $ipt->{lei} = $self->{lei};
+        $ipt->ipc_atfork_child;
 }
 
 sub each_mdir_fn { # maildir_each_file callback
diff --git a/lib/PublicInbox/LeiTag.pm b/lib/PublicInbox/LeiTag.pm
index 4b3ce7d8..e0532653 100644
--- a/lib/PublicInbox/LeiTag.pm
+++ b/lib/PublicInbox/LeiTag.pm
@@ -6,6 +6,7 @@ package PublicInbox::LeiTag;
 use strict;
 use v5.10.1;
 use parent qw(PublicInbox::IPC PublicInbox::LeiInput);
+use PublicInbox::InboxWritable qw(eml_from_path);
 
 sub input_eml_cb { # used by PublicInbox::LeiInput::input_fh
         my ($self, $eml) = @_;
@@ -24,8 +25,9 @@ sub input_mbox_cb {
         input_eml_cb($self, $eml);
 }
 
-sub input_maildir_cb { # maildir_each_eml cb
-        my ($f, $kw, $eml, $self) = @_;
+sub pmdir_cb { # called via wq_io_do from LeiPmdir->each_mdir_fn
+        my ($self, $f) = @_;
+        my $eml = eml_from_path($f) or return;
         input_eml_cb($self, $eml);
 }
 
@@ -42,12 +44,12 @@ sub lei_tag { # the "lei tag" method
         $lei->ale; # refresh and prepare
         my $vmd_mod = $self->vmd_mod_extract(\@argv);
         return $lei->fail(join("\n", @{$vmd_mod->{err}})) if $vmd_mod->{err};
+        $self->{vmd_mod} = $vmd_mod; # before LeiPmdir->new in prepare_inputs
         $self->prepare_inputs($lei, \@argv) or return;
         grep(defined, @$vmd_mod{qw(+kw +L -L -kw)}) or
                 return $lei->fail('no keywords or labels specified');
         my $ops = {};
         $lei->{auth}->op_merge($ops, $self) if $lei->{auth};
-        $self->{vmd_mod} = $vmd_mod;
         my $j = $self->{-wq_nr_workers} = 1; # locked for now
         (my $op_c, $ops) = $lei->workers_start($self, $j, $ops);
         $lei->{wq1} = $self;