about summary refs log tree commit homepage
diff options
context:
space:
mode:
authorEric Wong <e@yhbt.net>2020-06-27 10:03:47 +0000
committerEric Wong <e@yhbt.net>2020-06-28 22:27:23 +0000
commit14e3edd2646beb3ccde7f6f15b54fadf2da4bc7a (patch)
treee70eafa2929e5f2a35b88a4954b081cdfa84ee63
parent6e56fb09cbaec593f4ef3c2fff2140c201406b41 (diff)
downloadpublic-inbox-14e3edd2646beb3ccde7f6f15b54fadf2da4bc7a.tar.gz
With different polling intervals, multiple processes may
simultaneously write to IMAPtracker.  This ought to reduce
SQLite busy waiting and contention issues when importing
many inboxes in parallel.
-rw-r--r--lib/PublicInbox/WatchMaildir.pm16
1 files changed, 8 insertions, 8 deletions
diff --git a/lib/PublicInbox/WatchMaildir.pm b/lib/PublicInbox/WatchMaildir.pm
index e0caaa56..d492e5d6 100644
--- a/lib/PublicInbox/WatchMaildir.pm
+++ b/lib/PublicInbox/WatchMaildir.pm
@@ -335,12 +335,12 @@ sub mic_for ($$$) { # mic = Mail::IMAPClient
         $mic;
 }
 
-sub imap_import_msg ($$$$$) {
-        my ($self, $itrk, $r_uidval, $uid, $raw) = @_;
+sub imap_import_msg ($$$$) {
+        my ($self, $url, $uid, $raw) = @_;
         # our target audience expects LF-only, save storage
         $$raw =~ s/\r\n/\n/sg;
 
-        my $inboxes = $self->{imap}->{$itrk->{url}};
+        my $inboxes = $self->{imap}->{$url};
         if (ref($inboxes)) {
                 for my $ibx (@$inboxes) {
                         my $eml = PublicInbox::Eml->new($$raw);
@@ -348,12 +348,11 @@ sub imap_import_msg ($$$$$) {
                 }
         } elsif ($inboxes eq 'watchspam') {
                 my $eml = PublicInbox::Eml->new($raw);
-                my $arg = [ $self, $eml, "$itrk->{url} UID:$uid" ];
+                my $arg = [ $self, $eml, "$url UID:$uid" ];
                 $self->{config}->each_inbox(\&remove_eml_i, $arg);
         } else {
                 die "BUG: destination unknown $inboxes";
         }
-        $itrk->update_last($r_uidval, $uid);
 }
 
 sub imap_fetch_all ($$$) {
@@ -415,8 +414,8 @@ sub imap_fetch_all ($$$) {
                 return if $uids->[0] < $l_uid;
 
                 $l_uid = $uids->[-1] + 1; # for next search
+                my $last_uid;
 
-                $itrk->{dbh}->begin_work;
                 while (defined(($uid = shift(@$uids)))) {
                         local $0 = "UID:$uid $mbx $sec";
                         my $r = $mic->fetch_hash($uid, $req);
@@ -426,11 +425,12 @@ sub imap_fetch_all ($$$) {
                         }
                         # messages get deleted, so holes appear
                         defined(my $raw = delete $r->{$uid}->{$key}) or next;
-                        imap_import_msg($self, $itrk, $r_uidval, $uid, \$raw);
+                        imap_import_msg($self, $url, $uid, \$raw);
+                        $last_uid = $uid;
                         last if $self->{quit};
                 }
                 _done_for_now($self);
-                $itrk->{dbh}->commit;
+                $itrk->update_last($r_uidval, $last_uid) if defined $last_uid;
         } until ($err || $self->{quit});
         $err;
 }