From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on dcvr.yhbt.net X-Spam-Level: X-Spam-ASN: X-Spam-Status: No, score=-4.0 required=3.0 tests=ALL_TRUSTED,BAYES_00 shortcircuit=no autolearn=ham autolearn_force=no version=3.4.2 Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id 40F3E1FA19 for ; Sun, 2 May 2021 06:05:43 +0000 (UTC) From: Eric Wong To: meta@public-inbox.org Subject: [PATCH 6/6] lei : writes to Maildirs and IMAP use mail-sync Date: Sun, 2 May 2021 06:05:42 +0000 Message-Id: <20210502060542.11598-7-e@80x24.org> In-Reply-To: <20210502060542.11598-1-e@80x24.org> References: <20210502060542.11598-1-e@80x24.org> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit List-Id: This will allow keyword updates from other folders to propagate to folders where search results may be duplicated. --- lib/PublicInbox/LEI.pm | 2 +- lib/PublicInbox/LeiQuery.pm | 7 +++---- lib/PublicInbox/LeiStore.pm | 8 ++++---- lib/PublicInbox/LeiToMail.pm | 24 +++++++++++++++++------- 4 files changed, 25 insertions(+), 16 deletions(-) diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm index d5c6bd52..599cfab2 100644 --- a/lib/PublicInbox/LEI.pm +++ b/lib/PublicInbox/LEI.pm @@ -145,7 +145,7 @@ our %CMD = ( # sorted in order of importance/use: qw(save output|mfolder|o=s format|f=s dedupe|d=s threads|t+ sort|s=s reverse|r offset=i pretty jobs|j=s globoff|g augment|a import-before! lock=s@ rsyncable alert=s@ mua=s verbose|v+ - color!), @c_opt, opt_dash('limit|n=i', '[0-9]+') ], + color! mail-sync!), @c_opt, opt_dash('limit|n=i', '[0-9]+') ], 'up' => [ 'OUTPUT|--all', 'update saved search', qw(jobs|j=s lock=s@ alert=s@ mua=s verbose|v+ all:s), @c_opt ], diff --git a/lib/PublicInbox/LeiQuery.pm b/lib/PublicInbox/LeiQuery.pm index efe328cc..1999a534 100644 --- a/lib/PublicInbox/LeiQuery.pm +++ b/lib/PublicInbox/LeiQuery.pm @@ -29,10 +29,9 @@ sub _start_query { # used by "lei q" and "lei up" return $self->fail("`$mj' writer jobs must be >= 1"); } my $l2m = $self->{l2m}; - if ($l2m && ($opt->{'import-remote'} //= 1) | - # we use \1 (a ref) to distinguish between - # user-supplied and default value - (($opt->{'import-before'} //= \1) ? 1 : 0)) { + # we use \1 (a ref) to distinguish between default vs. user-supplied + if ($l2m && grep { $opt->{$_} //= \1 } (qw(mail-sync import-remote + import-before))) { $self->_lei_store(1)->write_prepare($self); } $l2m and $l2m->{-wq_nr_workers} = $mj // do { diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm index 8af740fd..29362b2e 100644 --- a/lib/PublicInbox/LeiStore.pm +++ b/lib/PublicInbox/LeiStore.pm @@ -193,15 +193,15 @@ sub remove_eml_vmd { \@docids; } -sub set_sync_info ($$$) { - my ($self, $oidhex, $sync_info) = @_; +sub set_sync_info { + my ($self, $oidhex, $folder, $id) = @_; ($self->{lms} //= do { require PublicInbox::LeiMailSync; my $f = "$self->{priv_eidx}->{topdir}/mail_sync.sqlite3"; my $lms = PublicInbox::LeiMailSync->new($f); $lms->lms_begin; $lms; - })->set_src($oidhex, @$sync_info); + })->set_src($oidhex, $folder, $id); } sub add_eml { @@ -212,7 +212,7 @@ sub add_eml { my $smsg = bless { -oidx => $oidx }, 'PublicInbox::Smsg'; my $im_mark = $im->add($eml, undef, $smsg); if ($vmd && $vmd->{sync_info}) { - set_sync_info($self, $smsg->{blob}, $vmd->{sync_info}); + set_sync_info($self, $smsg->{blob}, @{$vmd->{sync_info}}); } $im_mark or return; # duplicate blob returns undef diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm index ab4de378..71acf952 100644 --- a/lib/PublicInbox/LeiToMail.pm +++ b/lib/PublicInbox/LeiToMail.pm @@ -242,7 +242,7 @@ sub _buf2maildir { my $kw = $smsg->{kw} // []; my $sfx = join('', sort(map { $kw2char{$_} // () } @$kw)); my $rand = ''; # chosen by die roll :P - my ($tmp, $fh, $final, $ok); + my ($tmp, $fh, $base, $ok); my $common = $smsg->{blob} // _rand; if (defined(my $pct = $smsg->{pct})) { $common .= "=$pct" } do { @@ -257,11 +257,12 @@ sub _buf2maildir { $dst .= 'cur/'; $rand = ''; do { - $final = $dst.$rand.$common.':2,'.$sfx; - } while (!($ok = link($tmp, $final)) && $!{EEXIST} && + $base = $rand.$common.':2,'.$sfx + } while (!($ok = link($tmp, $dst.$base)) && $!{EEXIST} && ($rand = _rand.',')); - die "link($tmp, $final): $!" unless $ok; + die "link($tmp, $dst$base): $!" unless $ok; unlink($tmp) or warn "W: failed to unlink $tmp: $!\n"; + \$base; } else { my $err = "Error writing $smsg->{blob} to $dst: $!\n"; $_[0] = undef; # clobber dst @@ -276,13 +277,16 @@ sub _maildir_write_cb ($$) { $dedupe->prepare_dedupe if $dedupe; my $dst = $lei->{ovv}->{dst}; my $lse = $lei->{lse}; # may be undef + my $sto = $lei->{opt}->{'mail-sync'} ? $lei->{sto} : undef; + my $out = $sto ? 'maildir:'.$lei->rel2abs($dst) : undef; sub { # for git_to_mail my ($bref, $smsg, $eml) = @_; $dst // return $lei->fail; # dst may be undef-ed in last run return if $dedupe && $dedupe->is_dup($eml // PublicInbox::Eml->new($$bref)); $lse->xsmsg_vmd($smsg) if $lse; - _buf2maildir($dst, $bref // \($eml->as_string), $smsg); + my $n = _buf2maildir($dst, $bref // \($eml->as_string), $smsg); + $sto->ipc_do('set_sync_info', $smsg->{blob}, $out, $n) if $sto; ++$lei->{-nr_write}; } } @@ -291,21 +295,27 @@ sub _imap_write_cb ($$) { my ($self, $lei) = @_; my $dedupe = $lei->{dedupe}; $dedupe->prepare_dedupe if $dedupe; - my $imap_append = $lei->{net}->can('imap_append'); + my $append = $lei->{net}->can('imap_append'); my $mic = $lei->{net}->mic_get($self->{uri}); my $folder = $self->{uri}->mailbox; my $lse = $lei->{lse}; # may be undef + my $sto = $lei->{opt}->{'mail-sync'} ? $lei->{sto} : undef; + my $out = $lei->{ovv}->{dst}; sub { # for git_to_mail my ($bref, $smsg, $eml) = @_; $mic // return $lei->fail; # mic may be undef-ed in last run return if $dedupe && $dedupe->is_dup($eml // PublicInbox::Eml->new($$bref)); $lse->xsmsg_vmd($smsg) if $lse; - eval { $imap_append->($mic, $folder, $bref, $smsg, $eml) }; + my $uid = eval { $append->($mic, $folder, $bref, $smsg, $eml) }; if (my $err = $@) { undef $mic; die $err; } + # imap_append returns UID if IMAP server has UIDPLUS extension + ($sto && $uid =~ /\A[0-9]+\z/) and + $sto->ipc_do('set_sync_info', + $smsg->{blob}, $out, $uid + 0); ++$lei->{-nr_write}; } }