From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on dcvr.yhbt.net X-Spam-Level: X-Spam-ASN: X-Spam-Status: No, score=-4.0 required=3.0 tests=ALL_TRUSTED,BAYES_00 shortcircuit=no autolearn=ham autolearn_force=no version=3.4.2 Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id 23E5E1F9E5 for ; Sun, 23 May 2021 01:38:29 +0000 (UTC) From: Eric Wong To: meta@public-inbox.org Subject: [PATCH 2/3] lei export-kw: support exporting keywords to IMAP Date: Sun, 23 May 2021 01:38:27 +0000 Message-Id: <20210523013828.4888-3-e@80x24.org> In-Reply-To: <20210523013828.4888-1-e@80x24.org> References: <20210523013828.4888-1-e@80x24.org> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit List-Id: We support writing to IMAP stores in other places (just like Maildir), and it's actually less complex for us to write to IMAP. Neither usability nor performance is ideal, but usability will be addressed in the next commit to relax CLI argument checking. Performance is poor due to the synchronous Mail::IMAPClient API and will need to be addressed with pipelining sometime further in the future. --- lib/PublicInbox/LeiExportKw.pm | 31 +++++++++++++++++++++++-------- lib/PublicInbox/LeiToMail.pm | 9 +++++---- lib/PublicInbox/NetWriter.pm | 10 ++++++++++ xt/net_writer-imap.t | 18 +++++++++++++----- 4 files changed, 51 insertions(+), 17 deletions(-) diff --git a/lib/PublicInbox/LeiExportKw.pm b/lib/PublicInbox/LeiExportKw.pm index db4f7441..5ad33959 100644 --- a/lib/PublicInbox/LeiExportKw.pm +++ b/lib/PublicInbox/LeiExportKw.pm @@ -59,15 +59,28 @@ sub export_kw_md { # LeiMailSync->each_src callback $self->{lei}->child_error(1, "link($orig, $dst) ($oidhex): $e"); } +sub export_kw_imap { # LeiMailSync->each_src callback + my ($oidbin, $id, $self, $mic) = @_; + my $oidhex = unpack('H*', $oidbin); + my $sto_kw = $self->{lse}->oid_keywords($oidhex) or return; + $self->{imap_mod_kw}->($self->{nwr}, $mic, $id, [ keys %$sto_kw ]); +} + # overrides PublicInbox::LeiInput::input_path_url sub input_path_url { my ($self, $input, @args) = @_; my $lms = $self->{lms} //= $self->{lse}->lms; $lms->lms_begin; - if ($input =~ s/\Amaildir://i) { + if ($input =~ /\Amaildir:(.+)/i) { + my $mdir = $1; require PublicInbox::LeiToMail; # kw2suffix - $lms->each_src("maildir:$input", \&export_kw_md, $self, $input); - } + $lms->each_src($input, \&export_kw_md, $self, $mdir); + } elsif ($input =~ m!\Aimaps?://!) { + my $uri = PublicInbox::URIimap->new($input); + my $mic = $self->{nwr}->mic_for_folder($uri); + $lms->each_src($$uri, \&export_kw_imap, $self, $mic); + $mic->expunge; + } else { die "BUG: $input not supported" } $lms->lms_commit; } @@ -137,11 +150,6 @@ EOF if (my @ro = grep(!/\A(?:maildir|imaps?):/, @folders)) { return $lei->fail("cannot export to read-only folders: @ro"); } - if (my $net = $lei->{net}) { - require PublicInbox::NetWriter; - bless $net, 'PublicInbox::NetWriter'; - } - undef $lms; my $m = $opt->{mode} // 'merge'; if ($m eq 'merge') { # default $self->{-merge_kw} = 1; @@ -151,6 +159,13 @@ EOF --mode=$m not supported (`set' or `merge') EOM } + if (my $net = $lei->{net}) { + require PublicInbox::NetWriter; + $self->{nwr} = bless $net, 'PublicInbox::NetWriter'; + $self->{imap_mod_kw} = $net->can($self->{-merge_kw} ? + 'imap_add_kw' : 'imap_set_kw'); + } + undef $lms; my $ops = {}; $lei->{auth}->op_merge($ops, $self) if $lei->{auth}; $self->{-wq_nr_workers} = $j // 1; # locked diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm index b9d4c856..f3c03969 100644 --- a/lib/PublicInbox/LeiToMail.pm +++ b/lib/PublicInbox/LeiToMail.pm @@ -307,11 +307,12 @@ sub _imap_write_cb ($$) { my $dedupe = $lei->{dedupe}; $dedupe->prepare_dedupe if $dedupe; my $append = $lei->{net}->can('imap_append'); - my $mic = $lei->{net}->mic_get($self->{uri}); - my $folder = $self->{uri}->mailbox; + my $uri = $self->{uri}; + my $mic = $lei->{net}->mic_get($uri); + my $folder = $uri->mailbox; + $uri->uidvalidity($mic->uidvalidity($folder)); my $lse = $lei->{lse}; # may be undef my $sto = $lei->{opt}->{'mail-sync'} ? $lei->{sto} : undef; - my $out = $lei->{ovv}->{dst}; sub { # for git_to_mail my ($bref, $smsg, $eml) = @_; $mic // return $lei->fail; # mic may be undef-ed in last run @@ -327,7 +328,7 @@ sub _imap_write_cb ($$) { # imap_append returns UID if IMAP server has UIDPLUS extension ($sto && $uid =~ /\A[0-9]+\z/) and $sto->ipc_do('set_sync_info', - $smsg->{blob}, $out, $uid + 0); + $smsg->{blob}, $$uri, $uid + 0); ++$lei->{-nr_write}; } } diff --git a/lib/PublicInbox/NetWriter.pm b/lib/PublicInbox/NetWriter.pm index 2032a1fd..8ec7f85c 100644 --- a/lib/PublicInbox/NetWriter.pm +++ b/lib/PublicInbox/NetWriter.pm @@ -26,10 +26,20 @@ sub imap_append { die "APPEND $folder: $@"; } +# updates $uri with UIDVALIDITY sub mic_for_folder { my ($self, $uri) = @_; my $mic = $self->mic_get($uri) or die "E: not connected: $@"; $mic->select($uri->mailbox) or return; + my $uidval; + for ($mic->Results) { + /^\* OK \[UIDVALIDITY ([0-9]+)\].*/ or next; + $uidval = $1; + last; + } + $uidval //= $mic->uidvalidity($uri->mailbox) or + die "E: failed to get uidvalidity from <$uri>: $@"; + $uri->uidvalidity($uidval); $mic; } diff --git a/xt/net_writer-imap.t b/xt/net_writer-imap.t index 1298b958..0e6d4831 100644 --- a/xt/net_writer-imap.t +++ b/xt/net_writer-imap.t @@ -157,12 +157,20 @@ test_lei(sub { lei_ok qw(import -F eml), $f, \'import local copy w/o keywords'; + lei_ok 'ls-mail-sync'; diag $lei_out; + lei_ok 'import', $$folder_uri; # populate mail_sync.sqlite3 + lei_ok qw(tag +kw:seen +kw:answered +kw:flagged), $f; + lei_ok 'ls-mail-sync'; diag $lei_out; + chomp(my $uri_val = $lei_out); + lei_ok 'export-kw', $uri_val; $mic = $nwr->mic_for_folder($folder_uri); - # dummy set to ensure second set_kw clobbers - $nwr->imap_set_kw($mic, $uid[0], [ qw(seen answered flagged) ] - )->expunge or BAIL_OUT "expunge $@"; - $nwr->imap_set_kw($mic, $uid[0], [ 'seen' ] - )->expunge or BAIL_OUT "expunge $@"; + my $flags = $mic->flags($uid[0]); + is_deeply([sort @$flags], [ qw(\\Answered \\Flagged \\Seen) ], + 'IMAP flags set by export-kw') or diag explain($flags); + + # ensure this imap_set_kw clobbers + $nwr->imap_set_kw($mic, $uid[0], [ 'seen' ])->expunge or + BAIL_OUT "expunge $@"; $mic = undef; @res = (); $nwr->imap_each($folder_uri, $imap_slurp_all, \@res);