From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on dcvr.yhbt.net X-Spam-Level: X-Spam-ASN: X-Spam-Status: No, score=-2.6 required=3.0 tests=ALL_TRUSTED,AWL,BAYES_00, UNWANTED_LANGUAGE_BODY shortcircuit=no autolearn=ham autolearn_force=no version=3.4.2 Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id CB2991F9F3 for ; Sun, 13 Jun 2021 18:12:06 +0000 (UTC) From: Eric Wong To: meta@public-inbox.org Subject: [PATCH 2/2] lei index+import: reject keywords from R/O IMAP Date: Sun, 13 Jun 2021 18:12:06 +0000 Message-Id: <20210613181206.26754-3-e@80x24.org> In-Reply-To: <20210613181206.26754-1-e@80x24.org> References: <20210613181206.26754-1-e@80x24.org> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit List-Id: Since users can't set IMAP flags in read-only IMAP folders, we won't clobber local flags when importing from IMAP. This also enables the local_blob fallback used for lei-index to be used for index deduplication. --- lib/PublicInbox/LeiStore.pm | 3 ++- lib/PublicInbox/NetReader.pm | 11 ++++++----- t/lei-index.t | 13 +++++++++++++ 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm index 5446873e..f978288a 100644 --- a/lib/PublicInbox/LeiStore.pm +++ b/lib/PublicInbox/LeiStore.pm @@ -151,7 +151,8 @@ sub _docids_for ($$) { my $oid = $cur->{blob}; my $docid = $cur->{num}; my $bref = $im ? $im->cat_blob($oid) : undef; - $bref //= $eidx->git->cat_file($oid) // do { + $bref //= $eidx->git->cat_file($oid) // + _lms_rw($self)->local_blob($oid, 1) // do { warn "W: $oid (#$docid) <$mid> not found\n"; next; }; diff --git a/lib/PublicInbox/NetReader.pm b/lib/PublicInbox/NetReader.pm index 30784199..0c2288d8 100644 --- a/lib/PublicInbox/NetReader.pm +++ b/lib/PublicInbox/NetReader.pm @@ -420,7 +420,8 @@ sub _imap_do_msg ($$$$$) { my ($self, $uri, $uid, $raw, $flags) = @_; # our target audience expects LF-only, save storage $$raw =~ s/\r\n/\n/sg; - my $kw = flags2kw($self, $uri, $uid, $flags) // return; + my $kw = defined($flags) ? + (flags2kw($self, $uri, $uid, $flags) // return) : undef; my ($eml_cb, @args) = @{$self->{eml_each}}; $eml_cb->($uri, $uid, $kw, PublicInbox::Eml->new($raw), @args); } @@ -537,8 +538,8 @@ E: $uri strangely, UIDVALIDLITY matches ($l_uidval) EOF $mic->Uid(1); # the default, we hope my $err; - if (!defined($single_uid) && $self->{each_old} && - perm_fl_ok($perm_fl)) { + my $use_fl = perm_fl_ok($perm_fl); + if (!defined($single_uid) && $self->{each_old} && $use_fl) { $err = each_old_flags($self, $mic, $uri, $l_uid); return $err if $err; } @@ -593,8 +594,8 @@ EOF # messages get deleted, so holes appear my $per_uid = delete $r->{$uid} // next; my $raw = delete($per_uid->{$key}) // next; - _imap_do_msg($self, $uri, $uid, \$raw, - $per_uid->{FLAGS}); + my $fl = $use_fl ? $per_uid->{FLAGS} : undef; + _imap_do_msg($self, $uri, $uid, \$raw, $fl); $last_uid = $uid; last if $self->{quit}; } diff --git a/t/lei-index.t b/t/lei-index.t index c142e79c..eeda5196 100644 --- a/t/lei-index.t +++ b/t/lei-index.t @@ -80,6 +80,19 @@ test_lei({ tmpdir => $tmpdir }, sub { lei_ok('index', "nntp://$nntp_host_port/t.v2"); lei_ok('index', "imap://$imap_host_port/t.v2.0"); is_deeply([xqx($all_obj)], \@objs, 'no new objects from NNTP+IMAP'); + + lei_ok qw(q m:multipart-html-sucks@11); + $res_a = json_utf8->decode($lei_out)->[0]; + is_deeply($res_a->{'kw'}, ['seen'], + 'keywords still set after NNTP + IMAP import'); + + # ensure import works after lms->local_blob fallback in lei/store + lei_ok('import', 't/mda-mime.eml'); + lei_ok qw(q m:multipart-html-sucks@11); + $res_b = json_utf8->decode($lei_out)->[0]; + my $t = xqx(['git', "--git-dir=$store_path/ALL.git", + qw(cat-file -t), $res_b->{blob}]); + is($t, "blob\n", 'got blob'); }); done_testing;