From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on dcvr.yhbt.net X-Spam-Level: X-Spam-ASN: X-Spam-Status: No, score=-4.0 required=3.0 tests=ALL_TRUSTED,BAYES_00 shortcircuit=no autolearn=ham autolearn_force=no version=3.4.2 Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id 448BF1F9F3 for ; Fri, 21 May 2021 10:28:33 +0000 (UTC) From: Eric Wong To: meta@public-inbox.org Subject: [PATCH 3/8] lei tag: support tagging index-only messages Date: Fri, 21 May 2021 10:28:27 +0000 Message-Id: <20210521102832.10784-4-e@80x24.org> In-Reply-To: <20210521102832.10784-1-e@80x24.org> References: <20210521102832.10784-1-e@80x24.org> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit List-Id: This will make some of our tests faster and allow users to try more features of lei without high storage requirements. --- lib/PublicInbox/LeiSearch.pm | 8 ++++++-- lib/PublicInbox/LeiTag.pm | 10 ++++++++-- lib/PublicInbox/LeiToMail.pm | 4 +++- t/lei-index.t | 12 +++++++++++- 4 files changed, 28 insertions(+), 6 deletions(-) diff --git a/lib/PublicInbox/LeiSearch.pm b/lib/PublicInbox/LeiSearch.pm index c2b12146..fb19229f 100644 --- a/lib/PublicInbox/LeiSearch.pm +++ b/lib/PublicInbox/LeiSearch.pm @@ -63,7 +63,9 @@ sub content_key ($) { } sub _cmp_1st { # git->cat_async callback - my ($bref, $oid, $type, $size, $cmp) = @_; # cmp: [chash, xoids, smsg] + my ($bref, $oid, $type, $size, $cmp) = @_; + # cmp: [chash, xoids, smsg, lms] + $bref //= $cmp->[3] ? $cmp->[3]->local_blob($oid, 1) : undef; if ($bref && content_hash(PublicInbox::Eml->new($bref)) eq $cmp->[0]) { $cmp->[1]->{$oid} = $cmp->[2]->{num}; } @@ -78,6 +80,8 @@ sub xoids_for { my @overs = ($self->over // $self->overs_all); my $git = $self->git; my $xoids = {}; + # no lms when used via {ale}: + my $lms = $self->{-lms_ro} //= lms($self) if defined($self->{topdir}); for my $mid (@$mids) { for my $o (@overs) { my ($id, $prev); @@ -85,7 +89,7 @@ sub xoids_for { next if $cur->{bytes} == 0 || $xoids->{$cur->{blob}}; $git->cat_async($cur->{blob}, \&_cmp_1st, - [ $chash, $xoids, $cur ]); + [$chash, $xoids, $cur, $lms]); if ($min && scalar(keys %$xoids) >= $min) { $git->cat_async_wait; return $xoids; diff --git a/lib/PublicInbox/LeiTag.pm b/lib/PublicInbox/LeiTag.pm index c650e886..b6abd533 100644 --- a/lib/PublicInbox/LeiTag.pm +++ b/lib/PublicInbox/LeiTag.pm @@ -9,7 +9,8 @@ use parent qw(PublicInbox::IPC PublicInbox::LeiInput); sub input_eml_cb { # used by PublicInbox::LeiInput::input_fh my ($self, $eml) = @_; - if (my $xoids = $self->{lei}->{ale}->xoids_for($eml)) { + if (my $xoids = $self->{lse}->xoids_for($eml) // # tries LeiMailSync + $self->{lei}->{ale}->xoids_for($eml)) { $self->{lei}->{sto}->ipc_do('update_xvmd', $xoids, $eml, $self->{vmd_mod}); } else { @@ -17,7 +18,11 @@ sub input_eml_cb { # used by PublicInbox::LeiInput::input_fh } } -sub input_mbox_cb { input_eml_cb($_[1], $_[0]) } +sub input_mbox_cb { + my ($eml, $self) = @_; + $eml->header_set($_) for (qw(X-Status Status)); + input_eml_cb($self, $eml); +} sub input_maildir_cb { # maildir_each_eml cb my ($f, $kw, $eml, $self) = @_; @@ -60,6 +65,7 @@ sub note_missing { sub ipc_atfork_child { my ($self) = @_; PublicInbox::LeiInput::input_only_atfork_child($self); + $self->{lse} = $self->{lei}->{sto}->search; # this goes out-of-scope at worker process exit: PublicInbox::OnDestroy->new($$, \¬e_missing, $self); } diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm index da3a95d2..0cbdff8b 100644 --- a/lib/PublicInbox/LeiToMail.pm +++ b/lib/PublicInbox/LeiToMail.pm @@ -650,7 +650,9 @@ sub ipc_atfork_child { my ($self) = @_; my $lei = $self->{lei}; $lei->_lei_atfork_child; - $self->{-lms_ro} = $lei->{lse}->lms if $lei->{lse}; + if (my $lse = $lei->{lse}) { + $self->{-lms_ro} = $lse->{-lms_ro} //= $lse->lms; + } $lei->{auth}->do_auth_atfork($self) if $lei->{auth}; $SIG{__WARN__} = PublicInbox::Eml::warn_ignore_cb(); $self->SUPER::ipc_atfork_child; diff --git a/t/lei-index.t b/t/lei-index.t index b7dafb71..9a45d885 100644 --- a/t/lei-index.t +++ b/t/lei-index.t @@ -40,7 +40,7 @@ test_lei({ tmpdir => $tmpdir }, sub { my $res_a = json_utf8->decode($lei_out); my $blob = $res_a->[0]->{'blob'}; like($blob, qr/\A[0-9a-f]{40,}\z/, 'got blob from qp@example'); - lei_ok('blob', $blob); + lei_ok(qw(-C / blob), $blob); is($lei_out, $expect, 'got expected blob via Maildir'); lei_ok(qw(q mid:qp@example.com -f text)); like($lei_out, qr/^hi = bye/sm, 'lei2mail fallback'); @@ -58,6 +58,16 @@ test_lei({ tmpdir => $tmpdir }, sub { my $res_b = json_utf8->decode($lei_out); is_deeply($res_b, $res_a, 'no extra DB entries'); + # ensure tag works on index-only messages: + lei_ok(qw(tag +kw:seen t/utf8.eml)); + lei_ok(qw(q mid:testmessage@example.com)); + is_deeply(json_utf8->decode($lei_out)->[0]->{kw}, + ['seen'], 'seen kw can be set on index-only message'); + + lei_ok(qw(q z:0.. -o), "$tmpdir/all-results") for (1..2); + is_deeply([xqx($all_obj)], \@objs, + 'no new objects after 2x q to trigger implicit import'); + lei_ok('index', "nntp://$nntp_host_port/t.v2"); lei_ok('index', "imap://$imap_host_port/t.v2.0"); is_deeply([xqx($all_obj)], \@objs, 'no new objects from NNTP+IMAP');