user/dev discussion of public-inbox itself
 help / color / mirror / code / Atom feed
From: Eric Wong <e@80x24.org>
To: meta@public-inbox.org
Subject: [PATCH 3/8] lei tag: support tagging index-only messages
Date: Fri, 21 May 2021 10:28:27 +0000	[thread overview]
Message-ID: <20210521102832.10784-4-e@80x24.org> (raw)
In-Reply-To: <20210521102832.10784-1-e@80x24.org>

This will make some of our tests faster and allow users to try
more features of lei without high storage requirements.
---
 lib/PublicInbox/LeiSearch.pm |  8 ++++++--
 lib/PublicInbox/LeiTag.pm    | 10 ++++++++--
 lib/PublicInbox/LeiToMail.pm |  4 +++-
 t/lei-index.t                | 12 +++++++++++-
 4 files changed, 28 insertions(+), 6 deletions(-)

diff --git a/lib/PublicInbox/LeiSearch.pm b/lib/PublicInbox/LeiSearch.pm
index c2b12146..fb19229f 100644
--- a/lib/PublicInbox/LeiSearch.pm
+++ b/lib/PublicInbox/LeiSearch.pm
@@ -63,7 +63,9 @@ sub content_key ($) {
 }
 
 sub _cmp_1st { # git->cat_async callback
-	my ($bref, $oid, $type, $size, $cmp) = @_; # cmp: [chash, xoids, smsg]
+	my ($bref, $oid, $type, $size, $cmp) = @_;
+	# cmp: [chash, xoids, smsg, lms]
+	$bref //= $cmp->[3] ? $cmp->[3]->local_blob($oid, 1) : undef;
 	if ($bref && content_hash(PublicInbox::Eml->new($bref)) eq $cmp->[0]) {
 		$cmp->[1]->{$oid} = $cmp->[2]->{num};
 	}
@@ -78,6 +80,8 @@ sub xoids_for {
 	my @overs = ($self->over // $self->overs_all);
 	my $git = $self->git;
 	my $xoids = {};
+	# no lms when used via {ale}:
+	my $lms = $self->{-lms_ro} //= lms($self) if defined($self->{topdir});
 	for my $mid (@$mids) {
 		for my $o (@overs) {
 			my ($id, $prev);
@@ -85,7 +89,7 @@ sub xoids_for {
 				next if $cur->{bytes} == 0 ||
 					$xoids->{$cur->{blob}};
 				$git->cat_async($cur->{blob}, \&_cmp_1st,
-						[ $chash, $xoids, $cur ]);
+						[$chash, $xoids, $cur, $lms]);
 				if ($min && scalar(keys %$xoids) >= $min) {
 					$git->cat_async_wait;
 					return $xoids;
diff --git a/lib/PublicInbox/LeiTag.pm b/lib/PublicInbox/LeiTag.pm
index c650e886..b6abd533 100644
--- a/lib/PublicInbox/LeiTag.pm
+++ b/lib/PublicInbox/LeiTag.pm
@@ -9,7 +9,8 @@ use parent qw(PublicInbox::IPC PublicInbox::LeiInput);
 
 sub input_eml_cb { # used by PublicInbox::LeiInput::input_fh
 	my ($self, $eml) = @_;
-	if (my $xoids = $self->{lei}->{ale}->xoids_for($eml)) {
+	if (my $xoids = $self->{lse}->xoids_for($eml) // # tries LeiMailSync
+			$self->{lei}->{ale}->xoids_for($eml)) {
 		$self->{lei}->{sto}->ipc_do('update_xvmd', $xoids, $eml,
 						$self->{vmd_mod});
 	} else {
@@ -17,7 +18,11 @@ sub input_eml_cb { # used by PublicInbox::LeiInput::input_fh
 	}
 }
 
-sub input_mbox_cb { input_eml_cb($_[1], $_[0]) }
+sub input_mbox_cb {
+	my ($eml, $self) = @_;
+	$eml->header_set($_) for (qw(X-Status Status));
+	input_eml_cb($self, $eml);
+}
 
 sub input_maildir_cb { # maildir_each_eml cb
 	my ($f, $kw, $eml, $self) = @_;
@@ -60,6 +65,7 @@ sub note_missing {
 sub ipc_atfork_child {
 	my ($self) = @_;
 	PublicInbox::LeiInput::input_only_atfork_child($self);
+	$self->{lse} = $self->{lei}->{sto}->search;
 	# this goes out-of-scope at worker process exit:
 	PublicInbox::OnDestroy->new($$, \&note_missing, $self);
 }
diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm
index da3a95d2..0cbdff8b 100644
--- a/lib/PublicInbox/LeiToMail.pm
+++ b/lib/PublicInbox/LeiToMail.pm
@@ -650,7 +650,9 @@ sub ipc_atfork_child {
 	my ($self) = @_;
 	my $lei = $self->{lei};
 	$lei->_lei_atfork_child;
-	$self->{-lms_ro} = $lei->{lse}->lms if $lei->{lse};
+	if (my $lse = $lei->{lse}) {
+		$self->{-lms_ro} = $lse->{-lms_ro} //= $lse->lms;
+	}
 	$lei->{auth}->do_auth_atfork($self) if $lei->{auth};
 	$SIG{__WARN__} = PublicInbox::Eml::warn_ignore_cb();
 	$self->SUPER::ipc_atfork_child;
diff --git a/t/lei-index.t b/t/lei-index.t
index b7dafb71..9a45d885 100644
--- a/t/lei-index.t
+++ b/t/lei-index.t
@@ -40,7 +40,7 @@ test_lei({ tmpdir => $tmpdir }, sub {
 	my $res_a = json_utf8->decode($lei_out);
 	my $blob = $res_a->[0]->{'blob'};
 	like($blob, qr/\A[0-9a-f]{40,}\z/, 'got blob from qp@example');
-	lei_ok('blob', $blob);
+	lei_ok(qw(-C / blob), $blob);
 	is($lei_out, $expect, 'got expected blob via Maildir');
 	lei_ok(qw(q mid:qp@example.com -f text));
 	like($lei_out, qr/^hi = bye/sm, 'lei2mail fallback');
@@ -58,6 +58,16 @@ test_lei({ tmpdir => $tmpdir }, sub {
 	my $res_b = json_utf8->decode($lei_out);
 	is_deeply($res_b, $res_a, 'no extra DB entries');
 
+	# ensure tag works on index-only messages:
+	lei_ok(qw(tag +kw:seen t/utf8.eml));
+	lei_ok(qw(q mid:testmessage@example.com));
+	is_deeply(json_utf8->decode($lei_out)->[0]->{kw},
+		['seen'], 'seen kw can be set on index-only message');
+
+	lei_ok(qw(q z:0.. -o), "$tmpdir/all-results") for (1..2);
+	is_deeply([xqx($all_obj)], \@objs,
+		'no new objects after 2x q to trigger implicit import');
+
 	lei_ok('index', "nntp://$nntp_host_port/t.v2");
 	lei_ok('index', "imap://$imap_host_port/t.v2.0");
 	is_deeply([xqx($all_obj)], \@objs, 'no new objects from NNTP+IMAP');

  parent reply	other threads:[~2021-05-21 10:28 UTC|newest]

Thread overview: 9+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2021-05-21 10:28 [PATCH 0/8] lei: export-kw, IMAP import incompatibility Eric Wong
2021-05-21 10:28 ` [PATCH 1/8] treewide: favor open(..., '+<&=', $fd) Eric Wong
2021-05-21 10:28 ` [PATCH 2/8] lei: drop EOFpipe in favor of PktOp Eric Wong
2021-05-21 10:28 ` Eric Wong [this message]
2021-05-21 10:28 ` [PATCH 4/8] lei_input: fix canonicalization of Maildirs for sync Eric Wong
2021-05-21 10:28 ` [PATCH 5/8] lei index: support command-line options Eric Wong
2021-05-21 10:28 ` [PATCH 6/8] lei export-kw: new command to export keywords to Maildirs Eric Wong
2021-05-21 10:28 ` [PATCH 7/8] uri_imap: support uid/auth/user as full accessors Eric Wong
2021-05-21 10:28 ` [PATCH 8/8] lei import: store IMAP user+auth in mail_sync folder URI Eric Wong

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

  List information: https://public-inbox.org/README

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20210521102832.10784-4-e@80x24.org \
    --to=e@80x24.org \
    --cc=meta@public-inbox.org \
    --subject='Re: [PATCH 3/8] lei tag: support tagging index-only messages' \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link

user/dev discussion of public-inbox itself

This inbox may be cloned and mirrored by anyone:

	git clone --mirror https://public-inbox.org/meta
	git clone --mirror http://czquwvybam4bgbro.onion/meta
	git clone --mirror http://hjrcffqmbrq6wope.onion/meta
	git clone --mirror http://ou63pmih66umazou.onion/meta

	# If you have public-inbox 1.1+ installed, you may
	# initialize and index your mirror using the following commands:
	public-inbox-init -V1 meta meta/ https://public-inbox.org/meta \
		meta@public-inbox.org
	public-inbox-index meta

Example config snippet for mirrors.
Newsgroups are available over NNTP:
	nntp://news.public-inbox.org/inbox.comp.mail.public-inbox.meta
	nntp://7fh6tueqddpjyxjmgtdiueylzoqt6pt7hec3pukyptlmohoowvhde4yd.onion/inbox.comp.mail.public-inbox.meta
	nntp://ie5yzdi7fg72h7s4sdcztq5evakq23rdt33mfyfcddc5u3ndnw24ogqd.onion/inbox.comp.mail.public-inbox.meta
	nntp://4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/inbox.comp.mail.public-inbox.meta
	nntp://news.gmane.io/gmane.mail.public-inbox.general
 note: .onion URLs require Tor: https://www.torproject.org/

code repositories for project(s) associated with this inbox:

	https://80x24.org/public-inbox.git

AGPL code for this site: git clone https://public-inbox.org/public-inbox.git