* [PATCH 3/8] lei tag: support tagging index-only messages
2021-05-21 10:28 7% [PATCH 0/8] lei: export-kw, IMAP import incompatibility Eric Wong
@ 2021-05-21 10:28 6% ` Eric Wong
0 siblings, 0 replies; 2+ results
From: Eric Wong @ 2021-05-21 10:28 UTC (permalink / raw)
To: meta
This will make some of our tests faster and allow users to try
more features of lei without high storage requirements.
---
lib/PublicInbox/LeiSearch.pm | 8 ++++++--
lib/PublicInbox/LeiTag.pm | 10 ++++++++--
lib/PublicInbox/LeiToMail.pm | 4 +++-
t/lei-index.t | 12 +++++++++++-
4 files changed, 28 insertions(+), 6 deletions(-)
diff --git a/lib/PublicInbox/LeiSearch.pm b/lib/PublicInbox/LeiSearch.pm
index c2b12146..fb19229f 100644
--- a/lib/PublicInbox/LeiSearch.pm
+++ b/lib/PublicInbox/LeiSearch.pm
@@ -63,7 +63,9 @@ sub content_key ($) {
}
sub _cmp_1st { # git->cat_async callback
- my ($bref, $oid, $type, $size, $cmp) = @_; # cmp: [chash, xoids, smsg]
+ my ($bref, $oid, $type, $size, $cmp) = @_;
+ # cmp: [chash, xoids, smsg, lms]
+ $bref //= $cmp->[3] ? $cmp->[3]->local_blob($oid, 1) : undef;
if ($bref && content_hash(PublicInbox::Eml->new($bref)) eq $cmp->[0]) {
$cmp->[1]->{$oid} = $cmp->[2]->{num};
}
@@ -78,6 +80,8 @@ sub xoids_for {
my @overs = ($self->over // $self->overs_all);
my $git = $self->git;
my $xoids = {};
+ # no lms when used via {ale}:
+ my $lms = $self->{-lms_ro} //= lms($self) if defined($self->{topdir});
for my $mid (@$mids) {
for my $o (@overs) {
my ($id, $prev);
@@ -85,7 +89,7 @@ sub xoids_for {
next if $cur->{bytes} == 0 ||
$xoids->{$cur->{blob}};
$git->cat_async($cur->{blob}, \&_cmp_1st,
- [ $chash, $xoids, $cur ]);
+ [$chash, $xoids, $cur, $lms]);
if ($min && scalar(keys %$xoids) >= $min) {
$git->cat_async_wait;
return $xoids;
diff --git a/lib/PublicInbox/LeiTag.pm b/lib/PublicInbox/LeiTag.pm
index c650e886..b6abd533 100644
--- a/lib/PublicInbox/LeiTag.pm
+++ b/lib/PublicInbox/LeiTag.pm
@@ -9,7 +9,8 @@ use parent qw(PublicInbox::IPC PublicInbox::LeiInput);
sub input_eml_cb { # used by PublicInbox::LeiInput::input_fh
my ($self, $eml) = @_;
- if (my $xoids = $self->{lei}->{ale}->xoids_for($eml)) {
+ if (my $xoids = $self->{lse}->xoids_for($eml) // # tries LeiMailSync
+ $self->{lei}->{ale}->xoids_for($eml)) {
$self->{lei}->{sto}->ipc_do('update_xvmd', $xoids, $eml,
$self->{vmd_mod});
} else {
@@ -17,7 +18,11 @@ sub input_eml_cb { # used by PublicInbox::LeiInput::input_fh
}
}
-sub input_mbox_cb { input_eml_cb($_[1], $_[0]) }
+sub input_mbox_cb {
+ my ($eml, $self) = @_;
+ $eml->header_set($_) for (qw(X-Status Status));
+ input_eml_cb($self, $eml);
+}
sub input_maildir_cb { # maildir_each_eml cb
my ($f, $kw, $eml, $self) = @_;
@@ -60,6 +65,7 @@ sub note_missing {
sub ipc_atfork_child {
my ($self) = @_;
PublicInbox::LeiInput::input_only_atfork_child($self);
+ $self->{lse} = $self->{lei}->{sto}->search;
# this goes out-of-scope at worker process exit:
PublicInbox::OnDestroy->new($$, \¬e_missing, $self);
}
diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm
index da3a95d2..0cbdff8b 100644
--- a/lib/PublicInbox/LeiToMail.pm
+++ b/lib/PublicInbox/LeiToMail.pm
@@ -650,7 +650,9 @@ sub ipc_atfork_child {
my ($self) = @_;
my $lei = $self->{lei};
$lei->_lei_atfork_child;
- $self->{-lms_ro} = $lei->{lse}->lms if $lei->{lse};
+ if (my $lse = $lei->{lse}) {
+ $self->{-lms_ro} = $lse->{-lms_ro} //= $lse->lms;
+ }
$lei->{auth}->do_auth_atfork($self) if $lei->{auth};
$SIG{__WARN__} = PublicInbox::Eml::warn_ignore_cb();
$self->SUPER::ipc_atfork_child;
diff --git a/t/lei-index.t b/t/lei-index.t
index b7dafb71..9a45d885 100644
--- a/t/lei-index.t
+++ b/t/lei-index.t
@@ -40,7 +40,7 @@ test_lei({ tmpdir => $tmpdir }, sub {
my $res_a = json_utf8->decode($lei_out);
my $blob = $res_a->[0]->{'blob'};
like($blob, qr/\A[0-9a-f]{40,}\z/, 'got blob from qp@example');
- lei_ok('blob', $blob);
+ lei_ok(qw(-C / blob), $blob);
is($lei_out, $expect, 'got expected blob via Maildir');
lei_ok(qw(q mid:qp@example.com -f text));
like($lei_out, qr/^hi = bye/sm, 'lei2mail fallback');
@@ -58,6 +58,16 @@ test_lei({ tmpdir => $tmpdir }, sub {
my $res_b = json_utf8->decode($lei_out);
is_deeply($res_b, $res_a, 'no extra DB entries');
+ # ensure tag works on index-only messages:
+ lei_ok(qw(tag +kw:seen t/utf8.eml));
+ lei_ok(qw(q mid:testmessage@example.com));
+ is_deeply(json_utf8->decode($lei_out)->[0]->{kw},
+ ['seen'], 'seen kw can be set on index-only message');
+
+ lei_ok(qw(q z:0.. -o), "$tmpdir/all-results") for (1..2);
+ is_deeply([xqx($all_obj)], \@objs,
+ 'no new objects after 2x q to trigger implicit import');
+
lei_ok('index', "nntp://$nntp_host_port/t.v2");
lei_ok('index', "imap://$imap_host_port/t.v2.0");
is_deeply([xqx($all_obj)], \@objs, 'no new objects from NNTP+IMAP');
^ permalink raw reply related [relevance 6%]
* [PATCH 0/8] lei: export-kw, IMAP import incompatibility
@ 2021-05-21 10:28 7% Eric Wong
2021-05-21 10:28 6% ` [PATCH 3/8] lei tag: support tagging index-only messages Eric Wong
0 siblings, 1 reply; 2+ results
From: Eric Wong @ 2021-05-21 10:28 UTC (permalink / raw)
To: meta
"lei export-kw" is a new command. I'm not sure exactly how
it'll be used but it's probably more of a plumbing command,
for now. My brain hurts thinking about synchronization
and merge/conflict resolution when it comes to propagating
keywords assignments/clearing.
(I frequently mark messages as Unread in my MUA so I know to
reread them in the future, and I suspect it's a common thing).
mail_sync.sqlite3 now tracks AUTH=ANONYMOUS or username in the
folder name to account for lei(Unix) users having multiple IMAP
accounts on the same host with the same folders+UIDVALIDITY.
"lei import imap(s)://" users will waste a bit of bandwidth
resyncing as a result.
Eric Wong (8):
treewide: favor open(..., '+<&=', $fd)
lei: drop EOFpipe in favor of PktOp
lei tag: support tagging index-only messages
lei_input: fix canonicalization of Maildirs for sync
lei index: support command-line options
lei export-kw: new command to export keywords to Maildirs
uri_imap: support uid/auth/user as full accessors
lei import: store IMAP user+auth in mail_sync folder URI
MANIFEST | 2 +
examples/unsubscribe.milter | 3 +-
lib/PublicInbox/DS.pm | 3 +-
lib/PublicInbox/Daemon.pm | 2 +-
lib/PublicInbox/LEI.pm | 18 +++-
lib/PublicInbox/LeiExportKw.pm | 180 +++++++++++++++++++++++++++++++++
lib/PublicInbox/LeiInput.pm | 3 +-
lib/PublicInbox/LeiMailSync.pm | 10 ++
lib/PublicInbox/LeiOverview.pm | 2 +-
lib/PublicInbox/LeiSearch.pm | 22 +++-
lib/PublicInbox/LeiTag.pm | 10 +-
lib/PublicInbox/LeiToMail.pm | 12 ++-
lib/PublicInbox/MdirReader.pm | 14 +++
lib/PublicInbox/NetReader.pm | 42 +++++---
lib/PublicInbox/Sigfd.pm | 3 +-
lib/PublicInbox/URIimap.pm | 82 +++++++++++----
t/epoll.t | 7 +-
t/lei-export-kw.t | 35 +++++++
t/lei-import-imap.t | 9 +-
t/lei-index.t | 12 ++-
t/mdir_reader.t | 5 +
t/uri_imap.t | 60 ++++++++---
22 files changed, 461 insertions(+), 75 deletions(-)
create mode 100644 lib/PublicInbox/LeiExportKw.pm
create mode 100644 t/lei-export-kw.t
^ permalink raw reply [relevance 7%]
Results 1-2 of 2 | reverse | options above
-- pct% links below jump to the message on this page, permalinks otherwise --
2021-05-21 10:28 7% [PATCH 0/8] lei: export-kw, IMAP import incompatibility Eric Wong
2021-05-21 10:28 6% ` [PATCH 3/8] lei tag: support tagging index-only messages Eric Wong
Code repositories for project(s) associated with this public inbox
https://80x24.org/public-inbox.git
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).