* [PATCH 0/5] "lei q --save" + "lei up"
@ 2021-04-13 10:54 7% Eric Wong
2021-04-13 10:54 6% ` [PATCH 2/5] lei_dedupe: adjust to prepare for saved searches Eric Wong
0 siblings, 1 reply; 2+ results
From: Eric Wong @ 2021-04-13 10:54 UTC (permalink / raw)
To: meta
"--save" may become the default when writing to a
pathname or URL. "lei up" will be used to update
the results of "--save".
This only supports local externals at the moment,
remote externals won't be able to avoid excess
traffic easily.
Usability improvements are coming...
Eric Wong (5):
lei_xsearch: use per-external queries when not sorting
lei_dedupe: adjust to prepare for saved searches
lei_query: rearrange internals to capture query early
lei q: start wiring up saved search
lei: add "lei up" to complement "lei q --save"
MANIFEST | 4 +
lib/PublicInbox/LEI.pm | 6 +-
lib/PublicInbox/LeiDedupe.pm | 16 ++--
lib/PublicInbox/LeiQuery.pm | 59 +++++++------
lib/PublicInbox/LeiSavedSearch.pm | 142 ++++++++++++++++++++++++++++++
lib/PublicInbox/LeiToMail.pm | 18 ++--
lib/PublicInbox/LeiUp.pm | 46 ++++++++++
lib/PublicInbox/LeiXSearch.pm | 94 +++++++++++++-------
t/lei-q-save.t | 25 ++++++
t/lei.t | 2 +-
t/lei_dedupe.t | 11 ++-
t/lei_saved_search.t | 10 +++
12 files changed, 356 insertions(+), 77 deletions(-)
create mode 100644 lib/PublicInbox/LeiSavedSearch.pm
create mode 100644 lib/PublicInbox/LeiUp.pm
create mode 100644 t/lei-q-save.t
create mode 100644 t/lei_saved_search.t
^ permalink raw reply [relevance 7%]
* [PATCH 2/5] lei_dedupe: adjust to prepare for saved searches
2021-04-13 10:54 7% [PATCH 0/5] "lei q --save" + "lei up" Eric Wong
@ 2021-04-13 10:54 6% ` Eric Wong
0 siblings, 0 replies; 2+ results
From: Eric Wong @ 2021-04-13 10:54 UTC (permalink / raw)
To: meta
LeiSavedSearch will use a LeiDedupe-like internal API,
so we won't have to make as many changes to callsites
between saved and unsaved searches.
---
lib/PublicInbox/LeiDedupe.pm | 16 ++++++++--------
lib/PublicInbox/LeiToMail.pm | 6 +++---
t/lei_dedupe.t | 11 +++++++----
3 files changed, 18 insertions(+), 15 deletions(-)
diff --git a/lib/PublicInbox/LeiDedupe.pm b/lib/PublicInbox/LeiDedupe.pm
index a62b3a7c..378f748e 100644
--- a/lib/PublicInbox/LeiDedupe.pm
+++ b/lib/PublicInbox/LeiDedupe.pm
@@ -41,8 +41,8 @@ sub smsg_hash ($) {
sub dedupe_oid ($) {
my ($skv) = @_;
(sub { # may be called in a child process
- my ($eml, $oid) = @_;
- $skv->set_maybe(_oidbin($oid) // _regen_oid($eml), '');
+ my ($eml, $oidhex) = @_;
+ $skv->set_maybe(_oidbin($oidhex) // _regen_oid($eml), '');
}, sub {
my ($smsg) = @_;
$skv->set_maybe(_oidbin($smsg->{blob}), '');
@@ -53,9 +53,9 @@ sub dedupe_oid ($) {
sub dedupe_mid ($) {
my ($skv) = @_;
(sub { # may be called in a child process
- my ($eml, $oid) = @_;
- # TODO: lei will support non-public messages w/o Message-ID
- my $mid = $eml->header_raw('Message-ID') // _oidbin($oid) //
+ my ($eml, $oidhex) = @_;
+ # lei supports non-public drafts w/o Message-ID
+ my $mid = $eml->header_raw('Message-ID') // _oidbin($oidhex) //
content_hash($eml);
$skv->set_maybe($mid, '');
}, sub {
@@ -71,7 +71,7 @@ sub dedupe_mid ($) {
sub dedupe_content ($) {
my ($skv) = @_;
(sub { # may be called in a child process
- my ($eml) = @_; # oid = $_[1], ignored
+ my ($eml) = @_; # $oidhex = $_[1], ignored
$skv->set_maybe(content_hash($eml), '');
}, sub {
my ($smsg) = @_;
@@ -104,8 +104,8 @@ sub new {
# returns true on seen messages according to the deduplication strategy,
# returns false if unseen
sub is_dup {
- my ($self, $eml, $oid) = @_;
- !$self->[1]->($eml, $oid);
+ my ($self, $eml, $smsg) = @_;
+ !$self->[1]->($eml, $smsg ? $smsg->{blob} : undef);
}
sub is_smsg_dup {
diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm
index 70164e40..7adbffe7 100644
--- a/lib/PublicInbox/LeiToMail.pm
+++ b/lib/PublicInbox/LeiToMail.pm
@@ -194,7 +194,7 @@ sub _mbox_write_cb ($$) {
sub { # for git_to_mail
my ($buf, $smsg, $eml) = @_;
$eml //= PublicInbox::Eml->new($buf);
- return if $dedupe->is_dup($eml, $smsg->{blob});
+ return if $dedupe->is_dup($eml, $smsg);
$lse->xsmsg_vmd($smsg) if $lse;
$buf = $eml2mbox->($eml, $smsg);
return atomic_append($lei, $buf) if $atomic_append;
@@ -280,7 +280,7 @@ sub _maildir_write_cb ($$) {
$lse->xsmsg_vmd($smsg) if $lse;
return _buf2maildir($dst, $buf, $smsg) if !$dedupe;
$eml //= PublicInbox::Eml->new($$buf); # copy buf
- return if $dedupe->is_dup($eml, $smsg->{blob});
+ return if $dedupe->is_dup($eml, $smsg);
undef $eml;
_buf2maildir($dst, $buf, $smsg);
}
@@ -299,7 +299,7 @@ sub _imap_write_cb ($$) {
$mic // return $lei->fail; # mic may be undef-ed in last run
if ($dedupe) {
$eml //= PublicInbox::Eml->new($$bref); # copy bref
- return if $dedupe->is_dup($eml, $smsg->{blob});
+ return if $dedupe->is_dup($eml, $smsg);
}
$lse->xsmsg_vmd($smsg) if $lse;
eval { $imap_append->($mic, $folder, $bref, $smsg, $eml) };
diff --git a/t/lei_dedupe.t b/t/lei_dedupe.t
index bcb06a0a..e1944d02 100644
--- a/t/lei_dedupe.t
+++ b/t/lei_dedupe.t
@@ -74,10 +74,13 @@ ok(!$dd->is_dup($different), 'different is_dup with mid dedupe (augment)');
$different->header_set('Status', 'RO');
ok($dd->is_dup($different), 'different seen with oid dedupe Status removed');
-ok(!$dd->is_dup($eml, '01d'), '1st is_dup with oid dedupe');
-ok($dd->is_dup($different, '01d'), 'different content ignored if oid matches');
-ok($dd->is_dup($eml, '01D'), 'case insensitive oid comparison :P');
-ok(!$dd->is_dup($eml, '01dbad'), 'case insensitive oid comparison :P');
+$smsg = { blob => '01d' };
+ok(!$dd->is_dup($eml, $smsg), '1st is_dup with oid dedupe');
+ok($dd->is_dup($different, $smsg), 'different content ignored if oid matches');
+$smsg->{blob} = uc($smsg->{blob});
+ok($dd->is_dup($eml, $smsg), 'case insensitive oid comparison :P');
+$smsg->{blob} = '01dbad';
+ok(!$dd->is_dup($eml, $smsg), 'case insensitive oid comparison :P');
$smsg->{blob} = 'dead';
ok(!$dd->is_smsg_dup($smsg), 'smsg dedupe pass');
^ permalink raw reply related [relevance 6%]
Results 1-2 of 2 | reverse | options above
-- pct% links below jump to the message on this page, permalinks otherwise --
2021-04-13 10:54 7% [PATCH 0/5] "lei q --save" + "lei up" Eric Wong
2021-04-13 10:54 6% ` [PATCH 2/5] lei_dedupe: adjust to prepare for saved searches Eric Wong
Code repositories for project(s) associated with this public inbox
https://80x24.org/public-inbox.git
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).