* [PATCH 3/4] lei: use maildir_each_eml in more places
2021-03-03 13:48 6% [PATCH 0/4] lei q: avoiding accidental data loss Eric Wong
@ 2021-03-03 13:48 7% ` Eric Wong
0 siblings, 0 replies; 2+ results
From: Eric Wong @ 2021-03-03 13:48 UTC (permalink / raw)
To: meta
This saves us some code and redundant callsites for
eml_from_path. We'll change maildir_each_eml to include the
filename to facilitate an upcoming change to "lei q" without
--augment
---
lib/PublicInbox/LeiConvert.pm | 3 +--
lib/PublicInbox/LeiToMail.pm | 18 ++++++------------
lib/PublicInbox/MdirReader.pm | 10 ++++++----
t/lei-convert.t | 2 +-
4 files changed, 14 insertions(+), 19 deletions(-)
diff --git a/lib/PublicInbox/LeiConvert.pm b/lib/PublicInbox/LeiConvert.pm
index 4c0bbd88..0c705ba4 100644
--- a/lib/PublicInbox/LeiConvert.pm
+++ b/lib/PublicInbox/LeiConvert.pm
@@ -7,7 +7,6 @@ use strict;
use v5.10.1;
use parent qw(PublicInbox::IPC);
use PublicInbox::Eml;
-use PublicInbox::InboxWritable qw(eml_from_path);
use PublicInbox::LeiStore;
use PublicInbox::LeiOverview;
@@ -24,7 +23,7 @@ sub net_cb { # callback for ->imap_each, ->nntp_each
}
sub mdir_cb {
- my ($kw, $eml, $self) = @_;
+ my ($f, $kw, $eml, $self) = @_;
$self->{wcb}->(undef, { kw => $kw }, $eml);
}
diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm
index de640657..31b8aba8 100644
--- a/lib/PublicInbox/LeiToMail.pm
+++ b/lib/PublicInbox/LeiToMail.pm
@@ -19,7 +19,6 @@ use IO::Handle; # ->autoflush
use Fcntl qw(SEEK_SET SEEK_END O_CREAT O_EXCL O_WRONLY);
use Errno qw(EEXIST ESPIPE ENOENT EPIPE);
use Digest::SHA qw(sha256_hex);
-my ($maildir_each_file);
# struggles with short-lived repos, Gcf2Client makes little sense with lei;
# but we may use in-process libgit2 in the future.
@@ -268,8 +267,8 @@ sub _mbox_write_cb ($$) {
}
}
-sub _augment_file { # maildir_each_file cb
- my ($f, $lei, $mod, $shard) = @_;
+sub _augment_file { # maildir_each_eml cb
+ my ($f, undef, $eml, $lei, $mod, $shard) = @_;
if ($mod) {
# can't get dirent.d_ino w/ pure Perl, so we extract the OID
# if it looks like one:
@@ -278,7 +277,6 @@ sub _augment_file { # maildir_each_file cb
my $recno = hex(substr($hex, 0, 8));
return if ($recno % $mod) != $shard;
}
- my $eml = PublicInbox::InboxWritable::eml_from_path($f) or return;
_augment($eml, $lei);
}
@@ -375,12 +373,7 @@ sub new {
my $dst = $lei->{ovv}->{dst};
my $self = bless {}, $cls;
if ($fmt eq 'maildir') {
- $maildir_each_file //= do {
- require PublicInbox::MdirReader;
- PublicInbox::MdirReader->can('maildir_each_file');
- };
- $lei->{opt}->{augment} and
- require PublicInbox::InboxWritable; # eml_from_path
+ require PublicInbox::MdirReader;
$self->{base_type} = 'maildir';
-e $dst && !-d _ and die
"$dst exists and is not a directory\n";
@@ -430,12 +423,13 @@ sub _do_augment_maildir {
my $dedupe = $lei->{dedupe};
if ($dedupe && $dedupe->prepare_dedupe) {
my ($mod, $shard) = @{$self->{shard_info} // []};
- $maildir_each_file->($dst, \&_augment_file,
+ PublicInbox::MdirReader::maildir_each_eml($dst,
+ \&_augment_file,
$lei, $mod, $shard);
$dedupe->pause_dedupe;
}
} else { # clobber existing Maildir
- $maildir_each_file->($dst, \&_unlink);
+ PublicInbox::MdirReader::maildir_each_file($dst, \&_unlink);
}
}
diff --git a/lib/PublicInbox/MdirReader.pm b/lib/PublicInbox/MdirReader.pm
index 5fa534f5..44724af1 100644
--- a/lib/PublicInbox/MdirReader.pm
+++ b/lib/PublicInbox/MdirReader.pm
@@ -48,17 +48,19 @@ sub maildir_each_eml ($$;@) {
next if substr($bn, 0, 1) eq '.';
my @f = split(/:/, $bn, -1);
next if scalar(@f) != 1;
- my $eml = eml_from_path($pfx.$bn) or next;
- $cb->([], $eml, @arg);
+ my $f = $pfx.$bn;
+ my $eml = eml_from_path($f) or next;
+ $cb->($f, [], $eml, @arg);
}
}
$pfx = "$dir/cur/";
opendir my $dh, $pfx or return;
while (defined(my $bn = readdir($dh))) {
my $fl = maildir_basename_flags($bn) // next;
- my $eml = eml_from_path($pfx.$bn) or next;
+ my $f = $pfx.$bn;
+ my $eml = eml_from_path($f) or next;
my @kw = sort(map { $c2kw{$_} // () } split(//, $fl));
- $cb->(\@kw, $eml, @arg);
+ $cb->($f, \@kw, $eml, @arg);
}
}
diff --git a/t/lei-convert.t b/t/lei-convert.t
index 20099f65..186cfb13 100644
--- a/t/lei-convert.t
+++ b/t/lei-convert.t
@@ -58,7 +58,7 @@ test_lei({ tmpdir => $tmpdir }, sub {
ok(-d "$d/md", 'Maildir created');
my @md;
PublicInbox::MdirReader::maildir_each_eml("$d/md", sub {
- push @md, $_[1];
+ push @md, $_[2];
});
is(scalar(@md), scalar(@mboxrd), 'got expected emails in Maildir');
@md = sort { ${$a->{bdy}} cmp ${$b->{bdy}} } @md;
^ permalink raw reply related [relevance 7%]
* [PATCH 0/4] lei q: avoiding accidental data loss
@ 2021-03-03 13:48 6% Eric Wong
2021-03-03 13:48 7% ` [PATCH 3/4] lei: use maildir_each_eml in more places Eric Wong
0 siblings, 1 reply; 2+ results
From: Eric Wong @ 2021-03-03 13:48 UTC (permalink / raw)
To: meta
"lei q" mimicking mairix(1) could cause some grief if somebody
accidentally sets the output destination to one containing
precious mail or keyword changes. Start by stashing keyword
changes into our store during the augment/unlink phase.
Eric Wong (4):
eml: each_part: document IMAP user of the $all parameter
lei_xsearch: add_eml for remote mboxrd, not set_eml
lei: use maildir_each_eml in more places
lei q: import flags when clobbering/augmenting Maildirs
MANIFEST | 1 +
lib/PublicInbox/Eml.pm | 1 +
lib/PublicInbox/ExtSearchIdx.pm | 1 +
lib/PublicInbox/LEI.pm | 2 +-
lib/PublicInbox/LeiConvert.pm | 3 +--
lib/PublicInbox/LeiQuery.pm | 5 +++-
lib/PublicInbox/LeiSearch.pm | 47 +++++++++++++++++++++++++++++++++
lib/PublicInbox/LeiStore.pm | 27 +++++++++----------
lib/PublicInbox/LeiToMail.pm | 45 ++++++++++++++++++++-----------
lib/PublicInbox/LeiXSearch.pm | 2 +-
lib/PublicInbox/MdirReader.pm | 10 ++++---
t/lei-convert.t | 2 +-
t/lei-q-kw.t | 33 +++++++++++++++++++++++
t/lei.t | 3 ++-
t/lei_store.t | 10 ++++++-
15 files changed, 149 insertions(+), 43 deletions(-)
create mode 100644 t/lei-q-kw.t
^ permalink raw reply [relevance 6%]
Results 1-2 of 2 | reverse | options above
-- pct% links below jump to the message on this page, permalinks otherwise --
2021-03-03 13:48 6% [PATCH 0/4] lei q: avoiding accidental data loss Eric Wong
2021-03-03 13:48 7% ` [PATCH 3/4] lei: use maildir_each_eml in more places Eric Wong
Code repositories for project(s) associated with this public inbox
https://80x24.org/public-inbox.git
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).