* [PATCH 1/8] lei: dump errors to syslog, and not to CLI
2021-09-03 8:54 [PATCH 0/8] lei: fix IMAP R/W; L/kw false positives Eric Wong
@ 2021-09-03 8:54 ` Eric Wong
2021-09-03 8:54 ` [PATCH 2/8] lei/store: quiet down link(2) warnings Eric Wong
` (6 subsequent siblings)
7 siblings, 0 replies; 9+ messages in thread
From: Eric Wong @ 2021-09-03 8:54 UTC (permalink / raw)
To: meta
Dumping errors from the previous run can often get lost, so just
spew to syslog since it's a standard place to put errors that
don't make it to a client. Note: we don't rely on $SIG{__WARN__}
since some of the Net:: stuff will write directly to STDERR
(as will external processes).
---
lib/PublicInbox/LEI.pm | 16 +++++++++-------
lib/PublicInbox/LeiStoreErr.pm | 4 ++--
t/lei-daemon.t | 5 -----
3 files changed, 11 insertions(+), 14 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 41e811ca..9e9aa165 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -27,6 +27,7 @@ use PublicInbox::Eml;
use Time::HiRes qw(stat); # ctime comparisons for config cache
use File::Path qw(mkpath);
use File::Spec;
+use Sys::Syslog qw(openlog syslog closelog);
our $quit = \&CORE::exit;
our ($current_lei, $errors_log, $listener, $oldset, $dir_idle,
$recv_cmd, $send_cmd);
@@ -464,7 +465,6 @@ sub x_it ($$) {
my ($self, $code) = @_;
# make sure client sees stdout before exit
$self->{1}->autoflush(1) if $self->{1};
- dump_and_clear_log();
stop_pager($self);
if ($self->{pkt_op_p}) { # to top lei-daemon
$self->{pkt_op_p}->pkt_do('x_it', $code);
@@ -765,7 +765,6 @@ sub dispatch {
my ($self, $cmd, @argv) = @_;
local $current_lei = $self; # for __WARN__
$self->{2}->autoflush(1); # keep stdout buffered until x_it|DESTROY
- dump_and_clear_log("from previous run\n");
return _help($self, 'no command given') unless defined($cmd);
# do not support Getopt bundling for this
while ($cmd eq '-C' || $cmd eq '-c') {
@@ -1147,10 +1146,12 @@ sub oldset { $oldset }
sub dump_and_clear_log {
if (defined($errors_log) && -s STDIN && seek(STDIN, 0, SEEK_SET)) {
- my @pfx = @_;
- unshift(@pfx, "$errors_log ") if @pfx;
- warn @pfx, do { local $/; <STDIN> };
- truncate(STDIN, 0) or warn "ftruncate ($errors_log): $!";
+ openlog('lei-daemon', 'pid,nowait,nofatal,ndelay', 'user');
+ chomp(my @lines = <STDIN>);
+ truncate(STDIN, 0) or
+ syslog('warning', "ftruncate (%s): %m", $errors_log);
+ for my $l (@lines) { syslog('warning', '%s', $l) }
+ closelog(); # don't share across fork
}
}
@@ -1243,7 +1244,7 @@ sub lazy_start {
(-p STDOUT) or die "E: stdout must be a pipe\n";
open(STDIN, '+>>', $errors_log) or die "open($errors_log): $!";
STDIN->autoflush(1);
- dump_and_clear_log("from previous daemon process:\n");
+ dump_and_clear_log();
POSIX::setsid() > 0 or die "setsid: $!";
my $pid = fork // die "fork: $!";
return if $pid;
@@ -1345,6 +1346,7 @@ sub DESTROY {
}
$self->{1}->autoflush(1) if $self->{1};
stop_pager($self);
+ dump_and_clear_log();
# preserve $? for ->fail or ->x_it code
}
diff --git a/lib/PublicInbox/LeiStoreErr.pm b/lib/PublicInbox/LeiStoreErr.pm
index 5f9ba24d..cc085fdc 100644
--- a/lib/PublicInbox/LeiStoreErr.pm
+++ b/lib/PublicInbox/LeiStoreErr.pm
@@ -2,7 +2,7 @@
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
# forwards stderr from lei/store process to any lei clients using
-# the same store
+# the same store, falls back to syslog if no matching clients exist.
package PublicInbox::LeiStoreErr;
use strict;
use v5.10.1;
@@ -31,7 +31,7 @@ sub event_step {
print $err $$rbuf and $printed = 1;
}
if (!$printed) {
- openlog('lei-store', 'pid,nowait,nofatal,ndelay', 'user');
+ openlog('lei/store', 'pid,nowait,nofatal,ndelay', 'user');
for my $l (split(/\n/, $$rbuf)) { syslog('warning', '%s', $l) }
closelog(); # don't share across fork
}
diff --git a/t/lei-daemon.t b/t/lei-daemon.t
index a7c4b799..b0c94a87 100644
--- a/t/lei-daemon.t
+++ b/t/lei-daemon.t
@@ -21,14 +21,9 @@ test_lei({ daemon_only => 1 }, sub {
ok(kill(0, $pid), 'pid is valid');
ok(-S $sock, 'sock created');
is(-s $err_log, 0, 'nothing in errors.log');
- open my $efh, '>>', $err_log or BAIL_OUT $!;
- print $efh "phail\n" or BAIL_OUT $!;
- close $efh or BAIL_OUT $!;
-
lei_ok('daemon-pid');
chomp(my $pid_again = $lei_out);
is($pid, $pid_again, 'daemon-pid idempotent');
- like($lei_err, qr/phail/, 'got mock "phail" error previous run');
SKIP: {
skip 'only testing open files on Linux', 1 if $^O ne 'linux';
^ permalink raw reply related [flat|nested] 9+ messages in thread
* [PATCH 2/8] lei/store: quiet down link(2) warnings
2021-09-03 8:54 [PATCH 0/8] lei: fix IMAP R/W; L/kw false positives Eric Wong
2021-09-03 8:54 ` [PATCH 1/8] lei: dump errors to syslog, and not to CLI Eric Wong
@ 2021-09-03 8:54 ` Eric Wong
2021-09-03 8:54 ` [PATCH 3/8] lei: ->child_error less error-prone Eric Wong
` (5 subsequent siblings)
7 siblings, 0 replies; 9+ messages in thread
From: Eric Wong @ 2021-09-03 8:54 UTC (permalink / raw)
To: meta
ENOENT can be too common due to timing and concurrent access
from MUAs and "lei export-kw", and other mail synchronization
tools (e.g. mbsync and offlineimap).
---
lib/PublicInbox/LeiStore.pm | 10 +++-------
1 file changed, 3 insertions(+), 7 deletions(-)
diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm
index a91b30f7..f81a8dae 100644
--- a/lib/PublicInbox/LeiStore.pm
+++ b/lib/PublicInbox/LeiStore.pm
@@ -215,14 +215,10 @@ sub export1_kw_md ($$$$$) {
return;
} elsif ($! == EEXIST) { # lost race with "lei export-kw"?
return;
- } elsif ($! == ENOENT) {
- syslog('warning', "link($src -> $dst): $!")
- } # else loop @try
+ } elsif ($! != ENOENT) {
+ syslog('warning', "link($src -> $dst): $!");
+ }
}
- my $e = $!;
- my $src = "$mdir/{".join(',', @try)."}/$orig";
- my $oidhex = unpack('H*', $oidbin);
- syslog('warning', "link($src -> $dst) ($oidhex): $e");
for (@try) { return if -e "$mdir/$_/$orig" };
lms_clear_src($self, "maildir:$mdir", \$orig);
}
^ permalink raw reply related [flat|nested] 9+ messages in thread
* [PATCH 3/8] lei: ->child_error less error-prone
2021-09-03 8:54 [PATCH 0/8] lei: fix IMAP R/W; L/kw false positives Eric Wong
2021-09-03 8:54 ` [PATCH 1/8] lei: dump errors to syslog, and not to CLI Eric Wong
2021-09-03 8:54 ` [PATCH 2/8] lei/store: quiet down link(2) warnings Eric Wong
@ 2021-09-03 8:54 ` Eric Wong
2021-09-03 8:54 ` [PATCH 4/8] lei: use lei->lms in place of lse->lms in a few places Eric Wong
` (4 subsequent siblings)
7 siblings, 0 replies; 9+ messages in thread
From: Eric Wong @ 2021-09-03 8:54 UTC (permalink / raw)
To: meta
I was calling "child_error(1, ...)" in a few places where I meant
to be calling "child_error(1 << 8, ...)" and inadvertantly
triggering SIGHUP in script/lei. Since giving a zero exit code
to child_error makes no sense, just allow falsy values to
default to 1 << 8.
---
lib/PublicInbox/LEI.pm | 7 ++++---
lib/PublicInbox/LeiBlob.pm | 2 +-
lib/PublicInbox/LeiIndex.pm | 2 +-
lib/PublicInbox/LeiInput.pm | 4 ++--
lib/PublicInbox/LeiLcat.pm | 2 +-
lib/PublicInbox/LeiRediff.pm | 2 +-
lib/PublicInbox/LeiUp.pm | 2 +-
7 files changed, 11 insertions(+), 10 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 9e9aa165..8b6c1d36 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -532,6 +532,7 @@ sub puts ($;@) { out(shift, map { "$_\n" } @_) }
sub child_error { # passes non-fatal curl exit codes to user
my ($self, $child_error, $msg) = @_; # child_error is $?
+ $child_error ||= 1 << 8;
$self->err($msg) if $msg;
if ($self->{pkt_op_p}) { # to top lei-daemon
$self->{pkt_op_p}->pkt_do('child_error', $child_error);
@@ -1341,7 +1342,7 @@ sub DESTROY {
if (my $counters = delete $self->{counters}) {
for my $k (sort keys %$counters) {
my $nr = $counters->{$k};
- $self->child_error(1 << 8, "$nr $k messages");
+ $self->child_error(0, "$nr $k messages");
}
}
$self->{1}->autoflush(1) if $self->{1};
@@ -1417,7 +1418,7 @@ sub refresh_watches {
add_maildir_watch($d, $cfg_f);
}
} else { # TODO: imap/nntp/jmap
- $lei->child_error(1, "E: watch $url not supported, yet")
+ $lei->child_error(0, "E: watch $url not supported, yet")
}
}
@@ -1452,7 +1453,7 @@ sub refresh_watches {
my $d = canonpath_harder($1);
cancel_maildir_watch($d, $cfg_f);
} else { # TODO: imap/nntp/jmap
- $lei->child_error(1, "E: watch $url TODO");
+ $lei->child_error(0, "E: watch $url TODO");
}
}
}
diff --git a/lib/PublicInbox/LeiBlob.pm b/lib/PublicInbox/LeiBlob.pm
index 21003894..b94f67a0 100644
--- a/lib/PublicInbox/LeiBlob.pm
+++ b/lib/PublicInbox/LeiBlob.pm
@@ -32,7 +32,7 @@ sub solver_user_cb { # called by solver when done
my $lei = $self->{lei};
my $log_buf = delete $lei->{'log_buf'};
$$log_buf =~ s/^/# /sgm;
- ref($res) eq 'ARRAY' or return $lei->child_error(1 << 8, $$log_buf);
+ ref($res) eq 'ARRAY' or return $lei->child_error(0, $$log_buf);
$lei->qerr($$log_buf);
my ($git, $oid, $type, $size, $di) = @$res;
my $gd = $git->{git_dir};
diff --git a/lib/PublicInbox/LeiIndex.pm b/lib/PublicInbox/LeiIndex.pm
index 5b545998..ef3e4d0b 100644
--- a/lib/PublicInbox/LeiIndex.pm
+++ b/lib/PublicInbox/LeiIndex.pm
@@ -21,7 +21,7 @@ sub input_eml_cb { # used by input_maildir_cb and input_net_cb
sub input_fh { # overrides PublicInbox::LeiInput::input_fh
my ($self, $ifmt, $fh, $input, @args) = @_;
- $self->{lei}->child_error(1<<8, <<EOM);
+ $self->{lei}->child_error(0, <<EOM);
$input ($ifmt) not yet supported, try `lei import'
EOM
}
diff --git a/lib/PublicInbox/LeiInput.pm b/lib/PublicInbox/LeiInput.pm
index 1b28f36f..cb71e97c 100644
--- a/lib/PublicInbox/LeiInput.pm
+++ b/lib/PublicInbox/LeiInput.pm
@@ -63,7 +63,7 @@ sub input_fh {
my ($self, $ifmt, $fh, $name, @args) = @_;
if ($ifmt eq 'eml') {
my $buf = do { local $/; <$fh> } //
- return $self->{lei}->child_error(1 << 8, <<"");
+ return $self->{lei}->child_error(0, <<"");
error reading $name: $!
# mutt pipes single RFC822 messages with a "From " line,
@@ -104,7 +104,7 @@ sub handle_http_input ($$@) {
my $err = $@;
waitpid($pid, 0);
$? || $err and
- $lei->child_error($? || 1, "@$cmd failed".$err ? " $err" : '');
+ $lei->child_error($?, "@$cmd failed".$err ? " $err" : '');
}
sub input_path_url {
diff --git a/lib/PublicInbox/LeiLcat.pm b/lib/PublicInbox/LeiLcat.pm
index 9d95e899..1e54c3bf 100644
--- a/lib/PublicInbox/LeiLcat.pm
+++ b/lib/PublicInbox/LeiLcat.pm
@@ -18,7 +18,7 @@ sub lcat_folder ($$$) {
my $err = $lms->arg2folder($lei, $folders);
$lei->qerr(@{$err->{qerr}}) if $err && $err->{qerr};
if ($err && $err->{fail}) {
- $lei->child_error(1 << 8, "# unknown folder: $folder");
+ $lei->child_error(0, "# unknown folder: $folder");
} else {
for my $f (@$folders) {
my $fid = $lms->fid_for($f);
diff --git a/lib/PublicInbox/LeiRediff.pm b/lib/PublicInbox/LeiRediff.pm
index 0ba5897c..60286b06 100644
--- a/lib/PublicInbox/LeiRediff.pm
+++ b/lib/PublicInbox/LeiRediff.pm
@@ -23,7 +23,7 @@ sub rediff_user_cb { # called by solver when done
my $lei = $self->{lei};
my $log_buf = delete $lei->{log_buf};
$$log_buf =~ s/^/# /sgm;
- ref($res) eq 'ARRAY' or return $lei->child_error(1 << 8, $$log_buf);
+ ref($res) eq 'ARRAY' or return $lei->child_error(0, $$log_buf);
$lei->qerr($$log_buf);
my ($git, $oid, $type, $size, $di) = @$res;
my $oid_want = delete $self->{cur_oid_want};
diff --git a/lib/PublicInbox/LeiUp.pm b/lib/PublicInbox/LeiUp.pm
index 85efd9f5..e1da64aa 100644
--- a/lib/PublicInbox/LeiUp.pm
+++ b/lib/PublicInbox/LeiUp.pm
@@ -59,7 +59,7 @@ sub up1_redispatch {
up1($l, $out);
$l->qerr("# $out done");
};
- $l->child_error(1 << 8, $@) if $@;
+ $l->child_error(0, $@) if $@;
}
sub lei_up {
^ permalink raw reply related [flat|nested] 9+ messages in thread
* [PATCH 4/8] lei: use lei->lms in place of lse->lms in a few places
2021-09-03 8:54 [PATCH 0/8] lei: fix IMAP R/W; L/kw false positives Eric Wong
` (2 preceding siblings ...)
2021-09-03 8:54 ` [PATCH 3/8] lei: ->child_error less error-prone Eric Wong
@ 2021-09-03 8:54 ` Eric Wong
2021-09-03 8:54 ` [PATCH 5/8] lei up --all: avoid double-close on shared STDOUT Eric Wong
` (3 subsequent siblings)
7 siblings, 0 replies; 9+ messages in thread
From: Eric Wong @ 2021-09-03 8:54 UTC (permalink / raw)
To: meta
We can golf out some code and refcounts this way.
---
lib/PublicInbox/LeiInspect.pm | 3 +--
lib/PublicInbox/LeiPruneMailSync.pm | 26 +++++++++++++-------------
2 files changed, 14 insertions(+), 15 deletions(-)
diff --git a/lib/PublicInbox/LeiInspect.pm b/lib/PublicInbox/LeiInspect.pm
index 2d2ff1a0..2ade17af 100644
--- a/lib/PublicInbox/LeiInspect.pm
+++ b/lib/PublicInbox/LeiInspect.pm
@@ -206,8 +206,7 @@ sub lei_inspect {
sub _complete_inspect {
my ($lei, @argv) = @_;
- my $sto = $lei->_lei_store or return;
- my $lms = $sto->search->lms or return;
+ my $lms = $lei->lms or return;
my $match_cb = $lei->complete_url_prepare(\@argv);
map { $match_cb->($_) } $lms->folders;
}
diff --git a/lib/PublicInbox/LeiPruneMailSync.pm b/lib/PublicInbox/LeiPruneMailSync.pm
index 79f3325d..98239a13 100644
--- a/lib/PublicInbox/LeiPruneMailSync.pm
+++ b/lib/PublicInbox/LeiPruneMailSync.pm
@@ -40,7 +40,7 @@ sub prune_imap { # lms->each_src callback
sub input_path_url { # overrides PublicInbox::LeiInput::input_path_url
my ($self, $input, @args) = @_;
- my $lms = $self->{-lms_ro} //= $self->{lse}->lms;
+ my $lms = $self->{-lms_ro} //= $self->{lei}->lms;
if ($input =~ /\Amaildir:(.+)/i) {
my $mdir = $1;
$lms->each_src($input, \&prune_mdir, $self, $mdir);
@@ -59,24 +59,24 @@ sub lei_prune_mail_sync {
my $sto = $lei->_lei_store or return $lei->fail(<<EOM);
lei/store uninitialized, see lei-import(1)
EOM
- my $lse = $sto->search;
- my $lms = $lse->lms or return $lei->fail(<<EOM);
-lei mail_sync uninitialized, see lei-import(1)
-EOM
- if (defined(my $all = $lei->{opt}->{all})) {
- $lms->group2folders($lei, $all, \@folders) or return;
+ if (my $lms = $lei->lms) {
+ if (defined(my $all = $lei->{opt}->{all})) {
+ $lms->group2folders($lei, $all, \@folders) or return;
+ } else {
+ my $err = $lms->arg2folder($lei, \@folders);
+ $lei->qerr(@{$err->{qerr}}) if $err->{qerr};
+ return $lei->fail($err->{fail}) if $err->{fail};
+ }
} else {
- my $err = $lms->arg2folder($lei, \@folders);
- $lei->qerr(@{$err->{qerr}}) if $err->{qerr};
- return $lei->fail($err->{fail}) if $err->{fail};
+ return $lei->fail(<<EOM);
+lei mail_sync.sqlite3 uninitialized, see lei-import(1)
+EOM
}
- delete $lms->{dbh};
$sto->write_prepare($lei);
- my $self = bless { lse => $lse }, __PACKAGE__;
+ my $self = bless {}, __PACKAGE__;
$lei->{opt}->{'mail-sync'} = 1; # for prepare_inputs
$self->prepare_inputs($lei, \@folders) or return;
my $j = $lei->{opt}->{jobs} || scalar(@{$self->{inputs}}) || 1;
- undef $lms; # for fork
my $ops = {};
$sto->write_prepare($lei);
$lei->{auth}->op_merge($ops, $self) if $lei->{auth};
^ permalink raw reply related [flat|nested] 9+ messages in thread
* [PATCH 5/8] lei up --all: avoid double-close on shared STDOUT
2021-09-03 8:54 [PATCH 0/8] lei: fix IMAP R/W; L/kw false positives Eric Wong
` (3 preceding siblings ...)
2021-09-03 8:54 ` [PATCH 4/8] lei: use lei->lms in place of lse->lms in a few places Eric Wong
@ 2021-09-03 8:54 ` Eric Wong
2021-09-03 8:54 ` [PATCH 6/8] lei inspect: support reading eml from --stdin Eric Wong
` (2 subsequent siblings)
7 siblings, 0 replies; 9+ messages in thread
From: Eric Wong @ 2021-09-03 8:54 UTC (permalink / raw)
To: meta
This is merely to avoid perl setting errors internally which
were not user visible. The double-close wasn't a problem in
practice since we open a new file hanlde for the mbox or
mbox.gz anyways, so the new t/lei-up.t test case shows no
regressions nor fixes.
---
MANIFEST | 1 +
lib/PublicInbox/LeiUp.pm | 4 ++++
t/lei-up.t | 39 +++++++++++++++++++++++++++++++++++++++
3 files changed, 44 insertions(+)
create mode 100644 t/lei-up.t
diff --git a/MANIFEST b/MANIFEST
index be6ec927..fad29622 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -443,6 +443,7 @@ t/lei-q-save.t
t/lei-q-thread.t
t/lei-sigpipe.t
t/lei-tag.t
+t/lei-up.t
t/lei-watch.t
t/lei.t
t/lei_dedupe.t
diff --git a/lib/PublicInbox/LeiUp.pm b/lib/PublicInbox/LeiUp.pm
index e1da64aa..a39d6047 100644
--- a/lib/PublicInbox/LeiUp.pm
+++ b/lib/PublicInbox/LeiUp.pm
@@ -54,6 +54,10 @@ sub up1_redispatch {
$l->{opt} = { %{$l->{opt}} };
delete $l->{sock};
$l->{''} = $op_p; # daemon only
+
+ # make close($l->{1}) happy in lei->dclose
+ open my $fh, '>&', $l->{1} or return $l->child_error(0, "dup: $!");
+ $l->{1} = $fh;
eval {
$l->qerr("# updating $out");
up1($l, $out);
diff --git a/t/lei-up.t b/t/lei-up.t
new file mode 100644
index 00000000..c6f31c74
--- /dev/null
+++ b/t/lei-up.t
@@ -0,0 +1,39 @@
+#!perl -w
+# Copyright all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+use strict; use v5.10.1; use PublicInbox::TestCommon;
+my ($ro_home, $cfg_path) = setup_public_inboxes;
+use IO::Uncompress::Gunzip qw(gunzip $GunzipError);
+test_lei(sub {
+ my $s = eml_load('t/plack-qp.eml')->as_string;
+ lei_ok [qw(import -q -F eml -)], undef, { 0 => \$s, %$lei_opt };
+ lei_ok qw(q z:0.. -f mboxcl2 -o), "$ENV{HOME}/a.mbox.gz";
+ lei_ok qw(q z:0.. -f mboxcl2 -o), "$ENV{HOME}/b.mbox.gz";
+ lei_ok qw(q z:0.. -f mboxcl2 -o), "$ENV{HOME}/a";
+ lei_ok qw(q z:0.. -f mboxcl2 -o), "$ENV{HOME}/b";
+ lei_ok qw(ls-search);
+ $s = eml_load('t/utf8.eml')->as_string;
+ lei_ok [qw(import -q -F eml -)], undef, { 0 => \$s, %$lei_opt };
+ lei_ok qw(up --all=local);
+ open my $fh, "$ENV{HOME}/a.mbox.gz" or xbail "open: $!";
+ my $gz = do { local $/; <$fh> };
+ my $uc;
+ gunzip(\$gz => \$uc, MultiStream => 1) or xbail "gunzip $GunzipError";
+ open $fh, "$ENV{HOME}/a" or xbail "open: $!";
+
+ my $exp = do { local $/; <$fh> };
+ is($uc, $exp, 'compressed and uncompressed match (a.gz)');
+ like($exp, qr/testmessage\@example.com/, '2nd message added');
+ open $fh, "$ENV{HOME}/b.mbox.gz" or xbail "open: $!";
+
+ $gz = do { local $/; <$fh> };
+ undef $uc;
+ gunzip(\$gz => \$uc, MultiStream => 1) or xbail "gunzip $GunzipError";
+ is($uc, $exp, 'compressed and uncompressed match (b.gz)');
+
+ open $fh, "$ENV{HOME}/b" or xbail "open: $!";
+ $uc = do { local $/; <$fh> };
+ is($uc, $exp, 'uncompressed both match');
+});
+
+done_testing;
^ permalink raw reply related [flat|nested] 9+ messages in thread
* [PATCH 6/8] lei inspect: support reading eml from --stdin
2021-09-03 8:54 [PATCH 0/8] lei: fix IMAP R/W; L/kw false positives Eric Wong
` (4 preceding siblings ...)
2021-09-03 8:54 ` [PATCH 5/8] lei up --all: avoid double-close on shared STDOUT Eric Wong
@ 2021-09-03 8:54 ` Eric Wong
2021-09-03 8:54 ` [PATCH 7/8] lei_xsearch: avoid false-positives on externals w/ L: and kw: Eric Wong
2021-09-03 8:54 ` [PATCH 8/8] lei: fix read/write IMAP access Eric Wong
7 siblings, 0 replies; 9+ messages in thread
From: Eric Wong @ 2021-09-03 8:54 UTC (permalink / raw)
To: meta
This can be useful inside mutt since I was diagnosing why
a label ("L:$FOO") search was giving me a false-positive
search result...
---
lib/PublicInbox/LEI.pm | 4 ++--
lib/PublicInbox/LeiInspect.pm | 40 ++++++++++++++++++++++++++++++-----
2 files changed, 37 insertions(+), 7 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 8b6c1d36..098a45ba 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -279,8 +279,8 @@ our %CMD = ( # sorted in order of importance/use:
'git-config(1) wrapper for '._config_path($_[0]);
}, qw(config-file|system|global|file|f=s), # for conflict detection
qw(c=s@ C=s@), pass_through('git config') ],
-'inspect' => [ 'ITEMS...', 'inspect lei/store and/or local external',
- qw(pretty ascii dir=s), @c_opt ],
+'inspect' => [ 'ITEMS...|--stdin', 'inspect lei/store and/or local external',
+ qw(stdin| pretty ascii dir=s), @c_opt ],
'init' => [ '[DIRNAME]', sub {
"initialize storage, default: ".store_path($_[0]);
diff --git a/lib/PublicInbox/LeiInspect.pm b/lib/PublicInbox/LeiInspect.pm
index 2ade17af..25bd47e7 100644
--- a/lib/PublicInbox/LeiInspect.pm
+++ b/lib/PublicInbox/LeiInspect.pm
@@ -9,6 +9,7 @@ package PublicInbox::LeiInspect;
use strict;
use v5.10.1;
use PublicInbox::Config;
+use PublicInbox::MID qw(mids);
sub inspect_blob ($$) {
my ($lei, $oidhex) = @_;
@@ -184,6 +185,32 @@ sub inspect1 ($$$) {
1;
}
+sub _inspect_argv ($$) {
+ my ($lei, $argv) = @_;
+ my $multi = scalar(@$argv) > 1;
+ $lei->out('[') if $multi;
+ while (defined(my $x = shift @$argv)) {
+ inspect1($lei, $x, scalar(@$argv)) or return;
+ }
+ $lei->out(']') if $multi;
+}
+
+sub ins_add { # InputPipe->consume callback
+ my ($lei) = @_; # $_[1] = $rbuf
+ if (defined $_[1]) {
+ $_[1] eq '' and return eval {
+ my $str = delete $lei->{istr};
+ $str =~ s/\A[\r\n]*From [^\r\n]*\r?\n//s;
+ my $eml = PublicInbox::Eml->new(\$str);
+ _inspect_argv($lei, [ 'blob:'.$lei->git_blob_id($eml),
+ map { "mid:$_" } @{mids($eml)} ]);
+ };
+ $lei->{istr} .= $_[1];
+ } else {
+ $lei->fail("error reading stdin: $!");
+ }
+}
+
sub lei_inspect {
my ($lei, @argv) = @_;
$lei->{json} = ref(PublicInbox::Config::json())->new->utf8->canonical;
@@ -196,12 +223,15 @@ sub lei_inspect {
}
$lei->start_pager if -t $lei->{1};
$lei->{1}->autoflush(0);
- my $multi = scalar(@argv) > 1;
- $lei->out('[') if $multi;
- while (defined(my $x = shift @argv)) {
- inspect1($lei, $x, scalar(@argv)) or return;
+ if ($lei->{opt}->{stdin}) {
+ return $lei->fail(<<'') if @argv;
+no args allowed on command-line with --stdin
+
+ require PublicInbox::InputPipe;
+ PublicInbox::InputPipe::consume($lei->{0}, \&ins_add, $lei);
+ return;
}
- $lei->out(']') if $multi;
+ _inspect_argv($lei, \@argv);
}
sub _complete_inspect {
^ permalink raw reply related [flat|nested] 9+ messages in thread
* [PATCH 7/8] lei_xsearch: avoid false-positives on externals w/ L: and kw:
2021-09-03 8:54 [PATCH 0/8] lei: fix IMAP R/W; L/kw false positives Eric Wong
` (5 preceding siblings ...)
2021-09-03 8:54 ` [PATCH 6/8] lei inspect: support reading eml from --stdin Eric Wong
@ 2021-09-03 8:54 ` Eric Wong
2021-09-03 8:54 ` [PATCH 8/8] lei: fix read/write IMAP access Eric Wong
7 siblings, 0 replies; 9+ messages in thread
From: Eric Wong @ 2021-09-03 8:54 UTC (permalink / raw)
To: meta
We need to use LeiSearch->qparse_new to handle (and filter out)
"L:" and "kw:" search prefixes to avoid hitting false positives
when externals are involved. Unfortunately, this doesn't work
for remote HTTP(S) externals, but those aren't enabled by
default.
---
lib/PublicInbox/LeiXSearch.pm | 1 +
1 file changed, 1 insertion(+)
diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index b9f0d692..b6d7bf2b 100644
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -161,6 +161,7 @@ sub query_one_mset { # for --threads and l2m w/o sort
my ($srch, $over) = ($ibxish->search, $ibxish->over);
my $dir = $ibxish->{inboxdir} // $ibxish->{topdir};
return warn("$dir not indexed by Xapian\n") unless ($srch && $over);
+ bless $srch, 'PublicInbox::LeiSearch'; # for ->qparse_new
my $mo = { %{$lei->{mset_opt}} }; # copy
my $mset;
my $each_smsg = $lei->{ovv}->ovv_each_smsg_cb($lei);
^ permalink raw reply related [flat|nested] 9+ messages in thread
* [PATCH 8/8] lei: fix read/write IMAP access
2021-09-03 8:54 [PATCH 0/8] lei: fix IMAP R/W; L/kw false positives Eric Wong
` (6 preceding siblings ...)
2021-09-03 8:54 ` [PATCH 7/8] lei_xsearch: avoid false-positives on externals w/ L: and kw: Eric Wong
@ 2021-09-03 8:54 ` Eric Wong
7 siblings, 0 replies; 9+ messages in thread
From: Eric Wong @ 2021-09-03 8:54 UTC (permalink / raw)
To: meta
xt/net_writer-imap.t was completely broken in recent months and
I completely forgot this test. net->add_url still only accepts
bare scalars (and not scalar refs), so we must set that up
properly. Furthermore, our changes to do FLAGS-only
synchronization in lei of old messages was causing us to not
handle FLAGS properly for the test.
---
lib/PublicInbox/LeiToMail.pm | 2 +-
lib/PublicInbox/NetReader.pm | 5 ++++-
lib/PublicInbox/NetWriter.pm | 2 ++
lib/PublicInbox/Watch.pm | 2 ++
4 files changed, 9 insertions(+), 2 deletions(-)
diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm
index be6e178f..6e102a1d 100644
--- a/lib/PublicInbox/LeiToMail.pm
+++ b/lib/PublicInbox/LeiToMail.pm
@@ -406,7 +406,7 @@ sub new {
my $net = PublicInbox::NetWriter->new;
$net->{quiet} = $lei->{opt}->{quiet};
my $uri = PublicInbox::URIimap->new($dst)->canonical;
- $net->add_url($uri);
+ $net->add_url($$uri);
my $err = $net->errors($lei);
return $lei->fail($err) if $err;
$uri->mailbox or return $lei->fail("No mailbox: $dst");
diff --git a/lib/PublicInbox/NetReader.pm b/lib/PublicInbox/NetReader.pm
index 23445e7a..c050c60f 100644
--- a/lib/PublicInbox/NetReader.pm
+++ b/lib/PublicInbox/NetReader.pm
@@ -493,6 +493,9 @@ sub perm_fl_ok ($) {
undef;
}
+# may be overridden in NetWriter or Watch
+sub folder_select { $_[0]->{each_old} ? 'select' : 'examine' }
+
sub _imap_fetch_all ($$$) {
my ($self, $mic, $orig_uri) = @_;
my $sec = uri_section($orig_uri);
@@ -501,7 +504,7 @@ sub _imap_fetch_all ($$$) {
# we need to check for mailbox writability to see if we care about
# FLAGS from already-imported messages.
- my $cmd = $self->{each_old} ? 'select' : 'examine';
+ my $cmd = $self->folder_select;
$mic->$cmd($mbx) or return "E: \U$cmd\E $mbx ($sec) failed: $!";
my ($r_uidval, $r_uidnext, $perm_fl);
diff --git a/lib/PublicInbox/NetWriter.pm b/lib/PublicInbox/NetWriter.pm
index 82288e6b..629a752a 100644
--- a/lib/PublicInbox/NetWriter.pm
+++ b/lib/PublicInbox/NetWriter.pm
@@ -26,6 +26,8 @@ sub imap_append {
die "APPEND $folder: $@";
}
+sub folder_select { 'select' } # for PublicInbox::NetReader
+
sub imap_delete_all {
my ($self, $uri) = @_;
my $mic = $self->mic_for_folder($uri) or return;
diff --git a/lib/PublicInbox/Watch.pm b/lib/PublicInbox/Watch.pm
index 86dae91f..482d35c4 100644
--- a/lib/PublicInbox/Watch.pm
+++ b/lib/PublicInbox/Watch.pm
@@ -682,4 +682,6 @@ EOF
undef;
}
+sub folder_select { 'select' } # for PublicInbox::NetReader
+
1;
^ permalink raw reply related [flat|nested] 9+ messages in thread