* [PATCH 1/3] lei p2q: patch-to-query generator for "lei q --stdin"
2021-02-28 12:25 [PATCH 0/3] lei p2q (patch-to-query) Eric Wong
@ 2021-02-28 12:25 ` Eric Wong
2021-02-28 21:40 ` Kyle Meyer
2021-02-28 12:25 ` [PATCH 2/3] lei q: fix "-" shortcut for --stdin Eric Wong
2021-02-28 12:25 ` [PATCH 3/3] lei q: improve early aborts w/ remote externals Eric Wong
2 siblings, 1 reply; 6+ messages in thread
From: Eric Wong @ 2021-02-28 12:25 UTC (permalink / raw)
To: meta
Instead of teaching the to-be-implemented "lei show" to search
threads/messages based commits, this orthogonal sub-command is
designed to generate queries for use with "lei q --stdin".
URI-escaped query parameters may be generated with --uri for
HTTP(S) public-inbox instances, but otherwise the output is
designed for "lei q --stdin".
To find threads for a given git commit from a git worktree:
lei p2q $COMMIT_OID | lei q --stdin -t ...
It can also read via --stdin|-
curl $INBOX_URL/$MSGID/raw | lei p2q - | lei q --stdin -t
Or from the filesystem:
lei p2q $(git format-patch -1) | lei q --stdin -t
This defaults to only generating "dfpost:"-prefixed terms since
I've found those most useful for finding messages relating to a
commit. This is subject to change.
--want=s@ is a comma-separated or multi-value list of prefixes
that defaults to "dfpost7". Not all are implemented, yet, but
s, dfn, dfpre, and dfpost all seem to mostly work. Phrase
handling may need to be tweaked to work with Xapian.
OR, NEAR, ADJ, AND, NOT may be used with --want
(e.g. --want=dfpost,OR,dfn)
Prefixing the field prefix with '+' or '-' (e.g. --want=+dfpost)
generates "+dfpost:$EXTRACTED_OID" for Xapian. For non-boolean
search prefixes, wildcard (*) may also be supplied: (--want=dfn*)
For boolean search prefixes, suffixing the field prefix with a
digit (e.g. --want=dfpost7) provides a minimum length, allowing
truncated variations to be searched. This is helpful for
finding older messages as git chooses longer dfpost|dfpre
abbreviations as repos get larger.
Automatic date range generation is not implemented, yet.
---
MANIFEST | 2 +
lib/PublicInbox/LEI.pm | 30 +++++-
lib/PublicInbox/LeiP2q.pm | 197 ++++++++++++++++++++++++++++++++++++++
t/lei-p2q.t | 29 ++++++
4 files changed, 257 insertions(+), 1 deletion(-)
create mode 100644 lib/PublicInbox/LeiP2q.pm
create mode 100644 t/lei-p2q.t
diff --git a/MANIFEST b/MANIFEST
index 11ec5c01..5044e21c 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -187,6 +187,7 @@ lib/PublicInbox/LeiHelp.pm
lib/PublicInbox/LeiImport.pm
lib/PublicInbox/LeiMirror.pm
lib/PublicInbox/LeiOverview.pm
+lib/PublicInbox/LeiP2q.pm
lib/PublicInbox/LeiQuery.pm
lib/PublicInbox/LeiSearch.pm
lib/PublicInbox/LeiStore.pm
@@ -373,6 +374,7 @@ t/lei-import-maildir.t
t/lei-import-nntp.t
t/lei-import.t
t/lei-mirror.t
+t/lei-p2q.t
t/lei-q-remote-import.t
t/lei-q-thread.t
t/lei.t
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 0da24499..a2f8ffe7 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -179,6 +179,9 @@ our %CMD = ( # sorted in order of importance/use:
qw(stdin| in-format|F=s out-format|f=s output|mfolder|o=s quiet|q
lock=s@ kw|keywords|flags! C=s@),
],
+'p2q' => [ 'FILE|COMMIT_OID|--stdin',
+ "use a patch to generate a query for `lei q --stdin'",
+ qw(stdin| want|w=s@ uri debug) ],
'config' => [ '[...]', sub {
'git-config(1) wrapper for '._config_path($_[0]);
}, qw(config-file|system|global|file|f=s), # for conflict detection
@@ -238,6 +241,10 @@ my %OPTDESC = (
'show threads|t' => 'display entire thread a message belongs to',
'q threads|t+' =>
'return all messages in the same threads as the actual match(es)',
+
+'want|w=s@' => [ 'PREFIX|dfpost|dfn', # common ones in help...
+ 'search prefixes to extract (default: dfpost7)' ],
+
'alert=s@' => ['CMD,:WINCH,:bell,<any command>',
'run command(s) or perform ops when done writing to output ' .
'(default: ":WINCH,:bell" with --mua and Maildir/IMAP output, ' .
@@ -331,7 +338,7 @@ my %CONFIG_KEYS = (
'leistore.dir' => 'top-level storage location',
);
-my @WQ_KEYS = qw(lxs l2m imp mrr cnv); # internal workers
+my @WQ_KEYS = qw(lxs l2m imp mrr cnv p2q); # internal workers
# pronounced "exit": x_it(1 << 8) => exit(1); x_it(13) => SIGPIPE
sub x_it ($$) {
@@ -673,6 +680,11 @@ sub lei_convert {
PublicInbox::LeiConvert->call(@_);
}
+sub lei_p2q {
+ require PublicInbox::LeiP2q;
+ PublicInbox::LeiP2q->call(@_);
+}
+
sub lei_init {
my ($self, $dir) = @_;
my $cfg = _lei_cfg($self, 1);
@@ -854,6 +866,22 @@ sub poke_mua { # forces terminal MUAs to wake up and hopefully notice new mail
}
}
+my %path_to_fd = ('/dev/stdin' => 0, '/dev/stdout' => 1, '/dev/stderr' => 2);
+$path_to_fd{"/dev/fd/$_"} = $path_to_fd{"/proc/self/fd/$_"} for (0..2);
+sub fopen {
+ my ($self, $mode, $path) = @_;
+ rel2abs($self, $path);
+ $path =~ tr!/!/!s;
+ if (defined(my $fd = $path_to_fd{$path})) {
+ return $self->{$fd};
+ }
+ if ($path =~ m!\A/(?:dev|proc/self)/fd/[0-9]+\z!) {
+ return fail($self, "cannot open $path from daemon");
+ }
+ open my $fh, $mode, $path or return;
+ $fh;
+}
+
# caller needs to "-t $self->{1}" to check if tty
sub start_pager {
my ($self) = @_;
diff --git a/lib/PublicInbox/LeiP2q.pm b/lib/PublicInbox/LeiP2q.pm
new file mode 100644
index 00000000..d1dd125e
--- /dev/null
+++ b/lib/PublicInbox/LeiP2q.pm
@@ -0,0 +1,197 @@
+# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# front-end for the "lei patch-to-query" sub-command
+package PublicInbox::LeiP2q;
+use strict;
+use v5.10.1;
+use parent qw(PublicInbox::IPC);
+use PublicInbox::Eml;
+use PublicInbox::Smsg;
+use PublicInbox::MsgIter qw(msg_part_text);
+use PublicInbox::Git qw(git_unquote);
+use PublicInbox::Spawn qw(popen_rd);
+use URI::Escape qw(uri_escape_utf8);
+
+sub xphrase ($) {
+ my ($s) = @_;
+ return () unless $s =~ /\S/;
+ # cf. xapian-core/queryparser/queryparser.lemony
+ # [\./:\\\@] - is_phrase_generator (implicit phrase search)
+ # FIXME not really sure about these..., we basically want to
+ # extract the longest phrase possible that Xapian can handle
+ map {
+ s/\A\s*//;
+ s/\s+\z//;
+ /[\|=><,\sA-Z]/ && !m![\./:\\\@]! ? qq("$_") : $_;
+ } ($s =~ m!(\w[\|=><,\./:\\\@\-\w\s]+)!g);
+}
+
+sub extract_terms { # eml->each_part callback
+ my ($p, $lei) = @_;
+ my $part = $p->[0]; # ignore $depth and @idx;
+ my $ct = $part->content_type || 'text/plain';
+ my ($s, undef) = msg_part_text($part, $ct);
+ defined $s or return;
+ my $in_diff;
+ # TODO: b: nq: q:
+ for (split(/\n/, $s)) {
+ if ($in_diff && s/^ //) { # diff context
+ push @{$lei->{qterms}->{dfctx}}, xphrase($_);
+ } elsif (/^-- $/) { # email signature begins
+ $in_diff = undef;
+ } elsif (m!^diff --git "?[^/]+/.+ "?[^/]+/.+\z!) {
+ # wait until "---" and "+++" to capture filenames
+ $in_diff = 1;
+ } elsif (/^index ([a-f0-9]+)\.\.([a-f0-9]+)\b/) {
+ my ($oa, $ob) = ($1, $2);
+ push @{$lei->{qterms}->{dfpre}}, $oa;
+ push @{$lei->{qterms}->{dfpost}}, $ob;
+ # who uses dfblob?
+ } elsif (m!^(?:---|\+{3}) ("?[^/]+/.+)!) {
+ my $fn = (split(m!/!, git_unquote($1.''), 2))[1];
+ push @{$lei->{qterms}->{dfn}}, xphrase($fn);
+ } elsif ($in_diff && s/^\+//) { # diff added
+ push @{$lei->{qterms}->{dfb}}, xphrase($_);
+ } elsif ($in_diff && s/^-//) { # diff removed
+ push @{$lei->{qterms}->{dfa}}, xphrase($_);
+ } elsif (/^@@ (?:\S+) (?:\S+) @@\s*(\S+.*)/) {
+ push @{$lei->{qterms}->{dfhh}}, xphrase($1);
+ } elsif (/^(?:dis)similarity index/ ||
+ /^(?:old|new) mode/ ||
+ /^(?:deleted|new) file mode/ ||
+ /^(?:copy|rename) (?:from|to) / ||
+ /^(?:dis)?similarity index / ||
+ /^\\ No newline at end of file/ ||
+ /^Binary files .* differ/) {
+ } elsif ($_ eq '') {
+ # possible to be in diff context, some mail may be
+ # stripped by MUA or even GNU diff(1). "git apply"
+ # treats a bare "\n" as diff context, too
+ } else {
+ $in_diff = undef;
+ }
+ }
+}
+
+my %pfx2smsg = (
+ t => [ qw(to) ],
+ c => [ qw(cc) ],
+ f => [ qw(from) ],
+ tc => [ qw(to cc) ],
+ tcf => [ qw(to cc from) ],
+ a => [ qw(to cc from) ],
+ s => [ qw(subject) ],
+ bs => [ qw(subject) ], # body handled elsewhere
+ d => [ qw(ds) ], # nonsense?
+ dt => [ qw(ds) ], # ditto...
+ rt => [ qw(ts) ], # ditto...
+);
+
+sub do_p2q { # via wq_do
+ my ($self) = @_;
+ my $lei = $self->{lei};
+ my $want = $lei->{opt}->{want} // [ qw(dfpost7) ];
+ my @want = split(/[, ]+/, "@$want");
+ for (@want) {
+ /\A(?:(d|dt|rt):)?([0-9]+)(\.(?:day|weeks)s?)?\z/ or next;
+ my ($pfx, $n, $unit) = ($1, $2, $3);
+ $n *= 86400 * ($unit =~ /week/i ? 7 : 1);
+ $_ = [ $pfx, $n ];
+ }
+ my $smsg = bless {}, 'PublicInbox::Smsg';
+ my $in = $self->{0};
+ unless ($in) {
+ my $input = $self->{input};
+ if (-e $input) {
+ $in = $lei->fopen('<', $input) or
+ return $lei->fail("open < $input: $!");
+ } else {
+ my @cmd = (qw(git format-patch --stdout -1), $input);
+ $in = popen_rd(\@cmd, undef, { 2 => $lei->{2} });
+ }
+ };
+ my $eml = PublicInbox::Eml->new(\(do { local $/; <$in> }));
+ $lei->{diff_want} = +{ map { $_ => 1 } @want };
+ $smsg->populate($eml);
+ while (my ($pfx, $fields) = each %pfx2smsg) {
+ next unless $lei->{diff_want}->{$pfx};
+ for my $f (@$fields) {
+ my $v = $smsg->{$f} // next;
+ push @{$lei->{qterms}->{$pfx}}, xphrase($v);
+ }
+ }
+ $eml->each_part(\&extract_terms, $lei, 1);
+ if ($lei->{opt}->{debug}) {
+ my $json = ref(PublicInbox::Config->json)->new;
+ $json->utf8->canonical->pretty;
+ $lei->err($json->encode($lei->{qterms}));
+ }
+ my (@q, %seen);
+ for my $pfx (@want) {
+ if (ref($pfx) eq 'ARRAY') {
+ my ($p, $t_range) = @$pfx; # TODO
+
+ } elsif ($pfx =~ m!\A(?:OR|XOR|AND|NOT)\z! ||
+ $pfx =~ m!\A(?:ADJ|NEAR)(?:/[0-9]+)?\z!) {
+ push @q, $pfx;
+ } else {
+ my $plusminus = ($pfx =~ s/\A([\+\-])//) ? $1 : '';
+ my $end = ($pfx =~ s/([0-9\*]+)\z//) ? $1 : '';
+ my $x = delete($lei->{qterms}->{$pfx}) or next;
+ my $star = $end =~ tr/*//d ? '*' : '';
+ my $min_len = ($end // 0) + 0;
+
+ # no wildcards for bool_pfx_external
+ $star = '' if $pfx =~ /\A(dfpre|dfpost|mid)\z/;
+ $pfx = "$plusminus$pfx:";
+ if ($min_len) {
+ push @q, map {
+ my @t = ($pfx.$_.$star);
+ while (length > $min_len) {
+ chop $_;
+ push @t, 'OR', $pfx.$_.$star;
+ }
+ @t;
+ } @$x;
+ } else {
+ push @q, map {
+ my $k = $pfx.$_.$star;
+ $seen{$k}++ ? () : $k
+ } @$x;
+ }
+ }
+ }
+ if ($lei->{opt}->{uri}) {
+ @q = (join('+', map { uri_escape_utf8($_) } @q));
+ } else {
+ @q = (join(' ', @q));
+ }
+ $lei->out(@q, "\n");
+}
+
+sub call { # the "lei patch-to-query" entry point
+ my ($cls, $lei, $input) = @_;
+ my $self = $lei->{p2q} = bless {}, $cls;
+ if ($lei->{opt}->{stdin}) {
+ $self->{0} = delete $lei->{0}; # guard from lei_atfork_child
+ } else {
+ $self->{input} = $input;
+ }
+ my $op = $lei->workers_start($self, 'lei patch2query', 1, {
+ '' => [ $lei->{p2q_done} // $lei->can('dclose'), $lei ]
+ });
+ $self->wq_io_do('do_p2q', []);
+ $self->wq_close(1);
+ while ($op && $op->{sock}) { $op->event_step }
+}
+
+sub ipc_atfork_child {
+ my ($self) = @_;
+ my $lei = $self->{lei};
+ $lei->lei_atfork_child;
+ $SIG{__WARN__} = PublicInbox::Eml::warn_ignore_cb();
+ $self->SUPER::ipc_atfork_child;
+}
+
+1;
diff --git a/t/lei-p2q.t b/t/lei-p2q.t
new file mode 100644
index 00000000..1a2c2e4f
--- /dev/null
+++ b/t/lei-p2q.t
@@ -0,0 +1,29 @@
+#!perl -w
+# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+use strict; use v5.10.1; use PublicInbox::TestCommon;
+require_git 2.6;
+require_mods(qw(json DBD::SQLite Search::Xapian));
+
+test_lei(sub {
+ lei_ok(qw(p2q -w dfpost t/data/0001.patch));
+ is($lei_out, "dfpost:6e006fd73b1d\n", 'pathname');
+ open my $fh, '+<', 't/data/0001.patch';
+ lei_ok([qw(p2q -w dfpost -)], undef, { %$lei_opt, 0 => $fh });
+ is($lei_out, "dfpost:6e006fd73b1d\n", '--stdin');
+
+ lei_ok(qw(p2q --uri t/data/0001.patch -w), 'dfpost,dfn');
+ is($lei_out, "dfpost%3A6e006fd73b1d+".
+ "dfn%3Alib%2FPublicInbox%2FSearch.pm\n",
+ '--uri -w dfpost,dfn');
+ lei_ok(qw(p2q t/data/0001.patch), '--want=dfpost,OR,dfn');
+ is($lei_out, "dfpost:6e006fd73b1d OR dfn:lib/PublicInbox/Search.pm\n",
+ '--want=OR');
+ lei_ok(qw(p2q t/data/0001.patch --want=dfpost9));
+ is($lei_out, "dfpost:6e006fd73b1d OR " .
+ "dfpost:6e006fd73b1 OR " .
+ "dfpost:6e006fd73b OR " .
+ "dfpost:6e006fd73\n",
+ '3-byte chop');
+});
+done_testing;
^ permalink raw reply related [flat|nested] 6+ messages in thread* [PATCH 3/3] lei q: improve early aborts w/ remote externals
2021-02-28 12:25 [PATCH 0/3] lei p2q (patch-to-query) Eric Wong
2021-02-28 12:25 ` [PATCH 1/3] lei p2q: patch-to-query generator for "lei q --stdin" Eric Wong
2021-02-28 12:25 ` [PATCH 2/3] lei q: fix "-" shortcut for --stdin Eric Wong
@ 2021-02-28 12:25 ` Eric Wong
2 siblings, 0 replies; 6+ messages in thread
From: Eric Wong @ 2021-02-28 12:25 UTC (permalink / raw)
To: meta
We must issue LeiStore->done if a client disconnects
while we're streaming from a remote external. This
can happen via SIGPIPE, or if a client process is
interrupted by any other means.
---
lib/PublicInbox/LEI.pm | 3 +++
lib/PublicInbox/LeiImport.pm | 3 ++-
lib/PublicInbox/LeiXSearch.pm | 4 +--
t/lei-externals.t | 49 ++++++++++++++++++++++++++++++-----
4 files changed, 50 insertions(+), 9 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index f5e42869..834e399f 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -970,6 +970,9 @@ sub dclose {
}
}
close(delete $self->{1}) if $self->{1}; # may reap_compress
+ if (my $sto = delete $self->{sto}) {
+ $sto->ipc_do('done');
+ }
$self->close if $self->{sock}; # PublicInbox::DS::close
}
diff --git a/lib/PublicInbox/LeiImport.pm b/lib/PublicInbox/LeiImport.pm
index c2c98030..23cecd53 100644
--- a/lib/PublicInbox/LeiImport.pm
+++ b/lib/PublicInbox/LeiImport.pm
@@ -18,7 +18,8 @@ sub import_done_wait { # dwaitpid callback
my ($arg, $pid) = @_;
my ($imp, $lei) = @$arg;
$lei->child_error($?, 'non-fatal errors during import') if $?;
- my $ign = $lei->{sto}->ipc_do('done'); # PublicInbox::LeiStore::done
+ my $sto = delete $lei->{sto};
+ my $wait = $sto->ipc_do('done') if $sto; # PublicInbox::LeiStore::done
$lei->dclose;
}
diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index 9a6457d7..d4607e16 100644
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -349,7 +349,7 @@ Error closing $lei->{ovv}->{dst}: $!
sub do_post_augment {
my ($lei) = @_;
- my $l2m = $lei->{l2m} or die 'BUG: unexpected do_post_augment';
+ my $l2m = $lei->{l2m} or return; # client disconnected
my $err;
eval { $l2m->post_augment($lei) };
$err = $@;
@@ -368,7 +368,7 @@ sub do_post_augment {
sub incr_post_augment { # called whenever an l2m shard finishes augment
my ($lei) = @_;
- my $l2m = $lei->{l2m} or die 'BUG: unexpected incr_post_augment';
+ my $l2m = $lei->{l2m} or return; # client disconnected
return if ++$lei->{nr_post_augment} != $l2m->{-wq_nr_workers};
do_post_augment($lei);
}
diff --git a/t/lei-externals.t b/t/lei-externals.t
index d422a9d1..b78b5580 100644
--- a/t/lei-externals.t
+++ b/t/lei-externals.t
@@ -6,7 +6,8 @@ use Fcntl qw(SEEK_SET);
use PublicInbox::Spawn qw(which);
use PublicInbox::OnDestroy;
require_git 2.6;
-require_mods(qw(DBD::SQLite Search::Xapian));
+require_mods(qw(json DBD::SQLite Search::Xapian));
+use POSIX qw(WTERMSIG WIFSIGNALED SIGPIPE);
my @onions = qw(http://hjrcffqmbrq6wope.onion/meta/
http://czquwvybam4bgbro.onion/meta/
@@ -15,19 +16,55 @@ my @onions = qw(http://hjrcffqmbrq6wope.onion/meta/
my $test_external_remote = sub {
my ($url, $k) = @_;
SKIP: {
- my $nr = 5;
- skip "$k unset", $nr if !$url;
- which('curl') or skip 'no curl', $nr;
- which('torsocks') or skip 'no torsocks', $nr if $url =~ m!\.onion/!;
+ skip "$k unset", 1 if !$url;
+ state $curl = which('curl');
+ $curl or skip 'no curl', 1;
+ which('torsocks') or skip 'no torsocks', 1 if $url =~ m!\.onion/!;
my $mid = '20140421094015.GA8962@dcvr.yhbt.net';
my @cmd = ('q', '--only', $url, '-q', "m:$mid");
lei_ok(@cmd, \"query $url");
is($lei_err, '', "no errors on $url");
my $res = json_utf8->decode($lei_out);
- is($res->[0]->{'m'}, "<$mid>", "got expected mid from $url");
+ is($res->[0]->{'m'}, "<$mid>", "got expected mid from $url") or
+ skip 'further remote tests', 1;
lei_ok(@cmd, 'd:..20101002', \'no results, no error');
is($lei_err, '', 'no output on 404, matching local FS behavior');
is($lei_out, "[null]\n", 'got null results');
+ my ($pid_before, $pid_after);
+ if (-d $ENV{XDG_RUNTIME_DIR} && -w _) {
+ lei_ok 'daemon-pid';
+ chomp($pid_before = $lei_out);
+ ok($pid_before, 'daemon is live');
+ }
+ for my $out ([], [qw(-f mboxcl2)]) {
+ pipe(my ($r, $w)) or BAIL_OUT $!;
+ open my $err, '+>', undef or BAIL_OUT $!;
+ my $opt = { run_mode => 0, 1 => $w, 2 => $err };
+ my $cmd = [qw(lei q -qt), @$out, 'bytes:1..'];
+ my $tp = start_script($cmd, undef, $opt);
+ close $w;
+ sysread($r, my $buf, 1);
+ close $r; # trigger SIGPIPE
+ $tp->join;
+ ok(WIFSIGNALED($?), "signaled @$out");
+ is(WTERMSIG($?), SIGPIPE, "got SIGPIPE @$out");
+ seek($err, 0, 0);
+ my @err = grep(!m{mkdir .*sun_path\b}, <$err>);
+ is_deeply(\@err, [], "no errors @$out");
+ }
+ if (-d $ENV{XDG_RUNTIME_DIR} && -w _) {
+ lei_ok 'daemon-pid';
+ chomp(my $pid_after = $lei_out);
+ is($pid_after, $pid_before, 'pid unchanged') or
+ skip 'daemon died', 1;
+ lei_ok 'daemon-kill';
+ my $alive = 1;
+ for (1..100) {
+ $alive = kill(0, $pid_after) or last;
+ tick();
+ }
+ ok(!$alive, 'daemon-kill worked');
+ }
} # /SKIP
}; # /sub
^ permalink raw reply related [flat|nested] 6+ messages in thread