* [PATCH 11/11] lei q: support reading queries from stdin
2021-02-03 8:11 7% [PATCH 00/11] lei q --stdin, shortcut names, etc Eric Wong
@ 2021-02-03 8:11 4% ` Eric Wong
0 siblings, 0 replies; 2+ results
From: Eric Wong @ 2021-02-03 8:11 UTC (permalink / raw)
To: meta
This will be useful on shared machines when a user doesn't want
search queries visible to other users looking at the ps(1)
output or similar.
---
MANIFEST | 1 +
lib/PublicInbox/InputPipe.pm | 37 ++++++++++++++++++++++++++++++++++
lib/PublicInbox/LEI.pm | 7 ++++---
lib/PublicInbox/LeiOverview.pm | 1 -
lib/PublicInbox/LeiQuery.pm | 32 ++++++++++++++++++++++-------
lib/PublicInbox/LeiXSearch.pm | 2 ++
t/lei.t | 19 +++++++++++++++++
7 files changed, 88 insertions(+), 11 deletions(-)
create mode 100644 lib/PublicInbox/InputPipe.pm
diff --git a/MANIFEST b/MANIFEST
index bcb9d08e..6922f9b1 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -173,6 +173,7 @@ lib/PublicInbox/In2Tie.pm
lib/PublicInbox/Inbox.pm
lib/PublicInbox/InboxIdle.pm
lib/PublicInbox/InboxWritable.pm
+lib/PublicInbox/InputPipe.pm
lib/PublicInbox/Isearch.pm
lib/PublicInbox/KQNotify.pm
lib/PublicInbox/LEI.pm
diff --git a/lib/PublicInbox/InputPipe.pm b/lib/PublicInbox/InputPipe.pm
new file mode 100644
index 00000000..a8bdf031
--- /dev/null
+++ b/lib/PublicInbox/InputPipe.pm
@@ -0,0 +1,37 @@
+# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# for reading pipes and sockets off the DS event loop
+package PublicInbox::InputPipe;
+use strict;
+use v5.10.1;
+use parent qw(PublicInbox::DS);
+use PublicInbox::Syscall qw(EPOLLIN EPOLLET);
+
+sub consume {
+ my ($in, $cb, @args) = @_;
+ my $self = bless { cb => $cb, sock => $in, args => \@args },__PACKAGE__;
+ if ($PublicInbox::DS::in_loop) {
+ eval { $self->SUPER::new($in, EPOLLIN|EPOLLET) };
+ return $in->blocking(0) unless $@; # regular file sets $@
+ }
+ event_step($self) while $self->{sock};
+}
+
+sub event_step {
+ my ($self) = @_;
+ my ($r, $rbuf);
+ while (($r = sysread($self->{sock}, $rbuf, 65536))) {
+ $self->{cb}->(@{$self->{args} // []}, $rbuf);
+ }
+ if (defined($r)) { # EOF
+ $self->{cb}->(@{$self->{args} // []}, '');
+ } elsif ($!{EAGAIN}) {
+ return;
+ } else {
+ $self->{cb}->(@{$self->{args} // []}, undef)
+ }
+ $self->{sock}->blocking ? delete($self->{sock}) : $self->close
+}
+
+1;
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 28dce0c5..49deed13 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -101,10 +101,10 @@ sub _config_path ($) {
# TODO: generate shell completion + help using %CMD and %OPTDESC
# command => [ positional_args, 1-line description, Getopt::Long option spec ]
our %CMD = ( # sorted in order of importance/use:
-'q' => [ 'SEARCH_TERMS...', 'search for messages matching terms', qw(
+'q' => [ '--stdin|SEARCH_TERMS...', 'search for messages matching terms', qw(
save-as=s output|mfolder|o=s format|f=s dedupe|d=s thread|t augment|a
sort|s=s reverse|r offset=i remote! local! external! pretty
- include|I=s@ exclude=s@ only=s@ jobs|j=s globoff|g
+ include|I=s@ exclude=s@ only=s@ jobs|j=s globoff|g stdin|
mua-cmd|mua=s no-torsocks torsocks=s verbose|v quiet|q
received-after=s received-before=s sent-after=s sent-since=s),
PublicInbox::LeiQuery::curl_opt(), opt_dash('limit|n=i', '[0-9]+') ],
@@ -554,12 +554,13 @@ sub optparse ($$$) {
} elsif ($var =~ /\A\[-?$POS_ARG\]\z/) { # one optional arg
$i++;
} elsif ($var =~ /\A.+?\|/) { # required FOO|--stdin
+ $inf = 1 if index($var, '...') > 0;
my @or = split(/\|/, $var);
my $ok;
for my $o (@or) {
if ($o =~ /\A--([a-z0-9\-]+)/) {
$ok = defined($OPT->{$1});
- last;
+ last if $ok;
} elsif (defined($argv->[$i])) {
$ok = 1;
$i++;
diff --git a/lib/PublicInbox/LeiOverview.pm b/lib/PublicInbox/LeiOverview.pm
index 88034ada..e33d63a2 100644
--- a/lib/PublicInbox/LeiOverview.pm
+++ b/lib/PublicInbox/LeiOverview.pm
@@ -81,7 +81,6 @@ sub new {
my ($isatty, $seekable);
if ($dst eq '/dev/stdout') {
$isatty = -t $lei->{1};
- $lei->start_pager if $isatty;
$opt->{pretty} //= $isatty;
if (!$isatty && -f _) {
my $fl = fcntl($lei->{1}, F_GETFL, 0) //
diff --git a/lib/PublicInbox/LeiQuery.pm b/lib/PublicInbox/LeiQuery.pm
index 8015ecec..4fe40400 100644
--- a/lib/PublicInbox/LeiQuery.pm
+++ b/lib/PublicInbox/LeiQuery.pm
@@ -12,6 +12,16 @@ sub prep_ext { # externals_each callback
$lxs->prepare_external($loc) unless $exclude->{$loc};
}
+sub qstr_add { # for --stdin
+ my ($self) = @_; # $_[1] = $rbuf
+ if (defined($_[1])) {
+ return eval { $self->{lxs}->do_query($self) } if $_[1] eq '';
+ $self->{mset_opt}->{qstr} .= $_[1];
+ } else {
+ $self->fail("error reading stdin: $!");
+ }
+}
+
# the main "lei q SEARCH_TERMS" method
sub lei_q {
my ($self, @argv) = @_;
@@ -84,12 +94,6 @@ sub lei_q {
my %mset_opt = map { $_ => $opt->{$_} } qw(thread limit offset);
$mset_opt{asc} = $opt->{'reverse'} ? 1 : 0;
$mset_opt{limit} //= 10000;
- $mset_opt{qstr} = join(' ', map {;
- # Consider spaces in argv to be for phrase search in Xapian.
- # In other words, the users should need only care about
- # normal shell quotes and not have to learn Xapian quoting.
- /\s/ ? (s/\A(\w+:)// ? qq{$1"$_"} : qq{"$_"}) : $_
- } @argv);
if (defined(my $sort = $opt->{'sort'})) {
if ($sort eq 'relevance') {
$mset_opt{relevance} = 1;
@@ -104,7 +108,21 @@ sub lei_q {
# descending docid order
$mset_opt{relevance} //= -2 if $opt->{thread};
$self->{mset_opt} = \%mset_opt;
- $self->{ovv}->ovv_begin($self);
+
+ if ($opt->{stdin}) {
+ return $self->fail(<<'') if @argv;
+no query allowed on command-line with --stdin
+
+ require PublicInbox::InputPipe;
+ PublicInbox::InputPipe::consume($self->{0}, \&qstr_add, $self);
+ return;
+ }
+ # Consider spaces in argv to be for phrase search in Xapian.
+ # In other words, the users should need only care about
+ # normal shell quotes and not have to learn Xapian quoting.
+ $mset_opt{qstr} = join(' ', map {;
+ /\s/ ? (s/\A(\w+:)// ? qq{$1"$_"} : qq{"$_"}) : $_
+ } @argv);
$lxs->do_query($self);
}
diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index d33064bb..965617b5 100644
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -402,6 +402,8 @@ sub sigpipe_handler { # handles SIGPIPE from l2m/lxs workers
sub do_query {
my ($self, $lei) = @_;
$lei->{1}->autoflush(1);
+ $lei->start_pager if -t $lei->{1};
+ $lei->{ovv}->ovv_begin($lei);
my ($au_done, $zpipe);
my $l2m = $lei->{l2m};
if ($l2m) {
diff --git a/t/lei.t b/t/lei.t
index 03bbb078..01eed1da 100644
--- a/t/lei.t
+++ b/t/lei.t
@@ -275,6 +275,25 @@ my $test_external = sub {
my $pretty = $json->decode($out);
is_deeply($res, $pretty, '--pretty is identical after decode');
+ {
+ open my $fh, '+>', undef or BAIL_OUT $!;
+ $fh->autoflush(1);
+ print $fh 's:use' or BAIL_OUT $!;
+ seek($fh, 0, SEEK_SET) or BAIL_OUT $!;
+ ok($lei->([qw(q -q --stdin)], undef, { %$opt, 0 => $fh }),
+ '--stdin on regular file works');
+ like($out, qr/use boolean prefix/, '--stdin on regular file');
+ }
+ {
+ pipe(my ($r, $w)) or BAIL_OUT $!;
+ print $w 's:use' or BAIL_OUT $!;
+ close $w or BAIL_OUT $!;
+ ok($lei->([qw(q -q --stdin)], undef, { %$opt, 0 => $r }),
+ '--stdin on pipe file works');
+ like($out, qr/use boolean prefix/, '--stdin on pipe');
+ }
+ ok(!$lei->(qw(q -q --stdin s:use)), "--stdin and argv don't mix");
+
for my $fmt (qw(ldjson ndjson jsonl)) {
$lei->('q', '-f', $fmt, 's:use boolean prefix');
is($out, $json->encode($pretty->[0])."\n", "-f $fmt");
^ permalink raw reply related [relevance 4%]
* [PATCH 00/11] lei q --stdin, shortcut names, etc
@ 2021-02-03 8:11 7% Eric Wong
2021-02-03 8:11 4% ` [PATCH 11/11] lei q: support reading queries from stdin Eric Wong
0 siblings, 1 reply; 2+ results
From: Eric Wong @ 2021-02-03 8:11 UTC (permalink / raw)
To: meta
Since externals tend to have common URL or pathname prefixes,
it's now possible to use -I/--only/--exclude with just the
basename of a URL or directory if that's unambiguous.
Wildcard matches are also supported with -I/--only/--exclude.
forget-external still requires the full path, but that's
rarely-used.
add-external bash completion now supports URL hostnames
and common base names.
"lei q" also supports reading queries from stdin.
FD use is slightly reduced, but still far from ideal
(it's bad when I have to bump "ulimit -n" to reattach
screen(1) while I'm running stress tests).
Eric Wong (11):
lei: reduce FD pressure from lei2mail worker
lei: further reduce lei2mail FD pressure
pkt_op: rely on DS::in_loop global
lei: err: avoid uninitialized variable warnings
lei: propagate curl errors, improve internal consistency
lei q: -I/--exclude/--only support globs and basenames
lei: complete basenames for include|exclude|only
lei: help starts pager
lei add-external: completion for existing URL basenames
lei: use sleep(1) loop for infinite sleep
lei q: support reading queries from stdin
MANIFEST | 1 +
contrib/completion/lei-completion.bash | 6 ++
lib/PublicInbox/InputPipe.pm | 37 ++++++++++++
lib/PublicInbox/LEI.pm | 37 +++++++-----
lib/PublicInbox/LeiExternal.pm | 82 +++++++++++++++++++++-----
lib/PublicInbox/LeiOverview.pm | 9 ++-
lib/PublicInbox/LeiQuery.pm | 59 ++++++++++++++----
lib/PublicInbox/LeiToMail.pm | 2 +-
lib/PublicInbox/LeiXSearch.pm | 20 +++----
lib/PublicInbox/PktOp.pm | 25 +++++---
script/lei | 2 +-
t/lei.t | 51 ++++++++++++----
12 files changed, 248 insertions(+), 83 deletions(-)
create mode 100644 lib/PublicInbox/InputPipe.pm
^ permalink raw reply [relevance 7%]
Results 1-2 of 2 | reverse | options above
-- pct% links below jump to the message on this page, permalinks otherwise --
2021-02-03 8:11 7% [PATCH 00/11] lei q --stdin, shortcut names, etc Eric Wong
2021-02-03 8:11 4% ` [PATCH 11/11] lei q: support reading queries from stdin Eric Wong
Code repositories for project(s) associated with this public inbox
https://80x24.org/public-inbox.git
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).