[PATCH 11/11] lei q: support reading queries from stdin

[Eric Wong <e@80x24.org>]

This will be useful on shared machines when a user doesn't want
search queries visible to other users looking at the ps(1)
output or similar.
---
 MANIFEST                       |  1 +
 lib/PublicInbox/InputPipe.pm   | 37 ++++++++++++++++++++++++++++++++++
 lib/PublicInbox/LEI.pm         |  7 ++++---
 lib/PublicInbox/LeiOverview.pm |  1 -
 lib/PublicInbox/LeiQuery.pm    | 32 ++++++++++++++++++++++-------
 lib/PublicInbox/LeiXSearch.pm  |  2 ++
 t/lei.t                        | 19 +++++++++++++++++
 7 files changed, 88 insertions(+), 11 deletions(-)
 create mode 100644 lib/PublicInbox/InputPipe.pm

diff --git a/MANIFEST b/MANIFEST
index bcb9d08e..6922f9b1 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -173,6 +173,7 @@ lib/PublicInbox/In2Tie.pm
 lib/PublicInbox/Inbox.pm
 lib/PublicInbox/InboxIdle.pm
 lib/PublicInbox/InboxWritable.pm
+lib/PublicInbox/InputPipe.pm
 lib/PublicInbox/Isearch.pm
 lib/PublicInbox/KQNotify.pm
 lib/PublicInbox/LEI.pm
diff --git a/lib/PublicInbox/InputPipe.pm b/lib/PublicInbox/InputPipe.pm
new file mode 100644
index 00000000..a8bdf031
--- /dev/null
+++ b/lib/PublicInbox/InputPipe.pm
@@ -0,0 +1,37 @@
+# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# for reading pipes and sockets off the DS event loop
+package PublicInbox::InputPipe;
+use strict;
+use v5.10.1;
+use parent qw(PublicInbox::DS);
+use PublicInbox::Syscall qw(EPOLLIN EPOLLET);
+
+sub consume {
+	my ($in, $cb, @args) = @_;
+	my $self = bless { cb => $cb, sock => $in, args => \@args },__PACKAGE__;
+	if ($PublicInbox::DS::in_loop) {
+		eval { $self->SUPER::new($in, EPOLLIN|EPOLLET) };
+		return $in->blocking(0) unless $@; # regular file sets $@
+	}
+	event_step($self) while $self->{sock};
+}
+
+sub event_step {
+	my ($self) = @_;
+	my ($r, $rbuf);
+	while (($r = sysread($self->{sock}, $rbuf, 65536))) {
+		$self->{cb}->(@{$self->{args} // []}, $rbuf);
+	}
+	if (defined($r)) { # EOF
+		$self->{cb}->(@{$self->{args} // []}, '');
+	} elsif ($!{EAGAIN}) {
+		return;
+	} else {
+		$self->{cb}->(@{$self->{args} // []}, undef)
+	}
+	$self->{sock}->blocking ? delete($self->{sock}) : $self->close
+}
+
+1;
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 28dce0c5..49deed13 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -101,10 +101,10 @@ sub _config_path ($) {
 # TODO: generate shell completion + help using %CMD and %OPTDESC
 # command => [ positional_args, 1-line description, Getopt::Long option spec ]
 our %CMD = ( # sorted in order of importance/use:
-'q' => [ 'SEARCH_TERMS...', 'search for messages matching terms', qw(
+'q' => [ '--stdin|SEARCH_TERMS...', 'search for messages matching terms', qw(
 	save-as=s output|mfolder|o=s format|f=s dedupe|d=s thread|t augment|a
 	sort|s=s reverse|r offset=i remote! local! external! pretty
-	include|I=s@ exclude=s@ only=s@ jobs|j=s globoff|g
+	include|I=s@ exclude=s@ only=s@ jobs|j=s globoff|g stdin|
 	mua-cmd|mua=s no-torsocks torsocks=s verbose|v quiet|q
 	received-after=s received-before=s sent-after=s sent-since=s),
 	PublicInbox::LeiQuery::curl_opt(), opt_dash('limit|n=i', '[0-9]+') ],
@@ -554,12 +554,13 @@ sub optparse ($$$) {
 		} elsif ($var =~ /\A\[-?$POS_ARG\]\z/) { # one optional arg
 			$i++;
 		} elsif ($var =~ /\A.+?\|/) { # required FOO|--stdin
+			$inf = 1 if index($var, '...') > 0;
 			my @or = split(/\|/, $var);
 			my $ok;
 			for my $o (@or) {
 				if ($o =~ /\A--([a-z0-9\-]+)/) {
 					$ok = defined($OPT->{$1});
-					last;
+					last if $ok;
 				} elsif (defined($argv->[$i])) {
 					$ok = 1;
 					$i++;
diff --git a/lib/PublicInbox/LeiOverview.pm b/lib/PublicInbox/LeiOverview.pm
index 88034ada..e33d63a2 100644
--- a/lib/PublicInbox/LeiOverview.pm
+++ b/lib/PublicInbox/LeiOverview.pm
@@ -81,7 +81,6 @@ sub new {
 	my ($isatty, $seekable);
 	if ($dst eq '/dev/stdout') {
 		$isatty = -t $lei->{1};
-		$lei->start_pager if $isatty;
 		$opt->{pretty} //= $isatty;
 		if (!$isatty && -f _) {
 			my $fl = fcntl($lei->{1}, F_GETFL, 0) //
diff --git a/lib/PublicInbox/LeiQuery.pm b/lib/PublicInbox/LeiQuery.pm
index 8015ecec..4fe40400 100644
--- a/lib/PublicInbox/LeiQuery.pm
+++ b/lib/PublicInbox/LeiQuery.pm
@@ -12,6 +12,16 @@ sub prep_ext { # externals_each callback
 	$lxs->prepare_external($loc) unless $exclude->{$loc};
 }
 
+sub qstr_add { # for --stdin
+	my ($self) = @_; # $_[1] = $rbuf
+	if (defined($_[1])) {
+		return eval { $self->{lxs}->do_query($self) } if $_[1] eq '';
+		$self->{mset_opt}->{qstr} .= $_[1];
+	} else {
+		$self->fail("error reading stdin: $!");
+	}
+}
+
 # the main "lei q SEARCH_TERMS" method
 sub lei_q {
 	my ($self, @argv) = @_;
@@ -84,12 +94,6 @@ sub lei_q {
 	my %mset_opt = map { $_ => $opt->{$_} } qw(thread limit offset);
 	$mset_opt{asc} = $opt->{'reverse'} ? 1 : 0;
 	$mset_opt{limit} //= 10000;
-	$mset_opt{qstr} = join(' ', map {;
-		# Consider spaces in argv to be for phrase search in Xapian.
-		# In other words, the users should need only care about
-		# normal shell quotes and not have to learn Xapian quoting.
-		/\s/ ? (s/\A(\w+:)// ? qq{$1"$_"} : qq{"$_"}) : $_
-	} @argv);
 	if (defined(my $sort = $opt->{'sort'})) {
 		if ($sort eq 'relevance') {
 			$mset_opt{relevance} = 1;
@@ -104,7 +108,21 @@ sub lei_q {
 	# descending docid order
 	$mset_opt{relevance} //= -2 if $opt->{thread};
 	$self->{mset_opt} = \%mset_opt;
-	$self->{ovv}->ovv_begin($self);
+
+	if ($opt->{stdin}) {
+		return $self->fail(<<'') if @argv;
+no query allowed on command-line with --stdin
+
+		require PublicInbox::InputPipe;
+		PublicInbox::InputPipe::consume($self->{0}, \&qstr_add, $self);
+		return;
+	}
+	# Consider spaces in argv to be for phrase search in Xapian.
+	# In other words, the users should need only care about
+	# normal shell quotes and not have to learn Xapian quoting.
+	$mset_opt{qstr} = join(' ', map {;
+		/\s/ ? (s/\A(\w+:)// ? qq{$1"$_"} : qq{"$_"}) : $_
+	} @argv);
 	$lxs->do_query($self);
 }
 
diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index d33064bb..965617b5 100644
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -402,6 +402,8 @@ sub sigpipe_handler { # handles SIGPIPE from l2m/lxs workers
 sub do_query {
 	my ($self, $lei) = @_;
 	$lei->{1}->autoflush(1);
+	$lei->start_pager if -t $lei->{1};
+	$lei->{ovv}->ovv_begin($lei);
 	my ($au_done, $zpipe);
 	my $l2m = $lei->{l2m};
 	if ($l2m) {
diff --git a/t/lei.t b/t/lei.t
index 03bbb078..01eed1da 100644
--- a/t/lei.t
+++ b/t/lei.t
@@ -275,6 +275,25 @@ my $test_external = sub {
 	my $pretty = $json->decode($out);
 	is_deeply($res, $pretty, '--pretty is identical after decode');
 
+	{
+		open my $fh, '+>', undef or BAIL_OUT $!;
+		$fh->autoflush(1);
+		print $fh 's:use' or BAIL_OUT $!;
+		seek($fh, 0, SEEK_SET) or BAIL_OUT $!;
+		ok($lei->([qw(q -q --stdin)], undef, { %$opt, 0 => $fh }),
+				'--stdin on regular file works');
+		like($out, qr/use boolean prefix/, '--stdin on regular file');
+	}
+	{
+		pipe(my ($r, $w)) or BAIL_OUT $!;
+		print $w 's:use' or BAIL_OUT $!;
+		close $w or BAIL_OUT $!;
+		ok($lei->([qw(q -q --stdin)], undef, { %$opt, 0 => $r }),
+				'--stdin on pipe file works');
+		like($out, qr/use boolean prefix/, '--stdin on pipe');
+	}
+	ok(!$lei->(qw(q -q --stdin s:use)), "--stdin and argv don't mix");
+
 	for my $fmt (qw(ldjson ndjson jsonl)) {
 		$lei->('q', '-f', $fmt, 's:use boolean prefix');
 		is($out, $json->encode($pretty->[0])."\n", "-f $fmt");