[PATCH] lei ls-search: command to list saved searches

[PATCH] lei ls-search: command to list saved searches

[Eric Wong]

Going forward, we'll probably support JSON for all the "ls-*"
subcommands.  This also provides the basis for "lei up" shell
completion.
---
 MANIFEST                          |   1 +
 lib/PublicInbox/LEI.pm            |  10 +--
 lib/PublicInbox/LeiExternal.pm    |  11 +--
 lib/PublicInbox/LeiLsSearch.pm    | 109 ++++++++++++++++++++++++++++++
 lib/PublicInbox/LeiSavedSearch.pm |  37 ++++++++--
 lib/PublicInbox/LeiUp.pm          |   6 ++
 t/lei-q-save.t                    |  12 ++++
 7 files changed, 173 insertions(+), 13 deletions(-)
 create mode 100644 lib/PublicInbox/LeiLsSearch.pm

diff --git a/MANIFEST b/MANIFEST
index 1b7d16ee..f35c514c 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -196,6 +196,7 @@ lib/PublicInbox/LeiImport.pm
 lib/PublicInbox/LeiInit.pm
 lib/PublicInbox/LeiInput.pm
 lib/PublicInbox/LeiLsLabel.pm
+lib/PublicInbox/LeiLsSearch.pm
 lib/PublicInbox/LeiMirror.pm
 lib/PublicInbox/LeiOverview.pm
 lib/PublicInbox/LeiP2q.pm
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index f223b3de..56640be1 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -157,8 +157,8 @@ our %CMD = ( # sorted in order of importance/use:
 	'exclude further results from a publicinbox|extindex',
 	qw(prune), @c_opt ],
 
-'ls-query' => [ '[FILTER...]', 'list saved search queries',
-		qw(name-only format|f=s), @c_opt ],
+'ls-search' => [ '[PREFIX]', 'list saved search queries',
+		qw(format|f=s pretty l ascii z|0), @c_opt ],
 'rm-query' => [ 'QUERY_NAME', 'remove a saved search', @c_opt ],
 'mv-query' => [ qw(OLD_NAME NEW_NAME), 'rename a saved search', @c_opt ],
 
@@ -312,7 +312,9 @@ my %OPTDESC = (
 'jobs|j=i	add-external' => 'set parallelism when indexing after --mirror',
 
 'in-format|F=s' => $stdin_formats,
-'format|f=s	ls-query' => $ls_format,
+'format|f=s	ls-search' => ['OUT|json|jsonl|concatjson',
+			'listing output format' ],
+'l	ls-search' => 'long listing format',
 'format|f=s	ls-external' => $ls_format,
 
 'limit|n=i@' => ['NUM', 'limit on number of matches (default: 10000)' ],
@@ -353,7 +355,7 @@ my %CONFIG_KEYS = (
 	'leistore.dir' => 'top-level storage location',
 );
 
-my @WQ_KEYS = qw(lxs l2m imp mrr cnv p2q tag sol); # internal workers
+my @WQ_KEYS = qw(lxs l2m imp mrr cnv p2q tag sol lsss); # internal workers
 
 sub _drop_wq {
 	my ($self) = @_;
diff --git a/lib/PublicInbox/LeiExternal.pm b/lib/PublicInbox/LeiExternal.pm
index 5e8dc71a..b0ebe947 100644
--- a/lib/PublicInbox/LeiExternal.pm
+++ b/lib/PublicInbox/LeiExternal.pm
@@ -215,8 +215,8 @@ sub lei_forget_external {
 	}
 }
 
-sub _complete_url_common ($) {
-	my ($argv) = @_;
+sub complete_url_common {
+	my $argv = $_[-1];
 	# Workaround bash word-splitting URLs to ['https', ':', '//' ...]
 	# Maybe there's a better way to go about this in
 	# contrib/completion/lei-completion.bash
@@ -228,7 +228,8 @@ sub _complete_url_common ($) {
 			push @x, $cur;
 			$cur = '';
 		}
-		while (@x > 2 && $x[0] !~ /\Ahttps?\z/ && $x[1] ne ':') {
+		while (@x > 2 && $x[0] !~ /\A(?:http|nntp|imap)s?\z/i &&
+				$x[1] ne ':') {
 			shift @x;
 		}
 		if (@x >= 2) { # qw(https : hostname : 443) or qw(http :)
@@ -245,7 +246,7 @@ sub _complete_url_common ($) {
 sub _complete_forget_external {
 	my ($self, @argv) = @_;
 	my $cfg = $self->_lei_cfg;
-	my ($cur, $re) = _complete_url_common(\@argv);
+	my ($cur, $re) = complete_url_common(\@argv);
 	# FIXME: bash completion off "http:" or "https:" when the last
 	# character is a colon doesn't work properly even if we're
 	# returning "//$HTTP_HOST/$PATH_INFO/", not sure why, could
@@ -261,7 +262,7 @@ sub _complete_forget_external {
 sub _complete_add_external { # for bash, this relies on "compopt -o nospace"
 	my ($self, @argv) = @_;
 	my $cfg = $self->_lei_cfg;
-	my ($cur, $re) = _complete_url_common(\@argv);
+	my ($cur, $re) = complete_url_common(\@argv);
 	require URI;
 	map {
 		my $u = URI->new(substr($_, length('external.')));
diff --git a/lib/PublicInbox/LeiLsSearch.pm b/lib/PublicInbox/LeiLsSearch.pm
new file mode 100644
index 00000000..2aa457c0
--- /dev/null
+++ b/lib/PublicInbox/LeiLsSearch.pm
@@ -0,0 +1,109 @@
+# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# "lei ls-search" to display results saved via "lei q --save"
+package PublicInbox::LeiLsSearch;
+use strict;
+use v5.10.1;
+use PublicInbox::LeiSavedSearch;
+use parent qw(PublicInbox::IPC);
+
+sub do_ls_search_long {
+	my ($self, $pfx) = @_;
+	# TODO: share common JSON output code with LeiOverview
+	my $json = $self->{json}->new->utf8->canonical;
+	my $lei = $self->{lei};
+	$json->ascii(1) if $lei->{opt}->{ascii};
+	my $fmt = $lei->{opt}->{'format'};
+	$lei->{1}->autoflush(0);
+	my $ORS = "\n";
+	my $pretty = $lei->{opt}->{pretty};
+	my $EOR;  # TODO: compact pretty like "lei q"
+	if ($fmt =~ /\A(concat)?json\z/ && $pretty) {
+		$EOR = ($1//'') eq 'concat' ? "\n}" : "\n},";
+	}
+	if ($fmt eq 'json') {
+		$lei->out('[');
+		$ORS = ",\n";
+	}
+	my @x = sort(grep(/\A\Q$pfx/, PublicInbox::LeiSavedSearch::list($lei)));
+	while (my $x = shift @x) {
+		$ORS = '' if !scalar(@x);
+		my $lss = PublicInbox::LeiSavedSearch->new($lei, $x) or next;
+		my $cfg = $lss->{-cfg};
+		my $ent = {
+			q => $cfg->get_all('lei.q'),
+			output => $cfg->{'lei.q.output'},
+		};
+		for my $k ($lss->ARRAY_FIELDS) {
+			my $ary = $cfg->get_all("lei.q.$k") // next;
+			$ent->{$k} = $ary;
+		}
+		for my $k ($lss->BOOL_FIELDS) {
+			my $val = $cfg->{"lei.q.$k"} // next;
+			$ent->{$k} = $val;
+		}
+		if (defined $EOR) { # pretty, but compact
+			$EOR = "\n}" if !scalar(@x);
+			my $buf = "{\n";
+			$buf .= join(",\n", map {;
+				my $f = $_;
+				if (my $v = $ent->{$f}) {
+					$v = $json->encode([$v]);
+					qq{  "$f": }.substr($v, 1, -1);
+				} else {
+					();
+				}
+			# key order by importance
+			} (qw(output q), $lss->ARRAY_FIELDS,
+				$lss->BOOL_FIELDS) );
+			$lei->out($buf .= $EOR);
+		} else {
+			$lei->out($json->encode($ent), $ORS);
+		}
+	}
+	if ($fmt eq 'json') {
+		$lei->out("]\n");
+	} elsif ($fmt eq 'concatjson') {
+		$lei->out("\n");
+	}
+}
+
+sub bg_worker ($$$) {
+	my ($lei, $pfx, $json) = @_;
+	my $self = bless { -wq_nr_workers => 1, json => $json }, __PACKAGE__;
+	my ($op_c, $ops) = $lei->workers_start($self, 'ls-search', 1);
+	$lei->{lsss} = $self;
+	$self->wq_io_do('do_ls_search_long', [], $pfx);
+	$self->wq_close(1);
+	$op_c->op_wait_event($ops);
+}
+
+sub lei_ls_search {
+	my ($lei, $pfx) = @_;
+	my $fmt = $lei->{opt}->{'format'} // '';
+	if ($lei->{opt}->{l}) {
+		$lei->{opt}->{'format'} //= $fmt = 'json';
+	}
+	my $json;
+	my $tty = -t $lei->{1};
+	$lei->start_pager if $tty;
+	if ($fmt =~ /\A(ldjson|ndjson|jsonl|(?:concat)?json)\z/) {
+		$lei->{opt}->{pretty} //= $tty;
+		$json = ref(PublicInbox::Config->json);
+	} elsif ($fmt ne '') {
+		return $lei->fail("unknown format: $fmt");
+	}
+	my $ORS = "\n";
+	if ($lei->{opt}->{z}) {
+		return $lei->fail('-z and --format do not mix') if $json;
+		$ORS = "\0";
+	}
+	$pfx //= '';
+	return bg_worker($lei, $pfx, $json) if $json;
+	for (sort(grep(/\A\Q$pfx/, PublicInbox::LeiSavedSearch::list($lei)))) {
+		$lei->out($_, $ORS);
+	}
+}
+
+1;
diff --git a/lib/PublicInbox/LeiSavedSearch.pm b/lib/PublicInbox/LeiSavedSearch.pm
index 3076d14c..d67622c9 100644
--- a/lib/PublicInbox/LeiSavedSearch.pm
+++ b/lib/PublicInbox/LeiSavedSearch.pm
@@ -21,6 +21,11 @@ sub cquote_val ($) { # cf. git-config(1)
 	$val;
 }
 
+sub ARRAY_FIELDS () { qw(only include exclude) }
+sub BOOL_FIELDS () {
+	qw(external local remote import-remote import-before threads)
+}
+
 sub lss_dir_for ($$) {
 	my ($lei, $dstref) = @_;
 	my @n;
@@ -39,6 +44,31 @@ sub lss_dir_for ($$) {
 	$lei->share_path . '/saved-searches/' . join('-', @n);
 }
 
+sub list {
+	my ($lei, $pfx) = @_;
+	my $lss_dir = $lei->share_path.'/saved-searches/';
+	return () unless -d $lss_dir;
+	# TODO: persist the cache?  Use another format?
+	my $f = $lei->cache_dir."/saved-tmp.$$.".time.'.config';
+	open my $fh, '>', $f or die "open $f: $!";
+	print $fh "[include]\n";
+	for my $p (glob("$lss_dir/*/lei.saved-search")) {
+		print $fh "\tpath = ", cquote_val($p), "\n";
+	}
+	close $fh or die "close $f: $!";
+	my $cfg = PublicInbox::Config::git_config_dump($f);
+	unlink($f);
+	bless $cfg, 'PublicInbox::Config';
+	my $out = $cfg->get_all('lei.q.output') or return ();
+	map {;
+		if (s!\A(?:maildir|mh|mbox.+|mmdf):!!i) {
+			-e $_ ? $_ : (); # TODO auto-prune somewhere?
+		} else { # IMAP, maybe JMAP
+			$_;
+		}
+	} @$out
+}
+
 sub new {
 	my ($cls, $lei, $dst) = @_;
 	my $self = bless { ale => $lei->ale }, $cls;
@@ -74,16 +104,15 @@ $q
 [lei "q"]
 	output = $dst
 EOM
-		for my $k (qw(only include exclude)) {
+		for my $k (ARRAY_FIELDS) {
 			my $ary = $lei->{opt}->{$k} // next;
 			for my $x (@$ary) {
 				print $fh "\t$k = ".cquote_val($x)."\n";
 			}
 		}
-		for my $k (qw(external local remote import-remote
-				import-before threads)) {
+		for my $k (BOOL_FIELDS) {
 			my $val = $lei->{opt}->{$k} // next;
-			print $fh "\t$k = ".cquote_val($val)."\n";
+			print $fh "\t$k = ".($val ? 1 : 0)."\n";
 		}
 		close($fh) or return $lei->fail("close $f: $!");
 	}
diff --git a/lib/PublicInbox/LeiUp.pm b/lib/PublicInbox/LeiUp.pm
index 9fe4901b..73286ea2 100644
--- a/lib/PublicInbox/LeiUp.pm
+++ b/lib/PublicInbox/LeiUp.pm
@@ -42,4 +42,10 @@ sub lei_up {
 	$lei->_start_query;
 }
 
+sub _complete_up {
+	my ($lei, @argv) = @_;
+	my ($cur, $re) = $lei->complete_url_common(\@argv);
+	grep(/\A$re\Q$cur/, PublicInbox::LeiSavedSearch::list($lei));
+}
+
 1;
diff --git a/t/lei-q-save.t b/t/lei-q-save.t
index a8eda41e..761814b4 100644
--- a/t/lei-q-save.t
+++ b/t/lei-q-save.t
@@ -55,5 +55,17 @@ test_lei(sub {
 	ok(-s "$home/mbcl2" > $size, 'size increased after up');
 
 	ok(!lei(qw(up -q), $home), 'up fails w/o --save');
+
+	lei_ok qw(ls-search); my @d = split(/\n/, $lei_out);
+	lei_ok qw(ls-search -z); my @z = split(/\0/, $lei_out);
+	is_deeply(\@d, \@z, '-z output matches non-z');
+	is_deeply(\@d, [ "$home/mbcl2", "$home/md/" ],
+		'ls-search output alphabetically sorted');
+	lei_ok qw(ls-search -l);
+	my $json = PublicInbox::Config->json->decode($lei_out);
+	ok($json && $json->[0]->{output}, 'JSON has output');
+	lei_ok qw(_complete lei up);
+	like($lei_out, qr!^\Q$home/mbcl2\E$!sm, 'complete got mbcl2 output');
+	like($lei_out, qr!^\Q$home/md/\E$!sm, 'complete got maildir output');
 });
 done_testing;