lei ls-search: command to list saved searches

Going forward, we'll probably support JSON for all the "ls-*"
subcommands.  This also provides the basis for "lei up" shell
completion.

7 files changed, 173 insertions, 13 deletions

diff --git a/MANIFEST b/MANIFEST
index 1b7d16ee..f35c514c 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -196,6 +196,7 @@ lib/PublicInbox/LeiImport.pm
 lib/PublicInbox/LeiInit.pm
 lib/PublicInbox/LeiInput.pm
 lib/PublicInbox/LeiLsLabel.pm
+lib/PublicInbox/LeiLsSearch.pm
 lib/PublicInbox/LeiMirror.pm
 lib/PublicInbox/LeiOverview.pm
 lib/PublicInbox/LeiP2q.pm
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index f223b3de..56640be1 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -157,8 +157,8 @@ our %CMD = ( # sorted in order of importance/use:
         'exclude further results from a publicinbox|extindex',
         qw(prune), @c_opt ],
 
-'ls-query' => [ '[FILTER...]', 'list saved search queries',
-                qw(name-only format|f=s), @c_opt ],
+'ls-search' => [ '[PREFIX]', 'list saved search queries',
+                qw(format|f=s pretty l ascii z|0), @c_opt ],
 'rm-query' => [ 'QUERY_NAME', 'remove a saved search', @c_opt ],
 'mv-query' => [ qw(OLD_NAME NEW_NAME), 'rename a saved search', @c_opt ],
 
@@ -312,7 +312,9 @@ my %OPTDESC = (
 'jobs|j=i        add-external' => 'set parallelism when indexing after --mirror',
 
 'in-format|F=s' => $stdin_formats,
-'format|f=s        ls-query' => $ls_format,
+'format|f=s        ls-search' => ['OUT|json|jsonl|concatjson',
+                        'listing output format' ],
+'l        ls-search' => 'long listing format',
 'format|f=s        ls-external' => $ls_format,
 
 'limit|n=i@' => ['NUM', 'limit on number of matches (default: 10000)' ],
@@ -353,7 +355,7 @@ my %CONFIG_KEYS = (
         'leistore.dir' => 'top-level storage location',
 );
 
-my @WQ_KEYS = qw(lxs l2m imp mrr cnv p2q tag sol); # internal workers
+my @WQ_KEYS = qw(lxs l2m imp mrr cnv p2q tag sol lsss); # internal workers
 
 sub _drop_wq {
         my ($self) = @_;
diff --git a/lib/PublicInbox/LeiExternal.pm b/lib/PublicInbox/LeiExternal.pm
index 5e8dc71a..b0ebe947 100644
--- a/lib/PublicInbox/LeiExternal.pm
+++ b/lib/PublicInbox/LeiExternal.pm
@@ -215,8 +215,8 @@ sub lei_forget_external {
         }
 }
 
-sub _complete_url_common ($) {
-        my ($argv) = @_;
+sub complete_url_common {
+        my $argv = $_[-1];
         # Workaround bash word-splitting URLs to ['https', ':', '//' ...]
         # Maybe there's a better way to go about this in
         # contrib/completion/lei-completion.bash
@@ -228,7 +228,8 @@ sub _complete_url_common ($) {
                         push @x, $cur;
                         $cur = '';
                 }
-                while (@x > 2 && $x[0] !~ /\Ahttps?\z/ && $x[1] ne ':') {
+                while (@x > 2 && $x[0] !~ /\A(?:http|nntp|imap)s?\z/i &&
+                                $x[1] ne ':') {
                         shift @x;
                 }
                 if (@x >= 2) { # qw(https : hostname : 443) or qw(http :)
@@ -245,7 +246,7 @@ sub _complete_url_common ($) {
 sub _complete_forget_external {
         my ($self, @argv) = @_;
         my $cfg = $self->_lei_cfg;
-        my ($cur, $re) = _complete_url_common(\@argv);
+        my ($cur, $re) = complete_url_common(\@argv);
         # FIXME: bash completion off "http:" or "https:" when the last
         # character is a colon doesn't work properly even if we're
         # returning "//$HTTP_HOST/$PATH_INFO/", not sure why, could
@@ -261,7 +262,7 @@ sub _complete_forget_external {
 sub _complete_add_external { # for bash, this relies on "compopt -o nospace"
         my ($self, @argv) = @_;
         my $cfg = $self->_lei_cfg;
-        my ($cur, $re) = _complete_url_common(\@argv);
+        my ($cur, $re) = complete_url_common(\@argv);
         require URI;
         map {
                 my $u = URI->new(substr($_, length('external.')));
diff --git a/lib/PublicInbox/LeiLsSearch.pm b/lib/PublicInbox/LeiLsSearch.pm
new file mode 100644
index 00000000..2aa457c0
--- /dev/null
+++ b/lib/PublicInbox/LeiLsSearch.pm
@@ -0,0 +1,109 @@
+# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# "lei ls-search" to display results saved via "lei q --save"
+package PublicInbox::LeiLsSearch;
+use strict;
+use v5.10.1;
+use PublicInbox::LeiSavedSearch;
+use parent qw(PublicInbox::IPC);
+
+sub do_ls_search_long {
+        my ($self, $pfx) = @_;
+        # TODO: share common JSON output code with LeiOverview
+        my $json = $self->{json}->new->utf8->canonical;
+        my $lei = $self->{lei};
+        $json->ascii(1) if $lei->{opt}->{ascii};
+        my $fmt = $lei->{opt}->{'format'};
+        $lei->{1}->autoflush(0);
+        my $ORS = "\n";
+        my $pretty = $lei->{opt}->{pretty};
+        my $EOR;  # TODO: compact pretty like "lei q"
+        if ($fmt =~ /\A(concat)?json\z/ && $pretty) {
+                $EOR = ($1//'') eq 'concat' ? "\n}" : "\n},";
+        }
+        if ($fmt eq 'json') {
+                $lei->out('[');
+                $ORS = ",\n";
+        }
+        my @x = sort(grep(/\A\Q$pfx/, PublicInbox::LeiSavedSearch::list($lei)));
+        while (my $x = shift @x) {
+                $ORS = '' if !scalar(@x);
+                my $lss = PublicInbox::LeiSavedSearch->new($lei, $x) or next;
+                my $cfg = $lss->{-cfg};
+                my $ent = {
+                        q => $cfg->get_all('lei.q'),
+                        output => $cfg->{'lei.q.output'},
+                };
+                for my $k ($lss->ARRAY_FIELDS) {
+                        my $ary = $cfg->get_all("lei.q.$k") // next;
+                        $ent->{$k} = $ary;
+                }
+                for my $k ($lss->BOOL_FIELDS) {
+                        my $val = $cfg->{"lei.q.$k"} // next;
+                        $ent->{$k} = $val;
+                }
+                if (defined $EOR) { # pretty, but compact
+                        $EOR = "\n}" if !scalar(@x);
+                        my $buf = "{\n";
+                        $buf .= join(",\n", map {;
+                                my $f = $_;
+                                if (my $v = $ent->{$f}) {
+                                        $v = $json->encode([$v]);
+                                        qq{  "$f": }.substr($v, 1, -1);
+                                } else {
+                                        ();
+                                }
+                        # key order by importance
+                        } (qw(output q), $lss->ARRAY_FIELDS,
+                                $lss->BOOL_FIELDS) );
+                        $lei->out($buf .= $EOR);
+                } else {
+                        $lei->out($json->encode($ent), $ORS);
+                }
+        }
+        if ($fmt eq 'json') {
+                $lei->out("]\n");
+        } elsif ($fmt eq 'concatjson') {
+                $lei->out("\n");
+        }
+}
+
+sub bg_worker ($$$) {
+        my ($lei, $pfx, $json) = @_;
+        my $self = bless { -wq_nr_workers => 1, json => $json }, __PACKAGE__;
+        my ($op_c, $ops) = $lei->workers_start($self, 'ls-search', 1);
+        $lei->{lsss} = $self;
+        $self->wq_io_do('do_ls_search_long', [], $pfx);
+        $self->wq_close(1);
+        $op_c->op_wait_event($ops);
+}
+
+sub lei_ls_search {
+        my ($lei, $pfx) = @_;
+        my $fmt = $lei->{opt}->{'format'} // '';
+        if ($lei->{opt}->{l}) {
+                $lei->{opt}->{'format'} //= $fmt = 'json';
+        }
+        my $json;
+        my $tty = -t $lei->{1};
+        $lei->start_pager if $tty;
+        if ($fmt =~ /\A(ldjson|ndjson|jsonl|(?:concat)?json)\z/) {
+                $lei->{opt}->{pretty} //= $tty;
+                $json = ref(PublicInbox::Config->json);
+        } elsif ($fmt ne '') {
+                return $lei->fail("unknown format: $fmt");
+        }
+        my $ORS = "\n";
+        if ($lei->{opt}->{z}) {
+                return $lei->fail('-z and --format do not mix') if $json;
+                $ORS = "\0";
+        }
+        $pfx //= '';
+        return bg_worker($lei, $pfx, $json) if $json;
+        for (sort(grep(/\A\Q$pfx/, PublicInbox::LeiSavedSearch::list($lei)))) {
+                $lei->out($_, $ORS);
+        }
+}
+
+1;
diff --git a/lib/PublicInbox/LeiSavedSearch.pm b/lib/PublicInbox/LeiSavedSearch.pm
index 3076d14c..d67622c9 100644
--- a/lib/PublicInbox/LeiSavedSearch.pm
+++ b/lib/PublicInbox/LeiSavedSearch.pm
@@ -21,6 +21,11 @@ sub cquote_val ($) { # cf. git-config(1)
         $val;
 }
 
+sub ARRAY_FIELDS () { qw(only include exclude) }
+sub BOOL_FIELDS () {
+        qw(external local remote import-remote import-before threads)
+}
+
 sub lss_dir_for ($$) {
         my ($lei, $dstref) = @_;
         my @n;
@@ -39,6 +44,31 @@ sub lss_dir_for ($$) {
         $lei->share_path . '/saved-searches/' . join('-', @n);
 }
 
+sub list {
+        my ($lei, $pfx) = @_;
+        my $lss_dir = $lei->share_path.'/saved-searches/';
+        return () unless -d $lss_dir;
+        # TODO: persist the cache?  Use another format?
+        my $f = $lei->cache_dir."/saved-tmp.$$.".time.'.config';
+        open my $fh, '>', $f or die "open $f: $!";
+        print $fh "[include]\n";
+        for my $p (glob("$lss_dir/*/lei.saved-search")) {
+                print $fh "\tpath = ", cquote_val($p), "\n";
+        }
+        close $fh or die "close $f: $!";
+        my $cfg = PublicInbox::Config::git_config_dump($f);
+        unlink($f);
+        bless $cfg, 'PublicInbox::Config';
+        my $out = $cfg->get_all('lei.q.output') or return ();
+        map {;
+                if (s!\A(?:maildir|mh|mbox.+|mmdf):!!i) {
+                        -e $_ ? $_ : (); # TODO auto-prune somewhere?
+                } else { # IMAP, maybe JMAP
+                        $_;
+                }
+        } @$out
+}
+
 sub new {
         my ($cls, $lei, $dst) = @_;
         my $self = bless { ale => $lei->ale }, $cls;
@@ -74,16 +104,15 @@ $q
 [lei "q"]
         output = $dst
 EOM
-                for my $k (qw(only include exclude)) {
+                for my $k (ARRAY_FIELDS) {
                         my $ary = $lei->{opt}->{$k} // next;
                         for my $x (@$ary) {
                                 print $fh "\t$k = ".cquote_val($x)."\n";
                         }
                 }
-                for my $k (qw(external local remote import-remote
-                                import-before threads)) {
+                for my $k (BOOL_FIELDS) {
                         my $val = $lei->{opt}->{$k} // next;
-                        print $fh "\t$k = ".cquote_val($val)."\n";
+                        print $fh "\t$k = ".($val ? 1 : 0)."\n";
                 }
                 close($fh) or return $lei->fail("close $f: $!");
         }
diff --git a/lib/PublicInbox/LeiUp.pm b/lib/PublicInbox/LeiUp.pm
index 9fe4901b..73286ea2 100644
--- a/lib/PublicInbox/LeiUp.pm
+++ b/lib/PublicInbox/LeiUp.pm
@@ -42,4 +42,10 @@ sub lei_up {
         $lei->_start_query;
 }
 
+sub _complete_up {
+        my ($lei, @argv) = @_;
+        my ($cur, $re) = $lei->complete_url_common(\@argv);
+        grep(/\A$re\Q$cur/, PublicInbox::LeiSavedSearch::list($lei));
+}
+
 1;
diff --git a/t/lei-q-save.t b/t/lei-q-save.t
index a8eda41e..761814b4 100644
--- a/t/lei-q-save.t
+++ b/t/lei-q-save.t
@@ -55,5 +55,17 @@ test_lei(sub {
         ok(-s "$home/mbcl2" > $size, 'size increased after up');
 
         ok(!lei(qw(up -q), $home), 'up fails w/o --save');
+
+        lei_ok qw(ls-search); my @d = split(/\n/, $lei_out);
+        lei_ok qw(ls-search -z); my @z = split(/\0/, $lei_out);
+        is_deeply(\@d, \@z, '-z output matches non-z');
+        is_deeply(\@d, [ "$home/mbcl2", "$home/md/" ],
+                'ls-search output alphabetically sorted');
+        lei_ok qw(ls-search -l);
+        my $json = PublicInbox::Config->json->decode($lei_out);
+        ok($json && $json->[0]->{output}, 'JSON has output');
+        lei_ok qw(_complete lei up);
+        like($lei_out, qr!^\Q$home/mbcl2\E$!sm, 'complete got mbcl2 output');
+        like($lei_out, qr!^\Q$home/md/\E$!sm, 'complete got maildir output');
 });
 done_testing;