From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on dcvr.yhbt.net X-Spam-Level: X-Spam-ASN: X-Spam-Status: No, score=-4.0 required=3.0 tests=ALL_TRUSTED,BAYES_00 shortcircuit=no autolearn=ham autolearn_force=no version=3.4.2 Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id F155D1F4B4 for ; Sun, 18 Apr 2021 08:40:14 +0000 (UTC) From: Eric Wong To: meta@public-inbox.org Subject: [PATCH] lei ls-search: command to list saved searches Date: Sun, 18 Apr 2021 08:40:14 +0000 Message-Id: <20210418084014.23377-1-e@80x24.org> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit List-Id: Going forward, we'll probably support JSON for all the "ls-*" subcommands. This also provides the basis for "lei up" shell completion. --- MANIFEST | 1 + lib/PublicInbox/LEI.pm | 10 +-- lib/PublicInbox/LeiExternal.pm | 11 +-- lib/PublicInbox/LeiLsSearch.pm | 109 ++++++++++++++++++++++++++++++ lib/PublicInbox/LeiSavedSearch.pm | 37 ++++++++-- lib/PublicInbox/LeiUp.pm | 6 ++ t/lei-q-save.t | 12 ++++ 7 files changed, 173 insertions(+), 13 deletions(-) create mode 100644 lib/PublicInbox/LeiLsSearch.pm diff --git a/MANIFEST b/MANIFEST index 1b7d16ee..f35c514c 100644 --- a/MANIFEST +++ b/MANIFEST @@ -196,6 +196,7 @@ lib/PublicInbox/LeiImport.pm lib/PublicInbox/LeiInit.pm lib/PublicInbox/LeiInput.pm lib/PublicInbox/LeiLsLabel.pm +lib/PublicInbox/LeiLsSearch.pm lib/PublicInbox/LeiMirror.pm lib/PublicInbox/LeiOverview.pm lib/PublicInbox/LeiP2q.pm diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm index f223b3de..56640be1 100644 --- a/lib/PublicInbox/LEI.pm +++ b/lib/PublicInbox/LEI.pm @@ -157,8 +157,8 @@ our %CMD = ( # sorted in order of importance/use: 'exclude further results from a publicinbox|extindex', qw(prune), @c_opt ], -'ls-query' => [ '[FILTER...]', 'list saved search queries', - qw(name-only format|f=s), @c_opt ], +'ls-search' => [ '[PREFIX]', 'list saved search queries', + qw(format|f=s pretty l ascii z|0), @c_opt ], 'rm-query' => [ 'QUERY_NAME', 'remove a saved search', @c_opt ], 'mv-query' => [ qw(OLD_NAME NEW_NAME), 'rename a saved search', @c_opt ], @@ -312,7 +312,9 @@ my %OPTDESC = ( 'jobs|j=i add-external' => 'set parallelism when indexing after --mirror', 'in-format|F=s' => $stdin_formats, -'format|f=s ls-query' => $ls_format, +'format|f=s ls-search' => ['OUT|json|jsonl|concatjson', + 'listing output format' ], +'l ls-search' => 'long listing format', 'format|f=s ls-external' => $ls_format, 'limit|n=i@' => ['NUM', 'limit on number of matches (default: 10000)' ], @@ -353,7 +355,7 @@ my %CONFIG_KEYS = ( 'leistore.dir' => 'top-level storage location', ); -my @WQ_KEYS = qw(lxs l2m imp mrr cnv p2q tag sol); # internal workers +my @WQ_KEYS = qw(lxs l2m imp mrr cnv p2q tag sol lsss); # internal workers sub _drop_wq { my ($self) = @_; diff --git a/lib/PublicInbox/LeiExternal.pm b/lib/PublicInbox/LeiExternal.pm index 5e8dc71a..b0ebe947 100644 --- a/lib/PublicInbox/LeiExternal.pm +++ b/lib/PublicInbox/LeiExternal.pm @@ -215,8 +215,8 @@ sub lei_forget_external { } } -sub _complete_url_common ($) { - my ($argv) = @_; +sub complete_url_common { + my $argv = $_[-1]; # Workaround bash word-splitting URLs to ['https', ':', '//' ...] # Maybe there's a better way to go about this in # contrib/completion/lei-completion.bash @@ -228,7 +228,8 @@ sub _complete_url_common ($) { push @x, $cur; $cur = ''; } - while (@x > 2 && $x[0] !~ /\Ahttps?\z/ && $x[1] ne ':') { + while (@x > 2 && $x[0] !~ /\A(?:http|nntp|imap)s?\z/i && + $x[1] ne ':') { shift @x; } if (@x >= 2) { # qw(https : hostname : 443) or qw(http :) @@ -245,7 +246,7 @@ sub _complete_url_common ($) { sub _complete_forget_external { my ($self, @argv) = @_; my $cfg = $self->_lei_cfg; - my ($cur, $re) = _complete_url_common(\@argv); + my ($cur, $re) = complete_url_common(\@argv); # FIXME: bash completion off "http:" or "https:" when the last # character is a colon doesn't work properly even if we're # returning "//$HTTP_HOST/$PATH_INFO/", not sure why, could @@ -261,7 +262,7 @@ sub _complete_forget_external { sub _complete_add_external { # for bash, this relies on "compopt -o nospace" my ($self, @argv) = @_; my $cfg = $self->_lei_cfg; - my ($cur, $re) = _complete_url_common(\@argv); + my ($cur, $re) = complete_url_common(\@argv); require URI; map { my $u = URI->new(substr($_, length('external.'))); diff --git a/lib/PublicInbox/LeiLsSearch.pm b/lib/PublicInbox/LeiLsSearch.pm new file mode 100644 index 00000000..2aa457c0 --- /dev/null +++ b/lib/PublicInbox/LeiLsSearch.pm @@ -0,0 +1,109 @@ +# Copyright (C) 2021 all contributors +# License: AGPL-3.0+ + +# "lei ls-search" to display results saved via "lei q --save" +package PublicInbox::LeiLsSearch; +use strict; +use v5.10.1; +use PublicInbox::LeiSavedSearch; +use parent qw(PublicInbox::IPC); + +sub do_ls_search_long { + my ($self, $pfx) = @_; + # TODO: share common JSON output code with LeiOverview + my $json = $self->{json}->new->utf8->canonical; + my $lei = $self->{lei}; + $json->ascii(1) if $lei->{opt}->{ascii}; + my $fmt = $lei->{opt}->{'format'}; + $lei->{1}->autoflush(0); + my $ORS = "\n"; + my $pretty = $lei->{opt}->{pretty}; + my $EOR; # TODO: compact pretty like "lei q" + if ($fmt =~ /\A(concat)?json\z/ && $pretty) { + $EOR = ($1//'') eq 'concat' ? "\n}" : "\n},"; + } + if ($fmt eq 'json') { + $lei->out('['); + $ORS = ",\n"; + } + my @x = sort(grep(/\A\Q$pfx/, PublicInbox::LeiSavedSearch::list($lei))); + while (my $x = shift @x) { + $ORS = '' if !scalar(@x); + my $lss = PublicInbox::LeiSavedSearch->new($lei, $x) or next; + my $cfg = $lss->{-cfg}; + my $ent = { + q => $cfg->get_all('lei.q'), + output => $cfg->{'lei.q.output'}, + }; + for my $k ($lss->ARRAY_FIELDS) { + my $ary = $cfg->get_all("lei.q.$k") // next; + $ent->{$k} = $ary; + } + for my $k ($lss->BOOL_FIELDS) { + my $val = $cfg->{"lei.q.$k"} // next; + $ent->{$k} = $val; + } + if (defined $EOR) { # pretty, but compact + $EOR = "\n}" if !scalar(@x); + my $buf = "{\n"; + $buf .= join(",\n", map {; + my $f = $_; + if (my $v = $ent->{$f}) { + $v = $json->encode([$v]); + qq{ "$f": }.substr($v, 1, -1); + } else { + (); + } + # key order by importance + } (qw(output q), $lss->ARRAY_FIELDS, + $lss->BOOL_FIELDS) ); + $lei->out($buf .= $EOR); + } else { + $lei->out($json->encode($ent), $ORS); + } + } + if ($fmt eq 'json') { + $lei->out("]\n"); + } elsif ($fmt eq 'concatjson') { + $lei->out("\n"); + } +} + +sub bg_worker ($$$) { + my ($lei, $pfx, $json) = @_; + my $self = bless { -wq_nr_workers => 1, json => $json }, __PACKAGE__; + my ($op_c, $ops) = $lei->workers_start($self, 'ls-search', 1); + $lei->{lsss} = $self; + $self->wq_io_do('do_ls_search_long', [], $pfx); + $self->wq_close(1); + $op_c->op_wait_event($ops); +} + +sub lei_ls_search { + my ($lei, $pfx) = @_; + my $fmt = $lei->{opt}->{'format'} // ''; + if ($lei->{opt}->{l}) { + $lei->{opt}->{'format'} //= $fmt = 'json'; + } + my $json; + my $tty = -t $lei->{1}; + $lei->start_pager if $tty; + if ($fmt =~ /\A(ldjson|ndjson|jsonl|(?:concat)?json)\z/) { + $lei->{opt}->{pretty} //= $tty; + $json = ref(PublicInbox::Config->json); + } elsif ($fmt ne '') { + return $lei->fail("unknown format: $fmt"); + } + my $ORS = "\n"; + if ($lei->{opt}->{z}) { + return $lei->fail('-z and --format do not mix') if $json; + $ORS = "\0"; + } + $pfx //= ''; + return bg_worker($lei, $pfx, $json) if $json; + for (sort(grep(/\A\Q$pfx/, PublicInbox::LeiSavedSearch::list($lei)))) { + $lei->out($_, $ORS); + } +} + +1; diff --git a/lib/PublicInbox/LeiSavedSearch.pm b/lib/PublicInbox/LeiSavedSearch.pm index 3076d14c..d67622c9 100644 --- a/lib/PublicInbox/LeiSavedSearch.pm +++ b/lib/PublicInbox/LeiSavedSearch.pm @@ -21,6 +21,11 @@ sub cquote_val ($) { # cf. git-config(1) $val; } +sub ARRAY_FIELDS () { qw(only include exclude) } +sub BOOL_FIELDS () { + qw(external local remote import-remote import-before threads) +} + sub lss_dir_for ($$) { my ($lei, $dstref) = @_; my @n; @@ -39,6 +44,31 @@ sub lss_dir_for ($$) { $lei->share_path . '/saved-searches/' . join('-', @n); } +sub list { + my ($lei, $pfx) = @_; + my $lss_dir = $lei->share_path.'/saved-searches/'; + return () unless -d $lss_dir; + # TODO: persist the cache? Use another format? + my $f = $lei->cache_dir."/saved-tmp.$$.".time.'.config'; + open my $fh, '>', $f or die "open $f: $!"; + print $fh "[include]\n"; + for my $p (glob("$lss_dir/*/lei.saved-search")) { + print $fh "\tpath = ", cquote_val($p), "\n"; + } + close $fh or die "close $f: $!"; + my $cfg = PublicInbox::Config::git_config_dump($f); + unlink($f); + bless $cfg, 'PublicInbox::Config'; + my $out = $cfg->get_all('lei.q.output') or return (); + map {; + if (s!\A(?:maildir|mh|mbox.+|mmdf):!!i) { + -e $_ ? $_ : (); # TODO auto-prune somewhere? + } else { # IMAP, maybe JMAP + $_; + } + } @$out +} + sub new { my ($cls, $lei, $dst) = @_; my $self = bless { ale => $lei->ale }, $cls; @@ -74,16 +104,15 @@ $q [lei "q"] output = $dst EOM - for my $k (qw(only include exclude)) { + for my $k (ARRAY_FIELDS) { my $ary = $lei->{opt}->{$k} // next; for my $x (@$ary) { print $fh "\t$k = ".cquote_val($x)."\n"; } } - for my $k (qw(external local remote import-remote - import-before threads)) { + for my $k (BOOL_FIELDS) { my $val = $lei->{opt}->{$k} // next; - print $fh "\t$k = ".cquote_val($val)."\n"; + print $fh "\t$k = ".($val ? 1 : 0)."\n"; } close($fh) or return $lei->fail("close $f: $!"); } diff --git a/lib/PublicInbox/LeiUp.pm b/lib/PublicInbox/LeiUp.pm index 9fe4901b..73286ea2 100644 --- a/lib/PublicInbox/LeiUp.pm +++ b/lib/PublicInbox/LeiUp.pm @@ -42,4 +42,10 @@ sub lei_up { $lei->_start_query; } +sub _complete_up { + my ($lei, @argv) = @_; + my ($cur, $re) = $lei->complete_url_common(\@argv); + grep(/\A$re\Q$cur/, PublicInbox::LeiSavedSearch::list($lei)); +} + 1; diff --git a/t/lei-q-save.t b/t/lei-q-save.t index a8eda41e..761814b4 100644 --- a/t/lei-q-save.t +++ b/t/lei-q-save.t @@ -55,5 +55,17 @@ test_lei(sub { ok(-s "$home/mbcl2" > $size, 'size increased after up'); ok(!lei(qw(up -q), $home), 'up fails w/o --save'); + + lei_ok qw(ls-search); my @d = split(/\n/, $lei_out); + lei_ok qw(ls-search -z); my @z = split(/\0/, $lei_out); + is_deeply(\@d, \@z, '-z output matches non-z'); + is_deeply(\@d, [ "$home/mbcl2", "$home/md/" ], + 'ls-search output alphabetically sorted'); + lei_ok qw(ls-search -l); + my $json = PublicInbox::Config->json->decode($lei_out); + ok($json && $json->[0]->{output}, 'JSON has output'); + lei_ok qw(_complete lei up); + like($lei_out, qr!^\Q$home/mbcl2\E$!sm, 'complete got mbcl2 output'); + like($lei_out, qr!^\Q$home/md/\E$!sm, 'complete got maildir output'); }); done_testing;