lei ls-mail-source: write through to URL folder cache

We'll be able to use this for shell completion for
lei import, lcat, tag, etc..

This also adds --url support for scripting purposes.

3 files changed, 59 insertions, 5 deletions

diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 122045ee..50a7fdad 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -116,6 +116,12 @@ sub cache_dir ($) {
                 .'/lei');
 }
 
+sub url_folder_cache {
+        my ($self) = @_;
+        require PublicInbox::SharedKV; # URI => updated_at_sec_
+        PublicInbox::SharedKV->new(cache_dir($self).'/uri_folder');
+}
+
 sub ale {
         my ($self) = @_;
         $self->{ale} //= do {
@@ -197,7 +203,7 @@ our %CMD = ( # sorted in order of importance/use:
 'ls-mail-sync' => [ '[FILTER]', 'list mail sync folders',
                 qw(z|0 globoff|g invert-match|v local remote), @c_opt ],
 'ls-mail-source' => [ 'URL', 'list IMAP or NNTP mail source folders',
-                qw(z|0 ascii l), @c_opt ],
+                qw(z|0 ascii l url), @c_opt ],
 'forget-external' => [ 'LOCATION...|--prune',
         'exclude further results from a publicinbox|extindex',
         qw(prune), @c_opt ],
@@ -384,6 +390,7 @@ my %OPTDESC = (
                         'listing output format' ],
 'l        ls-search' => 'long listing format',
 'l        ls-mail-source' => 'long listing format',
+'url        ls-mail-source' => 'show full URL of newsgroup or IMAP folder',
 'format|f=s        ls-external' => $ls_format,
 
 'limit|n=i@' => ['NUM', 'limit on number of matches (default: 10000)' ],
diff --git a/lib/PublicInbox/LeiLsMailSource.pm b/lib/PublicInbox/LeiLsMailSource.pm
index e95c0b34..cadc61ed 100644
--- a/lib/PublicInbox/LeiLsMailSource.pm
+++ b/lib/PublicInbox/LeiLsMailSource.pm
@@ -21,19 +21,27 @@ sub input_path_url { # overrides LeiInput version
         } elsif ($self->{lei}->{opt}->{z}) {
                 $ORS = "\0";
         }
+        my @f;
         if ($url =~ m!\Aimaps?://!i) {
                 my $uri = PublicInbox::URIimap->new($url);
+                my $sec = $lei->{net}->can('uri_section')->($uri);
                 my $mic = $lei->{net}->mic_get($uri);
                 my $l = $mic->folders_hash($uri->path); # server-side filter
+                @f = map { "$sec/$_->{name}" } @$l;
                 if ($json) {
+                        $_->{url} = "$sec/$_->{name}" for @$l;
                         $lei->puts($json->encode($l));
                 } else {
+                        if ($self->{lei}->{opt}->{url}) {
+                                $_->{name} = "$sec/$_->{name}" for @$l;
+                        }
                         $lei->out(join($ORS, (map { $_->{name} } @$l), ''));
                 }
         } elsif ($url =~ m!\A(?:nntps?|s?news)://!i) {
                 my $uri = PublicInbox::URInntps->new($url);
                 my $nn = $lei->{net}->nn_get($uri);
                 my $l = $nn->newsgroups($uri->group); # name => description
+                my $sec = $lei->{net}->can('uri_section')->($uri);
                 if ($json) {
                         my $all = $nn->list;
                         my @x;
@@ -46,15 +54,30 @@ sub input_path_url { # overrides LeiInput version
                                 $desc =~ s/\r\z//;
 
                                 my ($hwm, $lwm, $status) = @{$all->{$ng}};
-                                push @x, { name => $ng, lwm => $lwm + 0,
+                                push @x, { name => $ng, url => "$sec/$ng",
+                                        lwm => $lwm + 0,
                                         hwm => $hwm + 0, status => $status,
                                         description => $desc };
                         }
+                        @f = map { "$sec/$_" } keys %$all;
                         $lei->puts($json->encode(\@x));
                 } else {
-                        $lei->out(join($ORS, sort(keys %$l), ''));
+                        @f = map { "$sec/$_" } keys %$l;
+                        if ($self->{lei}->{opt}->{url}) {
+                                $lei->out(join($ORS, sort(@f), ''));
+                        } else {
+                                $lei->out(join($ORS, sort(keys %$l), ''));
+                        }
                 }
         } else { die "BUG: $url not supported" }
+        if (@f) {
+                my $fc = $lei->url_folder_cache;
+                my $lk = $fc->lock_for_scope;
+                $fc->dbh->begin_work;
+                my $now = time;
+                $fc->set($_, $now) for @f;
+                $fc->dbh->commit;
+        }
 }
 
 sub lei_ls_mail_source {
@@ -62,6 +85,7 @@ sub lei_ls_mail_source {
         $url =~ m!\A(?:imaps?|nntps?|s?news)://!i or return
                 $lei->fail('only NNTP and IMAP URLs supported');
         my $self = bless { pfx => $pfx, -ls_ok => 1 }, __PACKAGE__;
+        $self->{cfg} = $lei->_lei_cfg; # may be undef
         $self->prepare_inputs($lei, [ $url ]) or return;
         $lei->start_pager if -t $lei->{1};
         my $ops = {};
@@ -74,6 +98,20 @@ sub lei_ls_mail_source {
         $lei->wait_wq_events($op_c, $ops); # net_merge_all_done if !{auth}
 }
 
+sub _complete_ls_mail_source {
+        my ($lei, @argv) = @_;
+        my $match_cb = $lei->complete_url_prepare(\@argv);
+        my @m = map { $match_cb->($_) } $lei->url_folder_cache->keys;
+        my %f = map { $_ => 1 } @m;
+        my $sto = $lei->_lei_store;
+        if (my $lms = $sto ? $sto->search->lms : undef) {
+                @m = map { $match_cb->($_) } grep(
+                        m!\A(?:imaps?|nntps?|s?news)://!, $lms->folders);
+                @f{@m} = @m;
+        }
+        keys %f;
+}
+
 no warnings 'once';
 *ipc_atfork_child = \&PublicInbox::LeiInput::input_only_atfork_child;
 *net_merge_all_done = \&PublicInbox::LeiInput::input_only_net_merge_all_done;
diff --git a/lib/PublicInbox/SharedKV.pm b/lib/PublicInbox/SharedKV.pm
index d65c3158..8347b195 100644
--- a/lib/PublicInbox/SharedKV.pm
+++ b/lib/PublicInbox/SharedKV.pm
@@ -11,7 +11,7 @@ use parent qw(PublicInbox::Lock);
 use File::Temp qw(tempdir);
 use DBI ();
 use PublicInbox::Spawn;
-use File::Path qw(rmtree);
+use File::Path qw(rmtree make_path);
 
 sub dbh {
         my ($self, $lock) = @_;
@@ -46,8 +46,8 @@ CREATE TABLE IF NOT EXISTS kv (
 sub new {
         my ($cls, $dir, $base, $opt) = @_;
         my $self = bless { opt => $opt }, $cls;
+        make_path($dir) if defined($dir) && !-d $dir;
         $dir //= $self->{"tmp$$.$self"} = tempdir("skv.$$-XXXX", TMPDIR => 1);
-        -d $dir or mkdir($dir) or die "mkdir($dir): $!";
         $base //= '';
         my $f = $self->{filename} = "$dir/$base.sqlite3";
         $self->{lock_path} = $opt->{lock_path} // "$dir/$base.flock";
@@ -83,6 +83,15 @@ SELECT k,v FROM kv
         $sth
 }
 
+sub keys {
+        my ($self) = @_;
+        my $sth = $self->dbh->prepare_cached(<<'', undef, 1);
+SELECT k FROM kv
+
+        $sth->execute;
+        map { $_->[0] } @{$sth->fetchall_arrayref};
+}
+
 sub delete_by_val {
         my ($self, $val, $lock) = @_;
         $lock //= $self->lock_for_scope_fast;