lei q: support IMAP/IMAPS --output destinations

Augment (and dedupe) aren't parallel, yet, so its more sensitive to
high-latency networks.

6 files changed, 95 insertions, 7 deletions

diff --git a/lib/PublicInbox/LeiAuth.pm b/lib/PublicInbox/LeiAuth.pm
index 7acb9900..bf0110ed 100644
--- a/lib/PublicInbox/LeiAuth.pm
+++ b/lib/PublicInbox/LeiAuth.pm
@@ -63,7 +63,7 @@ sub ipc_atfork_child {
 }
 
 sub new {
-        my ($cls, $nrd) = @_;
+        my ($cls, $nrd) = @_; # nrd may be NetReader or descendant (NetWriter)
         bless { nrd => $nrd }, $cls;
 }
 
diff --git a/lib/PublicInbox/LeiOverview.pm b/lib/PublicInbox/LeiOverview.pm
index 3169bae6..4db1d8c8 100644
--- a/lib/PublicInbox/LeiOverview.pm
+++ b/lib/PublicInbox/LeiOverview.pm
@@ -59,7 +59,12 @@ sub new {
 
         my $fmt = $opt->{$ofmt_key};
         $fmt = lc($fmt) if defined $fmt;
-        if ($dst =~ s/\A([a-z0-9]+)://is) { # e.g. Maildir:/home/user/Mail/
+        if ($dst =~ m!\A([a-z0-9\+]+)://!is) {
+                defined($fmt) and return $lei->fail(<<"");
+--$ofmt_key=$fmt invalid with URL $dst
+
+                $fmt = lc $1;
+        } elsif ($dst =~ s/\A([a-z0-9]+)://is) { # e.g. Maildir:/home/user/Mail/
                 my $ofmt = lc $1;
                 $fmt //= $ofmt;
                 return $lei->fail(<<"") if $fmt ne $ofmt;
diff --git a/lib/PublicInbox/LeiQuery.pm b/lib/PublicInbox/LeiQuery.pm
index f71beae6..eaf91f2e 100644
--- a/lib/PublicInbox/LeiQuery.pm
+++ b/lib/PublicInbox/LeiQuery.pm
@@ -11,14 +11,26 @@ sub prep_ext { # externals_each callback
         $lxs->prepare_external($loc) unless $exclude->{$loc};
 }
 
-sub qstr_add { # for --stdin
+sub _start_query {
+        my ($self) = @_;
+        if (my $nwr = $self->{nwr}) {
+                require PublicInbox::LeiAuth;
+                my $auth = $self->{auth} = PublicInbox::LeiAuth->new($nwr);
+                my $lxs = $self->{lxs};
+                $auth->auth_start($self, $lxs->can('do_query'), $lxs, $self);
+        } else {
+                $self->{lxs}->do_query($self);
+        }
+}
+
+sub qstr_add { # PublicInbox::InputPipe::consume callback for --stdin
         my ($self) = @_; # $_[1] = $rbuf
         if (defined($_[1])) {
                 $_[1] eq '' and return eval {
                         my $lse = delete $self->{lse};
                         $lse->query_approxidate($lse->git,
                                                 $self->{mset_opt}->{qstr});
-                        $self->{lxs}->do_query($self);
+                        _start_query($self);
                 };
                 $self->{mset_opt}->{qstr} .= $_[1];
         } else {
@@ -115,7 +127,7 @@ no query allowed on command-line with --stdin
                 return;
         }
         $mset_opt{qstr} = $lse->query_argv_to_string($lse->git, \@argv);
-        $lxs->do_query($self);
+        _start_query($self);
 }
 
 # shell completion helper called by lei__complete
diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm
index e89cca71..0e0b0a43 100644
--- a/lib/PublicInbox/LeiToMail.pm
+++ b/lib/PublicInbox/LeiToMail.pm
@@ -331,9 +331,31 @@ sub _maildir_write_cb ($$) {
         }
 }
 
+sub _imap_write_cb ($$) {
+        my ($self, $lei) = @_;
+        my $dedupe = $lei->{dedupe};
+        $dedupe->prepare_dedupe if $dedupe;
+        my $imap_append = $lei->{nwr}->can('imap_append');
+        my $mic = $lei->{nwr}->mic_get($lei->{ovv}->{dst});
+        my $folder = $self->{uri}->mailbox;
+        sub { # for git_to_mail
+                my ($bref, $smsg, $eml) = @_;
+                $mic // return $lei->fail; # dst may be undef-ed in last run
+                if ($dedupe) {
+                        $eml //= PublicInbox::Eml->new($$bref); # copy bref
+                        return if $dedupe->is_dup($eml, $smsg->{blob});
+                }
+                eval { $imap_append->($mic, $folder, $bref, $smsg, $eml) };
+                if (my $err = $@) {
+                        undef $mic;
+                        die $err;
+                }
+        }
+}
+
 sub write_cb { # returns a callback for git_to_mail
         my ($self, $lei) = @_;
-        # _mbox_write_cb or _maildir_write_cb
+        # _mbox_write_cb, _maildir_write_cb or _imap_write_cb
         my $m = "_$self->{base_type}_write_cb";
         $self->$m($lei);
 }
@@ -360,6 +382,18 @@ sub new {
                         "$dst exists and is not a writable file\n";
                 $self->can("eml2$fmt") or die "bad mbox format: $fmt\n";
                 $self->{base_type} = 'mbox';
+        } elsif ($fmt =~ /\Aimaps?\z/) { # TODO .onion support
+                require PublicInbox::NetWriter;
+                my $nwr = PublicInbox::NetWriter->new;
+                $nwr->add_url($dst);
+                $nwr->{quiet} = $lei->{opt}->{quiet};
+                my $err = $nwr->errors($dst);
+                return $lei->fail($err) if $err;
+                require PublicInbox::URIimap; # TODO: URI cast early
+                $self->{uri} = PublicInbox::URIimap->new($dst);
+                $self->{uri}->mailbox or die "No mailbox: $dst";
+                $lei->{nwr} = $nwr;
+                $self->{base_type} = 'imap';
         } else {
                 die "bad mail --format=$fmt\n";
         }
@@ -394,6 +428,26 @@ sub _do_augment_maildir {
         }
 }
 
+sub _augment_imap { # PublicInbox::NetReader::imap_each cb
+        my ($url, $uid, $kw, $eml, $lei) = @_;
+        _augment($eml, $lei);
+}
+
+sub _do_augment_imap {
+        my ($self, $lei) = @_;
+        my $dst = $lei->{ovv}->{dst};
+        my $nwr = $lei->{nwr};
+        if ($lei->{opt}->{augment}) {
+                my $dedupe = $lei->{dedupe};
+                if ($dedupe && $dedupe->prepare_dedupe) {
+                        $nwr->imap_each($dst, \&_augment_imap, $lei);
+                        $dedupe->pause_dedupe;
+                }
+        } else { # clobber existing IMAP folder
+                $nwr->imap_delete_all($dst);
+        }
+}
+
 sub _pre_augment_mbox {
         my ($self, $lei) = @_;
         my $dst = $lei->{ovv}->{dst};
diff --git a/lib/PublicInbox/NetReader.pm b/lib/PublicInbox/NetReader.pm
index 92d004bc..541094a0 100644
--- a/lib/PublicInbox/NetReader.pm
+++ b/lib/PublicInbox/NetReader.pm
@@ -422,8 +422,13 @@ sub _imap_fetch_all ($$$) {
 # uses cached auth info prepared by mic_for
 sub mic_get {
         my ($self, $sec) = @_;
-        my $mic_arg = $self->{mic_arg}->{$sec} or
+        my $mic_arg = $self->{mic_arg}->{$sec};
+        unless ($mic_arg) {
+                my $uri = PublicInbox::URIimap->new($sec);
+                $sec = uri_section($uri);
+                $mic_arg = $self->{mic_arg}->{$sec} or
                         die "BUG: no Mail::IMAPClient->new arg for $sec";
+        }
         if (defined(my $cb_name = $mic_arg->{Authcallback})) {
                 if (ref($cb_name) ne 'CODE') {
                         $mic_arg->{Authcallback} = $self->can($cb_name);
diff --git a/lib/PublicInbox/NetWriter.pm b/lib/PublicInbox/NetWriter.pm
index 6f0a0b94..89f8662e 100644
--- a/lib/PublicInbox/NetWriter.pm
+++ b/lib/PublicInbox/NetWriter.pm
@@ -23,4 +23,16 @@ sub imap_append {
                 die "APPEND $folder: $@";
 }
 
+sub imap_delete_all {
+        my ($self, $url) = @_;
+        my $uri = PublicInbox::URIimap->new($url);
+        my $sec = $self->can('uri_section')->($uri);
+        local $0 = $uri->mailbox." $sec";
+        my $mic = $self->mic_get($sec) or die "E: not connected: $@";
+        $mic->select($uri->mailbox) or return; # non-existent
+        if ($mic->delete_message('1:*')) {
+                $mic->expunge;
+        }
+}
+
 1;