[PATCH 0/3] lei export-kw: IMAP support

[PATCH 0/3] lei export-kw: IMAP support

[Eric Wong]

Dealing with high-latency connections to be addressed at some
point, but probably not right away...

Eric Wong (3):
  net_reader|net_writer: pass URI refs deeper into callbacks
  lei export-kw: support exporting keywords to IMAP
  lei export-kw: relax IMAP URL matching

 lib/PublicInbox/LeiExportKw.pm | 72 ++++++++++++++++++++++++++++++----
 lib/PublicInbox/LeiImport.pm   |  4 +-
 lib/PublicInbox/LeiToMail.pm   | 13 +++---
 lib/PublicInbox/NetReader.pm   | 11 +++---
 lib/PublicInbox/NetWriter.pm   | 22 +++++++----
 xt/net_writer-imap.t           | 38 ++++++++++++------
 6 files changed, 117 insertions(+), 43 deletions(-)

[PATCH 1/3] net_reader|net_writer: pass URI refs deeper into callbacks

[Eric Wong]

This will give us more flexibility in the future w.r.t.
dealing with UIDVALIDITY and AUTH= info with IMAP.  The LoC
reduction is welcome, too.
---
 lib/PublicInbox/LeiImport.pm |  4 ++--
 lib/PublicInbox/LeiToMail.pm |  4 ++--
 lib/PublicInbox/NetReader.pm | 11 +++++------
 lib/PublicInbox/NetWriter.pm | 12 ++++--------
 4 files changed, 13 insertions(+), 18 deletions(-)

diff --git a/lib/PublicInbox/LeiImport.pm b/lib/PublicInbox/LeiImport.pm
index 55925cc5..01e6c93c 100644
--- a/lib/PublicInbox/LeiImport.pm
+++ b/lib/PublicInbox/LeiImport.pm
@@ -42,9 +42,9 @@ sub input_maildir_cb { # maildir_each_eml cb
 }
 
 sub input_net_cb { # imap_each / nntp_each
-	my ($url, $uid, $kw, $eml, $self) = @_;
+	my ($uri, $uid, $kw, $eml, $self) = @_;
 	my $vmd = $self->{-import_kw} ? { kw => $kw } : undef;
-	$vmd->{sync_info} = [ $url, $uid ] if $self->{-mail_sync};
+	$vmd->{sync_info} = [ $$uri, $uid ] if $self->{-mail_sync};
 	$self->input_eml_cb($eml, $vmd);
 }
 
diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm
index 96a1f881..b9d4c856 100644
--- a/lib/PublicInbox/LeiToMail.pm
+++ b/lib/PublicInbox/LeiToMail.pm
@@ -452,10 +452,10 @@ sub _do_augment_maildir {
 }
 
 sub _imap_augment_or_delete { # PublicInbox::NetReader::imap_each cb
-	my ($url, $uid, $kw, $eml, $lei, $lse, $delete_mic) = @_;
+	my ($uri, $uid, $kw, $eml, $lei, $lse, $delete_mic) = @_;
 	update_kw_maybe($lei, $lse, $eml, $kw);
 	if ($delete_mic) {
-		$lei->{net}->imap_delete_1($url, $uid, $delete_mic);
+		$lei->{net}->imap_delete_1($uri, $uid, $delete_mic);
 	} else {
 		_augment($eml, $lei);
 	}
diff --git a/lib/PublicInbox/NetReader.pm b/lib/PublicInbox/NetReader.pm
index a532b218..73b8b1cd 100644
--- a/lib/PublicInbox/NetReader.pm
+++ b/lib/PublicInbox/NetReader.pm
@@ -397,7 +397,7 @@ sub errors {
 }
 
 sub _imap_do_msg ($$$$$) {
-	my ($self, $url, $uid, $raw, $flags) = @_;
+	my ($self, $uri, $uid, $raw, $flags) = @_;
 	# our target audience expects LF-only, save storage
 	$$raw =~ s/\r\n/\n/sg;
 	my $kw = [];
@@ -408,12 +408,12 @@ sub _imap_do_msg ($$$$$) {
 		} elsif ($f eq "\\Deleted") { # not in JMAP
 			return;
 		} elsif ($self->{verbose}) {
-			warn "# unknown IMAP flag $f <$url/;UID=$uid>\n";
+			warn "# unknown IMAP flag $f <$uri/;UID=$uid>\n";
 		}
 	}
 	@$kw = sort @$kw; # for all UI/UX purposes
 	my ($eml_cb, @args) = @{$self->{eml_each}};
-	$eml_cb->($url, $uid, $kw, PublicInbox::Eml->new($raw), @args);
+	$eml_cb->($uri, $uid, $kw, PublicInbox::Eml->new($raw), @args);
 }
 
 sub run_commit_cb ($) {
@@ -532,7 +532,7 @@ EOF
 				# messages get deleted, so holes appear
 				my $per_uid = delete $r->{$uid} // next;
 				my $raw = delete($per_uid->{$key}) // next;
-				_imap_do_msg($self, $$uri, $uid, \$raw,
+				_imap_do_msg($self, $uri, $uid, \$raw,
 						$per_uid->{FLAGS});
 				$last_uid = $uid;
 				last if $self->{quit};
@@ -638,7 +638,6 @@ sub _nntp_fetch_all ($$$) {
 		warn "# $uri fetching ARTICLE $beg..$end\n";
 	}
 	my $n = $self->{max_batch};
-	my $url = $$uri;
 	for ($beg..$end) {
 		last if $self->{quit};
 		$art = $_;
@@ -661,7 +660,7 @@ sub _nntp_fetch_all ($$$) {
 		$raw = join('', @$raw);
 		$raw =~ s/\r\n/\n/sg;
 		my ($eml_cb, @args) = @{$self->{eml_each}};
-		$eml_cb->($url, $art, $kw, PublicInbox::Eml->new(\$raw), @args);
+		$eml_cb->($uri, $art, $kw, PublicInbox::Eml->new(\$raw), @args);
 		$last_art = $art;
 	}
 	run_commit_cb($self);
diff --git a/lib/PublicInbox/NetWriter.pm b/lib/PublicInbox/NetWriter.pm
index 06d69f27..2032a1fd 100644
--- a/lib/PublicInbox/NetWriter.pm
+++ b/lib/PublicInbox/NetWriter.pm
@@ -28,18 +28,14 @@ sub imap_append {
 
 sub mic_for_folder {
 	my ($self, $uri) = @_;
-	if (!ref($uri)) {
-		my $u = PublicInbox::URIimap->new($uri);
-		$_[1] = $uri = $u;
-	}
 	my $mic = $self->mic_get($uri) or die "E: not connected: $@";
 	$mic->select($uri->mailbox) or return;
 	$mic;
 }
 
 sub imap_delete_all {
-	my ($self, $url) = @_;
-	my $mic = mic_for_folder($self, my $uri = $url) or return;
+	my ($self, $uri) = @_;
+	my $mic = mic_for_folder($self, $uri) or return;
 	my $sec = $self->can('uri_section')->($uri);
 	local $0 = $uri->mailbox." $sec";
 	if ($mic->delete_message('1:*')) {
@@ -48,8 +44,8 @@ sub imap_delete_all {
 }
 
 sub imap_delete_1 {
-	my ($self, $url, $uid, $delete_mic) = @_;
-	$$delete_mic //= mic_for_folder($self, my $uri = $url) or return;
+	my ($self, $uri, $uid, $delete_mic) = @_;
+	$$delete_mic //= mic_for_folder($self, $uri) or return;
 	$$delete_mic->delete_message($uid);
 }
 

[PATCH 2/3] lei export-kw: support exporting keywords to IMAP

[Eric Wong]

We support writing to IMAP stores in other places (just like
Maildir), and it's actually less complex for us to write to
IMAP.  Neither usability nor performance is ideal, but usability
will be addressed in the next commit to relax CLI argument
checking.

Performance is poor due to the synchronous Mail::IMAPClient
API and will need to be addressed with pipelining sometime
further in the future.
---
 lib/PublicInbox/LeiExportKw.pm | 31 +++++++++++++++++++++++--------
 lib/PublicInbox/LeiToMail.pm   |  9 +++++----
 lib/PublicInbox/NetWriter.pm   | 10 ++++++++++
 xt/net_writer-imap.t           | 18 +++++++++++++-----
 4 files changed, 51 insertions(+), 17 deletions(-)

diff --git a/lib/PublicInbox/LeiExportKw.pm b/lib/PublicInbox/LeiExportKw.pm
index db4f7441..5ad33959 100644
--- a/lib/PublicInbox/LeiExportKw.pm
+++ b/lib/PublicInbox/LeiExportKw.pm
@@ -59,15 +59,28 @@ sub export_kw_md { # LeiMailSync->each_src callback
 	$self->{lei}->child_error(1, "link($orig, $dst) ($oidhex): $e");
 }
 
+sub export_kw_imap { # LeiMailSync->each_src callback
+	my ($oidbin, $id, $self, $mic) = @_;
+	my $oidhex = unpack('H*', $oidbin);
+	my $sto_kw = $self->{lse}->oid_keywords($oidhex) or return;
+	$self->{imap_mod_kw}->($self->{nwr}, $mic, $id, [ keys %$sto_kw ]);
+}
+
 # overrides PublicInbox::LeiInput::input_path_url
 sub input_path_url {
 	my ($self, $input, @args) = @_;
 	my $lms = $self->{lms} //= $self->{lse}->lms;
 	$lms->lms_begin;
-	if ($input =~ s/\Amaildir://i) {
+	if ($input =~ /\Amaildir:(.+)/i) {
+		my $mdir = $1;
 		require PublicInbox::LeiToMail; # kw2suffix
-		$lms->each_src("maildir:$input", \&export_kw_md, $self, $input);
-	}
+		$lms->each_src($input, \&export_kw_md, $self, $mdir);
+	} elsif ($input =~ m!\Aimaps?://!) {
+		my $uri = PublicInbox::URIimap->new($input);
+		my $mic = $self->{nwr}->mic_for_folder($uri);
+		$lms->each_src($$uri, \&export_kw_imap, $self, $mic);
+		$mic->expunge;
+	} else { die "BUG: $input not supported" }
 	$lms->lms_commit;
 }
 
@@ -137,11 +150,6 @@ EOF
 	if (my @ro = grep(!/\A(?:maildir|imaps?):/, @folders)) {
 		return $lei->fail("cannot export to read-only folders: @ro");
 	}
-	if (my $net = $lei->{net}) {
-		require PublicInbox::NetWriter;
-		bless $net, 'PublicInbox::NetWriter';
-	}
-	undef $lms;
 	my $m = $opt->{mode} // 'merge';
 	if ($m eq 'merge') { # default
 		$self->{-merge_kw} = 1;
@@ -151,6 +159,13 @@ EOF
 --mode=$m not supported (`set' or `merge')
 EOM
 	}
+	if (my $net = $lei->{net}) {
+		require PublicInbox::NetWriter;
+		$self->{nwr} = bless $net, 'PublicInbox::NetWriter';
+		$self->{imap_mod_kw} = $net->can($self->{-merge_kw} ?
+					'imap_add_kw' : 'imap_set_kw');
+	}
+	undef $lms;
 	my $ops = {};
 	$lei->{auth}->op_merge($ops, $self) if $lei->{auth};
 	$self->{-wq_nr_workers} = $j // 1; # locked
diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm
index b9d4c856..f3c03969 100644
--- a/lib/PublicInbox/LeiToMail.pm
+++ b/lib/PublicInbox/LeiToMail.pm
@@ -307,11 +307,12 @@ sub _imap_write_cb ($$) {
 	my $dedupe = $lei->{dedupe};
 	$dedupe->prepare_dedupe if $dedupe;
 	my $append = $lei->{net}->can('imap_append');
-	my $mic = $lei->{net}->mic_get($self->{uri});
-	my $folder = $self->{uri}->mailbox;
+	my $uri = $self->{uri};
+	my $mic = $lei->{net}->mic_get($uri);
+	my $folder = $uri->mailbox;
+	$uri->uidvalidity($mic->uidvalidity($folder));
 	my $lse = $lei->{lse}; # may be undef
 	my $sto = $lei->{opt}->{'mail-sync'} ? $lei->{sto} : undef;
-	my $out = $lei->{ovv}->{dst};
 	sub { # for git_to_mail
 		my ($bref, $smsg, $eml) = @_;
 		$mic // return $lei->fail; # mic may be undef-ed in last run
@@ -327,7 +328,7 @@ sub _imap_write_cb ($$) {
 		# imap_append returns UID if IMAP server has UIDPLUS extension
 		($sto && $uid =~ /\A[0-9]+\z/) and
 			$sto->ipc_do('set_sync_info',
-					$smsg->{blob}, $out, $uid + 0);
+					$smsg->{blob}, $$uri, $uid + 0);
 		++$lei->{-nr_write};
 	}
 }
diff --git a/lib/PublicInbox/NetWriter.pm b/lib/PublicInbox/NetWriter.pm
index 2032a1fd..8ec7f85c 100644
--- a/lib/PublicInbox/NetWriter.pm
+++ b/lib/PublicInbox/NetWriter.pm
@@ -26,10 +26,20 @@ sub imap_append {
 		die "APPEND $folder: $@";
 }
 
+# updates $uri with UIDVALIDITY
 sub mic_for_folder {
 	my ($self, $uri) = @_;
 	my $mic = $self->mic_get($uri) or die "E: not connected: $@";
 	$mic->select($uri->mailbox) or return;
+	my $uidval;
+	for ($mic->Results) {
+		/^\* OK \[UIDVALIDITY ([0-9]+)\].*/ or next;
+		$uidval = $1;
+		last;
+	}
+	$uidval //= $mic->uidvalidity($uri->mailbox) or
+		die "E: failed to get uidvalidity from <$uri>: $@";
+	$uri->uidvalidity($uidval);
 	$mic;
 }
 
diff --git a/xt/net_writer-imap.t b/xt/net_writer-imap.t
index 1298b958..0e6d4831 100644
--- a/xt/net_writer-imap.t
+++ b/xt/net_writer-imap.t
@@ -157,12 +157,20 @@ test_lei(sub {
 
 	lei_ok qw(import -F eml), $f, \'import local copy w/o keywords';
 
+	lei_ok 'ls-mail-sync'; diag $lei_out;
+	lei_ok 'import', $$folder_uri; # populate mail_sync.sqlite3
+	lei_ok qw(tag +kw:seen +kw:answered +kw:flagged), $f;
+	lei_ok 'ls-mail-sync'; diag $lei_out;
+	chomp(my $uri_val = $lei_out);
+	lei_ok 'export-kw', $uri_val;
 	$mic = $nwr->mic_for_folder($folder_uri);
-	# dummy set to ensure second set_kw clobbers
-	$nwr->imap_set_kw($mic, $uid[0], [ qw(seen answered flagged) ]
-			)->expunge or BAIL_OUT "expunge $@";
-	$nwr->imap_set_kw($mic, $uid[0], [ 'seen' ]
-			)->expunge or BAIL_OUT "expunge $@";
+	my $flags = $mic->flags($uid[0]);
+	is_deeply([sort @$flags], [ qw(\\Answered \\Flagged \\Seen) ],
+		'IMAP flags set by export-kw') or diag explain($flags);
+
+	# ensure this imap_set_kw clobbers
+	$nwr->imap_set_kw($mic, $uid[0], [ 'seen' ])->expunge or
+		BAIL_OUT "expunge $@";
 	$mic = undef;
 	@res = ();
 	$nwr->imap_each($folder_uri, $imap_slurp_all, \@res);

[PATCH 3/3] lei export-kw: relax IMAP URL matching

[Eric Wong]

It's unreasonable to expect UIDVALIDITY= to be specified in
command-line arguments.  We'll also check for cases without
"$USER@" or ";AUTH=", since we accept those forms on the
command-line.
---
 lib/PublicInbox/LeiExportKw.pm | 41 ++++++++++++++++++++++++++++++++++
 xt/net_writer-imap.t           | 30 ++++++++++++++-----------
 2 files changed, 58 insertions(+), 13 deletions(-)

diff --git a/lib/PublicInbox/LeiExportKw.pm b/lib/PublicInbox/LeiExportKw.pm
index 5ad33959..82a4db04 100644
--- a/lib/PublicInbox/LeiExportKw.pm
+++ b/lib/PublicInbox/LeiExportKw.pm
@@ -84,6 +84,37 @@ sub input_path_url {
 	$lms->lms_commit;
 }
 
+sub match_imap_url ($$) {
+	my ($all, $url) = @_; # $all = [ $lms->folders ];
+	require PublicInbox::URIimap;
+	my $cli = PublicInbox::URIimap->new($url)->canonical;
+	my ($s, $h, $mb) = ($cli->scheme, $cli->host, $cli->mailbox);
+	my @uri = map { PublicInbox::URIimap->new($_)->canonical }
+		grep(m!\A\Q$s\E://.*?\Q$h\E\b.*?/\Q$mb\E\b!, @$all);
+	my @match;
+	for my $x (@uri) {
+		next if $x->mailbox ne $cli->mailbox;
+		next if $x->host ne $cli->host;
+		next if $x->port != $cli->port;
+		my $x_uidval = $x->uidvalidity;
+		next if ($cli->uidvalidity // $x_uidval) != $x_uidval;
+
+		# allow nothing in CLI to possibly match ";AUTH=ANONYMOUS"
+		if (defined($x->auth) && !defined($cli->auth) &&
+				!defined($cli->user)) {
+			push @match, $x;
+		# or maybe user was forgotten on CLI:
+		} elsif (defined($x->user) && !defined($cli->user)) {
+			push @match, $x;
+		} elsif (($x->user//"\0") eq ($cli->user//"\0")) {
+			push @match, $x;
+		}
+	}
+	return $match[0] if scalar(@match) <= 1;
+	warn "E: `$url' is ambiguous:\n\t", join("\n\t", @match), "\n";
+	undef;
+}
+
 sub lei_export_kw {
 	my ($lei, @folders) = @_;
 	my $sto = $lei->_lei_store or return $lei->fail(<<EOM);
@@ -133,6 +164,16 @@ EOM
 				my $d = 'maildir:'.$lei->rel2abs($_);
 				push(@no, $_) unless $all{$d};
 				$_ = $d;
+			} elsif (m!\Aimaps?://!i) {
+				my $orig = $_;
+				if (my $canon = match_imap_url(\@all, $orig)) {
+					$lei->qerr(<<EOM);
+# using `$canon' instead of `$orig'
+EOM
+					$_ = $canon;
+				} else {
+					push @no, $orig;
+				}
 			} else {
 				push @no, $_;
 			}
diff --git a/xt/net_writer-imap.t b/xt/net_writer-imap.t
index 0e6d4831..ec8f80d1 100644
--- a/xt/net_writer-imap.t
+++ b/xt/net_writer-imap.t
@@ -23,7 +23,8 @@ my $folder = "INBOX.$base-$host-".strftime('%Y%m%d%H%M%S', gmtime(time)).
 		"-$$-".sprintf('%x', int(rand(0xffffffff)));
 my $nwr = PublicInbox::NetWriter->new;
 chop($imap_url) if substr($imap_url, -1) eq '/';
-my $folder_uri = PublicInbox::URIimap->new("$imap_url/$folder");
+my $folder_url = "$imap_url/$folder";
+my $folder_uri = PublicInbox::URIimap->new($folder_url);
 is($folder_uri->mailbox, $folder, 'folder correct') or
 		BAIL_OUT "BUG: bad $$uri";
 $nwr->add_url($$folder_uri);
@@ -120,13 +121,13 @@ test_lei(sub {
 	}
 	$set_cred_helper->("$ENV{HOME}/.gitconfig", $cred_set) if $cred_set;
 
-	lei_ok qw(q f:qp@example.com -o), $$folder_uri;
+	lei_ok qw(q f:qp@example.com -o), $folder_url;
 	$nwr->imap_each($folder_uri, $imap_slurp_all, my $res = []);
 	is(scalar(@$res), 1, 'got one deduped result') or diag explain($res);
 	is_deeply($res->[0]->[1], $plack_qp_eml,
 			'lei q wrote expected result');
 
-	lei_ok qw(q f:matz -a -o), $$folder_uri;
+	lei_ok qw(q f:matz -a -o), $folder_url;
 	$nwr->imap_each($folder_uri, $imap_slurp_all, my $aug = []);
 	is(scalar(@$aug), 2, '2 results after augment') or diag explain($aug);
 	my $exp = $res->[0]->[1]->as_string;
@@ -136,13 +137,13 @@ test_lei(sub {
 	is(scalar(grep { $_->[1]->as_string eq $exp } @$aug), 1,
 			'new result shown after augment');
 
-	lei_ok qw(q s:thisbetternotgiveanyresult -o), $folder_uri->as_string;
+	lei_ok qw(q s:thisbetternotgiveanyresult -o), $folder_url;
 	$nwr->imap_each($folder_uri, $imap_slurp_all, my $empty = []);
 	is(scalar(@$empty), 0, 'no results w/o augment');
 
 	my $f = 't/utf8.eml'; # <testmessage@example.com>
 	$exp = eml_load($f);
-	lei_ok qw(convert -F eml -o), $$folder_uri, $f;
+	lei_ok qw(convert -F eml -o), $folder_url, $f;
 	my (@uid, @res);
 	$nwr->imap_each($folder_uri, sub {
 		my ($u, $uid, $kw, $eml) = @_;
@@ -157,12 +158,15 @@ test_lei(sub {
 
 	lei_ok qw(import -F eml), $f, \'import local copy w/o keywords';
 
-	lei_ok 'ls-mail-sync'; diag $lei_out;
-	lei_ok 'import', $$folder_uri; # populate mail_sync.sqlite3
+	lei_ok 'import', $folder_url; # populate mail_sync.sqlite3
 	lei_ok qw(tag +kw:seen +kw:answered +kw:flagged), $f;
-	lei_ok 'ls-mail-sync'; diag $lei_out;
-	chomp(my $uri_val = $lei_out);
-	lei_ok 'export-kw', $uri_val;
+	lei_ok 'ls-mail-sync';
+	my @ls = split(/\n/, $lei_out);
+	is(scalar(@ls), 1, 'only one folder in ls-mail-sync') or xbail(\@ls);
+	for my $l (@ls) {
+		like($l, qr/;UIDVALIDITY=\d+\z/, 'UIDVALIDITY');
+	}
+	lei_ok 'export-kw', $folder_url;
 	$mic = $nwr->mic_for_folder($folder_uri);
 	my $flags = $mic->flags($uid[0]);
 	is_deeply([sort @$flags], [ qw(\\Answered \\Flagged \\Seen) ],
@@ -177,7 +181,7 @@ test_lei(sub {
 	is_deeply(\@res, [ [ ['seen'], $exp ] ], 'seen flag set') or
 		diag explain(\@res);
 
-	lei_ok qw(q s:thisbetternotgiveanyresult -o), $folder_uri->as_string,
+	lei_ok qw(q s:thisbetternotgiveanyresult -o), $folder_url,
 		\'clobber folder but import flag';
 	$nwr->imap_each($folder_uri, $imap_slurp_all, $empty = []);
 	is_deeply($empty, [], 'clobbered folder');
@@ -206,7 +210,7 @@ EOM
 	run_script(\@cmd) or BAIL_OUT "init wtest";
 	xsys(qw(git config), "--file=$ENV{HOME}/.public-inbox/config",
 			'publicinbox.wtest.watch',
-			$$folder_uri) == 0 or BAIL_OUT "git config $?";
+			$folder_url) == 0 or BAIL_OUT "git config $?";
 	my $watcherr = "$ENV{HOME}/watch.err";
 	open my $err_wr, '>>', $watcherr or BAIL_OUT $!;
 	my $pub_cfg = PublicInbox::Config->new;
@@ -231,7 +235,7 @@ EOM
 	ok(defined($mm->num_for('forwarded@test.example.com')),
 		'-watch takes forwarded message');
 	undef $w; # done with watch
-	lei_ok qw(import), $$folder_uri;
+	lei_ok qw(import), $folder_url;
 	lei_ok qw(q m:forwarded@test.example.com);
 	is_deeply(json_utf8->decode($lei_out)->[0]->{kw}, ['forwarded'],
 		'forwarded kw imported from IMAP');