[PATCH 37/82] xt: add imapd-validate and imapd-mbsync-oimap

[Eric Wong <e@yhbt.net>]

imapd-validate is a beefed up version of our nntpd-validate test
which hammers the server with parallel connections over regular
IMAP, IMAPS, IMAP+STARTTLS; and COMPRESS=DEFLATE variants of
each of those.  It uses $START_UID:$END_UID fetch ranges to
reduce requests and slurp many responses at once to saturate
"git cat-file --batch" processes.

mbsync(1) also uses pipelining extensively (but IMHO
unnecessarily), so it was able to shake out some bugs in
the async git code.

Finally, we remove xt/cmp-imapd-compress.t since it's
redundant now that we have PublicInbox::IMAPClient to work
around bugs in Mail::IMAPClient.
---
 MANIFEST                |   3 +-
 xt/cmp-imapd-compress.t |  83 -------------------
 xt/imapd-mbsync-oimap.t | 129 +++++++++++++++++++++++++++++
 xt/imapd-validate.t     | 175 ++++++++++++++++++++++++++++++++++++++++
 4 files changed, 306 insertions(+), 84 deletions(-)
 delete mode 100644 xt/cmp-imapd-compress.t
 create mode 100644 xt/imapd-mbsync-oimap.t
 create mode 100644 xt/imapd-validate.t

diff --git a/MANIFEST b/MANIFEST
index 6744a519efc..957228250a7 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -347,12 +347,13 @@ t/www_listing.t
 t/www_static.t
 t/x-unknown-alpine.eml
 t/xcpdb-reshard.t
-xt/cmp-imapd-compress.t
 xt/cmp-msgstr.t
 xt/cmp-msgview.t
 xt/eml_check_limits.t
 xt/git-http-backend.t
 xt/git_async_cmp.t
+xt/imapd-mbsync-oimap.t
+xt/imapd-validate.t
 xt/mem-msgview.t
 xt/msgtime_cmp.t
 xt/nntpd-validate.t
diff --git a/xt/cmp-imapd-compress.t b/xt/cmp-imapd-compress.t
deleted file mode 100644
index b12cf74e2d5..00000000000
--- a/xt/cmp-imapd-compress.t
+++ /dev/null
@@ -1,83 +0,0 @@
-#!perl -w
-# Copyright (C) 2020 all contributors <meta@public-inbox.org>
-# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
-use strict;
-use Test::More;
-use PublicInbox::TestCommon;
-require_mods('Data::Dumper');
-Data::Dumper->import('Dumper');
-my $inboxdir = $ENV{GIANT_INBOX_DIR};
-(defined($inboxdir) && -d $inboxdir) or
-	plan skip_all => "GIANT_INBOX_DIR not defined for $0";
-plan skip_all => "bad characters in $inboxdir" if $inboxdir =~ m![^\w\.\-/]!;
-my ($tmpdir, $for_destroy) = tmpdir();
-my $cfg = "$tmpdir/cfg";
-my $mailbox = 'inbox.test';
-{
-	open my $fh, '>', $cfg or BAIL_OUT "open: $!";
-	print $fh <<EOF or BAIL_OUT "print: $!";
-[publicinbox "test"]
-	newsgroup = $mailbox
-	address = test\@example.com
-	inboxdir = $inboxdir
-EOF
-	close $fh or BAIL_OUT "close: $!";
-}
-my ($out, $err) = ("$tmpdir/stdout.log", "$tmpdir/stderr.log");
-my $sock = tcp_server();
-my $cmd = [ '-imapd', '-W0', "--stdout=$out", "--stderr=$err"];
-my $env = { PI_CONFIG => $cfg };
-my $td = start_script($cmd, $env, { 3 => $sock }) or BAIL_OUT "-imapd: $?";
-my ($host, $port) = ($sock->sockhost, $sock->sockport);
-my $c = tcp_connect($sock);
-like(readline($c), qr/CAPABILITY /, 'got greeting');
-undef $c;
-
-SKIP: {
-	require_mods('Mail::IMAPClient', 3);
-	unless ($ENV{RT_132720_FIXED}) {
-		my $bug = 'https://rt.cpan.org/Ticket/Display.html?id=132720';
-		skip "<$bug>, RT_132720_FIXED not defined", 3;
-	}
-	my %opt = (Server => $host, Port => $port,
-			User => 'u', Password => 'p', Clear => 1);
-	my $uc = Mail::IMAPClient->new(%opt);
-	my $c = Mail::IMAPClient->new(%opt);
-	ok($c->compress, 'enabled compression');
-	ok $c->examine($mailbox), 'compressed EXAMINE-ed';
-	ok $uc->examine($mailbox), 'uncompress EXAMINE-ed';
-	my $range = $uc->search('all');
-	for my $uid (@$range) {
-		my $A = $uc->fetch_hash($uid, 'BODY[]');
-		my $B = $c->fetch_hash($uid, 'BODY[]');
-		if (!is_deeply($A, $B, "$uid identical")) {
-			diag Dumper([$A, $B]);
-			diag Dumper([$uc, $c]);
-			last;
-		}
-	}
-	$uc->logout;
-	$c->logout;
-}
-
-SKIP: {
-	require_mods('Mail::IMAPTalk', 3);
-	my %opt = (Server => $host, Port => $port, UseSSL => 0,
-		Username => 'u', Password => 'p', Uid => 1);
-	my $uc = Mail::IMAPTalk->new(%opt) or BAIL_OUT 'IMAPTalk->new';
-	my $c = Mail::IMAPTalk->new(%opt, UseCompress => 1) or
-		BAIL_OUT 'IMAPTalk->new(UseCompress => 1)';
-	ok $c->examine($mailbox), 'compressed EXAMINE-ed';
-	ok $uc->examine($mailbox), 'uncompress EXAMINE-ed';
-	my $range = $uc->search('all');
-	for my $uid (@$range) {
-		my $A = $uc->fetch($uid, 'rfc822');
-		my $B = $c->fetch($uid, 'rfc822');
-		if (!is_deeply($A, $B, "$uid identical")) {
-			diag Dumper([$A, $B]);
-			diag Dumper([$uc, $c]);
-			last;
-		}
-	}
-}
-done_testing;
diff --git a/xt/imapd-mbsync-oimap.t b/xt/imapd-mbsync-oimap.t
new file mode 100644
index 00000000000..d2237a24bef
--- /dev/null
+++ b/xt/imapd-mbsync-oimap.t
@@ -0,0 +1,129 @@
+#!perl -w
+# Copyright (C) 2020 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+# ensure mbsync and offlineimap compatibility
+use strict;
+use Test::More;
+use File::Path qw(mkpath);
+use PublicInbox::TestCommon;
+use PublicInbox::Spawn qw(which spawn);
+my $inboxdir = $ENV{GIANT_INBOX_DIR};
+(defined($inboxdir) && -d $inboxdir) or
+	plan skip_all => "GIANT_INBOX_DIR not defined for $0";
+plan skip_all => "bad characters in $inboxdir" if $inboxdir =~ m![^\w\.\-/]!;
+my ($tmpdir, $for_destroy) = tmpdir();
+my $cfg = "$tmpdir/cfg";
+my $mailbox = 'inbox.test';
+{
+	open my $fh, '>', $cfg or BAIL_OUT "open: $!";
+	print $fh <<EOF or BAIL_OUT "print: $!";
+[publicinbox "test"]
+	newsgroup = $mailbox
+	address = oimap\@example.com
+	inboxdir = $inboxdir
+EOF
+	close $fh or BAIL_OUT "close: $!";
+}
+my ($out, $err) = ("$tmpdir/stdout.log", "$tmpdir/stderr.log");
+my $sock = tcp_server();
+my $cmd = [ '-imapd', '-W0', "--stdout=$out", "--stderr=$err" ];
+my $env = { PI_CONFIG => $cfg };
+my $td = start_script($cmd, $env, { 3 => $sock }) or BAIL_OUT "-imapd: $?";
+{
+	my $c = tcp_connect($sock);
+	like(readline($c), qr/CAPABILITY /, 'got greeting');
+}
+my ($host, $port) = ($sock->sockhost, $sock->sockport);
+my %pids;
+
+SKIP: {
+	mkpath([map { "$tmpdir/oimapdir/$_" } qw(cur new tmp)]);
+	my $oimap = which('offlineimap') or skip 'no offlineimap(1)', 1;
+	open my $fh, '>', "$tmpdir/.offlineimaprc" or BAIL_OUT "open: $!";
+	print $fh <<EOF or BAIL_OUT "print: $!";
+[general]
+accounts = test
+socktimeout = 10
+fsync = false
+
+[Account test]
+localrepository = l.test
+remoterepository = r.test
+
+[Repository l.test]
+type = Maildir
+localfolders = ~/oimapdir
+
+[Repository r.test]
+type = IMAP
+ssl = no
+remotehost = $host
+remoteport = $port
+remoteuser = anonymous
+remotepass = Hunter2
+
+# python-imaplib2 times out on select/poll when compression is enabled
+# <https://bugs.debian.org/961713>
+usecompression = no
+EOF
+	close $fh or BAIL_OUT "close: $!";
+	my $cmd = [ $oimap, qw(-o -q -u quiet) ];
+	my $pid = spawn($cmd, { HOME => $tmpdir }, { 1 => 2 });
+	$pids{$pid} = $cmd;
+}
+
+SKIP: {
+	mkpath([map { "$tmpdir/mbsyncdir/test/$_" } qw(cur new tmp)]);
+	my $mbsync = which('mbsync') or skip 'no mbsync(1)', 1;
+	open my $fh, '>', "$tmpdir/.mbsyncrc" or BAIL_OUT "open: $!";
+	print $fh <<EOF or BAIL_OUT "print: $!";
+Create Slave
+SyncState *
+Remove None
+FSync no
+
+MaildirStore local
+Path ~/mbsyncdir/
+Inbox ~/mbsyncdir/test
+SubFolders verbatim
+
+IMAPStore remote
+Host $host
+Port $port
+User anonymous
+Pass Hunter2
+SSLType None
+UseNamespace no
+# DisableExtension COMPRESS=DEFLATE
+
+Channel "test"
+Master ":remote:inbox"
+Slave ":local:test"
+Expunge None
+Sync PullNew
+Patterns *
+EOF
+	close $fh or BAIL_OUT "close: $!";
+	my $cmd = [ $mbsync, qw(-aqq) ];
+	my $pid = spawn($cmd, { HOME => $tmpdir }, { 1 => 2 });
+	$pids{$pid} = $cmd;
+}
+
+while (scalar keys %pids) {
+	my $pid = waitpid(-1, 0) or next;
+	my $cmd = delete $pids{$pid} or next;
+	is($?, 0, join(' ', @$cmd, 'done'));
+}
+
+if (my $sec = $ENV{TEST_PERSIST}) {
+	diag "sleeping ${sec}s, imap://$host:$port/$mailbox available";
+	diag "tmpdir=$tmpdir (Maildirs available)";
+	diag "stdout=$out";
+	diag "stderr=$err";
+	diag "pid=$td->{pid}";
+	sleep $sec;
+}
+$td->kill;
+$td->join;
+is($?, 0, 'no error on -imapd exit');
+done_testing;
diff --git a/xt/imapd-validate.t b/xt/imapd-validate.t
new file mode 100644
index 00000000000..f96ec8791b9
--- /dev/null
+++ b/xt/imapd-validate.t
@@ -0,0 +1,175 @@
+#!perl -w
+# Copyright (C) 2020 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+# Expensive test to validate compression and TLS.
+use strict;
+use Test::More;
+use Symbol qw(gensym);
+use PublicInbox::DS qw(now);
+use POSIX qw(_exit);
+use PublicInbox::TestCommon;
+my $inbox_dir = $ENV{GIANT_INBOX_DIR};
+plan skip_all => "GIANT_INBOX_DIR not defined for $0" unless $inbox_dir;
+# how many emails to read into memory at once per-process
+my $BATCH = $ENV{TEST_BATCH} // 100;
+my $REPEAT = $ENV{TEST_REPEAT} // 1;
+
+require_mods(qw(Mail::IMAPClient));
+my $imap_client = 'Mail::IMAPClient';
+my $can_compress = $imap_client->can('compress');
+if ($can_compress) { # hope this gets fixed upstream, soon
+	require PublicInbox::IMAPClient;
+	$imap_client = 'PublicInbox::IMAPClient';
+}
+
+my $test_tls = $ENV{TEST_SKIP_TLS} ? 0 : eval { require IO::Socket::SSL };
+my ($cert, $key) = qw(certs/server-cert.pem certs/server-key.pem);
+if ($test_tls && !-r $key || !-r $cert) {
+	plan skip_all =>
+		"certs/ missing for $0, run $^X ./certs/create-certs.perl";
+}
+my ($tmpdir, $for_destroy) = tmpdir();
+my %OPT = qw(User u Password p);
+my (%STARTTLS_OPT, %IMAPS_OPT, $td, $mailbox, $make_local_server);
+if (($ENV{IMAP_TEST_URL} // '') =~ m!\Aimap://([^/]+)/(.+)\z!) {
+	($OPT{Server}, $mailbox) = ($1, $2);
+	$OPT{Server} =~ s/:([0-9]+)\z// and $OPT{Port} = $1 + 0;
+	%STARTTLS_OPT = %OPT;
+	%IMAPS_OPT = (%OPT, Port => 993) if $OPT{Port} == 143;
+} else {
+	require_mods(qw(DBD::SQLite));
+	$make_local_server->();
+}
+
+my %opts = (imap => \%OPT, 'imap+compress' => { %OPT, Compress => 1 });
+my $uid_max = do {
+	my $mic = $imap_client->new(%OPT) or BAIL_OUT "new $!";
+	$mic->examine($mailbox) or BAIL_OUT "examine: $!";
+	my $next = $mic->uidnext($mailbox) or BAIL_OUT "uidnext: $!";
+	$next - 1;
+};
+
+if (scalar keys %STARTTLS_OPT) {
+	$opts{starttls} = \%STARTTLS_OPT;
+	$opts{'starttls+compress'} = { %STARTTLS_OPT, Compress => 1 };
+}
+if (scalar keys %IMAPS_OPT) {
+	$opts{imaps} = \%IMAPS_OPT;
+	$opts{'imaps+compress'} = { %IMAPS_OPT, Compress => 1 };
+}
+
+my $do_get_all = sub {
+	my ($desc, $opt) = @_;
+	local $SIG{__DIE__} = sub { print STDERR $desc, ': ', @_; _exit(1) };
+	my $t0 = now();
+	my $dig = Digest::SHA->new(1);
+	my $mic = $imap_client->new(%$opt);
+	$mic->examine($mailbox) or die "examine: $!";
+	my $uid_base = 1;
+	my $bytes = 0;
+	my $nr = 0;
+	until ($uid_base > $uid_max) {
+		my $end = $uid_base + $BATCH;
+		my $ret = $mic->fetch_hash("$uid_base:$end", 'BODY[]') or last;
+		for my $uid ($uid_base..$end) {
+			$dig->add($uid);
+			my $h = delete $ret->{$uid} or next;
+			my $body = delete $h->{'BODY[]'} or
+						die "no BODY[] for UID=$uid";
+			$dig->add($body);
+			$bytes += length($body);
+			++$nr;
+		}
+		$uid_base = $end + 1;
+	}
+	$mic->logout or die "logout failed: $!";
+	my $elapsed = sprintf('%0.3f', now() - $t0);
+	my $res = $dig->hexdigest;
+	print STDERR "# $desc $res (${elapsed}s) $bytes bytes, NR=$nr\n";
+	$res;
+};
+
+my (%pids, %res);
+for (1..$REPEAT) {
+	while (my ($desc, $opt) = each %opts) {
+		pipe(my ($r, $w)) or die;
+		my $pid = fork;
+		if ($pid == 0) {
+			close $r or die;
+			my $res = $do_get_all->($desc, $opt);
+			print $w $res or die;
+			close $w or die;
+			_exit(0);
+		}
+		close $w or die;
+		$pids{$pid} = [ $desc, $r ];
+	}
+}
+
+while (scalar keys %pids) {
+	my $pid = waitpid(-1, 0) or next;
+	my $child = delete $pids{$pid} or next;
+	my ($desc, $rpipe) = @$child;
+	is($?, 0, "$desc done");
+	my $sum = do { local $/; <$rpipe> };
+	push @{$res{$sum}}, $desc;
+}
+is(scalar keys %res, 1, 'all got the same result');
+$td->kill;
+$td->join;
+is($?, 0, 'no error on -imapd exit');
+done_testing;
+
+BEGIN {
+
+$make_local_server = sub {
+	require PublicInbox::Inbox;
+	$mailbox = 'inbox.test';
+	my $ibx = { inboxdir => $inbox_dir, newsgroup => $mailbox };
+	$ibx = PublicInbox::Inbox->new($ibx);
+	my $pi_config = "$tmpdir/config";
+	{
+		open my $fh, '>', $pi_config or die "open($pi_config): $!";
+		print $fh <<"" or die "print $pi_config: $!";
+[publicinbox "test"]
+	newsgroup = $mailbox
+	inboxdir = $inbox_dir
+	address = test\@example.com
+
+		close $fh or die "close($pi_config): $!";
+	}
+	my ($out, $err) = ("$tmpdir/out", "$tmpdir/err");
+	for ($out, $err) {
+		open my $fh, '>', $_ or die "truncate: $!";
+	}
+	my $imap = tcp_server();
+	my $rdr = { 3 => $imap };
+	$OPT{Server} = $imap->sockhost;
+	$OPT{Port} = $imap->sockport;
+
+	# not using multiple workers, here, since we want to increase
+	# the chance of tripping concurrency bugs within PublicInbox/IMAP*.pm
+	my $cmd = [ '-imapd', "--stdout=$out", "--stderr=$err", '-W0' ];
+	push @$cmd, '-limap://'.$imap->sockhost.':'.$imap->sockport;
+	if ($test_tls) {
+		my $imaps = tcp_server();
+		$rdr->{4} = $imaps;
+		push @$cmd, '-limaps://'.$imaps->sockhost.':'.$imaps->sockport;
+		push @$cmd, "--cert=$cert", "--key=$key";
+		my $tls_opt = [
+			SSL_hostname => 'server.local',
+			SSL_verifycn_name => 'server.local',
+			SSL_verify_mode => IO::Socket::SSL::SSL_VERIFY_PEER(),
+			SSL_ca_file => 'certs/test-ca.pem',
+		];
+		%STARTTLS_OPT = (%OPT, Starttls => $tls_opt);
+		%IMAPS_OPT = (%OPT, Ssl => $tls_opt,
+			Server => $imaps->sockhost,
+			Port => $imaps->sockport
+		);
+	}
+	print STDERR "# CMD ". join(' ', @$cmd). "\n";
+	my $env = { PI_CONFIG => $pi_config };
+	$td = start_script($cmd, $env, $rdr);
+};
+} # BEGIN