[PATCH 01/35] test_common: add create_inbox helper sub

[Eric Wong <e@80x24.org>]

This saves over 100ms in t/lei-q-remote-import.t so far when
TMPDIR is on an SSD.  If we can memoize inbox creation to save a
few dozen milliseconds every test, this could add up to
noticeable savings across our entire test suite.
---
 MANIFEST                      |  1 +
 Makefile.PL                   |  3 +-
 lib/PublicInbox/TestCommon.pm | 55 ++++++++++++++++++++++++++++++++---
 t/data-gen/.gitignore         |  2 ++
 t/lei-q-remote-import.t       | 20 ++++---------
 5 files changed, 61 insertions(+), 20 deletions(-)
 create mode 100644 t/data-gen/.gitignore

diff --git a/MANIFEST b/MANIFEST
index 941a1f90..4d2cfb10 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -311,6 +311,7 @@ t/config.t
 t/config_limiter.t
 t/content_hash.t
 t/convert-compact.t
+t/data-gen/.gitignore
 t/data/0001.patch
 t/data/message_embed.eml
 t/dir_idle.t
diff --git a/Makefile.PL b/Makefile.PL
index 21d3d6ea..8165e601 100644
--- a/Makefile.PL
+++ b/Makefile.PL
@@ -176,7 +176,8 @@ WriteMakefile(
 	},
 	MAN3PODS => \%man3,
 	clean => {
-		FILES => 't/home*/setup* t/home*/t* t/home*/.public-inbox'
+		FILES => 't/home*/setup* t/home*/t* t/home*/.public-inbox '.
+			't/data-gen/*'
 	},
 );
 
diff --git a/lib/PublicInbox/TestCommon.pm b/lib/PublicInbox/TestCommon.pm
index af1b2e4f..77306a6e 100644
--- a/lib/PublicInbox/TestCommon.pm
+++ b/lib/PublicInbox/TestCommon.pm
@@ -14,9 +14,8 @@ our @EXPORT;
 BEGIN {
 	@EXPORT = qw(tmpdir tcp_server tcp_connect require_git require_mods
 		run_script start_script key2sub xsys xsys_e xqx eml_load tick
-		have_xapian_compact json_utf8 setup_public_inboxes
-		tcp_host_port test_lei lei lei_ok
-		$lei_out $lei_err $lei_opt);
+		have_xapian_compact json_utf8 setup_public_inboxes create_inbox
+		tcp_host_port test_lei lei lei_ok $lei_out $lei_err $lei_opt);
 	require Test::More;
 	my @methods = grep(!/\W/, @Test::More::EXPORT);
 	eval(join('', map { "*$_=\\&Test::More::$_;" } @methods));
@@ -587,7 +586,55 @@ sub setup_public_inboxes () {
 	$seen or BAIL_OUT 'no imports';
 	open my $fh, '>', $stamp or BAIL_OUT "open $stamp: $!";
 	@ret;
-};
+}
+
+sub create_inbox ($$;@) {
+	my $ident = shift;
+	my $cb = pop;
+	my %opt = @_;
+	require PublicInbox::Lock;
+	require PublicInbox::InboxWritable;
+	my ($base) = ($0 =~ m!\b([^/]+)\.[^\.]+\z!);
+	my $dir = "t/data-gen/$base.$ident";
+	unless (-d $dir) {
+		mkdir $dir; # may race
+		-d $dir or BAIL_OUT "$dir could not be created: $!";
+	}
+	my $lk = bless { lock_path => "$dir/creat.lock" }, 'PublicInbox::Lock';
+	$opt{inboxdir} = File::Spec->rel2abs($dir);
+	$opt{name} //= $ident;
+	$opt{-no_fsync} = 1;
+	my $no_gc = delete $opt{-no_gc};
+	my $tmpdir = delete $opt{tmpdir};
+	my $addr = $opt{address} // [];
+	$opt{-primary_address} //= $addr->[0] // "$ident\@example.com";
+	my $parallel = delete($opt{importer_parallel}) // 0;
+	my $creat_opt = { nproc => delete($opt{nproc}) // 1 };
+	my $ibx = PublicInbox::InboxWritable->new({ %opt }, $creat_opt);
+	my $scope = $lk->lock_for_scope;
+	if (!-f "$dir/creat.stamp") {
+		my $im = $ibx->importer($parallel);
+		$cb->($im, $ibx);
+		$im->done if $im;
+		unless ($no_gc) {
+			my @to_gc = $ibx->version == 1 ? ($ibx->{inboxdir}) :
+					glob("$ibx->{inboxdir}/git/*.git");
+			for my $dir (@to_gc) {
+				xsys_e([ qw(git gc -q) ], { GIT_DIR => $dir });
+			}
+		}
+		open my $s, '>', "$dir/creat.stamp" or
+			BAIL_OUT "error creating $dir/creat.stamp: $!";
+	}
+	if ($tmpdir) {
+		undef $ibx;
+		xsys([qw(/bin/cp -Rp), $dir, $tmpdir]) == 0 or
+			BAIL_OUT "cp $dir $tmpdir";
+		$opt{inboxdir} = $tmpdir;
+		$ibx = PublicInbox::InboxWritable->new(\%opt);
+	}
+	$ibx;
+}
 
 package PublicInboxTestProcess;
 use strict;
diff --git a/t/data-gen/.gitignore b/t/data-gen/.gitignore
new file mode 100644
index 00000000..11e8933b
--- /dev/null
+++ b/t/data-gen/.gitignore
@@ -0,0 +1,2 @@
+# read-only test data generated by create_inbox
+*
diff --git a/t/lei-q-remote-import.t b/t/lei-q-remote-import.t
index 8b82579c..2293489a 100644
--- a/t/lei-q-remote-import.t
+++ b/t/lei-q-remote-import.t
@@ -5,7 +5,6 @@ use strict; use v5.10.1; use PublicInbox::TestCommon;
 require_git 2.6;
 require_mods(qw(json DBD::SQLite Search::Xapian));
 use PublicInbox::MboxReader;
-use PublicInbox::InboxWritable;
 my ($ro_home, $cfg_path) = setup_public_inboxes;
 my $sock = tcp_server;
 my ($tmpdir, $for_destroy) = tmpdir;
@@ -61,20 +60,11 @@ test_lei({ tmpdir => $tmpdir }, sub {
 	lei_ok(@cmd, '--lock=dotlock,timeout=0.000001',
 		\'succeeds after lock removal');
 
-	# XXX memoize this external creation
-	my $inboxdir = "$ENV{HOME}/tmp_git";
-	my $ibx = PublicInbox::InboxWritable->new({
-		name => 'tmp',
-		-primary_address => 'lei@example.com',
-		inboxdir => $inboxdir,
-		indexlevel => 'medium',
-	}, { nproc => 1 });
-	my $im = $ibx->importer(0);
-	$im->add(eml_load('t/utf8.eml')) or BAIL_OUT '->add';
-	$im->done;
-
-	run_script(['-index', $inboxdir], undef) or BAIL_OUT '-init';
-	lei_ok(qw(add-external -q), $inboxdir);
+	my $ibx = create_inbox 'local-external', indexlevel => 'medium', sub {
+		my ($im) = @_;
+		$im->add(eml_load('t/utf8.eml')) or BAIL_OUT '->add';
+	};
+	lei_ok(qw(add-external -q), $ibx->{inboxdir});
 	lei_ok(qw(q -o), "mboxrd:$o", '--only', $url,
 		'm:testmessage@example.com');
 	ok(-s $o, 'got result from remote external');