user/dev discussion of public-inbox itself
 help / color / mirror / code / Atom feed
From: Eric Wong <e@80x24.org>
To: meta@public-inbox.org
Subject: [PATCH 01/35] test_common: add create_inbox helper sub
Date: Mon, 15 Mar 2021 12:57:52 +0100	[thread overview]
Message-ID: <20210315115826.17591-2-e@80x24.org> (raw)
In-Reply-To: <20210315115826.17591-1-e@80x24.org>

This saves over 100ms in t/lei-q-remote-import.t so far when
TMPDIR is on an SSD.  If we can memoize inbox creation to save a
few dozen milliseconds every test, this could add up to
noticeable savings across our entire test suite.
---
 MANIFEST                      |  1 +
 Makefile.PL                   |  3 +-
 lib/PublicInbox/TestCommon.pm | 55 ++++++++++++++++++++++++++++++++---
 t/data-gen/.gitignore         |  2 ++
 t/lei-q-remote-import.t       | 20 ++++---------
 5 files changed, 61 insertions(+), 20 deletions(-)
 create mode 100644 t/data-gen/.gitignore

diff --git a/MANIFEST b/MANIFEST
index 941a1f90..4d2cfb10 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -311,6 +311,7 @@ t/config.t
 t/config_limiter.t
 t/content_hash.t
 t/convert-compact.t
+t/data-gen/.gitignore
 t/data/0001.patch
 t/data/message_embed.eml
 t/dir_idle.t
diff --git a/Makefile.PL b/Makefile.PL
index 21d3d6ea..8165e601 100644
--- a/Makefile.PL
+++ b/Makefile.PL
@@ -176,7 +176,8 @@ WriteMakefile(
 	},
 	MAN3PODS => \%man3,
 	clean => {
-		FILES => 't/home*/setup* t/home*/t* t/home*/.public-inbox'
+		FILES => 't/home*/setup* t/home*/t* t/home*/.public-inbox '.
+			't/data-gen/*'
 	},
 );
 
diff --git a/lib/PublicInbox/TestCommon.pm b/lib/PublicInbox/TestCommon.pm
index af1b2e4f..77306a6e 100644
--- a/lib/PublicInbox/TestCommon.pm
+++ b/lib/PublicInbox/TestCommon.pm
@@ -14,9 +14,8 @@ our @EXPORT;
 BEGIN {
 	@EXPORT = qw(tmpdir tcp_server tcp_connect require_git require_mods
 		run_script start_script key2sub xsys xsys_e xqx eml_load tick
-		have_xapian_compact json_utf8 setup_public_inboxes
-		tcp_host_port test_lei lei lei_ok
-		$lei_out $lei_err $lei_opt);
+		have_xapian_compact json_utf8 setup_public_inboxes create_inbox
+		tcp_host_port test_lei lei lei_ok $lei_out $lei_err $lei_opt);
 	require Test::More;
 	my @methods = grep(!/\W/, @Test::More::EXPORT);
 	eval(join('', map { "*$_=\\&Test::More::$_;" } @methods));
@@ -587,7 +586,55 @@ sub setup_public_inboxes () {
 	$seen or BAIL_OUT 'no imports';
 	open my $fh, '>', $stamp or BAIL_OUT "open $stamp: $!";
 	@ret;
-};
+}
+
+sub create_inbox ($$;@) {
+	my $ident = shift;
+	my $cb = pop;
+	my %opt = @_;
+	require PublicInbox::Lock;
+	require PublicInbox::InboxWritable;
+	my ($base) = ($0 =~ m!\b([^/]+)\.[^\.]+\z!);
+	my $dir = "t/data-gen/$base.$ident";
+	unless (-d $dir) {
+		mkdir $dir; # may race
+		-d $dir or BAIL_OUT "$dir could not be created: $!";
+	}
+	my $lk = bless { lock_path => "$dir/creat.lock" }, 'PublicInbox::Lock';
+	$opt{inboxdir} = File::Spec->rel2abs($dir);
+	$opt{name} //= $ident;
+	$opt{-no_fsync} = 1;
+	my $no_gc = delete $opt{-no_gc};
+	my $tmpdir = delete $opt{tmpdir};
+	my $addr = $opt{address} // [];
+	$opt{-primary_address} //= $addr->[0] // "$ident\@example.com";
+	my $parallel = delete($opt{importer_parallel}) // 0;
+	my $creat_opt = { nproc => delete($opt{nproc}) // 1 };
+	my $ibx = PublicInbox::InboxWritable->new({ %opt }, $creat_opt);
+	my $scope = $lk->lock_for_scope;
+	if (!-f "$dir/creat.stamp") {
+		my $im = $ibx->importer($parallel);
+		$cb->($im, $ibx);
+		$im->done if $im;
+		unless ($no_gc) {
+			my @to_gc = $ibx->version == 1 ? ($ibx->{inboxdir}) :
+					glob("$ibx->{inboxdir}/git/*.git");
+			for my $dir (@to_gc) {
+				xsys_e([ qw(git gc -q) ], { GIT_DIR => $dir });
+			}
+		}
+		open my $s, '>', "$dir/creat.stamp" or
+			BAIL_OUT "error creating $dir/creat.stamp: $!";
+	}
+	if ($tmpdir) {
+		undef $ibx;
+		xsys([qw(/bin/cp -Rp), $dir, $tmpdir]) == 0 or
+			BAIL_OUT "cp $dir $tmpdir";
+		$opt{inboxdir} = $tmpdir;
+		$ibx = PublicInbox::InboxWritable->new(\%opt);
+	}
+	$ibx;
+}
 
 package PublicInboxTestProcess;
 use strict;
diff --git a/t/data-gen/.gitignore b/t/data-gen/.gitignore
new file mode 100644
index 00000000..11e8933b
--- /dev/null
+++ b/t/data-gen/.gitignore
@@ -0,0 +1,2 @@
+# read-only test data generated by create_inbox
+*
diff --git a/t/lei-q-remote-import.t b/t/lei-q-remote-import.t
index 8b82579c..2293489a 100644
--- a/t/lei-q-remote-import.t
+++ b/t/lei-q-remote-import.t
@@ -5,7 +5,6 @@ use strict; use v5.10.1; use PublicInbox::TestCommon;
 require_git 2.6;
 require_mods(qw(json DBD::SQLite Search::Xapian));
 use PublicInbox::MboxReader;
-use PublicInbox::InboxWritable;
 my ($ro_home, $cfg_path) = setup_public_inboxes;
 my $sock = tcp_server;
 my ($tmpdir, $for_destroy) = tmpdir;
@@ -61,20 +60,11 @@ test_lei({ tmpdir => $tmpdir }, sub {
 	lei_ok(@cmd, '--lock=dotlock,timeout=0.000001',
 		\'succeeds after lock removal');
 
-	# XXX memoize this external creation
-	my $inboxdir = "$ENV{HOME}/tmp_git";
-	my $ibx = PublicInbox::InboxWritable->new({
-		name => 'tmp',
-		-primary_address => 'lei@example.com',
-		inboxdir => $inboxdir,
-		indexlevel => 'medium',
-	}, { nproc => 1 });
-	my $im = $ibx->importer(0);
-	$im->add(eml_load('t/utf8.eml')) or BAIL_OUT '->add';
-	$im->done;
-
-	run_script(['-index', $inboxdir], undef) or BAIL_OUT '-init';
-	lei_ok(qw(add-external -q), $inboxdir);
+	my $ibx = create_inbox 'local-external', indexlevel => 'medium', sub {
+		my ($im) = @_;
+		$im->add(eml_load('t/utf8.eml')) or BAIL_OUT '->add';
+	};
+	lei_ok(qw(add-external -q), $ibx->{inboxdir});
 	lei_ok(qw(q -o), "mboxrd:$o", '--only', $url,
 		'm:testmessage@example.com');
 	ok(-s $o, 'got result from remote external');

  reply	other threads:[~2021-03-15 11:58 UTC|newest]

Thread overview: 37+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2021-03-15 11:57 [PATCH 00/35] memoize inbox creations Eric Wong
2021-03-15 11:57 ` Eric Wong [this message]
2021-03-15 11:57 ` [PATCH 02/35] t/lei_xsearch: use create_inbox Eric Wong
2021-03-15 11:57 ` [PATCH 03/35] test_common: minor simplifications to setup_public_inboxes Eric Wong
2021-03-15 11:57 ` [PATCH 04/35] t/imapd-tls: switch to create_inbox Eric Wong
2021-03-15 11:57 ` [PATCH 05/35] t/www_altid: use create_inbox Eric Wong
2021-03-15 11:57 ` [PATCH 06/35] t/xcpdb-reshard: " Eric Wong
2021-03-15 11:57 ` [PATCH 07/35] t/v2dupindex: create_inbox Eric Wong
2021-03-15 11:57 ` [PATCH 08/35] t/admin: switch to create_inbox Eric Wong
2021-03-15 19:51   ` [SQUASH] " Eric Wong
2021-03-15 11:58 ` [PATCH 09/35] t/html_index: remove now-worthless test Eric Wong
2021-03-15 11:58 ` [PATCH 10/35] t/plack: use create_inbox Eric Wong
2021-03-15 11:58 ` [PATCH 11/35] t/psgi_attach: convert to create_inbox Eric Wong
2021-03-15 11:58 ` [PATCH 12/35] t/httpd: " Eric Wong
2021-03-15 11:58 ` [PATCH 13/35] t/convert-compact: create_inbox Eric Wong
2021-03-15 11:58 ` [PATCH 14/35] t/altid: use create_inbox Eric Wong
2021-03-15 11:58 ` [PATCH 15/35] t/psgi_mount: switch to create_inbox Eric Wong
2021-03-15 11:58 ` [PATCH 16/35] t/feed: " Eric Wong
2021-03-15 11:58 ` [PATCH 17/35] t/psgi_bad_mids: use create_inbox Eric Wong
2021-03-15 11:58 ` [PATCH 18/35] t/psgi_multipart_not: " Eric Wong
2021-03-15 11:58 ` [PATCH 19/35] t/psgi_scan_all: create_inbox lots saved Eric Wong
2021-03-15 11:58 ` [PATCH 20/35] t/psgi_v2: create_inbox Eric Wong
2021-03-15 11:58 ` [PATCH 21/35] t/imapd: create_inbox (minor) Eric Wong
2021-03-15 11:58 ` [PATCH 22/35] t/solver_git: use create_inbox Eric Wong
2021-03-15 11:58 ` [PATCH 23/35] t/cgi: create_inbox Eric Wong
2021-03-15 11:58 ` [PATCH 24/35] t/edit: switch to create_inbox Eric Wong
2021-03-15 11:58 ` [PATCH 25/35] t/inbox_idle: " Eric Wong
2021-03-15 11:58 ` [PATCH 26/35] t/lei_xsearch: use create_inbox Eric Wong
2021-03-15 11:58 ` [PATCH 27/35] t/indexlevels-mirror: " Eric Wong
2021-03-15 11:58 ` [PATCH 28/35] t/multi-mid: " Eric Wong
2021-03-15 11:58 ` [PATCH 29/35] t/psgi_search: " Eric Wong
2021-03-15 11:58 ` [PATCH 30/35] t/miscsearch: " Eric Wong
2021-03-15 11:58 ` [PATCH 31/35] t/nntpd-tls: " Eric Wong
2021-03-15 11:58 ` [PATCH 32/35] t/nntpd: " Eric Wong
2021-03-15 11:58 ` [PATCH 33/35] t/thread-index-gap: create_inbox Eric Wong
2021-03-15 11:58 ` [PATCH 34/35] t/altid_v2: create_inbox Eric Wong
2021-03-15 11:58 ` [PATCH 35/35] t/*: disable fsync on tests were create_inbox isn't worth it Eric Wong

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

  List information: https://public-inbox.org/README

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20210315115826.17591-2-e@80x24.org \
    --to=e@80x24.org \
    --cc=meta@public-inbox.org \
    --subject='Re: [PATCH 01/35] test_common: add create_inbox helper sub' \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link

user/dev discussion of public-inbox itself

This inbox may be cloned and mirrored by anyone:

	git clone --mirror https://public-inbox.org/meta
	git clone --mirror http://czquwvybam4bgbro.onion/meta
	git clone --mirror http://hjrcffqmbrq6wope.onion/meta
	git clone --mirror http://ou63pmih66umazou.onion/meta

	# If you have public-inbox 1.1+ installed, you may
	# initialize and index your mirror using the following commands:
	public-inbox-init -V1 meta meta/ https://public-inbox.org/meta \
		meta@public-inbox.org
	public-inbox-index meta

Example config snippet for mirrors.
Newsgroups are available over NNTP:
	nntp://news.public-inbox.org/inbox.comp.mail.public-inbox.meta
	nntp://7fh6tueqddpjyxjmgtdiueylzoqt6pt7hec3pukyptlmohoowvhde4yd.onion/inbox.comp.mail.public-inbox.meta
	nntp://ie5yzdi7fg72h7s4sdcztq5evakq23rdt33mfyfcddc5u3ndnw24ogqd.onion/inbox.comp.mail.public-inbox.meta
	nntp://4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/inbox.comp.mail.public-inbox.meta
	nntp://news.gmane.io/gmane.mail.public-inbox.general
 note: .onion URLs require Tor: https://www.torproject.org/

code repositories for project(s) associated with this inbox:

	https://80x24.org/public-inbox.git

AGPL code for this site: git clone https://public-inbox.org/public-inbox.git