From: Eric Wong <e@80x24.org> To: meta@public-inbox.org Subject: [PATCH 01/35] test_common: add create_inbox helper sub Date: Mon, 15 Mar 2021 12:57:52 +0100 [thread overview] Message-ID: <20210315115826.17591-2-e@80x24.org> (raw) In-Reply-To: <20210315115826.17591-1-e@80x24.org> This saves over 100ms in t/lei-q-remote-import.t so far when TMPDIR is on an SSD. If we can memoize inbox creation to save a few dozen milliseconds every test, this could add up to noticeable savings across our entire test suite. --- MANIFEST | 1 + Makefile.PL | 3 +- lib/PublicInbox/TestCommon.pm | 55 ++++++++++++++++++++++++++++++++--- t/data-gen/.gitignore | 2 ++ t/lei-q-remote-import.t | 20 ++++--------- 5 files changed, 61 insertions(+), 20 deletions(-) create mode 100644 t/data-gen/.gitignore diff --git a/MANIFEST b/MANIFEST index 941a1f90..4d2cfb10 100644 --- a/MANIFEST +++ b/MANIFEST @@ -311,6 +311,7 @@ t/config.t t/config_limiter.t t/content_hash.t t/convert-compact.t +t/data-gen/.gitignore t/data/0001.patch t/data/message_embed.eml t/dir_idle.t diff --git a/Makefile.PL b/Makefile.PL index 21d3d6ea..8165e601 100644 --- a/Makefile.PL +++ b/Makefile.PL @@ -176,7 +176,8 @@ WriteMakefile( }, MAN3PODS => \%man3, clean => { - FILES => 't/home*/setup* t/home*/t* t/home*/.public-inbox' + FILES => 't/home*/setup* t/home*/t* t/home*/.public-inbox '. + 't/data-gen/*' }, ); diff --git a/lib/PublicInbox/TestCommon.pm b/lib/PublicInbox/TestCommon.pm index af1b2e4f..77306a6e 100644 --- a/lib/PublicInbox/TestCommon.pm +++ b/lib/PublicInbox/TestCommon.pm @@ -14,9 +14,8 @@ our @EXPORT; BEGIN { @EXPORT = qw(tmpdir tcp_server tcp_connect require_git require_mods run_script start_script key2sub xsys xsys_e xqx eml_load tick - have_xapian_compact json_utf8 setup_public_inboxes - tcp_host_port test_lei lei lei_ok - $lei_out $lei_err $lei_opt); + have_xapian_compact json_utf8 setup_public_inboxes create_inbox + tcp_host_port test_lei lei lei_ok $lei_out $lei_err $lei_opt); require Test::More; my @methods = grep(!/\W/, @Test::More::EXPORT); eval(join('', map { "*$_=\\&Test::More::$_;" } @methods)); @@ -587,7 +586,55 @@ sub setup_public_inboxes () { $seen or BAIL_OUT 'no imports'; open my $fh, '>', $stamp or BAIL_OUT "open $stamp: $!"; @ret; -}; +} + +sub create_inbox ($$;@) { + my $ident = shift; + my $cb = pop; + my %opt = @_; + require PublicInbox::Lock; + require PublicInbox::InboxWritable; + my ($base) = ($0 =~ m!\b([^/]+)\.[^\.]+\z!); + my $dir = "t/data-gen/$base.$ident"; + unless (-d $dir) { + mkdir $dir; # may race + -d $dir or BAIL_OUT "$dir could not be created: $!"; + } + my $lk = bless { lock_path => "$dir/creat.lock" }, 'PublicInbox::Lock'; + $opt{inboxdir} = File::Spec->rel2abs($dir); + $opt{name} //= $ident; + $opt{-no_fsync} = 1; + my $no_gc = delete $opt{-no_gc}; + my $tmpdir = delete $opt{tmpdir}; + my $addr = $opt{address} // []; + $opt{-primary_address} //= $addr->[0] // "$ident\@example.com"; + my $parallel = delete($opt{importer_parallel}) // 0; + my $creat_opt = { nproc => delete($opt{nproc}) // 1 }; + my $ibx = PublicInbox::InboxWritable->new({ %opt }, $creat_opt); + my $scope = $lk->lock_for_scope; + if (!-f "$dir/creat.stamp") { + my $im = $ibx->importer($parallel); + $cb->($im, $ibx); + $im->done if $im; + unless ($no_gc) { + my @to_gc = $ibx->version == 1 ? ($ibx->{inboxdir}) : + glob("$ibx->{inboxdir}/git/*.git"); + for my $dir (@to_gc) { + xsys_e([ qw(git gc -q) ], { GIT_DIR => $dir }); + } + } + open my $s, '>', "$dir/creat.stamp" or + BAIL_OUT "error creating $dir/creat.stamp: $!"; + } + if ($tmpdir) { + undef $ibx; + xsys([qw(/bin/cp -Rp), $dir, $tmpdir]) == 0 or + BAIL_OUT "cp $dir $tmpdir"; + $opt{inboxdir} = $tmpdir; + $ibx = PublicInbox::InboxWritable->new(\%opt); + } + $ibx; +} package PublicInboxTestProcess; use strict; diff --git a/t/data-gen/.gitignore b/t/data-gen/.gitignore new file mode 100644 index 00000000..11e8933b --- /dev/null +++ b/t/data-gen/.gitignore @@ -0,0 +1,2 @@ +# read-only test data generated by create_inbox +* diff --git a/t/lei-q-remote-import.t b/t/lei-q-remote-import.t index 8b82579c..2293489a 100644 --- a/t/lei-q-remote-import.t +++ b/t/lei-q-remote-import.t @@ -5,7 +5,6 @@ use strict; use v5.10.1; use PublicInbox::TestCommon; require_git 2.6; require_mods(qw(json DBD::SQLite Search::Xapian)); use PublicInbox::MboxReader; -use PublicInbox::InboxWritable; my ($ro_home, $cfg_path) = setup_public_inboxes; my $sock = tcp_server; my ($tmpdir, $for_destroy) = tmpdir; @@ -61,20 +60,11 @@ test_lei({ tmpdir => $tmpdir }, sub { lei_ok(@cmd, '--lock=dotlock,timeout=0.000001', \'succeeds after lock removal'); - # XXX memoize this external creation - my $inboxdir = "$ENV{HOME}/tmp_git"; - my $ibx = PublicInbox::InboxWritable->new({ - name => 'tmp', - -primary_address => 'lei@example.com', - inboxdir => $inboxdir, - indexlevel => 'medium', - }, { nproc => 1 }); - my $im = $ibx->importer(0); - $im->add(eml_load('t/utf8.eml')) or BAIL_OUT '->add'; - $im->done; - - run_script(['-index', $inboxdir], undef) or BAIL_OUT '-init'; - lei_ok(qw(add-external -q), $inboxdir); + my $ibx = create_inbox 'local-external', indexlevel => 'medium', sub { + my ($im) = @_; + $im->add(eml_load('t/utf8.eml')) or BAIL_OUT '->add'; + }; + lei_ok(qw(add-external -q), $ibx->{inboxdir}); lei_ok(qw(q -o), "mboxrd:$o", '--only', $url, 'm:testmessage@example.com'); ok(-s $o, 'got result from remote external');
next prev parent reply other threads:[~2021-03-15 11:58 UTC|newest] Thread overview: 37+ messages / expand[flat|nested] mbox.gz Atom feed top 2021-03-15 11:57 [PATCH 00/35] memoize inbox creations Eric Wong 2021-03-15 11:57 ` Eric Wong [this message] 2021-03-15 11:57 ` [PATCH 02/35] t/lei_xsearch: use create_inbox Eric Wong 2021-03-15 11:57 ` [PATCH 03/35] test_common: minor simplifications to setup_public_inboxes Eric Wong 2021-03-15 11:57 ` [PATCH 04/35] t/imapd-tls: switch to create_inbox Eric Wong 2021-03-15 11:57 ` [PATCH 05/35] t/www_altid: use create_inbox Eric Wong 2021-03-15 11:57 ` [PATCH 06/35] t/xcpdb-reshard: " Eric Wong 2021-03-15 11:57 ` [PATCH 07/35] t/v2dupindex: create_inbox Eric Wong 2021-03-15 11:57 ` [PATCH 08/35] t/admin: switch to create_inbox Eric Wong 2021-03-15 19:51 ` [SQUASH] " Eric Wong 2021-03-15 11:58 ` [PATCH 09/35] t/html_index: remove now-worthless test Eric Wong 2021-03-15 11:58 ` [PATCH 10/35] t/plack: use create_inbox Eric Wong 2021-03-15 11:58 ` [PATCH 11/35] t/psgi_attach: convert to create_inbox Eric Wong 2021-03-15 11:58 ` [PATCH 12/35] t/httpd: " Eric Wong 2021-03-15 11:58 ` [PATCH 13/35] t/convert-compact: create_inbox Eric Wong 2021-03-15 11:58 ` [PATCH 14/35] t/altid: use create_inbox Eric Wong 2021-03-15 11:58 ` [PATCH 15/35] t/psgi_mount: switch to create_inbox Eric Wong 2021-03-15 11:58 ` [PATCH 16/35] t/feed: " Eric Wong 2021-03-15 11:58 ` [PATCH 17/35] t/psgi_bad_mids: use create_inbox Eric Wong 2021-03-15 11:58 ` [PATCH 18/35] t/psgi_multipart_not: " Eric Wong 2021-03-15 11:58 ` [PATCH 19/35] t/psgi_scan_all: create_inbox lots saved Eric Wong 2021-03-15 11:58 ` [PATCH 20/35] t/psgi_v2: create_inbox Eric Wong 2021-03-15 11:58 ` [PATCH 21/35] t/imapd: create_inbox (minor) Eric Wong 2021-03-15 11:58 ` [PATCH 22/35] t/solver_git: use create_inbox Eric Wong 2021-03-15 11:58 ` [PATCH 23/35] t/cgi: create_inbox Eric Wong 2021-03-15 11:58 ` [PATCH 24/35] t/edit: switch to create_inbox Eric Wong 2021-03-15 11:58 ` [PATCH 25/35] t/inbox_idle: " Eric Wong 2021-03-15 11:58 ` [PATCH 26/35] t/lei_xsearch: use create_inbox Eric Wong 2021-03-15 11:58 ` [PATCH 27/35] t/indexlevels-mirror: " Eric Wong 2021-03-15 11:58 ` [PATCH 28/35] t/multi-mid: " Eric Wong 2021-03-15 11:58 ` [PATCH 29/35] t/psgi_search: " Eric Wong 2021-03-15 11:58 ` [PATCH 30/35] t/miscsearch: " Eric Wong 2021-03-15 11:58 ` [PATCH 31/35] t/nntpd-tls: " Eric Wong 2021-03-15 11:58 ` [PATCH 32/35] t/nntpd: " Eric Wong 2021-03-15 11:58 ` [PATCH 33/35] t/thread-index-gap: create_inbox Eric Wong 2021-03-15 11:58 ` [PATCH 34/35] t/altid_v2: create_inbox Eric Wong 2021-03-15 11:58 ` [PATCH 35/35] t/*: disable fsync on tests were create_inbox isn't worth it Eric Wong
Reply instructions: You may reply publicly to this message via plain-text email using any one of the following methods: * Save the following mbox file, import it into your mail client, and reply-to-all from there: mbox Avoid top-posting and favor interleaved quoting: https://en.wikipedia.org/wiki/Posting_style#Interleaved_style List information: https://public-inbox.org/README * Reply using the --to, --cc, and --in-reply-to switches of git-send-email(1): git send-email \ --in-reply-to=20210315115826.17591-2-e@80x24.org \ --to=e@80x24.org \ --cc=meta@public-inbox.org \ --subject='Re: [PATCH 01/35] test_common: add create_inbox helper sub' \ /path/to/YOUR_REPLY https://kernel.org/pub/software/scm/git/docs/git-send-email.html * If your mail client supports setting the In-Reply-To header via mailto: links, try the mailto: link
Code repositories for project(s) associated with this inbox: https://80x24.org/public-inbox.git This is a public inbox, see mirroring instructions for how to clone and mirror all data and code used for this inbox; as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).