* [PATCH 06/35] t/xcpdb-reshard: use create_inbox
2021-03-15 11:57 7% [PATCH 00/35] memoize inbox creations Eric Wong
@ 2021-03-15 11:57 4% ` Eric Wong
0 siblings, 0 replies; 2+ results
From: Eric Wong @ 2021-03-15 11:57 UTC (permalink / raw)
To: meta
Over 100ms saved and fewer LoC to boot
---
t/xcpdb-reshard.t | 54 ++++++++++++++++++++---------------------------
1 file changed, 23 insertions(+), 31 deletions(-)
diff --git a/t/xcpdb-reshard.t b/t/xcpdb-reshard.t
index 1b726f1a..8516b907 100644
--- a/t/xcpdb-reshard.t
+++ b/t/xcpdb-reshard.t
@@ -1,52 +1,50 @@
+#!perl -w
# Copyright (C) 2019-2021 all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
use strict;
-use warnings;
-use Test::More;
+use v5.10.1;
use PublicInbox::TestCommon;
require_mods(qw(DBD::SQLite Search::Xapian));
require_git('2.6');
use PublicInbox::Eml;
-use PublicInbox::InboxWritable;
require PublicInbox::Search;
-my $mime = PublicInbox::Eml->new(<<'EOF');
+my ($tmpdir, $for_destroy) = tmpdir();
+my $nproc = 8;
+my $ndoc = 13;
+my $ibx = create_inbox 'test', version => 2, indexlevel => 'medium',
+ tmpdir => "$tmpdir/testbox", nproc => $nproc, sub {
+ my ($im, $ibx) = @_;
+ my $eml = PublicInbox::Eml->new(<<'EOF');
From: a@example.com
To: test@example.com
Subject: this is a subject
Date: Fri, 02 Oct 1993 00:00:00 +0000
EOF
-my ($this) = (split('/', $0))[-1];
-my ($tmpdir, $for_destroy) = tmpdir();
-local $ENV{PI_CONFIG} = "$tmpdir/config";
-my $ibx = PublicInbox::Inbox->new({
- inboxdir => "$tmpdir/testbox",
- name => $this,
- version => 2,
- -primary_address => 'test@example.com',
- indexlevel => 'medium',
-});
-my @xcpdb = qw(-xcpdb -q);
-my $nproc = 8;
-my $ndoc = 13;
-my $im = PublicInbox::InboxWritable->new($ibx, {nproc => $nproc})->importer;
-for my $i (1..$ndoc) {
- $mime->header_set('Message-ID', "<m$i\@example.com>");
- ok($im->add($mime), "message $i added");
-}
-$im->done;
+ for my $i (1..$ndoc) {
+ $eml->header_set('Message-ID', "<m$i\@example.com>");
+ ok($im->add($eml), "message $i added");
+ }
+ open my $fh, '>', "$ibx->{inboxdir}/empty" or BAIL_OUT "open $!";
+};
+my $env = { PI_CONFIG => "$ibx->{inboxdir}/empty" };
my @shards = grep(m!/\d+\z!, glob("$ibx->{inboxdir}/xap*/*"));
is(scalar(@shards), $nproc - 1, 'got expected shards');
my $orig = $ibx->over->query_xover(1, $ndoc);
my %nums = map {; "$_->{num}" => 1 } @$orig;
+my @xcpdb = qw(-xcpdb -q);
+my $XapianDatabase = do {
+ no warnings 'once';
+ $PublicInbox::Search::X{Database};
+};
# ensure we can go up or down in shards, or stay the same:
for my $R (qw(2 4 1 3 3)) {
delete $ibx->{search}; # release old handles
my $cmd = [@xcpdb, "-R$R", $ibx->{inboxdir}];
push @$cmd, '--compact' if $R == 1 && have_xapian_compact;
- ok(run_script($cmd), "xcpdb -R$R");
+ ok(run_script($cmd, $env), "xcpdb -R$R");
my @new_shards = grep(m!/\d+\z!, glob("$ibx->{inboxdir}/xap*/*"));
is(scalar(@new_shards), $R, 'resharded to two shards');
my $mset = $ibx->search->mset('s:this');
@@ -60,10 +58,6 @@ for my $R (qw(2 4 1 3 3)) {
# ensure docids in Xapian match NNTP article numbers
my $tot = 0;
my %tmp = %nums;
- my $XapianDatabase = do {
- no warnings 'once';
- $PublicInbox::Search::X{Database};
- };
foreach my $d (@new_shards) {
my $xdb = $XapianDatabase->new($d);
$tot += $xdb->get_doccount;
@@ -78,6 +72,4 @@ for my $R (qw(2 4 1 3 3)) {
}
is(scalar keys %tmp, 0, 'all docids seen');
}
-
-done_testing();
-1;
+done_testing;
^ permalink raw reply related [relevance 4%]
* [PATCH 00/35] memoize inbox creations
@ 2021-03-15 11:57 7% Eric Wong
2021-03-15 11:57 4% ` [PATCH 06/35] t/xcpdb-reshard: use create_inbox Eric Wong
0 siblings, 1 reply; 2+ results
From: Eric Wong @ 2021-03-15 11:57 UTC (permalink / raw)
To: meta
This speeds up a bunch of tests by a few dozen to hundreds of
milliseconds. Part of it is due to disabling fsync/fdatasync in
a few more places.
Users with slower TMPDIR will notice more, and users with TMPDIR
pointed to tmpfs will notice it least.
Overall it's not hugely significant, but the diffstat
and LoC reduction still makes it nice even if there were
no performance improvements.
Eric Wong (35):
test_common: add create_inbox helper sub
t/lei_xsearch: use create_inbox
test_common: minor simplifications to setup_public_inboxes
t/imapd-tls: switch to create_inbox
t/www_altid: use create_inbox
t/xcpdb-reshard: use create_inbox
t/v2dupindex: create_inbox
t/admin: switch to create_inbox
t/html_index: remove now-worthless test
t/plack: use create_inbox
t/psgi_attach: convert to create_inbox
t/httpd: convert to create_inbox
t/convert-compact: create_inbox
t/altid: use create_inbox
t/psgi_mount: switch to create_inbox
t/feed: switch to create_inbox
t/psgi_bad_mids: use create_inbox
t/psgi_multipart_not: use create_inbox
t/psgi_scan_all: create_inbox lots saved
t/psgi_v2: create_inbox
t/imapd: create_inbox (minor)
t/solver_git: use create_inbox
t/cgi: create_inbox
t/edit: switch to create_inbox
t/inbox_idle: switch to create_inbox
t/lei_xsearch: use create_inbox
t/indexlevels-mirror: use create_inbox
t/multi-mid: use create_inbox
t/psgi_search: use create_inbox
t/miscsearch: use create_inbox
t/nntpd-tls: use create_inbox
t/nntpd: use create_inbox
t/thread-index-gap: create_inbox
t/altid_v2: create_inbox
t/*: disable fsync on tests were create_inbox isn't worth it
MANIFEST | 2 +-
Makefile.PL | 3 +-
lib/PublicInbox/TestCommon.pm | 67 ++++++++++++++---
t/admin.t | 38 ++++------
t/altid.t | 26 ++-----
t/altid_v2.t | 48 +++++-------
t/cgi.t | 84 +++++++++------------
t/convert-compact.t | 48 +++++-------
t/data-gen/.gitignore | 2 +
t/edit.t | 34 ++++-----
t/extsearch.t | 2 +
t/feed.t | 53 ++++----------
t/html_index.t | 56 --------------
t/httpd.t | 36 +++------
t/imapd-tls.t | 38 +++-------
t/imapd.t | 66 ++++++++---------
t/inbox_idle.t | 31 ++++----
t/indexlevels-mirror.t | 24 +++---
t/lei-q-remote-import.t | 20 ++---
t/lei_xsearch.t | 56 +++++---------
t/miscsearch.t | 28 +++----
t/multi-mid.t | 34 +++------
t/nntpd-tls.t | 52 ++++---------
t/nntpd.t | 102 ++++++++++++--------------
t/plack.t | 69 +++++++----------
t/psgi_attach.t | 56 ++++++--------
t/psgi_bad_mids.t | 46 ++++--------
t/psgi_mount.t | 35 ++++-----
t/psgi_multipart_not.t | 35 +++------
t/psgi_scan_all.t | 48 ++++--------
t/psgi_search.t | 37 +++-------
t/psgi_v2.t | 134 ++++++++++++++++------------------
t/purge.t | 1 +
t/replace.t | 1 +
t/solver_git.t | 88 +++++++++++-----------
t/thread-index-gap.t | 24 +++---
t/v1reindex.t | 1 +
t/v2-add-remove-add.t | 1 +
t/v2dupindex.t | 76 +++++++++----------
t/v2mirror.t | 1 +
t/v2reindex.t | 1 +
t/v2writable.t | 1 +
t/watch_filter_rubylang.t | 2 +
t/watch_maildir_v2.t | 1 +
t/www_altid.t | 47 +++++-------
t/xcpdb-reshard.t | 54 ++++++--------
46 files changed, 692 insertions(+), 1017 deletions(-)
create mode 100644 t/data-gen/.gitignore
delete mode 100644 t/html_index.t
^ permalink raw reply [relevance 7%]
Results 1-2 of 2 | reverse | options above
-- pct% links below jump to the message on this page, permalinks otherwise --
2021-03-15 11:57 7% [PATCH 00/35] memoize inbox creations Eric Wong
2021-03-15 11:57 4% ` [PATCH 06/35] t/xcpdb-reshard: use create_inbox Eric Wong
Code repositories for project(s) associated with this public inbox
https://80x24.org/public-inbox.git
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).