* [PATCH 02/10] xt: add create-many-inboxes helper test
2020-12-23 8:38 7% [PATCH 00/10] start optimizing startup w/ ALL->misc Eric Wong
@ 2020-12-23 8:38 7% ` Eric Wong
0 siblings, 0 replies; 2+ results
From: Eric Wong @ 2020-12-23 8:38 UTC (permalink / raw)
To: meta
I've been using something like this to mock out thousands
of inboxes for testing.
---
MANIFEST | 1 +
xt/create-many-inboxes.t | 99 ++++++++++++++++++++++++++++++++++++++++
2 files changed, 100 insertions(+)
create mode 100644 xt/create-many-inboxes.t
diff --git a/MANIFEST b/MANIFEST
index ac442606..a4cdedff 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -394,6 +394,7 @@ t/x-unknown-alpine.eml
t/xcpdb-reshard.t
xt/cmp-msgstr.t
xt/cmp-msgview.t
+xt/create-many-inboxes.t
xt/eml_check_limits.t
xt/git-http-backend.t
xt/git_async_cmp.t
diff --git a/xt/create-many-inboxes.t b/xt/create-many-inboxes.t
new file mode 100644
index 00000000..c92643b2
--- /dev/null
+++ b/xt/create-many-inboxes.t
@@ -0,0 +1,99 @@
+#!perl -w
+# Copyright (C) 2020 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+use strict;
+use Test::More;
+use PublicInbox::TestCommon;
+use PublicInbox::Eml;
+use File::Path qw(mkpath);
+use IO::Handle (); # autoflush
+use POSIX qw(_exit);
+use Cwd qw(getcwd abs_path);
+use File::Spec;
+my $many_root = $ENV{TEST_MANY_ROOT} or
+ plan skip_all => 'TEST_MANY_ROOT not defined';
+my $cwd = getcwd();
+mkpath($many_root);
+-d $many_root or BAIL_OUT "$many_root: $!";
+$many_root = abs_path($many_root);
+$many_root =~ m!\A\Q$cwd\E/! and BAIL_OUT "$many_root must not be in $cwd";
+require_git 2.6;
+require_mods(qw(DBD::SQLite Search::Xapian));
+use_ok 'PublicInbox::V2Writable';
+my $nr_inbox = $ENV{NR_INBOX} // 10;
+my $nproc = $ENV{NPROC} || PublicInbox::V2Writable::detect_nproc() || 2;
+my $indexlevel = $ENV{TEST_INDEXLEVEL} // 'basic';
+diag "NR_INBOX=$nr_inbox NPROC=$nproc TEST_INDEXLEVEL=$indexlevel";
+diag "TEST_MANY_ROOT=$many_root";
+my $level_cfg = $indexlevel eq 'full' ? '' : "\tindexlevel = $indexlevel\n";
+my $pfx = "$many_root/$nr_inbox-$indexlevel";
+mkpath($pfx);
+open my $cfg_fh, '>>', "$pfx/config" or BAIL_OUT $!;
+$cfg_fh->autoflush(1);
+my $v2_init_add = sub {
+ my ($i) = @_;
+ my $ibx = PublicInbox::Inbox->new({
+ inboxdir => "$pfx/test-$i",
+ name => "test-$i",
+ newsgroup => "inbox.comp.test.foo.test-$i",
+ address => [ "test-$i\@example.com" ],
+ url => [ "//example.com/test-$i" ],
+ version => 2,
+ });
+ $ibx->{indexlevel} = $indexlevel if $level_cfg ne '';
+ my $entry = <<EOF;
+[publicinbox "$ibx->{name}"]
+ address = $ibx->{-primary_address}
+ url = $ibx->{url}->[0]
+ newsgroup = $ibx->{newsgroup}
+ inboxdir = $ibx->{inboxdir}
+EOF
+ $entry .= $level_cfg;
+ print $cfg_fh $entry or die $!;
+ my $v2w = PublicInbox::V2Writable->new($ibx, { nproc => 0 });
+ $v2w->init_inbox(0);
+ $v2w->add(PublicInbox::Eml->new(<<EOM));
+Date: Sat, 02 Oct 2010 00:00:00 +0000
+From: Lorelei <l\@example.com>
+To: test-$i\@example.com
+Message-ID: <20101002-000000-$i\@example.com>
+Subject: hello world $i
+
+hi
+EOM
+ $v2w->done;
+};
+
+my @children;
+for my $i (1..$nproc) {
+ my ($r, $w);
+ pipe($r, $w) or BAIL_OUT $!;
+ my $pid = fork;
+ if ($pid == 0) {
+ close $w;
+ while (my $i = <$r>) {
+ chomp $i;
+ $v2_init_add->($i);
+ }
+ _exit(0);
+ }
+ defined $pid or BAIL_OUT "fork: $!";
+ close $r or BAIL_OUT $!;
+ push @children, [ $w, $pid ];
+ $w->autoflush(1);
+}
+
+for my $i (0..$nr_inbox) {
+ print { $children[$i % @children]->[0] } "$i\n" or BAIL_OUT $!;
+}
+
+for my $c (@children) {
+ close $c->[0] or BAIL_OUT "close $!";
+}
+my $i = 0;
+for my $c (@children) {
+ my $pid = waitpid($c->[1], 0);
+ is($?, 0, ++$i.' exited ok');
+}
+ok(close($cfg_fh), 'config written');
+done_testing;
^ permalink raw reply related [relevance 7%]
* [PATCH 00/10] start optimizing startup w/ ALL->misc
@ 2020-12-23 8:38 7% Eric Wong
2020-12-23 8:38 7% ` [PATCH 02/10] xt: add create-many-inboxes helper test Eric Wong
0 siblings, 1 reply; 2+ results
From: Eric Wong @ 2020-12-23 8:38 UTC (permalink / raw)
To: meta
-nntpd [PATCH 5/10] is the single most significant improvements.
And some cleanups, and some general improvements independent of
indexing in patches 7-10 (patch 8 is already superceded by 10,
but kept separate for documentation purposes).
PublicInbox::Config->new is over twice as fast, now.
Eric Wong (10):
miscsearch: load Xapian at initialization
xt: add create-many-inboxes helper test
inbox: git_epoch: correct false comment
inboxwritable: _init_v1: set created_at ASAP
miscsearch: index UIDVALIDITY, use as startup cache
extsearchidx: close SQLite handles after attaching
config: _fill: inbox name extraction optimization
config: git_config_dump: pre-compile RE for split
config: config_fh_parse: micro-optimize
config: config_fh_parse: micro-optimize even harder
MANIFEST | 1 +
lib/PublicInbox/Config.pm | 26 ++++-----
lib/PublicInbox/ExtSearchIdx.pm | 25 +++++---
lib/PublicInbox/Inbox.pm | 2 +-
lib/PublicInbox/InboxWritable.pm | 3 +-
lib/PublicInbox/MiscIdx.pm | 26 ++++++---
lib/PublicInbox/MiscSearch.pm | 57 ++++++++++++++++--
lib/PublicInbox/NNTPD.pm | 4 +-
lib/PublicInbox/Search.pm | 9 ++-
lib/PublicInbox/SearchIdx.pm | 7 ---
t/search.t | 4 +-
xt/create-many-inboxes.t | 99 ++++++++++++++++++++++++++++++++
12 files changed, 213 insertions(+), 50 deletions(-)
create mode 100644 xt/create-many-inboxes.t
^ permalink raw reply [relevance 7%]
Results 1-2 of 2 | reverse | options above
-- pct% links below jump to the message on this page, permalinks otherwise --
2020-12-23 8:38 7% [PATCH 00/10] start optimizing startup w/ ALL->misc Eric Wong
2020-12-23 8:38 7% ` [PATCH 02/10] xt: add create-many-inboxes helper test Eric Wong
Code repositories for project(s) associated with this public inbox
https://80x24.org/public-inbox.git
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).