[PATCH 00/29] speed up tests by preloading

[PATCH 00/29] speed up tests by preloading

[Eric Wong]

On my fastest system, this brings "make check" time down from
~17s to ~10s.  This also improves consistency of our test suite,
adds ENV{TAIL} support to all daemons, and removes the test-time
dependency on the IPC::Run module.

Several cleanups were necessary to limit the scope of some
references and minor bugs were found (and fixed) in preparation
for this.  Most of the changes were to explicitly pass global
variables into subs to avoid warnings.

TEST_RUN_MODE=0 can be set in the environment to restore
real-world behavior with (v)fork && execve.

Eric Wong (29):
  edit: pass global variables into subs
  edit: use OO API of File::Temp to shorten lifetime
  admin: get rid of singleton $CFG var
  index: pass global variables into subs
  init: pass global variables into subs
  mda: pass global variables into subs
  learn: pass global variables into subs
  inboxwritable: add ->cleanup method
  import: only pass Inbox object to SearchIdx->new
  xapcmd: do not fire END and DESTROY handlers in child
  spawn: which: allow embedded slash for relative path
  t/common: introduce run_script wrapper for t/cgi.t
  t/edit: switch to use run_script
  t/init: convert to using run_script
  t/purge: convert to run_script
  t/v2mirror: get rid of IPC::Run dependency
  t/mda: switch to run_script for testing
  t/mda_filter_rubylang: drop IPC::Run dependency
  doc: remove IPC::Run as a dev and test dependency
  t/v2mirror: switch to default run_mode for speedup
  t/convert-compact: convert to run_script
  t/httpd: use run_script for -init
  t/watch_maildir_v2: use run_script for -init
  t/nntpd: use run_script for -init
  t/watch_filter_rubylang: run_script for -init and -index
  t/v2mda: switch to run_script in many places
  t/indexlevels-mirror*: switch to run_script
  t/xcpdb-reshard: use run_script for -xcpdb
  t/common: start_script replaces spawn_listener

 INSTALL                          |   4 -
 ci/deps.perl                     |   1 -
 lib/PublicInbox/Admin.pm         |   9 +-
 lib/PublicInbox/Import.pm        |   4 +-
 lib/PublicInbox/InboxWritable.pm |   4 +
 lib/PublicInbox/Spawn.pm         |   2 +-
 lib/PublicInbox/Xapcmd.pm        |   5 +-
 script/public-inbox-edit         |  40 ++---
 script/public-inbox-index        |   3 +-
 script/public-inbox-init         |  27 +++-
 script/public-inbox-learn        |   8 +-
 script/public-inbox-mda          |  12 +-
 t/cgi.t                          |  16 +-
 t/common.perl                    | 248 ++++++++++++++++++++++++++++---
 t/convert-compact.t              |  18 +--
 t/edit.t                         |  65 ++++----
 t/git-http-backend.t             |  14 +-
 t/httpd-corner.t                 |  48 +++---
 t/httpd-https.t                  |  28 +---
 t/httpd-unix.t                   |  47 +++---
 t/httpd.t                        |  18 +--
 t/indexlevels-mirror.t           |  24 +--
 t/init.t                         |  85 +++++------
 t/mda.t                          |  53 ++++---
 t/mda_filter_rubylang.t          |  17 +--
 t/nntpd-tls.t                    |  29 +---
 t/nntpd-validate.t               |  27 ++--
 t/nntpd.t                        |  22 ++-
 t/perf-nntpd.t                   |  22 ++-
 t/purge.t                        |  20 +--
 t/v2mda.t                        |  38 ++---
 t/v2mirror.t                     |  55 +++----
 t/v2writable.t                   |   8 +-
 t/watch_filter_rubylang.t        |   6 +-
 t/watch_maildir_v2.t             |   4 +-
 t/www_listing.t                  |   8 +-
 t/xcpdb-reshard.t                |   5 +-
 37 files changed, 584 insertions(+), 460 deletions(-)

[PATCH 01/29] edit: pass global variables into subs

[Eric Wong]

Avoid 'Variable "%s" will not stay shared' warnings
when the contents of this script eval'ed into a sub.
---
 script/public-inbox-edit | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/script/public-inbox-edit b/script/public-inbox-edit
index 43ce9900..c9884053 100755
--- a/script/public-inbox-edit
+++ b/script/public-inbox-edit
@@ -46,9 +46,9 @@ PublicInbox::AdminEdit::check_editable(\@ibxs);
 
 my $found = {}; # cid => [ [ibx, smsg] [, [ibx, smsg] ] ]
 
-sub find_mid ($) {
-	my ($mid) = @_;
-	foreach my $ibx (@ibxs) {
+sub find_mid ($$$) {
+	my ($found, $mid, $ibxs) = @_;
+	foreach my $ibx (@$ibxs) {
 		my $over = $ibx->over;
 		my ($id, $prev);
 		while (my $smsg = $over->next_by_mid($mid, \$id, \$prev)) {
@@ -68,7 +68,8 @@ sub show_cmd ($$) {
 	" GIT_DIR=$ibx->{inboxdir}/all.git \\\n    git show $smsg->{blob}\n";
 }
 
-sub show_found () {
+sub show_found ($) {
+	my ($found) = @_;
 	foreach my $to_edit (values %$found) {
 		foreach my $tuple (@$to_edit) {
 			my ($ibx, $smsg) = @$tuple;
@@ -79,7 +80,7 @@ sub show_found () {
 
 if (defined($mid)) {
 	$mid = mid_clean($mid);
-	$found = find_mid($mid);
+	find_mid($found, $mid, \@ibxs);
 	my $nr = scalar(keys %$found);
 	die "No message found for <$mid>\n" unless $nr;
 	if ($nr > 1) {
@@ -87,7 +88,7 @@ if (defined($mid)) {
 Multiple messages with different content found matching
 <$mid>:
 
-		show_found();
+		show_found($found);
 		die "Use --force to edit all of them\n" if !$opt->{force};
 		warn "Will edit all of them\n";
 	}
@@ -96,7 +97,7 @@ Multiple messages with different content found matching
 	my $orig = do { local $/; <$fh> };
 	my $mime = PublicInbox::MIME->new(\$orig);
 	my $mids = mids($mime->header_obj);
-	find_mid($_) for (@$mids); # populates $found
+	find_mid($found, $_, \@ibxs) for (@$mids); # populates $found
 	my $cid = content_id($mime);
 	my $to_edit = $found->{$cid};
 	unless ($to_edit) {
@@ -106,7 +107,7 @@ Multiple messages with different content found matching
 $nr matches to Message-ID(s) in $file, but none matched content
 Partial matches below:
 
-			show_found();
+			show_found($found);
 		} elsif ($nr == 0) {
 			$mids = join('', map { "  <$_>\n" } @$mids);
 			warn <<"";

[PATCH 02/29] edit: use OO API of File::Temp to shorten lifetime

[Eric Wong]

Instead of relying on END{} blocks, rely on ->DESTROY
so the temporary files go out-of-scope and system
resources get released, sooner.
---
 script/public-inbox-edit | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/script/public-inbox-edit b/script/public-inbox-edit
index c9884053..0accd7c1 100755
--- a/script/public-inbox-edit
+++ b/script/public-inbox-edit
@@ -8,7 +8,7 @@ use strict;
 use warnings;
 use Getopt::Long qw(:config gnu_getopt no_ignore_case auto_abbrev);
 use PublicInbox::AdminEdit;
-use File::Temp qw(tempfile);
+use File::Temp ();
 use PublicInbox::ContentId qw(content_id);
 use PublicInbox::MID qw(mid_clean mids);
 PublicInbox::Admin::check_require('-index');
@@ -120,10 +120,16 @@ $mids
 	$found = { $cid => $to_edit };
 }
 
-my $tmpl = 'public-inbox-edit-XXXXXX';
+my %tmpopt = (
+	TEMPLATE => 'public-inbox-edit-XXXXXX',
+	TMPDIR => 1,
+	SUFFIX => $opt->{raw} ? '.eml' : '.mbox',
+);
+
 foreach my $to_edit (values %$found) {
-	my ($edit_fh, $edit_fn) = tempfile($tmpl, TMPDIR => 1, UNLINK => 1);
+	my $edit_fh = File::Temp->new(%tmpopt);
 	$edit_fh->autoflush(1);
+	my $edit_fn = $edit_fh->filename;
 	my ($ibx, $smsg) = @{$to_edit->[0]};
 	my $old_raw = $ibx->msg_by_smsg($smsg);
 	delete @$ibx{qw(over mm git search)}; # cleanup

[PATCH 03/29] admin: get rid of singleton $CFG var

[Eric Wong]

PublicInbox::Admin::config() just adds an extra layer of
indirection which we barely rely on.  So get rid of this
global variable and make it easier to run tests in the
future without relying on global state.
---
 lib/PublicInbox/Admin.pm | 9 +++------
 script/public-inbox-edit | 7 +++----
 2 files changed, 6 insertions(+), 10 deletions(-)

diff --git a/lib/PublicInbox/Admin.pm b/lib/PublicInbox/Admin.pm
index d2a0d06b..dddeeae9 100644
--- a/lib/PublicInbox/Admin.pm
+++ b/lib/PublicInbox/Admin.pm
@@ -9,7 +9,6 @@ use warnings;
 use Cwd 'abs_path';
 use base qw(Exporter);
 our @EXPORT_OK = qw(resolve_repo_dir);
-my $CFG; # all the admin stuff is a singleton
 require PublicInbox::Config;
 
 sub resolve_repo_dir {
@@ -80,14 +79,12 @@ sub unconfigured_ibx ($$) {
 	});
 }
 
-sub config () { $CFG //= eval { PublicInbox::Config->new } }
-
-sub resolve_inboxes ($;$) {
-	my ($argv, $opt) = @_;
+sub resolve_inboxes ($;$$) {
+	my ($argv, $opt, $cfg) = @_;
 	require PublicInbox::Inbox;
 	$opt ||= {};
 
-	my $cfg = config();
+	$cfg //= eval { PublicInbox::Config->new };
 	if ($opt->{all}) {
 		my $cfgfile = PublicInbox::Config::default_file();
 		$cfg or die "--all specified, but $cfgfile not readable\n";
diff --git a/script/public-inbox-edit b/script/public-inbox-edit
index 0accd7c1..b437b3c0 100755
--- a/script/public-inbox-edit
+++ b/script/public-inbox-edit
@@ -22,12 +22,11 @@ my @opt = qw(mid|m=s file|F=s raw);
 GetOptions($opt, @PublicInbox::AdminEdit::OPT, @opt) or
 	die "bad command-line args\n$usage\n";
 
+my $cfg = eval { PublicInbox::Config->new };
 my $editor = $ENV{MAIL_EDITOR}; # e.g. "mutt -f"
 unless (defined $editor) {
 	my $k = 'publicinbox.mailEditor';
-	if (my $cfg = PublicInbox::Admin::config()) {
-		$editor = $cfg->{lc($k)};
-	}
+	$editor = $cfg->{lc($k)} if $cfg;
 	unless (defined $editor) {
 		warn "\`$k' not configured, trying \`git var GIT_EDITOR'\n";
 		chomp($editor = `git var GIT_EDITOR`);
@@ -41,7 +40,7 @@ if (defined $mid && defined $file) {
 	die "the --mid and --file options are mutually exclusive\n";
 }
 
-my @ibxs = PublicInbox::Admin::resolve_inboxes(\@ARGV, $opt);
+my @ibxs = PublicInbox::Admin::resolve_inboxes(\@ARGV, $opt, $cfg);
 PublicInbox::AdminEdit::check_editable(\@ibxs);
 
 my $found = {}; # cid => [ [ibx, smsg] [, [ibx, smsg] ] ]

[PATCH 04/29] index: pass global variables into subs

[Eric Wong]

Avoid 'Variable "%s" will not stay shared' warnings
when the contents of this script eval'ed into a sub.
---
 script/public-inbox-index | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/script/public-inbox-index b/script/public-inbox-index
index 139b6e56..102381c3 100755
--- a/script/public-inbox-index
+++ b/script/public-inbox-index
@@ -18,11 +18,10 @@ GetOptions($opt, qw(verbose|v+ reindex jobs|j=i prune indexlevel|L=s))
 	or die "bad command-line args\n$usage";
 die "--jobs must be positive\n" if defined $opt->{jobs} && $opt->{jobs} <= 0;
 
-sub usage { print STDERR "Usage: $usage\n"; exit 1 }
 
 my @ibxs = PublicInbox::Admin::resolve_inboxes(\@ARGV);
 PublicInbox::Admin::require_or_die('-index');
-usage() unless @ibxs;
+unless (@ibxs) { print STDERR "Usage: $usage\n"; exit 1 }
 my $mods = {};
 foreach my $ibx (@ibxs) {
 	# XXX: users can shoot themselves in the foot, with opt->{indexlevel}

[PATCH 05/29] init: pass global variables into subs

[Eric Wong]

Avoid 'Variable "%s" will not stay shared' warnings
when the contents of this script eval'ed into a sub.

We also need to rely on ->DESTROY instead of END{}
to unlink the lock file on sub exit.
---
 script/public-inbox-init | 27 ++++++++++++++++++++++-----
 1 file changed, 22 insertions(+), 5 deletions(-)

diff --git a/script/public-inbox-init b/script/public-inbox-init
index 50711266..da683657 100755
--- a/script/public-inbox-init
+++ b/script/public-inbox-init
@@ -5,7 +5,12 @@
 # Initializes a public-inbox, basically a wrapper for git-init(1)
 use strict;
 use warnings;
-my $usage = "public-inbox-init NAME INBOX_DIR HTTP_URL ADDRESS [ADDRESS..]";
+sub usage {
+	print STDERR <<EOF;
+Usage: public-inbox-init NAME INBOX_DIR HTTP_URL ADDRESS [ADDRESS..]
+EOF
+	exit 1;
+}
 use Getopt::Long qw/:config gnu_getopt no_ignore_case auto_abbrev/;
 use PublicInbox::Admin;
 PublicInbox::Admin::require_or_die('-base');
@@ -19,7 +24,6 @@ use Fcntl qw(:DEFAULT);
 use Cwd qw/abs_path/;
 
 sub x { system(@_) and die join(' ', @_). " failed: $?\n" }
-sub usage { print STDERR "Usage: $usage\n"; exit 1 }
 my $version = undef;
 my $indexlevel = undef;
 my $skip_epoch;
@@ -57,12 +61,10 @@ my ($fh, $pi_config_tmp) = tempfile('pi-init-XXXXXXXX', DIR => $dir);
 my $lockfile = "$pi_config.lock";
 my $lockfh;
 sysopen($lockfh, $lockfile, O_RDWR|O_CREAT|O_EXCL) or do {
-	$lockfh = undef;
 	warn "could not open config file: $lockfile: $!\n";
 	exit(255);
 };
-END { unlink($lockfile) if $lockfh };
-
+my $auto_unlink = UnlinkMe->new($lockfile);
 my $perm;
 if (-e $pi_config) {
 	open(my $oh, '<', $pi_config) or die "unable to read $pi_config: $!\n";
@@ -166,3 +168,18 @@ if (defined $perm) {
 
 rename $pi_config_tmp, $pi_config or
 	die "failed to rename `$pi_config_tmp' to `$pi_config': $!\n";
+$auto_unlink->DESTROY;
+
+package UnlinkMe;
+use strict;
+
+sub new {
+	my ($klass, $file) = @_;
+	bless { file => $file }, $klass;
+}
+
+sub DESTROY {
+	my $f = delete($_[0]->{file});
+	unlink($f) if defined($f);
+}
+1;

[PATCH 06/29] mda: pass global variables into subs

[Eric Wong]

Avoid 'Variable "%s" will not stay shared' warnings
when the contents of this script eval'ed into a sub.
---
 script/public-inbox-mda | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/script/public-inbox-mda b/script/public-inbox-mda
index dca8a0ea..9da2d90f 100755
--- a/script/public-inbox-mda
+++ b/script/public-inbox-mda
@@ -9,11 +9,11 @@ my $usage = 'public-inbox-mda [OPTIONS] < rfc2822_message';
 my $precheck = grep(/\A--no-precheck\z/, @ARGV) ? 0 : 1;
 my ($ems, $emm);
 
-sub do_exit {
+my $do_exit = sub {
 	my ($code) = shift;
 	$emm = $ems = undef; # trigger DESTROY
 	exit $code;
-}
+};
 
 use Email::Simple;
 use PublicInbox::MIME;
@@ -48,7 +48,7 @@ if (!scalar(@$dests)) {
 	if (!scalar(@$dests) && !defined($recipient)) {
 		die "ORIGINAL_RECIPIENT not defined in ENV\n";
 	}
-	scalar(@$dests) or do_exit(67); # EX_NOUSER 5.1.1 user unknown
+	scalar(@$dests) or $do_exit->(67); # EX_NOUSER 5.1.1 user unknown
 }
 
 my $err;
@@ -67,7 +67,7 @@ my $err;
 	}
 } @$dests;
 
-do_exit(67) if $err && scalar(@$dests) == 0;
+$do_exit->(67) if $err && scalar(@$dests) == 0;
 
 $simple = undef;
 my $spam_ok;
@@ -84,7 +84,7 @@ if ($spamc) {
 	my $fh = $emm->fh;
 	read($fh, $str, -s $fh);
 }
-do_exit(0) unless $spam_ok;
+$do_exit->(0) unless $spam_ok;
 
 # -mda defaults to the strict base filter which we won't use anywhere else
 sub mda_filter_adjust ($) {
@@ -131,4 +131,4 @@ if (scalar(@rejects) && scalar(@rejects) == scalar(@$dests)) {
 	die join("\n", @rejects, '');
 }
 
-do_exit(0);
+$do_exit->(0);

[PATCH 07/29] learn: pass global variables into subs

[Eric Wong]

Avoid 'Variable "%s" will not stay shared' warnings
when the contents of this script eval'ed into a sub.
---
 script/public-inbox-learn | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/script/public-inbox-learn b/script/public-inbox-learn
index 3073294a..93aece2e 100644
--- a/script/public-inbox-learn
+++ b/script/public-inbox-learn
@@ -39,8 +39,8 @@ my $mime = PublicInbox::MIME->new(eval {
 	$data
 });
 
-sub remove_or_add ($$$) {
-	my ($ibx, $train, $addr) = @_;
+sub remove_or_add ($$$$) {
+	my ($ibx, $train, $mime, $addr) = @_;
 
 	# We do not touch GIT_COMMITTER_* env here so we can track
 	# who trained the message.
@@ -93,12 +93,12 @@ if ($train eq 'spam') {
 	while (my ($addr, $ibx) = each %dests) {
 		next unless ref($ibx); # $ibx may be 0
 		next if $seen{"$ibx"}++;
-		remove_or_add($ibx, $train, $addr);
+		remove_or_add($ibx, $train, $mime, $addr);
 	}
 	my $dests = PublicInbox::MDA->inboxes_for_list_id($pi_config, $mime);
 	for my $ibx (@$dests) {
 		next if !$seen{"$ibx"}++;
-		remove_or_add($ibx, $train, $ibx->{-primary_address});
+		remove_or_add($ibx, $train, $mime, $ibx->{-primary_address});
 	}
 }
 

[PATCH 08/29] inboxwritable: add ->cleanup method

[Eric Wong]

We've been using this in -edit, and will be using it in some
more scripts and tests to optimize for run_mode=2 with
run_script.

Keeping this in the *Writable modules since I don't see it being
useful for the WWW and NNTP read-only interfaces which use
PublicInbox::Inbox.
---
 lib/PublicInbox/InboxWritable.pm | 4 ++++
 lib/PublicInbox/Xapcmd.pm        | 2 +-
 script/public-inbox-edit         | 4 ++--
 3 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/lib/PublicInbox/InboxWritable.pm b/lib/PublicInbox/InboxWritable.pm
index c73910ac..d8391251 100644
--- a/lib/PublicInbox/InboxWritable.pm
+++ b/lib/PublicInbox/InboxWritable.pm
@@ -257,4 +257,8 @@ sub umask_prepare {
 	$self->{umask} = $umask;
 }
 
+sub cleanup ($) {
+	delete @{$_[0]}{qw(over mm git search)};
+}
+
 1;
diff --git a/lib/PublicInbox/Xapcmd.pm b/lib/PublicInbox/Xapcmd.pm
index c807bf10..77f0524e 100644
--- a/lib/PublicInbox/Xapcmd.pm
+++ b/lib/PublicInbox/Xapcmd.pm
@@ -234,7 +234,7 @@ sub run {
 			$im->lock_release;
 		}
 
-		delete($ibx->{$_}) for (qw(mm over search)); # cleanup
+		$ibx->cleanup;
 		process_queue(\@q, $cb, $max, $opt);
 		$im->lock_acquire if !$opt->{-coarse_lock};
 		commit_changes($ibx, $im, $tmp, $opt);
diff --git a/script/public-inbox-edit b/script/public-inbox-edit
index b437b3c0..1900b267 100755
--- a/script/public-inbox-edit
+++ b/script/public-inbox-edit
@@ -57,7 +57,7 @@ sub find_mid ($$$) {
 			my $tuple = [ $ibx, $smsg ];
 			push @{$found->{$cid} ||= []}, $tuple
 		}
-		delete @$ibx{qw(over mm git search)}; # cleanup
+		PublicInbox::InboxWritable::cleanup($ibx);
 	}
 	$found;
 }
@@ -131,7 +131,7 @@ foreach my $to_edit (values %$found) {
 	my $edit_fn = $edit_fh->filename;
 	my ($ibx, $smsg) = @{$to_edit->[0]};
 	my $old_raw = $ibx->msg_by_smsg($smsg);
-	delete @$ibx{qw(over mm git search)}; # cleanup
+	PublicInbox::InboxWritable::cleanup($ibx);
 
 	my $tmp = $$old_raw;
 	if (!$opt->{raw}) {

[PATCH 09/29] import: only pass Inbox object to SearchIdx->new

[Eric Wong]

SearchIdx->new no longer accepts a GIT_DIR path as its argument
since commit 585314673236d664729fe3ab2d4fb229d1c0f2d5
("searchidx: require PublicInbox::Inbox (or InboxWritable) ref")
---
 lib/PublicInbox/Import.pm | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/PublicInbox/Import.pm b/lib/PublicInbox/Import.pm
index e1f48771..cb25215d 100644
--- a/lib/PublicInbox/Import.pm
+++ b/lib/PublicInbox/Import.pm
@@ -179,9 +179,9 @@ sub _update_git_info ($$) {
 		run_die([@cmd, qw(read-tree -m -v -i), $self->{ref}], $env);
 	}
 	run_die([@cmd, 'update-server-info'], undef);
-	($self->{path_type} eq '2/38') and eval {
+	my $ibx = $self->{-inbox};
+	($ibx && $self->{path_type} eq '2/38') and eval {
 		require PublicInbox::SearchIdx;
-		my $ibx = $self->{-inbox} || $git_dir;
 		my $s = PublicInbox::SearchIdx->new($ibx);
 		$s->index_sync({ ref => $self->{ref} });
 	};

[PATCH 10/29] xapcmd: do not fire END and DESTROY handlers in child

[Eric Wong]

We need to bypass whatever Test::More does with END/DESTROY
handlers for use in lon-lived process.  This doesn't affect
any of our normal code since we don't use END/DESTROY for
Xapcmd and its callers.
---
 lib/PublicInbox/Xapcmd.pm | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/PublicInbox/Xapcmd.pm b/lib/PublicInbox/Xapcmd.pm
index 77f0524e..28285898 100644
--- a/lib/PublicInbox/Xapcmd.pm
+++ b/lib/PublicInbox/Xapcmd.pm
@@ -9,6 +9,7 @@ use PublicInbox::Search;
 use File::Temp qw(tempdir);
 use File::Path qw(remove_tree);
 use File::Basename qw(dirname);
+use POSIX ();
 
 # support testing with dev versions of Xapian which installs
 # commands with a version number suffix (e.g. "xapian-compact-1.5")
@@ -85,7 +86,7 @@ sub cb_spawn {
 	defined(my $pid = fork) or die "fork: $!";
 	return $pid if $pid > 0;
 	$cb->($args, $opt);
-	exit 0;
+	POSIX::_exit(0);
 }
 
 sub runnable_or_die ($) {

[PATCH 11/29] spawn: which: allow embedded slash for relative path

[Eric Wong]

This makes the subroutine behave more like which(1) command
and will make using spawn() in tests easier.
---
 lib/PublicInbox/Spawn.pm | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/PublicInbox/Spawn.pm b/lib/PublicInbox/Spawn.pm
index e2868a55..b946a663 100644
--- a/lib/PublicInbox/Spawn.pm
+++ b/lib/PublicInbox/Spawn.pm
@@ -178,7 +178,7 @@ unless (defined $vfork_spawn) {
 
 sub which ($) {
 	my ($file) = @_;
-	return $file if index($file, '/') == 0;
+	return $file if index($file, '/') >= 0;
 	foreach my $p (split(':', $ENV{PATH})) {
 		$p .= "/$file";
 		return $p if -x $p;

[PATCH 12/29] t/common: introduce run_script wrapper for t/cgi.t

[Eric Wong]

This will give us a consistent interface for running
test scripts in more performant ways while still giving
us a consistent interface to recreate real-world behavior
via spawn() (fork + execve), if needed.

The default run_mode (1) is faster and can run within the test
process with some minor adjustments to our code to avoid global
state.

This avoids the significante overhead of Perl code loading,
parsing and compilation phases.
---
 t/cgi.t       |  16 ++-----
 t/common.perl | 125 +++++++++++++++++++++++++++++++++++++++++++++++++-
 2 files changed, 129 insertions(+), 12 deletions(-)

diff --git a/t/cgi.t b/t/cgi.t
index 1b4b06cb..3c09ecd6 100644
--- a/t/cgi.t
+++ b/t/cgi.t
@@ -7,10 +7,7 @@ use warnings;
 use Test::More;
 use Email::MIME;
 use File::Temp qw/tempdir/;
-eval { require IPC::Run };
-plan skip_all => "missing IPC::Run for t/cgi.t" if $@;
-
-use constant CGI => "blib/script/public-inbox.cgi";
+require './t/common.perl';
 my $tmpdir = tempdir('pi-cgi-XXXXXX', TMPDIR => 1, CLEANUP => 1);
 my $home = "$tmpdir/pi-home";
 my $pi_home = "$home/.public-inbox";
@@ -145,11 +142,6 @@ EOF
 
 done_testing();
 
-sub run_with_env {
-	my ($env, @args) = @_;
-	IPC::Run::run(@args, init => sub { %ENV = (%ENV, %$env) });
-}
-
 sub cgi_run {
 	my %env = (
 		PATH_INFO => $_[0],
@@ -162,7 +154,9 @@ sub cgi_run {
 		HTTP_HOST => 'test.example.com',
 	);
 	my ($in, $out, $err) = ("", "", "");
-	my $rc = run_with_env(\%env, [CGI], \$in, \$out, \$err);
+	my $rdr = { 0 => \$in, 1 => \$out, 2 => \$err };
+	run_script(['.cgi'], \%env, $rdr);
+	die "unexpected error: \$?=$?" if $?;
 	my ($head, $body) = split(/\r\n\r\n/, $out, 2);
-	{ head => $head, body => $body, rc => $rc, err => $err }
+	{ head => $head, body => $body, err => $err }
 }
diff --git a/t/common.perl b/t/common.perl
index d4a0fcd2..c5693080 100644
--- a/t/common.perl
+++ b/t/common.perl
@@ -1,7 +1,7 @@
 # Copyright (C) 2015-2019 all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
-use Fcntl qw(FD_CLOEXEC F_SETFD F_GETFD);
+use Fcntl qw(FD_CLOEXEC F_SETFD F_GETFD :seek);
 use POSIX qw(dup2);
 use strict;
 use warnings;
@@ -68,4 +68,127 @@ sub require_git ($;$) {
 	1;
 }
 
+my %cached_scripts;
+sub key2script ($) {
+	my ($key) = @_;
+	return $key if $key =~ m!\A/!;
+	# n.b. we may have scripts which don't start with "public-inbox" in
+	# the future:
+	$key =~ s/\A([-\.])/public-inbox$1/;
+	'blib/script/'.$key;
+}
+
+sub _prepare_redirects ($) {
+	my ($fhref) = @_;
+	my @x = ([ \*STDIN, '<&' ], [ \*STDOUT, '>&' ], [ \*STDERR, '>&' ]);
+	for (my $fd = 0; $fd <= $#x; $fd++) {
+		my $fh = $fhref->[$fd] or next;
+		my ($oldfh, $mode) = @{$x[$fd]};
+		open $oldfh, $mode, $fh or die "$$oldfh $mode redirect: $!";
+	}
+}
+
+# $opt->{run_mode} (or $ENV{TEST_RUN_MODE}) allows chosing between
+# three ways to spawn our own short-lived Perl scripts for testing:
+#
+# 0 - (fork|vfork) + execve, the most realistic but slowest
+# 1 - preloading and running in a forked subprocess (fast)
+# 2 - preloading and running in current process (slightly faster than 1)
+#
+# 2 is not compatible with scripts which use "exit" (which we'll try to
+# avoid in the future).
+# The default is 2.
+our $run_script_exit_code;
+sub RUN_SCRIPT_EXIT () { "RUN_SCRIPT_EXIT\n" };
+sub run_script_exit (;$) {
+	$run_script_exit_code = $_[0] // 0;
+	die RUN_SCRIPT_EXIT;
+}
+
+sub run_script ($;$$) {
+	my ($cmd, $env, $opt) = @_;
+	my ($key, @argv) = @$cmd;
+	my $run_mode = $ENV{TEST_RUN_MODE} // $opt->{run_mode} // 1;
+	my $sub = $run_mode == 0 ? undef : ($cached_scripts{$key} //= do {
+		my $f = key2script($key);
+		open my $fh, '<', $f or die "open $f: $!";
+		my $str = do { local $/; <$fh> };
+		my ($fc, $rest) = ($key =~ m/([a-z])([a-z0-9]+)\z/);
+		$fc = uc($fc);
+		my $pkg = "PublicInbox::TestScript::$fc$rest";
+		eval <<EOF;
+package $pkg;
+use strict;
+use subs qw(exit);
+
+*exit = *::run_script_exit;
+sub main {
+$str
+	0;
+}
+1;
+EOF
+		$pkg->can('main');
+	}); # do
+
+	my $fhref = [];
+	my $spawn_opt = {};
+	for my $fd (0..2) {
+		my $redir = $opt->{$fd};
+		next unless ref($redir);
+		open my $fh, '+>', undef or die "open: $!";
+		$fhref->[$fd] = $fh;
+		$spawn_opt->{$fd} = fileno($fh);
+		next if $fd > 0;
+		$fh->autoflush(1);
+		print $fh $$redir or die "print: $!";
+		seek($fh, 0, SEEK_SET) or die "seek: $!";
+	}
+	if ($run_mode == 0) {
+		# spawn an independent new process, like real-world use cases:
+		require PublicInbox::Spawn;
+		my $cmd = [ key2script($key), @argv ];
+		my $pid = PublicInbox::Spawn::spawn($cmd, $env, $spawn_opt);
+		defined($pid) or die "spawn: $!";
+		if (defined $pid) {
+			my $r = waitpid($pid, 0);
+			defined($r) or die "waitpid: $!";
+			$r == $pid or die "waitpid: expected $pid, got $r";
+		}
+	} else { # localize and run everything in the same process:
+		local *STDIN = *STDIN;
+		local *STDOUT = *STDOUT;
+		local *STDERR = *STDERR;
+		local %ENV = $env ? (%ENV, %$env) : %ENV;
+		local %SIG = %SIG;
+		_prepare_redirects($fhref);
+		local @ARGV = @argv;
+		$run_script_exit_code = undef;
+		my $exit_code = eval { $sub->(@argv) };
+		if ($@ eq RUN_SCRIPT_EXIT) {
+			$@ = '';
+			$exit_code = $run_script_exit_code;
+			$? = ($exit_code << 8);
+		} elsif (defined($exit_code)) {
+			$? = ($exit_code << 8);
+		} elsif ($@) { # mimic die() behavior when uncaught
+			warn "E: eval-ed $key: $@\n";
+			$? = ($! << 8) if $!;
+			$? = (255 << 8) if $? == 0;
+		} else {
+			die "BUG: eval-ed $key: no exit code or \$@\n";
+		}
+	}
+
+	# slurp the redirects back into user-supplied strings
+	for my $fd (1..2) {
+		my $fh = $fhref->[$fd] or next;
+		seek($fh, 0, SEEK_SET) or die "seek: $!";
+		my $redir = $opt->{$fd};
+		local $/;
+		$$redir = <$fh>;
+	}
+	$? == 0;
+}
+
 1;

[PATCH 13/29] t/edit: switch to use run_script

[Eric Wong]

Perl parsing is slow, and run_script default behavior allows
this to speed up t/edit.t by over 100% in my case.
---
 t/edit.t | 65 ++++++++++++++++++++++++++++----------------------------
 1 file changed, 32 insertions(+), 33 deletions(-)

diff --git a/t/edit.t b/t/edit.t
index 5cb66a65..09e0cddd 100644
--- a/t/edit.t
+++ b/t/edit.t
@@ -12,14 +12,12 @@ require PublicInbox::InboxWritable;
 require PublicInbox::Config;
 use PublicInbox::MID qw(mid_clean);
 
-my @mods = qw(IPC::Run DBI DBD::SQLite);
+my @mods = qw(DBI DBD::SQLite);
 foreach my $mod (@mods) {
 	eval "require $mod";
 	plan skip_all => "missing $mod for $0" if $@;
 };
-IPC::Run->import(qw(run));
 
-my $cmd_pfx = 'blib/script/public-inbox';
 my $tmpdir = tempdir('pi-edit-XXXXXX', TMPDIR => 1, CLEANUP => 1);
 my $inboxdir = "$tmpdir/v2";
 my $ibx = PublicInbox::Inbox->new({
@@ -42,12 +40,13 @@ ok($im->add($mime), 'add message to be edited');
 $im->done;
 my ($in, $out, $err, $cmd, $cur, $t);
 my $git = PublicInbox::Git->new("$ibx->{inboxdir}/git/0.git");
+my $opt = { 0 => \$in, 1 => \$out, 2 => \$err };
 
 $t = '-F FILE'; {
 	$in = $out = $err = '';
 	local $ENV{MAIL_EDITOR} = "$^X -i -p -e 's/boolean prefix/bool pfx/'";
-	$cmd = [ "$cmd_pfx-edit", "-F$file", $inboxdir ];
-	ok(run($cmd, \$in, \$out, \$err), "$t edit OK");
+	$cmd = [ '-edit', "-F$file", $inboxdir ];
+	ok(run_script($cmd, undef, $opt), "$t edit OK");
 	$cur = PublicInbox::MIME->new($ibx->msg_by_mid($mid));
 	like($cur->header('Subject'), qr/bool pfx/, "$t message edited");
 	like($out, qr/[a-f0-9]{40}/, "$t shows commit on success");
@@ -56,8 +55,8 @@ $t = '-F FILE'; {
 $t = '-m MESSAGE_ID'; {
 	$in = $out = $err = '';
 	local $ENV{MAIL_EDITOR} = "$^X -i -p -e 's/bool pfx/boolean prefix/'";
-	$cmd = [ "$cmd_pfx-edit", "-m$mid", $inboxdir ];
-	ok(run($cmd, \$in, \$out, \$err), "$t edit OK");
+	$cmd = [ '-edit', "-m$mid", $inboxdir ];
+	ok(run_script($cmd, undef, $opt), "$t edit OK");
 	$cur = PublicInbox::MIME->new($ibx->msg_by_mid($mid));
 	like($cur->header('Subject'), qr/boolean prefix/, "$t message edited");
 	like($out, qr/[a-f0-9]{40}/, "$t shows commit on success");
@@ -67,8 +66,8 @@ $t = 'no-op -m MESSAGE_ID'; {
 	$in = $out = $err = '';
 	my $before = $git->qx(qw(rev-parse HEAD));
 	local $ENV{MAIL_EDITOR} = "$^X -i -p -e 's/bool pfx/boolean prefix/'";
-	$cmd = [ "$cmd_pfx-edit", "-m$mid", $inboxdir ];
-	ok(run($cmd, \$in, \$out, \$err), "$t succeeds");
+	$cmd = [ '-edit', "-m$mid", $inboxdir ];
+	ok(run_script($cmd, undef, $opt), "$t succeeds");
 	my $prev = $cur;
 	$cur = PublicInbox::MIME->new($ibx->msg_by_mid($mid));
 	is_deeply($cur, $prev, "$t makes no change");
@@ -82,10 +81,9 @@ $t = 'no-op -m MESSAGE_ID'; {
 $t = 'no-op -m MESSAGE_ID w/Status: header'; { # because mutt does it
 	$in = $out = $err = '';
 	my $before = $git->qx(qw(rev-parse HEAD));
-	local $ENV{MAIL_EDITOR} =
-			"$^X -i -p -e 's/^Subject:.*/Status: RO\\n\$&/'";
-	$cmd = [ "$cmd_pfx-edit", "-m$mid", $inboxdir ];
-	ok(run($cmd, \$in, \$out, \$err), "$t succeeds");
+	local $ENV{MAIL_EDITOR} = "$^X -i -p -e 's/^Subject:.*/Status: RO\\n\$&/'";
+	$cmd = [ '-edit', "-m$mid", $inboxdir ];
+	ok(run_script($cmd, undef, $opt), "$t succeeds");
 	my $prev = $cur;
 	$cur = PublicInbox::MIME->new($ibx->msg_by_mid($mid));
 	is_deeply($cur, $prev, "$t makes no change");
@@ -99,10 +97,9 @@ $t = 'no-op -m MESSAGE_ID w/Status: header'; { # because mutt does it
 
 $t = '-m MESSAGE_ID can change Received: headers'; {
 	$in = $out = $err = '';
-	local $ENV{MAIL_EDITOR} =
-			"$^X -i -p -e 's/^Subject:.*/Received: x\\n\$&/'";
-	$cmd = [ "$cmd_pfx-edit", "-m$mid", $inboxdir ];
-	ok(run($cmd, \$in, \$out, \$err), "$t succeeds");
+	local $ENV{MAIL_EDITOR} = "$^X -i -p -e 's/^Subject:.*/Received: x\\n\$&/'";
+	$cmd = [ '-edit', "-m$mid", $inboxdir ];
+	ok(run_script($cmd, undef, $opt), "$t succeeds");
 	$cur = PublicInbox::MIME->new($ibx->msg_by_mid($mid));
 	like($cur->header('Subject'), qr/boolean prefix/,
 		"$t does not change Subject");
@@ -112,16 +109,16 @@ $t = '-m MESSAGE_ID can change Received: headers'; {
 $t = '-m miss'; {
 	$in = $out = $err = '';
 	local $ENV{MAIL_EDITOR} = "$^X -i -p -e 's/boolean/FAIL/'";
-	$cmd = [ "$cmd_pfx-edit", "-m$mid-miss", $inboxdir ];
-	ok(!run($cmd, \$in, \$out, \$err), "$t fails on invalid MID");
+	$cmd = [ '-edit', "-m$mid-miss", $inboxdir ];
+	ok(!run_script($cmd, undef, $opt), "$t fails on invalid MID");
 	like($err, qr/No message found/, "$t shows error");
 }
 
 $t = 'non-interactive editor failure'; {
 	$in = $out = $err = '';
 	local $ENV{MAIL_EDITOR} = "$^X -i -p -e 'END { exit 1 }'";
-	$cmd = [ "$cmd_pfx-edit", "-m$mid", $inboxdir ];
-	ok(!run($cmd, \$in, \$out, \$err), "$t detected");
+	$cmd = [ '-edit', "-m$mid", $inboxdir ];
+	ok(!run_script($cmd, undef, $opt), "$t detected");
 	like($err, qr/END \{ exit 1 \}' failed:/, "$t shows error");
 }
 
@@ -132,9 +129,11 @@ $t = 'mailEditor set in config'; {
 			"$^X -i -p -e 's/boolean prefix/bool pfx/'");
 	is($rc, 0, 'set publicinbox.mailEditor');
 	local $ENV{MAIL_EDITOR};
+	delete $ENV{MAIL_EDITOR};
+	delete local $ENV{MAIL_EDITOR};
 	local $ENV{GIT_EDITOR} = 'echo should not run';
-	$cmd = [ "$cmd_pfx-edit", "-m$mid", $inboxdir ];
-	ok(run($cmd, \$in, \$out, \$err), "$t edited message");
+	$cmd = [ '-edit', "-m$mid", $inboxdir ];
+	ok(run_script($cmd, undef, $opt), "$t edited message");
 	$cur = PublicInbox::MIME->new($ibx->msg_by_mid($mid));
 	like($cur->header('Subject'), qr/bool pfx/, "$t message edited");
 	unlike($out, qr/should not run/, 'did not run GIT_EDITOR');
@@ -143,21 +142,21 @@ $t = 'mailEditor set in config'; {
 $t = '--raw and mbox escaping'; {
 	$in = $out = $err = '';
 	local $ENV{MAIL_EDITOR} = "$^X -i -p -e 's/^\$/\\nFrom not mbox\\n/'";
-	$cmd = [ "$cmd_pfx-edit", "-m$mid", '--raw', $inboxdir ];
-	ok(run($cmd, \$in, \$out, \$err), "$t succeeds");
+	$cmd = [ '-edit', "-m$mid", '--raw', $inboxdir ];
+	ok(run_script($cmd, undef, $opt), "$t succeeds");
 	$cur = PublicInbox::MIME->new($ibx->msg_by_mid($mid));
 	like($cur->body, qr/^From not mbox/sm, 'put "From " line into body');
 
 	local $ENV{MAIL_EDITOR} = "$^X -i -p -e 's/^>From not/\$& an/'";
-	$cmd = [ "$cmd_pfx-edit", "-m$mid", $inboxdir ];
-	ok(run($cmd, \$in, \$out, \$err), "$t succeeds with mbox escaping");
+	$cmd = [ '-edit', "-m$mid", $inboxdir ];
+	ok(run_script($cmd, undef, $opt), "$t succeeds with mbox escaping");
 	$cur = PublicInbox::MIME->new($ibx->msg_by_mid($mid));
 	like($cur->body, qr/^From not an mbox/sm,
 		'changed "From " line unescaped');
 
 	local $ENV{MAIL_EDITOR} = "$^X -i -p -e 's/^From not an mbox\\n//s'";
-	$cmd = [ "$cmd_pfx-edit", "-m$mid", '--raw', $inboxdir ];
-	ok(run($cmd, \$in, \$out, \$err), "$t succeeds again");
+	$cmd = [ '-edit', "-m$mid", '--raw', $inboxdir ];
+	ok(run_script($cmd, undef, $opt), "$t succeeds again");
 	$cur = PublicInbox::MIME->new($ibx->msg_by_mid($mid));
 	unlike($cur->body, qr/^From not an mbox/sm, "$t restored body");
 }
@@ -173,8 +172,8 @@ $t = 'reuse Message-ID'; {
 $t = 'edit ambiguous Message-ID with -m'; {
 	$in = $out = $err = '';
 	local $ENV{MAIL_EDITOR} = "$^X -i -p -e 's/bool pfx/boolean prefix/'";
-	$cmd = [ "$cmd_pfx-edit", "-m$mid", $inboxdir ];
-	ok(!run($cmd, \$in, \$out, \$err), "$t fails w/o --force");
+	$cmd = [ '-edit', "-m$mid", $inboxdir ];
+	ok(!run_script($cmd, undef, $opt), "$t fails w/o --force");
 	like($err, qr/Multiple messages with different content found matching/,
 		"$t shows matches");
 	like($err, qr/GIT_DIR=.*git show/is, "$t shows git commands");
@@ -183,8 +182,8 @@ $t = 'edit ambiguous Message-ID with -m'; {
 $t .= ' and --force'; {
 	$in = $out = $err = '';
 	local $ENV{MAIL_EDITOR} = "$^X -i -p -e 's/^Subject:.*/Subject:x/i'";
-	$cmd = [ "$cmd_pfx-edit", "-m$mid", '--force', $inboxdir ];
-	ok(run($cmd, \$in, \$out, \$err), "$t succeeds");
+	$cmd = [ '-edit', "-m$mid", '--force', $inboxdir ];
+	ok(run_script($cmd, undef, $opt), "$t succeeds");
 	like($err, qr/Will edit all of them/, "$t notes all will be edited");
 	my @dump = $git->qx(qw(cat-file --batch --batch-all-objects));
 	chomp @dump;

[PATCH 14/29] t/init: convert to using run_script

[Eric Wong]

This gives a 2-3x speedup on the test with the default
run_mode=1.
---
 t/init.t | 85 ++++++++++++++++++++++++--------------------------------
 1 file changed, 37 insertions(+), 48 deletions(-)

diff --git a/t/init.t b/t/init.t
index 0cd6f31f..2442eeec 100644
--- a/t/init.t
+++ b/t/init.t
@@ -7,55 +7,44 @@ use PublicInbox::Config;
 use File::Temp qw/tempdir/;
 require './t/common.perl';
 my $tmpdir = tempdir('pi-init-XXXXXX', TMPDIR => 1, CLEANUP => 1);
-use constant pi_init => 'blib/script/public-inbox-init';
-use PublicInbox::Import;
 use File::Basename;
-use PublicInbox::Spawn qw(spawn);
-use Cwd qw(getcwd);
-open my $null, '>>', '/dev/null';
-my $rdr = { 2 => fileno($null) };
 sub quiet_fail {
 	my ($cmd, $msg) = @_;
-	# run_die doesn't take absolute paths:
-	my $path = $ENV{PATH};
-	if (index($cmd->[0], '/') >= 0) {
-		my ($dir, $base) = ($cmd->[0] =~ m!\A(.+)/([^/]+)\z!);
-		$path = "$dir:$path";
-		$cmd->[0] = $base;
-	}
-	local $ENV{PATH} = $path;
-	eval { PublicInbox::Import::run_die($cmd, undef, $rdr) };
-	isnt($@, '', $msg);
+	my $err = '';
+	ok(!run_script($cmd, undef, { 2 => \$err, 1 => \$err }), $msg);
 }
 
 {
 	local $ENV{PI_DIR} = "$tmpdir/.public-inbox/";
 	my $cfgfile = "$ENV{PI_DIR}/config";
-	my @cmd = (pi_init, 'blist', "$tmpdir/blist",
-		   qw(http://example.com/blist blist@example.com));
-	is(system(@cmd), 0, 'public-inbox-init OK');
+	my $cmd = [ '-init', 'blist', "$tmpdir/blist",
+		   qw(http://example.com/blist blist@example.com) ];
+	ok(run_script($cmd), 'public-inbox-init OK');
 
 	is(read_indexlevel('blist'), '', 'indexlevel unset by default');
 
 	ok(-e $cfgfile, "config exists, now");
-	is(system(@cmd), 0, 'public-inbox-init OK (idempotent)');
+	ok(run_script($cmd), 'public-inbox-init OK (idempotent)');
 
 	chmod 0666, $cfgfile or die "chmod failed: $!";
-	@cmd = (pi_init, 'clist', "$tmpdir/clist",
-		   qw(http://example.com/clist clist@example.com));
-	is(system(@cmd), 0, 'public-inbox-init clist OK');
+	$cmd = [ '-init', 'clist', "$tmpdir/clist",
+		   qw(http://example.com/clist clist@example.com)];
+	ok(run_script($cmd), 'public-inbox-init clist OK');
 	is((stat($cfgfile))[2] & 07777, 0666, "permissions preserved");
 
-	@cmd = (pi_init, 'clist', '-V2', "$tmpdir/clist",
-		   qw(http://example.com/clist clist@example.com));
-	quiet_fail(\@cmd, 'attempting to init V2 from V1 fails');
+	$cmd = [ '-init', 'clist', '-V2', "$tmpdir/clist",
+		   qw(http://example.com/clist clist@example.com) ];
+	quiet_fail($cmd, 'attempting to init V2 from V1 fails');
+	ok(!-e "$cfgfile.lock", 'no lock leftover after init');
 
 	open my $lock, '+>', "$cfgfile.lock" or die;
-	@cmd = (getcwd(). '/'. pi_init, 'lock', "$tmpdir/lock",
-		qw(http://example.com/lock lock@example.com));
+	$cmd = [ '-init', 'lock', "$tmpdir/lock",
+		qw(http://example.com/lock lock@example.com) ];
 	ok(-e "$cfgfile.lock", 'lock exists');
-	my $pid = spawn(\@cmd, undef, $rdr);
-	is(waitpid($pid, 0), $pid, 'lock init failed');
+
+	# this calls exit():
+	my $err = '';
+	ok(!run_script($cmd, undef, {2 => \$err}), 'lock init failed');
 	is($? >> 8, 255, 'got expected exit code on lock failure');
 	ok(unlink("$cfgfile.lock"),
 		'-init did not unlink lock on failure');
@@ -69,44 +58,44 @@ SKIP: {
 	require_git(2.6, 1) or skip "git 2.6+ required", 2;
 	local $ENV{PI_DIR} = "$tmpdir/.public-inbox/";
 	my $cfgfile = "$ENV{PI_DIR}/config";
-	my @cmd = (pi_init, '-V2', 'v2list', "$tmpdir/v2list",
-		   qw(http://example.com/v2list v2list@example.com));
-	is(system(@cmd), 0, 'public-inbox-init -V2 OK');
+	my $cmd = [ '-init', '-V2', 'v2list', "$tmpdir/v2list",
+		   qw(http://example.com/v2list v2list@example.com) ];
+	ok(run_script($cmd), 'public-inbox-init -V2 OK');
 	ok(-d "$tmpdir/v2list", 'v2list directory exists');
 	ok(-f "$tmpdir/v2list/msgmap.sqlite3", 'msgmap exists');
 	ok(-d "$tmpdir/v2list/all.git", 'catch-all.git directory exists');
-	@cmd = (pi_init, 'v2list', "$tmpdir/v2list",
-		   qw(http://example.com/v2list v2list@example.com));
-	is(system(@cmd), 0, 'public-inbox-init is idempotent');
+	$cmd = [ '-init', 'v2list', "$tmpdir/v2list",
+		   qw(http://example.com/v2list v2list@example.com) ];
+	ok(run_script($cmd), 'public-inbox-init is idempotent');
 	ok(! -d "$tmpdir/public-inbox" && !-d "$tmpdir/objects",
 		'idempotent invocation w/o -V2 does not make inbox v1');
 	is(read_indexlevel('v2list'), '', 'indexlevel unset by default');
 
-	@cmd = (pi_init, 'v2list', "-V1", "$tmpdir/v2list",
-		   qw(http://example.com/v2list v2list@example.com));
-	quiet_fail(\@cmd, 'initializing V2 as V1 fails');
+	$cmd = [ '-init', 'v2list', "-V1", "$tmpdir/v2list",
+		   qw(http://example.com/v2list v2list@example.com) ];
+	quiet_fail($cmd, 'initializing V2 as V1 fails');
 
 	foreach my $lvl (qw(medium basic)) {
-		@cmd = (pi_init, "v2$lvl", '-V2', '-L', $lvl,
+		$cmd = [ '-init', "v2$lvl", '-V2', '-L', $lvl,
 			"$tmpdir/v2$lvl", "http://example.com/v2$lvl",
-			"v2$lvl\@example.com");
-		is(system(@cmd), 0, "-init -L $lvl");
+			"v2$lvl\@example.com" ];
+		ok(run_script($cmd), "-init -L $lvl");
 		is(read_indexlevel("v2$lvl"), $lvl, "indexlevel set to '$lvl'");
 	}
 
 	# loop for idempotency
 	for (1..2) {
-		@cmd = (pi_init, '-V2', '-S1', 'skip1', "$tmpdir/skip1",
-			   qw(http://example.com/skip1 skip1@example.com));
-		is(system(@cmd), 0, "--skip-epoch 1");
+		$cmd = [ '-init', '-V2', '-S1', 'skip1', "$tmpdir/skip1",
+			   qw(http://example.com/skip1 skip1@example.com) ];
+		ok(run_script($cmd), "--skip-epoch 1");
 		my $gits = [ glob("$tmpdir/skip1/git/*.git") ];
 		is_deeply($gits, ["$tmpdir/skip1/git/1.git"], 'skip OK');
 	}
 
 
-	@cmd = (pi_init, '-V2', '--skip-epoch=2', 'skip2', "$tmpdir/skip2",
-		   qw(http://example.com/skip2 skip2@example.com));
-	is(system(@cmd), 0, "--skip-epoch 2");
+	$cmd = [ '-init', '-V2', '--skip-epoch=2', 'skip2', "$tmpdir/skip2",
+		   qw(http://example.com/skip2 skip2@example.com) ];
+	ok(run_script($cmd), "--skip-epoch 2");
 	my $gits = [ glob("$tmpdir/skip2/git/*.git") ];
 	is_deeply($gits, ["$tmpdir/skip2/git/2.git"], 'skipping 2 works, too');
 }

[PATCH 15/29] t/purge: convert to run_script

[Eric Wong]

This nets us another sizeable speedup.
---
 t/purge.t | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/t/purge.t b/t/purge.t
index 67c4e58d..bcdbad52 100644
--- a/t/purge.t
+++ b/t/purge.t
@@ -6,12 +6,12 @@ use Test::More;
 use File::Temp qw/tempdir/;
 require './t/common.perl';
 require_git(2.6);
-my @mods = qw(IPC::Run DBI DBD::SQLite);
+my @mods = qw(DBI DBD::SQLite);
 foreach my $mod (@mods) {
 	eval "require $mod";
 	plan skip_all => "missing $mod for t/purge.t" if $@;
 };
-use Cwd qw(abs_path);
+use Cwd qw(abs_path); # we need this since we chdir below
 my $purge = abs_path('blib/script/public-inbox-purge');
 my $tmpdir = tempdir('pi-purge-XXXXXX', TMPDIR => 1, CLEANUP => 1);
 use_ok 'PublicInbox::V2Writable';
@@ -47,16 +47,16 @@ $v2w->done;
 # failing cases, first:
 my $in = "$raw\nMOAR\n";
 my ($out, $err) = ('', '');
-ok(IPC::Run::run([$purge, '-f', $inboxdir], \$in, \$out, \$err),
-	'purge -f OK');
+my $opt = { 0 => \$in, 1 => \$out, 2 => \$err };
+ok(run_script([$purge, '-f', $inboxdir], undef, $opt), 'purge -f OK');
 
 $out = $err = '';
-ok(!IPC::Run::run([$purge, $inboxdir], \$in, \$out, \$err),
-	'mismatch fails without -f');
+ok(!run_script([$purge, $inboxdir], undef, $opt), 'mismatch fails without -f');
 is($? >> 8, 1, 'missed purge exits with 1');
 
 # a successful case:
-ok(IPC::Run::run([$purge, $inboxdir], \$raw, \$out, \$err), 'match OK');
+$opt->{0} = \$raw;
+ok(run_script([$purge, $inboxdir], undef, $opt), 'match OK');
 like($out, qr/\b[a-f0-9]{40,}/m, 'removed commit noted');
 
 # add (old) vger filter to config file
@@ -83,13 +83,13 @@ EOF
 
 $out = $err = '';
 ok(chdir('/'), "chdir / OK for --all test");
-ok(IPC::Run::run([$purge, '--all'], \$pre_scrub, \$out, \$err),
-	'scrub purge OK');
+$opt->{0} = \$pre_scrub;
+ok(run_script([$purge, '--all'], undef, $opt), 'scrub purge OK');
 like($out, qr/\b[a-f0-9]{40,}/m, 'removed commit noted');
 # diag "out: $out"; diag "err: $err";
 
 $out = $err = '';
-ok(!IPC::Run::run([$purge, '--all' ], \$pre_scrub, \$out, \$err),
+ok(!run_script([$purge, '--all' ], undef, $opt),
 	'scrub purge not idempotent without -f');
 # diag "out: $out"; diag "err: $err";
 

[PATCH 16/29] t/v2mirror: get rid of IPC::Run dependency

[Eric Wong]

Not taking advantage of faster run modes in run_script, yet
since some lifetime problems need to be sorted.
---
 t/v2mirror.t | 25 +++++++++++++------------
 1 file changed, 13 insertions(+), 12 deletions(-)

diff --git a/t/v2mirror.t b/t/v2mirror.t
index f826775c..3c238093 100644
--- a/t/v2mirror.t
+++ b/t/v2mirror.t
@@ -8,8 +8,7 @@ require_git(2.6);
 
 # Integration tests for HTTP cloning + mirroring
 foreach my $mod (qw(Plack::Util Plack::Builder
-			HTTP::Date HTTP::Status Search::Xapian DBD::SQLite
-			IPC::Run)) {
+			HTTP::Date HTTP::Status Search::Xapian DBD::SQLite)) {
 	eval "require $mod";
 	plan skip_all => "$mod missing for v2mirror.t" if $@;
 }
@@ -84,10 +83,10 @@ foreach my $i (0..$epoch_max) {
 	ok(-d "$tmpdir/m/git/$i.git", "mirror $i OK");
 }
 
-@cmd = ("$script-init", '-V2', 'm', "$tmpdir/m", 'http://example.com/m',
+@cmd = ("-init", '-V2', 'm', "$tmpdir/m", 'http://example.com/m',
 	'alt@example.com');
-is(system(@cmd), 0, 'initialized public-inbox -V2');
-is(system("$script-index", "$tmpdir/m"), 0, 'indexed');
+ok(run_script(\@cmd, undef, {run_mode => 0}), 'initialized public-inbox -V2');
+ok(run_script(['-index', "$tmpdir/m"], undef, { run_mode => 0}), 'indexed');
 
 my $mibx = { inboxdir => "$tmpdir/m", address => 'alt@example.com' };
 $mibx = PublicInbox::Inbox->new($mibx);
@@ -113,7 +112,7 @@ fetch_each_epoch();
 
 my $mset = $mibx->search->reopen->query('m:15@example.com', {mset => 1});
 is(scalar($mset->items), 0, 'new message not found in mirror, yet');
-is(system("$script-index", "$tmpdir/m"), 0, 'index updated');
+ok(run_script(["-index", "$tmpdir/m"], undef, {run_mode=>0}), 'index updated');
 is_deeply([$mibx->mm->minmax], [$ibx->mm->minmax], 'index synched minmax');
 $mset = $mibx->search->reopen->query('m:15@example.com', {mset => 1});
 is(scalar($mset->items), 1, 'found message in mirror');
@@ -141,9 +140,10 @@ is(scalar($mset->items), 0, 'purged message gone from origin');
 
 fetch_each_epoch();
 {
-	my $cmd = [ "$script-index", '--prune', "$tmpdir/m" ];
-	my ($in, $out, $err) = ('', '', '');
-	ok(IPC::Run::run($cmd, \$in, \$out, \$err), '-index --prune');
+	my $cmd = [ '-index', '--prune', "$tmpdir/m" ];
+	my ($out, $err) = ('', '');
+	my $opt = { 1 => \$out, 2 => \$err, run_mode => 0 };
+	ok(run_script($cmd, undef, $opt), '-index --prune');
 	like($err, qr/discontiguous range/, 'warned about discontiguous range');
 	unlike($err, qr/fatal/, 'no scary fatal error shown');
 }
@@ -174,9 +174,10 @@ is($mibx->git->check($to_purge), undef, 'unindex+prune successful in mirror');
 	$v2w->done;
 	fetch_each_epoch();
 
-	my ($in, $out, $err) = ('', '', '');
-	my $cmd = [ "$script-index", "$tmpdir/m" ];
-	ok(IPC::Run::run($cmd, \$in, \$out, \$err), 'index ran');
+	my $cmd = [ "-index", "$tmpdir/m" ];
+	my ($out, $err) = ('', '');
+	my $opt = { 1 => \$out, 2 => \$err, run_mode => 0 };
+	ok(run_script($cmd, undef, $opt), 'index ran');
 	is($err, '', 'no errors reported by index');
 	$mset = $mibx->search->reopen->query('m:1@example.com', {mset => 1});
 	is(scalar($mset->items), 0, '1@example.com no longer visible in mirror');

[PATCH 17/29] t/mda: switch to run_script for testing

[Eric Wong]

Another noticeable speedup, this test is roughly ~3x faster now.
---
 t/mda.t | 53 ++++++++++++++++++++++++++---------------------------
 1 file changed, 26 insertions(+), 27 deletions(-)

diff --git a/t/mda.t b/t/mda.t
index 3e03a25a..89dedd4a 100644
--- a/t/mda.t
+++ b/t/mda.t
@@ -8,11 +8,7 @@ use File::Temp qw/tempdir/;
 use Cwd qw(getcwd);
 use PublicInbox::MID qw(mid2path);
 use PublicInbox::Git;
-eval { require IPC::Run };
-plan skip_all => "missing IPC::Run for t/mda.t" if $@;
-
-my $mda = "blib/script/public-inbox-mda";
-my $learn = "blib/script/public-inbox-learn";
+require './t/common.perl';
 my $tmpdir = tempdir('pi-mda-XXXXXX', TMPDIR => 1, CLEANUP => 1);
 my $home = "$tmpdir/pi-home";
 my $pi_home = "$home/.public-inbox";
@@ -33,7 +29,6 @@ my $git = PublicInbox::Git->new($maindir);
 		"spamc ham mock found (run in top of source tree");
 	ok(-x "$fail_bin/spamc",
 		"spamc mock found (run in top of source tree");
-	ok(-x $mda, "$mda is executable");
 	is(1, mkdir($home, 0755), "setup ~/ for testing");
 	is(1, mkdir($pi_home, 0755), "setup ~/.public-inbox");
 	is(0, system(qw(git init -q --bare), $maindir), "git init (main)");
@@ -92,7 +87,7 @@ EOF
 	# ensure successful message delivery
 	{
 		local $ENV{PATH} = $main_path;
-		IPC::Run::run([$mda], \$in);
+		ok(run_script(['-mda'], undef, { 0 => \$in }));
 		my $rev = $git->qx(qw(rev-list HEAD));
 		like($rev, qr/\A[a-f0-9]{40}/, "good revision committed");
 		chomp $rev;
@@ -109,7 +104,7 @@ EOF
 		my @prev = <$faildir/new/*>;
 		is(scalar @prev, 0 , "nothing in PI_EMERGENCY before");
 		local $ENV{PATH} = $fail_path;
-		IPC::Run::run([$mda], \$in);
+		ok(run_script(['-mda'], undef, { 0 => \$in }));
 		my @revs = $git->qx(qw(rev-list HEAD));
 		is(scalar @revs, 1, "bad revision not committed");
 		my @new = <$faildir/new/*>;
@@ -181,7 +176,7 @@ EOF
 
 	{
 		# deliver the spam message, first
-		IPC::Run::run([$mda], \$in);
+		ok(run_script(['-mda'], undef, { 0 => \$in }));
 		my $path = mid2path($mid);
 		my $msg = $git->cat_file("HEAD:$path");
 		like($$msg, qr/\Q$mid\E/, "message delivered");
@@ -189,11 +184,12 @@ EOF
 		# now train it
 		local $ENV{GIT_AUTHOR_EMAIL} = 'trainer@example.com';
 		local $ENV{GIT_COMMITTER_EMAIL} = 'trainer@example.com';
-		local $ENV{GIT_COMMITTER_NAME} = undef;
-		IPC::Run::run([$learn, "spam"], $msg);
-		is($?, 0, "no failure from learning spam");
-		IPC::Run::run([$learn, "spam"], $msg);
-		is($?, 0, "no failure from learning spam idempotently");
+		local $ENV{GIT_COMMITTER_NAME};
+		delete $ENV{GIT_COMMITTER_NAME};
+		ok(run_script(['-learn', 'spam'], undef, { 0 => $msg }),
+			"no failure from learning spam");
+		ok(run_script(['-learn', 'spam'], undef, { 0 => $msg }),
+			"no failure from learning spam idempotently");
 	}
 }
 
@@ -220,13 +216,13 @@ EOF
 	local $ENV{GIT_AUTHOR_EMAIL} = 'trainer@example.com';
 	local $ENV{GIT_COMMITTER_EMAIL} = 'trainer@example.com';
 
-	IPC::Run::run([$learn, "ham"], \$in);
-	is($?, 0, "learned ham without failure");
+	ok(run_script(['-learn', 'ham'], undef, { 0 => \$in }),
+		"learned ham without failure");
 	my $path = mid2path($mid);
 	my $msg = $git->cat_file("HEAD:$path");
 	like($$msg, qr/\Q$mid\E/, "ham message delivered");
-	IPC::Run::run([$learn, "ham"], \$in);
-	is($?, 0, "learned ham idempotently ");
+	ok(run_script(['-learn', 'ham'], undef, { 0 => \$in }),
+		"learned ham idempotently ");
 
 	# ensure trained email is filtered, too
 	my $html_body = "<html><body>hi</body></html>";
@@ -260,8 +256,8 @@ EOF
 
 	{
 		$in = $mime->as_string;
-		IPC::Run::run([$learn, "ham"], \$in);
-		is($?, 0, "learned ham without failure");
+		ok(run_script(['-learn', 'ham'], undef, { 0 => \$in }),
+			"learned ham without failure");
 		my $path = mid2path($mid);
 		$msg = $git->cat_file("HEAD:$path");
 		like($$msg, qr/<\Q$mid\E>/, "ham message delivered");
@@ -291,8 +287,8 @@ EOF
 	system(qw(git config --file), $pi_config, "$cfgpfx.listid", $list_id);
 	$? == 0 or die "failed to set listid $?";
 	my $in = $simple->as_string;
-	IPC::Run::run([$mda], \$in);
-	is($?, 0, 'mda OK with List-Id match');
+	ok(run_script(['-mda'], undef, { 0 => \$in }),
+		'mda OK with List-Id match');
 	my $path = mid2path($mid);
 	my $msg = $git->cat_file("HEAD:$path");
 	like($$msg, qr/\Q$list_id\E/, 'delivered message w/ List-ID matches');
@@ -306,8 +302,9 @@ this message would not be accepted without --no-precheck
 EOF
 	$in = $simple->as_string;
 	my ($out, $err) = ('', '');
-	IPC::Run::run([$mda, '--no-precheck'], \$in, \$out, \$err);
-	is($?, 0, 'mda OK with List-Id match and --no-precheck');
+	my $rdr = { 0 => \$in, 1 => \$out, 2 => \$err };
+	ok(run_script(['-mda', '--no-precheck'], undef, $rdr),
+		'mda OK with List-Id match and --no-precheck');
 	my $cur = $git->qx(qw(diff HEAD~1..HEAD));
 	like($cur, qr/this message would not be accepted without --no-precheck/,
 		'--no-precheck delivered message anyways');
@@ -324,8 +321,8 @@ Date: Fri, 02 Oct 1993 00:00:00 +0000
 
 EOF
 	($out, $err) = ('', '');
-	IPC::Run::run([$mda], \$in, \$out, \$err);
-	is($?, 0, 'mda OK with multiple List-Id matches');
+	ok(run_script(['-mda'], undef, $rdr),
+		'mda OK with multiple List-Id matches');
 	$cur = $git->qx(qw(diff HEAD~1..HEAD));
 	like($cur, qr/Message-ID: <2lids\@example>/,
 		'multi List-ID match delivered');
@@ -339,8 +336,10 @@ sub fail_bad_header {
 	my @f = glob("$faildir/*/*");
 	unlink @f if @f;
 	my ($out, $err) = ("", "");
+	my $opt = { 0 => \$in, 1 => \$out, 2 => \$err };
 	local $ENV{PATH} = $main_path;
-	IPC::Run::run([$mda], \$in, \$out, \$err);
+	ok(run_script(['-mda'], undef, $opt),
+		"no error on undeliverable ($msg)");
 	my $rev = $git->qx(qw(rev-list HEAD));
 	chomp $rev;
 	is($rev, $good_rev, "bad revision not commited ($msg)");

[PATCH 18/29] t/mda_filter_rubylang: drop IPC::Run dependency

[Eric Wong]

This test runs more than twice as fast, now.
---
 t/mda_filter_rubylang.t | 17 ++++++++---------
 1 file changed, 8 insertions(+), 9 deletions(-)

diff --git a/t/mda_filter_rubylang.t b/t/mda_filter_rubylang.t
index f7d872c9..e971b440 100644
--- a/t/mda_filter_rubylang.t
+++ b/t/mda_filter_rubylang.t
@@ -8,7 +8,7 @@ use PublicInbox::MIME;
 use PublicInbox::Config;
 require './t/common.perl';
 require_git(2.6);
-my @mods = qw(DBD::SQLite Search::Xapian IPC::Run);
+my @mods = qw(DBD::SQLite Search::Xapian);
 foreach my $mod (@mods) {
 	eval "require $mod";
 	plan skip_all => "$mod missing for mda_filter_rubylang.t" if $@;
@@ -19,7 +19,6 @@ my $tmpdir = tempdir('mda-XXXXXX', TMPDIR => 1, CLEANUP => 1);
 my $pi_config = "$tmpdir/pi_config";
 local $ENV{PI_CONFIG} = $pi_config;
 local $ENV{PI_EMERGENCY} = "$tmpdir/emergency";
-my $mda = 'blib/script/public-inbox-mda';
 my @cfg = ('git', 'config', "--file=$pi_config");
 is(system(@cfg, 'publicinboxmda.spamcheck', 'none'), 0);
 
@@ -29,17 +28,17 @@ for my $v (qw(V1 V2)) {
 	my $cfgpfx = "publicinbox.$v";
 	my $inboxdir = "$tmpdir/$v";
 	my $addr = "test-$v\@example.com";
-	my @cmd = ('blib/script/public-inbox-init', "-$v", $v, $inboxdir,
-		"http://example.com/$v", $addr);
-	is(system(@cmd), 0, 'public-inbox init OK');
-	is(system('blib/script/public-inbox-index', $inboxdir), 0);
+	my $cmd = [ '-init', "-$v", $v, $inboxdir,
+		"http://example.com/$v", $addr ];
+	ok(run_script($cmd), 'public-inbox-init');
+	ok(run_script(['-index', $inboxdir]), 'public-inbox-index');
 	is(system(@cfg, "$cfgpfx.filter", 'PublicInbox::Filter::RubyLang'), 0);
 	is(system(@cfg, "$cfgpfx.altid",
 		'serial:alerts:file=msgmap.sqlite3'), 0);
 
 	for my $i (1..2) {
-		local $ENV{ORIGINAL_RECIPIENT} = $addr;
-		my $msg = <<EOF;
+		my $env = { ORIGINAL_RECIPIENT => $addr };
+		my $opt = { 0 => \(<<EOF) };
 From: user\@example.com
 To: $addr
 Subject: blah $i
@@ -49,7 +48,7 @@ Date: Sat, 05 Jan 2019 04:19:17 +0000
 
 something
 EOF
-		ok(IPC::Run::run([$mda], \"$msg"), 'message delivered');
+		ok(run_script(['-mda'], $env, $opt), 'message delivered');
 	}
 	my $config = PublicInbox::Config->new;
 	my $ibx = $config->lookup_name($v);

[PATCH 19/29] doc: remove IPC::Run as a dev and test dependency

[Eric Wong]

It's no longer needed and we're able to speed up some
of our tests as a result.
---
 INSTALL      | 4 ----
 ci/deps.perl | 1 -
 2 files changed, 5 deletions(-)

diff --git a/INSTALL b/INSTALL
index aad52c7b..4d54e6a0 100644
--- a/INSTALL
+++ b/INSTALL
@@ -162,10 +162,6 @@ Uncommonly needed modules:
 
 Optional packages testing and development:
 
-- IPC::Run                         deb: libipc-run-perl
-                                   pkg: p5-IPC-Run
-                                   rpm: perl-IPC-Run
-
 - Plack::Test                      deb: libplack-test-perl
                                    pkg: p5-Plack
                                    rpm: perl-Plack-Test
diff --git a/ci/deps.perl b/ci/deps.perl
index 62870c1f..ae6083b9 100755
--- a/ci/deps.perl
+++ b/ci/deps.perl
@@ -50,7 +50,6 @@ my $profiles = {
 
 	# optional developer stuff
 	devtest => [ qw(
-		IPC::Run
 		XML::Feed
 		curl
 		w3m

[PATCH 20/29] t/v2mirror: switch to default run_mode for speedup

[Eric Wong]

We need to be careful and explicitly close FDs before doing
-index, since we can't rely on FD_CLOEXEC without execve(2)
syscalls.
---
 t/v2mirror.t | 27 +++++++++++++++++++--------
 1 file changed, 19 insertions(+), 8 deletions(-)

diff --git a/t/v2mirror.t b/t/v2mirror.t
index 3c238093..2c7f6a84 100644
--- a/t/v2mirror.t
+++ b/t/v2mirror.t
@@ -58,9 +58,13 @@ for my $i (1..9) {
 my $epoch_max = $v2w->{epoch_max};
 ok($epoch_max > 0, "multiple epochs");
 $v2w->done;
+$ibx->cleanup;
 
 my ($sock, $pid);
-END { kill 'TERM', $pid if defined $pid };
+
+# TODO: replace this with ->DESTROY:
+my $owner_pid = $$;
+END { kill('TERM', $pid) if defined($pid) && $owner_pid == $$ };
 
 $! = 0;
 $sock = tcp_server();
@@ -85,8 +89,9 @@ foreach my $i (0..$epoch_max) {
 
 @cmd = ("-init", '-V2', 'm', "$tmpdir/m", 'http://example.com/m',
 	'alt@example.com');
-ok(run_script(\@cmd, undef, {run_mode => 0}), 'initialized public-inbox -V2');
-ok(run_script(['-index', "$tmpdir/m"], undef, { run_mode => 0}), 'indexed');
+ok(run_script(\@cmd), 'initialized public-inbox -V2');
+
+ok(run_script(['-index', "$tmpdir/m"]), 'indexed');
 
 my $mibx = { inboxdir => "$tmpdir/m", address => 'alt@example.com' };
 $mibx = PublicInbox::Inbox->new($mibx);
@@ -98,7 +103,8 @@ for my $i (10..15) {
 	$mime->header_set('Subject', "subject = $i");
 	ok($v2w->add($mime), "add msg $i OK");
 }
-$v2w->barrier;
+$v2w->done;
+$ibx->cleanup;
 
 sub fetch_each_epoch {
 	foreach my $i (0..$epoch_max) {
@@ -112,7 +118,7 @@ fetch_each_epoch();
 
 my $mset = $mibx->search->reopen->query('m:15@example.com', {mset => 1});
 is(scalar($mset->items), 0, 'new message not found in mirror, yet');
-ok(run_script(["-index", "$tmpdir/m"], undef, {run_mode=>0}), 'index updated');
+ok(run_script(["-index", "$tmpdir/m"]), 'index updated');
 is_deeply([$mibx->mm->minmax], [$ibx->mm->minmax], 'index synched minmax');
 $mset = $mibx->search->reopen->query('m:15@example.com', {mset => 1});
 is(scalar($mset->items), 1, 'found message in mirror');
@@ -130,7 +136,7 @@ $mime->header_set('Subject', 'subject = 10');
 	is_deeply(\@subj, ["# subject = 10"], "only rewrote one");
 }
 
-$v2w->barrier;
+$v2w->done;
 
 my $msgs = $mibx->search->{over_ro}->get_thread('10@example.com');
 my $to_purge = $msgs->[0]->{blob};
@@ -140,9 +146,12 @@ is(scalar($mset->items), 0, 'purged message gone from origin');
 
 fetch_each_epoch();
 {
+	$ibx->cleanup;
+	PublicInbox::InboxWritable::cleanup($mibx);
+	$v2w->done;
 	my $cmd = [ '-index', '--prune', "$tmpdir/m" ];
 	my ($out, $err) = ('', '');
-	my $opt = { 1 => \$out, 2 => \$err, run_mode => 0 };
+	my $opt = { 1 => \$out, 2 => \$err };
 	ok(run_script($cmd, undef, $opt), '-index --prune');
 	like($err, qr/discontiguous range/, 'warned about discontiguous range');
 	unlike($err, qr/fatal/, 'no scary fatal error shown');
@@ -172,11 +181,13 @@ is($mibx->git->check($to_purge), undef, 'unindex+prune successful in mirror');
 	$mime->header_set('Subject', 'subject = 1');
 	ok($v2w->remove($mime), 'removed <1@example.com> from source');
 	$v2w->done;
+	$ibx->cleanup;
 	fetch_each_epoch();
+	PublicInbox::InboxWritable::cleanup($mibx);
 
 	my $cmd = [ "-index", "$tmpdir/m" ];
 	my ($out, $err) = ('', '');
-	my $opt = { 1 => \$out, 2 => \$err, run_mode => 0 };
+	my $opt = { 1 => \$out, 2 => \$err };
 	ok(run_script($cmd, undef, $opt), 'index ran');
 	is($err, '', 'no errors reported by index');
 	$mset = $mibx->search->reopen->query('m:1@example.com', {mset => 1});

[PATCH 21/29] t/convert-compact: convert to run_script

[Eric Wong]

While this didn't use IPC::Run, having to reload several Perl
modules and scripts is slow and inefficient, so roughly
double the speed of this test.
---
 t/convert-compact.t | 18 ++++++++----------
 1 file changed, 8 insertions(+), 10 deletions(-)

diff --git a/t/convert-compact.t b/t/convert-compact.t
index dbccfbad..0661ed14 100644
--- a/t/convert-compact.t
+++ b/t/convert-compact.t
@@ -62,22 +62,20 @@ foreach (@xdir) {
 		'sharedRepository respected on file after convert');
 }
 
-local $ENV{PATH} = "blib/script:$ENV{PATH}";
 local $ENV{PI_CONFIG} = '/dev/null';
-open my $err, '>>', "$tmpdir/err.log" or die "open: err.log $!\n";
-open my $out, '>>', "$tmpdir/out.log" or die "open: out.log $!\n";
-my $rdr = { 1 => fileno($out), 2 => fileno($err) };
+my ($out, $err) = ('', '');
+my $rdr = { 1 => \$out, 2 => \$err };
 
-my $cmd = [ 'public-inbox-compact', $ibx->{inboxdir} ];
-ok(PublicInbox::Import::run_die($cmd, undef, $rdr), 'v1 compact works');
+my $cmd = [ '-compact', $ibx->{inboxdir} ];
+ok(run_script($cmd, undef, $rdr), 'v1 compact works');
 
 @xdir = glob("$ibx->{inboxdir}/public-inbox/xap*");
 is(scalar(@xdir), 1, 'got one xapian directory after compact');
 is(((stat($xdir[0]))[2]) & 07777, 0755,
 	'sharedRepository respected on v1 compact');
 
-$cmd = [ 'public-inbox-convert', $ibx->{inboxdir}, "$tmpdir/v2" ];
-ok(PublicInbox::Import::run_die($cmd, undef, $rdr), 'convert works');
+$cmd = [ '-convert', $ibx->{inboxdir}, "$tmpdir/v2" ];
+ok(run_script($cmd, undef, $rdr), 'convert works');
 @xdir = glob("$tmpdir/v2/xap*/*");
 foreach (@xdir) {
 	my @st = stat($_);
@@ -85,9 +83,9 @@ foreach (@xdir) {
 		'sharedRepository respected after convert');
 }
 
-$cmd = [ 'public-inbox-compact', "$tmpdir/v2" ];
+$cmd = [ '-compact', "$tmpdir/v2" ];
 my $env = { NPROC => 2 };
-ok(PublicInbox::Import::run_die($cmd, $env, $rdr), 'v2 compact works');
+ok(run_script($cmd, $env, $rdr), 'v2 compact works');
 $ibx->{inboxdir} = "$tmpdir/v2";
 $ibx->{version} = 2;
 

[PATCH 22/29] t/httpd: use run_script for -init

[Eric Wong]

This only gives a small ~10% speedup, since -httpd still
needs execve, but any speedup is welcome.
---
 t/httpd.t | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/t/httpd.t b/t/httpd.t
index 15984a78..e7527ed6 100644
--- a/t/httpd.t
+++ b/t/httpd.t
@@ -22,7 +22,6 @@ my $group = 'test-httpd';
 my $addr = $group . '@example.com';
 my $cfgpfx = "publicinbox.$group";
 my $httpd = 'blib/script/public-inbox-httpd';
-my $init = 'blib/script/public-inbox-init';
 my $sock = tcp_server();
 my $pid;
 use_ok 'PublicInbox::Git';
@@ -31,8 +30,8 @@ use_ok 'Email::MIME';
 END { kill 'TERM', $pid if defined $pid };
 {
 	local $ENV{HOME} = $home;
-	ok(!system($init, $group, $maindir, 'http://example.com/', $addr),
-		'init ran properly');
+	my $cmd = [ '-init', $group, $maindir, 'http://example.com/', $addr ];
+	ok(run_script($cmd), 'init ran properly');
 
 	# ensure successful message delivery
 	{
@@ -53,7 +52,7 @@ EOF
 		$im->done($mime);
 	}
 	ok($sock, 'sock created');
-	my $cmd = [ $httpd, '-W0', "--stdout=$out", "--stderr=$err" ];
+	$cmd = [ $httpd, '-W0', "--stdout=$out", "--stderr=$err" ];
 	$pid = spawn_listener(undef, $cmd, [$sock]);
 	my $host = $sock->sockhost;
 	my $port = $sock->sockport;

[PATCH 23/29] t/watch_maildir_v2: use run_script for -init

[Eric Wong]

This only gives a small 10% speedup or so, but anything helps.
---
 t/watch_maildir_v2.t | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/t/watch_maildir_v2.t b/t/watch_maildir_v2.t
index ccc85c17..7fe23521 100644
--- a/t/watch_maildir_v2.t
+++ b/t/watch_maildir_v2.t
@@ -21,10 +21,10 @@ use_ok 'PublicInbox::WatchMaildir';
 use_ok 'PublicInbox::Emergency';
 my $cfgpfx = "publicinbox.test";
 my $addr = 'test-public@example.com';
-my @cmd = ('blib/script/public-inbox-init', '-V2', 'test', $inboxdir,
+my @cmd = ('-init', '-V2', 'test', $inboxdir,
 	'http://example.com/v2list', $addr);
 local $ENV{PI_CONFIG} = "$tmpdir/pi_config";
-is(system(@cmd), 0, 'public-inbox init OK');
+ok(run_script(\@cmd), 'public-inbox init OK');
 
 my $msg = <<EOF;
 From: user\@example.com

[PATCH 24/29] t/nntpd: use run_script for -init

[Eric Wong]

This only gives a 5% speedup or so, but anything helps.
---
 t/nntpd.t | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/t/nntpd.t b/t/nntpd.t
index 462e2da9..b516ffd1 100644
--- a/t/nntpd.t
+++ b/t/nntpd.t
@@ -30,7 +30,6 @@ my $inboxdir = "$tmpdir/main.git";
 my $group = 'test-nntpd';
 my $addr = $group . '@example.com';
 my $nntpd = 'blib/script/public-inbox-nntpd';
-my $init = 'blib/script/public-inbox-init';
 SKIP: {
 	skip "git 2.6+ required for V2Writable", 1 if $version == 1;
 	use_ok 'PublicInbox::V2Writable';
@@ -52,9 +51,9 @@ my $ibx = {
 $ibx = PublicInbox::Inbox->new($ibx);
 {
 	local $ENV{HOME} = $home;
-	my @cmd = ($init, $group, $inboxdir, 'http://example.com/', $addr);
+	my @cmd = ('-init', $group, $inboxdir, 'http://example.com/', $addr);
 	push @cmd, "-V$version", '-Lbasic';
-	is(system(@cmd), 0, 'init OK');
+	ok(run_script(\@cmd), 'init OK');
 	is(system(qw(git config), "--file=$home/.public-inbox/config",
 			"publicinbox.$group.newsgroup", $group),
 		0, 'enabled newsgroup');

[PATCH 25/29] t/watch_filter_rubylang: run_script for -init and -index

[Eric Wong]

This nets us a 20% speedup or so.
---
 t/watch_filter_rubylang.t | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/t/watch_filter_rubylang.t b/t/watch_filter_rubylang.t
index 4b88d670..57ab3b91 100644
--- a/t/watch_filter_rubylang.t
+++ b/t/watch_filter_rubylang.t
@@ -36,11 +36,11 @@ for my $v (@v) {
 	my $maildir = "$tmpdir/md-$v";
 	my $spamdir = "$tmpdir/spam-$v";
 	my $addr = "test-$v\@example.com";
-	my @cmd = ('blib/script/public-inbox-init', "-$v", $v, $inboxdir,
+	my @cmd = ('-init', "-$v", $v, $inboxdir,
 		"http://example.com/$v", $addr);
-	is(system(@cmd), 0, 'public-inbox init OK');
+	ok(run_script(\@cmd), 'public-inbox init OK');
 	if ($v eq 'V1') {
-		is(system('blib/script/public-inbox-index', $inboxdir), 0);
+		ok(run_script(['-index', $inboxdir]), 'v1 indexed');
 	}
 	PublicInbox::Emergency->new($spamdir);
 

[PATCH 26/29] t/v2mda: switch to run_script in many places

[Eric Wong]

This more than doubles the speed of the test.
---
 t/v2mda.t | 38 ++++++++++----------------------------
 1 file changed, 10 insertions(+), 28 deletions(-)

diff --git a/t/v2mda.t b/t/v2mda.t
index ebcbd1f4..0cd852b1 100644
--- a/t/v2mda.t
+++ b/t/v2mda.t
@@ -34,8 +34,6 @@ my $mime = PublicInbox::MIME->create(
 	body => "hello world\n",
 );
 
-my $mda = "blib/script/public-inbox-mda";
-ok(-f "blib/script/public-inbox-mda", '-mda exists');
 my $main_bin = getcwd()."/t/main-bin";
 my $fail_bin = getcwd()."/t/fail-bin";
 local $ENV{PI_DIR} = "$tmpdir/foo";
@@ -44,26 +42,19 @@ local $ENV{PATH} = "$main_bin:blib/script:$ENV{PATH}";
 my $faildir = "$tmpdir/fail";
 local $ENV{PI_EMERGENCY} = $faildir;
 ok(mkdir $faildir);
-my @cmd = (qw(public-inbox-init), "-V$V", $ibx->{name},
+my @cmd = (qw(-init), "-V$V", $ibx->{name},
 		$ibx->{inboxdir}, 'http://localhost/test',
 		$ibx->{address}->[0]);
-ok(PublicInbox::Import::run_die(\@cmd), 'initialized v2 inbox');
+ok(run_script(\@cmd), 'initialized v2 inbox');
 
-open my $tmp, '+>', undef or die "failed to open anonymous tempfile: $!";
-ok($tmp->print($mime->as_string), 'wrote to temporary file');
-ok($tmp->flush, 'flushed temporary file');
-ok($tmp->sysseek(0, SEEK_SET), 'seeked');
-
-my $rdr = { 0 => fileno($tmp) };
+my $rdr = { 0 => \($mime->as_string) };
 local $ENV{ORIGINAL_RECIPIENT} = 'test@example.com';
-ok(PublicInbox::Import::run_die(['public-inbox-mda'], undef, $rdr),
-	'mda delivered a message');
+ok(run_script(['-mda'], undef, $rdr), 'mda delivered a message');
 
 $ibx = PublicInbox::Inbox->new($ibx);
 
 if ($V == 1) {
-	my $cmd = [ 'public-inbox-index', "$tmpdir/inbox" ];
-	ok(PublicInbox::Import::run_die($cmd, undef, $rdr), 'v1 indexed');
+	ok(run_script([ '-index', "$tmpdir/inbox" ]), 'v1 indexed');
 }
 my $msgs = $ibx->search->query('');
 is(scalar(@$msgs), 1, 'only got one message');
@@ -75,15 +66,8 @@ is($saved->{mime}->as_string, $mime->as_string, 'injected message');
 	is_deeply(\@new, [], 'nothing in faildir');
 	local $ENV{PATH} = $fail_path;
 	$mime->header_set('Message-ID', '<bar@foo>');
-	ok($tmp->sysseek(0, SEEK_SET) &&
-			$tmp->truncate(0) &&
-			$tmp->print($mime->as_string) &&
-			$tmp->flush &&
-			$tmp->sysseek(0, SEEK_SET),
-		'rewound and rewrite temporary file');
-	my $cmd = ['public-inbox-mda'];
-	ok(PublicInbox::Import::run_die($cmd, undef, $rdr),
-		'mda did not die on "spam"');
+	$rdr->{0} = \($mime->as_string);
+	ok(run_script(['-mda'], undef, $rdr), 'mda did not die on "spam"');
 	@new = glob("$faildir/new/*");
 	is(scalar(@new), 1, 'got a message in faildir');
 	$msgs = $ibx->search->reopen->query('');
@@ -94,9 +78,8 @@ is($saved->{mime}->as_string, $mime->as_string, 'injected message');
 	my $k = 'publicinboxmda.spamcheck';
 	is(system('git', 'config', "--file=$config", $k, 'none'), 0,
 		'disabled spamcheck for mda');
-	ok($tmp->sysseek(0, SEEK_SET), 'rewound input file');
 
-	ok(PublicInbox::Import::run_die($cmd, undef, $rdr), 'mda did not die');
+	ok(run_script(['-mda'], undef, $rdr), 'mda did not die');
 	my @again = glob("$faildir/new/*");
 	is_deeply(\@again, \@new, 'no new message in faildir');
 	$msgs = $ibx->search->reopen->query('');
@@ -106,9 +89,8 @@ is($saved->{mime}->as_string, $mime->as_string, 'injected message');
 {
 	my $patch = 't/data/0001.patch';
 	open my $fh, '<', $patch or die "failed to open $patch: $!\n";
-	$rdr = { 0 => fileno($fh) };
-	ok(PublicInbox::Import::run_die(['public-inbox-mda'], undef, $rdr),
-		'mda delivered a patch');
+	$rdr->{0} = \(do { local $/; <$fh> });
+	ok(run_script(['-mda'], undef, $rdr), 'mda delivered a patch');
 	my $post = $ibx->search->reopen->query('dfpost:6e006fd7');
 	is(scalar(@$post), 1, 'got one result for dfpost');
 	my $pre = $ibx->search->query('dfpre:090d998');

[PATCH 27/29] t/indexlevels-mirror*: switch to run_script

[Eric Wong]

This more than doubles the speed of these tests
---
 t/indexlevels-mirror.t | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/t/indexlevels-mirror.t b/t/indexlevels-mirror.t
index 40afe4e9..d129237e 100644
--- a/t/indexlevels-mirror.t
+++ b/t/indexlevels-mirror.t
@@ -17,9 +17,7 @@ foreach my $mod (qw(DBD::SQLite)) {
 	plan skip_all => "$mod missing for $0" if $@;
 }
 
-my $path = 'blib/script';
-my $index = "$path/public-inbox-index";
-my @xcpdb = ("$path/public-inbox-xcpdb", '-q');
+my @xcpdb = qw(-xcpdb -q);
 
 my $mime = PublicInbox::MIME->create(
 	header => [
@@ -48,7 +46,8 @@ sub import_index_incremental {
 	$im->done;
 
 	# index master (required for v1)
-	is(system($index, $ibx->{inboxdir}, "-L$level"), 0, 'index master OK');
+	ok(run_script(['-index', $ibx->{inboxdir}, "-L$level"]),
+		'index master OK');
 	my $ro_master = PublicInbox::Inbox->new({
 		inboxdir => $ibx->{inboxdir},
 		indexlevel => $level
@@ -70,13 +69,13 @@ sub import_index_incremental {
 
 	# inbox init
 	local $ENV{PI_CONFIG} = "$tmpdir/.picfg";
-	@cmd = ("$path/public-inbox-init", '-L', $level,
+	@cmd = ('-init', '-L', $level,
 		'mirror', $mirror, '//example.com/test', 'test@example.com');
 	push @cmd, '-V2' if $v == 2;
-	is(system(@cmd), 0, "v$v init OK");
+	ok(run_script(\@cmd), "v$v init OK");
 
 	# index mirror
-	is(system($index, $mirror), 0, "v$v index mirror OK");
+	ok(run_script(['-index', $mirror]), "v$v index mirror OK");
 
 	# read-only access
 	my $ro_mirror = PublicInbox::Inbox->new({
@@ -94,14 +93,15 @@ sub import_index_incremental {
 
 	# mirror updates
 	is(system('git', "--git-dir=$fetch_dir", qw(fetch -q)), 0, 'fetch OK');
-	is(system($index, $mirror), 0, "v$v index mirror again OK");
+	ok(run_script(['-index', $mirror]), "v$v index mirror again OK");
 	($nr, $msgs) = $ro_mirror->recent;
 	is($nr, 2, '2nd message seen in mirror');
 	is_deeply([sort { $a cmp $b } map { $_->{mid} } @$msgs],
 		['m@1','m@2'], 'got both messages in mirror');
 
 	# incremental index master (required for v1)
-	is(system($index, $ibx->{inboxdir}, "-L$level"), 0, 'index master OK');
+	ok(run_script(['-index', $ibx->{inboxdir}, "-L$level"]),
+		'index master OK');
 	($nr, $msgs) = $ro_master->recent;
 	is($nr, 2, '2nd message seen in master');
 	is_deeply([sort { $a cmp $b } map { $_->{mid} } @$msgs],
@@ -120,7 +120,7 @@ sub import_index_incremental {
 	is_deeply(\@rw_nums, [1], 'unindex NNTP article'.$v.$level);
 
 	if ($level ne 'basic') {
-		is(system(@xcpdb, $mirror), 0, "v$v xcpdb OK");
+		ok(run_script([@xcpdb, $mirror]), "v$v xcpdb OK");
 		is(PublicInbox::Admin::detect_indexlevel($ro_mirror), $level,
 		   'indexlevel detectable by Admin after xcpdb v' .$v.$level);
 		delete $ro_mirror->{$_} for (qw(over search));
@@ -130,7 +130,7 @@ sub import_index_incremental {
 
 	# sync the mirror
 	is(system('git', "--git-dir=$fetch_dir", qw(fetch -q)), 0, 'fetch OK');
-	is(system($index, $mirror), 0, "v$v index mirror again OK");
+	ok(run_script(['-index', $mirror]), "v$v index mirror again OK");
 	($nr, $msgs) = $ro_mirror->recent;
 	is($nr, 1, '2nd message gone from mirror');
 	is_deeply([map { $_->{mid} } @$msgs], ['m@1'],
@@ -155,7 +155,7 @@ sub import_index_incremental {
 	}
 	$im->done;
 	is(system('git', "--git-dir=$fetch_dir", qw(fetch -q)), 0, 'fetch OK');
-	is(system($index, '--reindex', $mirror), 0,
+	ok(run_script(['-index', '--reindex', $mirror]),
 		"v$v index --reindex mirror OK");
 	@ro_nums = map { $_->{num} } @{$ro_mirror->over->query_ts(0, 0)};
 	@rw_nums = map { $_->{num} } @{$ibx->over->query_ts(0, 0)};

[PATCH 28/29] t/xcpdb-reshard: use run_script for -xcpdb

[Eric Wong]

This more than doubles the speed of the test, since we make
many invocations of -xcpdb.
---
 t/xcpdb-reshard.t | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/t/xcpdb-reshard.t b/t/xcpdb-reshard.t
index 43e08639..88e6c3dc 100644
--- a/t/xcpdb-reshard.t
+++ b/t/xcpdb-reshard.t
@@ -33,8 +33,7 @@ my $ibx = PublicInbox::Inbox->new({
 	-primary_address => 'test@example.com',
 	indexlevel => 'medium',
 });
-my $path = 'blib/script';
-my @xcpdb = ("$path/public-inbox-xcpdb", '-q');
+my @xcpdb = qw(-xcpdb -q);
 my $nproc = 8;
 my $ndoc = 13;
 my $im = PublicInbox::InboxWritable->new($ibx, {nproc => $nproc})->importer(1);
@@ -51,7 +50,7 @@ my %nums = map {; "$_->{num}" => 1 } @$orig;
 # ensure we can go up or down in shards, or stay the same:
 for my $R (qw(2 4 1 3 3)) {
 	delete $ibx->{search}; # release old handles
-	is(system(@xcpdb, "-R$R", $ibx->{inboxdir}), 0, "xcpdb -R$R");
+	ok(run_script([@xcpdb, "-R$R", $ibx->{inboxdir}]), "xcpdb -R$R");
 	my @new_shards = grep(m!/\d+\z!, glob("$ibx->{inboxdir}/xap*/*"));
 	is(scalar(@new_shards), $R, 'resharded to two shards');
 	my $msgs = $ibx->search->query('s:this');

[PATCH 29/29] t/common: start_script replaces spawn_listener

[Eric Wong]

We can shave several hundred milliseconds off tests which spawn
daemons by preloading and avoiding startup time for common
modules which are already loaded in the parent process.

This also gives ENV{TAIL} support to all tests which support
daemons which log to stdout/stderr.
---
 t/common.perl        | 173 +++++++++++++++++++++++++++++++------------
 t/git-http-backend.t |  14 ++--
 t/httpd-corner.t     |  48 +++++++-----
 t/httpd-https.t      |  28 ++-----
 t/httpd-unix.t       |  47 ++++++------
 t/httpd.t            |  13 ++--
 t/nntpd-tls.t        |  29 ++------
 t/nntpd-validate.t   |  27 +++----
 t/nntpd.t            |  17 ++---
 t/perf-nntpd.t       |  22 +++---
 t/v2mirror.t         |  21 ++----
 t/v2writable.t       |   8 +-
 t/www_listing.t      |   8 +-
 13 files changed, 234 insertions(+), 221 deletions(-)

diff --git a/t/common.perl b/t/common.perl
index c5693080..29254fef 100644
--- a/t/common.perl
+++ b/t/common.perl
@@ -30,30 +30,6 @@ sub tcp_connect {
 	$s;
 }
 
-sub spawn_listener {
-	my ($env, $cmd, $socks) = @_;
-	my $pid = fork;
-	defined $pid or die "fork failed: $!\n";
-	if ($pid == 0) {
-		# pretend to be systemd (cf. sd_listen_fds(3))
-		my $fd = 3; # 3 == SD_LISTEN_FDS_START
-		foreach my $s (@$socks) {
-			my $fl = fcntl($s, F_GETFD, 0);
-			if (($fl & FD_CLOEXEC) != FD_CLOEXEC) {
-				warn "got FD:".fileno($s)." w/o CLOEXEC\n";
-			}
-			fcntl($s, F_SETFD, $fl &= ~FD_CLOEXEC);
-			dup2(fileno($s), $fd++) or die "dup2 failed: $!\n";
-		}
-		$ENV{LISTEN_PID} = $$;
-		$ENV{LISTEN_FDS} = scalar @$socks;
-		%ENV = (%ENV, %$env) if $env;
-		exec @$cmd;
-		die "FAIL: ",join(' ', @$cmd), ": $!\n";
-	}
-	$pid;
-}
-
 sub require_git ($;$) {
 	my ($req, $maybe) = @_;
 	my ($req_maj, $req_min) = split(/\./, $req);
@@ -68,7 +44,6 @@ sub require_git ($;$) {
 	1;
 }
 
-my %cached_scripts;
 sub key2script ($) {
 	my ($key) = @_;
 	return $key if $key =~ m!\A/!;
@@ -105,11 +80,10 @@ sub run_script_exit (;$) {
 	die RUN_SCRIPT_EXIT;
 }
 
-sub run_script ($;$$) {
-	my ($cmd, $env, $opt) = @_;
-	my ($key, @argv) = @$cmd;
-	my $run_mode = $ENV{TEST_RUN_MODE} // $opt->{run_mode} // 1;
-	my $sub = $run_mode == 0 ? undef : ($cached_scripts{$key} //= do {
+my %cached_scripts;
+sub key2sub ($) {
+	my ($key) = @_;
+	$cached_scripts{$key} //= do {
 		my $f = key2script($key);
 		open my $fh, '<', $f or die "open $f: $!";
 		my $str = do { local $/; <$fh> };
@@ -129,8 +103,34 @@ $str
 1;
 EOF
 		$pkg->can('main');
-	}); # do
+	}
+}
 
+sub _run_sub ($$$) {
+	my ($sub, $key, $argv) = @_;
+	local @ARGV = @$argv;
+	$run_script_exit_code = undef;
+	my $exit_code = eval { $sub->(@$argv) };
+	if ($@ eq RUN_SCRIPT_EXIT) {
+		$@ = '';
+		$exit_code = $run_script_exit_code;
+		$? = ($exit_code << 8);
+	} elsif (defined($exit_code)) {
+		$? = ($exit_code << 8);
+	} elsif ($@) { # mimic die() behavior when uncaught
+		warn "E: eval-ed $key: $@\n";
+		$? = ($! << 8) if $!;
+		$? = (255 << 8) if $? == 0;
+	} else {
+		die "BUG: eval-ed $key: no exit code or \$@\n";
+	}
+}
+
+sub run_script ($;$$) {
+	my ($cmd, $env, $opt) = @_;
+	my ($key, @argv) = @$cmd;
+	my $run_mode = $ENV{TEST_RUN_MODE} // $opt->{run_mode} // 1;
+	my $sub = $run_mode == 0 ? undef : key2sub($key);
 	my $fhref = [];
 	my $spawn_opt = {};
 	for my $fd (0..2) {
@@ -162,22 +162,7 @@ EOF
 		local %ENV = $env ? (%ENV, %$env) : %ENV;
 		local %SIG = %SIG;
 		_prepare_redirects($fhref);
-		local @ARGV = @argv;
-		$run_script_exit_code = undef;
-		my $exit_code = eval { $sub->(@argv) };
-		if ($@ eq RUN_SCRIPT_EXIT) {
-			$@ = '';
-			$exit_code = $run_script_exit_code;
-			$? = ($exit_code << 8);
-		} elsif (defined($exit_code)) {
-			$? = ($exit_code << 8);
-		} elsif ($@) { # mimic die() behavior when uncaught
-			warn "E: eval-ed $key: $@\n";
-			$? = ($! << 8) if $!;
-			$? = (255 << 8) if $? == 0;
-		} else {
-			die "BUG: eval-ed $key: no exit code or \$@\n";
-		}
+		_run_sub($sub, $key, \@argv);
 	}
 
 	# slurp the redirects back into user-supplied strings
@@ -191,4 +176,98 @@ EOF
 	$? == 0;
 }
 
+sub wait_for_tail () { sleep(2) }
+
+sub start_script {
+	my ($cmd, $env, $opt) = @_;
+	my ($key, @argv) = @$cmd;
+	my $run_mode = $ENV{TEST_RUN_MODE} // $opt->{run_mode} // 1;
+	my $sub = $run_mode == 0 ? undef : key2sub($key);
+	my $tail_pid;
+	if (my $tail_cmd = $ENV{TAIL}) {
+		my @paths;
+		for (@argv) {
+			next unless /\A--std(?:err|out)=(.+)\z/;
+			push @paths, $1;
+		}
+		if (@paths) {
+			defined($tail_pid = fork) or die "fork: $!\n";
+			if ($tail_pid == 0) {
+				# make sure files exist, first
+				open my $fh, '>>', $_ for @paths;
+				open(STDOUT, '>&STDERR') or die "1>&2: $!";
+				exec(split(' ', $tail_cmd), @paths);
+				die "$tail_cmd failed: $!";
+			}
+			wait_for_tail();
+		}
+	}
+	defined(my $pid = fork) or die "fork: $!\n";
+	if ($pid == 0) {
+		# pretend to be systemd (cf. sd_listen_fds(3))
+		# 3 == SD_LISTEN_FDS_START
+		my $fd;
+		for ($fd = 0; 1; $fd++) {
+			my $s = $opt->{$fd};
+			last if $fd >= 3 && !defined($s);
+			next unless $s;
+			my $fl = fcntl($s, F_GETFD, 0);
+			if (($fl & FD_CLOEXEC) != FD_CLOEXEC) {
+				warn "got FD:".fileno($s)." w/o CLOEXEC\n";
+			}
+			fcntl($s, F_SETFD, $fl &= ~FD_CLOEXEC);
+			dup2(fileno($s), $fd) or die "dup2 failed: $!\n";
+		}
+		%ENV = (%ENV, %$env) if $env;
+		my $fds = $fd - 3;
+		if ($fds > 0) {
+			$ENV{LISTEN_PID} = $$;
+			$ENV{LISTEN_FDS} = $fds;
+		}
+		if ($sub) {
+			_run_sub($sub, $key, \@argv);
+			POSIX::_exit($? >> 8);
+		} else {
+			exec(key2script($key), @argv);
+			die "FAIL: ",join(' ', $key, @argv), ": $!\n";
+		}
+	}
+	TestProcess->new($pid, $tail_pid);
+}
+
+package TestProcess;
+use strict;
+
+# prevent new threads from inheriting these objects
+sub CLONE_SKIP { 1 }
+
+sub new {
+	my ($klass, $pid, $tail_pid) = @_;
+	bless { pid => $pid, tail_pid => $tail_pid, owner => $$ }, $klass;
+}
+
+sub kill {
+	my ($self, $sig) = @_;
+	CORE::kill($sig // 'TERM', $self->{pid});
+}
+
+sub join {
+	my ($self) = @_;
+	my $pid = delete $self->{pid} or return;
+	my $ret = waitpid($pid, 0);
+	defined($ret) or die "waitpid($pid): $!";
+	$ret == $pid or die "waitpid($pid) != $ret";
+}
+
+sub DESTROY {
+	my ($self) = @_;
+	return if $self->{owner} != $$;
+	if (my $tail = delete $self->{tail_pid}) {
+		::wait_for_tail();
+		CORE::kill('TERM', $tail);
+	}
+	my $pid = delete $self->{pid} or return;
+	CORE::kill('TERM', $pid);
+}
+
 1;
diff --git a/t/git-http-backend.t b/t/git-http-backend.t
index c2a04653..c4dc09a1 100644
--- a/t/git-http-backend.t
+++ b/t/git-http-backend.t
@@ -22,12 +22,10 @@ my $psgi = "./t/git-http-backend.psgi";
 my $tmpdir = tempdir('pi-git-http-backend-XXXXXX', TMPDIR => 1, CLEANUP => 1);
 my $err = "$tmpdir/stderr.log";
 my $out = "$tmpdir/stdout.log";
-my $httpd = 'blib/script/public-inbox-httpd';
 my $sock = tcp_server();
 my $host = $sock->sockhost;
 my $port = $sock->sockport;
-my $pid;
-END { kill 'TERM', $pid if defined $pid };
+my $td;
 
 my $get_maxrss = sub {
         my $http = Net::HTTP->new(Host => "$host:$port");
@@ -44,9 +42,8 @@ my $get_maxrss = sub {
 
 {
 	ok($sock, 'sock created');
-	my $cmd = [ $httpd, '-W0', "--stdout=$out", "--stderr=$err", $psgi ];
-	ok(defined($pid = spawn_listener(undef, $cmd, [$sock])),
-	   'forked httpd process successfully');
+	my $cmd = [ '-httpd', '-W0', "--stdout=$out", "--stderr=$err", $psgi ];
+	$td = start_script($cmd, undef, { 3 => $sock });
 }
 my $mem_a = $get_maxrss->();
 
@@ -113,9 +110,8 @@ SKIP: {
 }
 
 {
-	ok(kill('TERM', $pid), 'killed httpd');
-	$pid = undef;
-	waitpid(-1, 0);
+	ok($td->kill, 'killed httpd');
+	$td->join;
 }
 
 done_testing();
diff --git a/t/httpd-corner.t b/t/httpd-corner.t
index b063d9fa..5efa6ab2 100644
--- a/t/httpd-corner.t
+++ b/t/httpd-corner.t
@@ -26,7 +26,6 @@ my $fifo = "$tmpdir/fifo";
 ok(defined mkfifo($fifo, 0777), 'created FIFO');
 my $err = "$tmpdir/stderr.log";
 my $out = "$tmpdir/stdout.log";
-my $httpd = 'blib/script/public-inbox-httpd';
 my $psgi = "./t/httpd-corner.psgi";
 my $sock = tcp_server() or die;
 
@@ -64,13 +63,11 @@ sub unix_server ($) {
 my $upath = "$tmpdir/s";
 my $unix = unix_server($upath);
 ok($unix, 'UNIX socket created');
-my $pid;
-END { kill 'TERM', $pid if defined $pid };
+my $td;
 my $spawn_httpd = sub {
 	my (@args) = @_;
-	my $cmd = [ $httpd, @args, "--stdout=$out", "--stderr=$err", $psgi ];
-	$pid = spawn_listener(undef, $cmd, [ $sock, $unix ]);
-	ok(defined $pid, 'forked httpd process successfully');
+	my $cmd = [ '-httpd', @args, "--stdout=$out", "--stderr=$err", $psgi ];
+	$td = start_script($cmd, undef, { 3 => $sock, 4 => $unix });
 };
 
 $spawn_httpd->();
@@ -208,16 +205,14 @@ sub conn_for {
 	open my $f, '>', $fifo or die "open $fifo: $!\n";
 	$f->autoflush(1);
 	ok(print($f "hello\n"), 'wrote something to fifo');
-	my $kpid = $pid;
-	$pid = undef;
-	is(kill('TERM', $kpid), 1, 'started graceful shutdown');
+	is($td->kill, 1, 'started graceful shutdown');
 	ok(print($f "world\n"), 'wrote else to fifo');
 	close $f or die "close fifo: $!\n";
 	$conn->read(my $buf, 8192);
 	my ($head, $body) = split(/\r\n\r\n/, $buf, 2);
 	like($head, qr!\AHTTP/1\.[01] 200 OK!, 'got 200 for slow-header');
 	is($body, "hello\nworld\n", 'read expected body');
-	is(waitpid($kpid, 0), $kpid, 'reaped httpd');
+	$td->join;
 	is($?, 0, 'no error');
 	$spawn_httpd->('-W0');
 }
@@ -239,15 +234,13 @@ sub conn_for {
 		$conn->sysread($buf, 8192);
 		is($buf, $c, 'got trickle for reading');
 	}
-	my $kpid = $pid;
-	$pid = undef;
-	is(kill('TERM', $kpid), 1, 'started graceful shutdown');
+	is($td->kill, 1, 'started graceful shutdown');
 	ok(print($f "world\n"), 'wrote else to fifo');
 	close $f or die "close fifo: $!\n";
 	$conn->sysread($buf, 8192);
 	is($buf, "world\n", 'read expected body');
 	is($conn->sysread($buf, 8192), 0, 'got EOF from server');
-	is(waitpid($kpid, 0), $kpid, 'reaped httpd');
+	$td->join;
 	is($?, 0, 'no error');
 	$spawn_httpd->('-W0');
 }
@@ -341,9 +334,7 @@ SKIP: {
 	$conn->write("Content-Length: $len\r\n");
 	delay();
 	$conn->write("\r\n");
-	my $kpid = $pid;
-	$pid = undef;
-	is(kill('TERM', $kpid), 1, 'started graceful shutdown');
+	is($td->kill, 1, 'started graceful shutdown');
 	delay();
 	my $n = 0;
 	foreach my $c ('a'..'z') {
@@ -351,7 +342,7 @@ SKIP: {
 	}
 	is($n, $len, 'wrote alphabet');
 	$check_self->($conn);
-	is(waitpid($kpid, 0), $kpid, 'reaped httpd');
+	$td->join;
 	is($?, 0, 'no error');
 	$spawn_httpd->('-W0');
 }
@@ -548,12 +539,29 @@ SKIP: {
 	defined(my $x = getsockopt($sock, SOL_SOCKET, $var)) or die;
 	is($x, $accf_arg, 'SO_ACCEPTFILTER unchanged if previously set');
 };
+
 SKIP: {
 	skip 'only testing lsof(8) output on Linux', 1 if $^O ne 'linux';
 	skip 'no lsof in PATH', 1 unless which('lsof');
-	my @lsof = `lsof -p $pid`;
+	my @lsof = `lsof -p $td->{pid}`;
 	is_deeply([grep(/\bdeleted\b/, @lsof)], [], 'no lingering deleted inputs');
-	is_deeply([grep(/\bpipe\b/, @lsof)], [], 'no extra pipes with -W0');
+
+	# filter out pipes inherited from the parent
+	my @this = `lsof -p $$`;
+	my $bad;
+	sub extract_inodes {
+		map {;
+			my @f = split(' ', $_);
+			my $inode = $f[-2];
+			$bad = $_ if $inode !~ /\A[0-9]+\z/;
+			$inode => 1;
+		} grep (/\bpipe\b/, @_);
+	}
+	my %child = extract_inodes(@lsof);
+	my %parent = extract_inodes(@this);
+	skip("inode not in expected format: $bad", 1) if defined($bad);
+	delete @child{(keys %parent)};
+	is_deeply([], [keys %child], 'no extra pipes with -W0');
 };
 
 done_testing();
diff --git a/t/httpd-https.t b/t/httpd-https.t
index 22c62bf4..81a11108 100644
--- a/t/httpd-https.t
+++ b/t/httpd-https.t
@@ -23,14 +23,8 @@ my $psgi = "./t/httpd-corner.psgi";
 my $tmpdir = tempdir('pi-httpd-https-XXXXXX', TMPDIR => 1, CLEANUP => 1);
 my $err = "$tmpdir/stderr.log";
 my $out = "$tmpdir/stdout.log";
-my $httpd = 'blib/script/public-inbox-httpd';
 my $https = tcp_server();
-my ($pid, $tail_pid);
-END {
-	foreach ($pid, $tail_pid) {
-		kill 'TERM', $_ if defined $_;
-	}
-};
+my $td;
 my $https_addr = $https->sockhost . ':' . $https->sockport;
 
 for my $args (
@@ -39,15 +33,9 @@ for my $args (
 	for ($out, $err) {
 		open my $fh, '>', $_ or die "truncate: $!";
 	}
-	if (my $tail_cmd = $ENV{TAIL}) { # don't assume GNU tail
-		$tail_pid = fork;
-		if (defined $tail_pid && $tail_pid == 0) {
-			exec(split(' ', $tail_cmd), $out, $err);
-		}
-	}
-	my $cmd = [ $httpd, '-W0', @$args,
+	my $cmd = [ '-httpd', '-W0', @$args,
 			"--stdout=$out", "--stderr=$err", $psgi ];
-	$pid = spawn_listener(undef, $cmd, [ $https ]);
+	$td = start_script($cmd, undef, { 3 => $https });
 	my %o = (
 		SSL_hostname => 'server.local',
 		SSL_verifycn_name => 'server.local',
@@ -119,15 +107,9 @@ for my $args (
 	};
 
 	$c = undef;
-	kill('TERM', $pid);
-	is($pid, waitpid($pid, 0), 'httpd exited successfully');
+	$td->kill;
+	$td->join;
 	is($?, 0, 'no error in exited process');
-	$pid = undef;
-	if (defined $tail_pid) {
-		kill 'TERM', $tail_pid;
-		waitpid($tail_pid, 0);
-		$tail_pid = undef;
-	}
 }
 done_testing();
 1;
diff --git a/t/httpd-unix.t b/t/httpd-unix.t
index d0c70a72..81626497 100644
--- a/t/httpd-unix.t
+++ b/t/httpd-unix.t
@@ -4,6 +4,8 @@
 use strict;
 use warnings;
 use Test::More;
+require './t/common.perl';
+use Errno qw(EADDRINUSE);
 
 foreach my $mod (qw(Plack::Util Plack::Builder HTTP::Date HTTP::Status)) {
 	eval "require $mod";
@@ -14,22 +16,15 @@ use File::Temp qw/tempdir/;
 use IO::Socket::UNIX;
 my $tmpdir = tempdir('httpd-unix-XXXXXX', TMPDIR => 1, CLEANUP => 1);
 my $unix = "$tmpdir/unix.sock";
-my $httpd = 'blib/script/public-inbox-httpd';
 my $psgi = './t/httpd-corner.psgi';
 my $out = "$tmpdir/out.log";
 my $err = "$tmpdir/err.log";
-
-my $pid;
-END { kill 'TERM', $pid if defined $pid };
+my $td;
 
 my $spawn_httpd = sub {
 	my (@args) = @_;
-	$pid = fork;
-	if ($pid == 0) {
-		exec $httpd, @args, "--stdout=$out", "--stderr=$err", $psgi;
-		die "FAIL: $!\n";
-	}
-	ok(defined $pid, 'forked httpd process successfully');
+	my $cmd = [ '-httpd', @args, "--stdout=$out", "--stderr=$err", $psgi ];
+	$td = start_script($cmd);
 };
 
 {
@@ -64,15 +59,18 @@ sub check_sock ($) {
 check_sock($unix);
 
 { # do not clobber existing socket
-	my $fpid = fork;
-	if ($fpid == 0) {
-		open STDOUT, '>>', "$tmpdir/1" or die "redirect failed: $!";
-		open STDERR, '>>', "$tmpdir/2" or die "redirect failed: $!";
-		exec $httpd, '-l', $unix, '-W0', $psgi;
-		die "FAIL: $!\n";
-	}
-	is($fpid, waitpid($fpid, 0), 'second httpd exits');
-	isnt($?, 0, 'httpd failed with failure to bind');
+	my %err = ( 'linux' => EADDRINUSE );
+	open my $out, '>>', "$tmpdir/1" or die "redirect failed: $!";
+	open my $err, '>>', "$tmpdir/2" or die "redirect failed: $!";
+	my $cmd = ['-httpd', '-l', $unix, '-W0', $psgi];
+	my $ftd = start_script($cmd, undef, { 1 => $out, 2 => $err });
+	$ftd->join;
+	isnt($?, 0, 'httpd failure set $?');
+	SKIP: {
+		my $ec = $err{$^O} or
+			skip("not sure if $^O fails with EADDRINUSE", 1);
+		is($? >> 8, $ec, 'httpd failed with EADDRINUSE');
+	};
 	open my $fh, "$tmpdir/2" or die "failed to open $tmpdir/2: $!";
 	local $/;
 	my $e = <$fh>;
@@ -81,10 +79,8 @@ check_sock($unix);
 }
 
 {
-	my $kpid = $pid;
-	$pid = undef;
-	is(kill('TERM', $kpid), 1, 'terminate existing process');
-	is(waitpid($kpid, 0), $kpid, 'existing httpd terminated');
+	is($td->kill, 1, 'terminate existing process');
+	$td->join;
 	is($?, 0, 'existing httpd exited successfully');
 	ok(-S $unix, 'unix socket still exists');
 }
@@ -95,9 +91,8 @@ SKIP: {
 
 	# wait for daemonization
 	$spawn_httpd->("-l$unix", '-D', '-P', "$tmpdir/pid");
-	my $kpid = $pid;
-	$pid = undef;
-	is(waitpid($kpid, 0), $kpid, 'existing httpd terminated');
+	$td->join;
+	is($?, 0, 'daemonized process OK');
 	check_sock($unix);
 
 	ok(-f "$tmpdir/pid", 'pid file written');
diff --git a/t/httpd.t b/t/httpd.t
index e7527ed6..ce8063b2 100644
--- a/t/httpd.t
+++ b/t/httpd.t
@@ -21,13 +21,11 @@ my $maindir = "$tmpdir/main.git";
 my $group = 'test-httpd';
 my $addr = $group . '@example.com';
 my $cfgpfx = "publicinbox.$group";
-my $httpd = 'blib/script/public-inbox-httpd';
 my $sock = tcp_server();
-my $pid;
+my $td;
 use_ok 'PublicInbox::Git';
 use_ok 'PublicInbox::Import';
 use_ok 'Email::MIME';
-END { kill 'TERM', $pid if defined $pid };
 {
 	local $ENV{HOME} = $home;
 	my $cmd = [ '-init', $group, $maindir, 'http://example.com/', $addr ];
@@ -52,8 +50,8 @@ EOF
 		$im->done($mime);
 	}
 	ok($sock, 'sock created');
-	$cmd = [ $httpd, '-W0', "--stdout=$out", "--stderr=$err" ];
-	$pid = spawn_listener(undef, $cmd, [$sock]);
+	$cmd = [ '-httpd', '-W0', "--stdout=$out", "--stderr=$err" ];
+	$td = start_script($cmd, undef, { 3 => $sock });
 	my $host = $sock->sockhost;
 	my $port = $sock->sockport;
 	my $conn = tcp_connect($sock);
@@ -78,9 +76,8 @@ EOF
 			"http://$host:$port/$group", "$tmpdir/dumb.git"),
 		0, 'clone successful');
 
-	ok(kill('TERM', $pid), 'killed httpd');
-	$pid = undef;
-	waitpid(-1, 0);
+	ok($td->kill, 'killed httpd');
+	$td->join;
 
 	is(system('git', "--git-dir=$tmpdir/clone.git",
 		  qw(fsck --no-verbose)), 0,
diff --git a/t/nntpd-tls.t b/t/nntpd-tls.t
index 0b6afcef..9f2173ce 100644
--- a/t/nntpd-tls.t
+++ b/t/nntpd-tls.t
@@ -41,16 +41,8 @@ my $inboxdir = "$tmpdir";
 my $pi_config = "$tmpdir/pi_config";
 my $group = 'test-nntpd-tls';
 my $addr = $group . '@example.com';
-my $nntpd = 'blib/script/public-inbox-nntpd';
 my $starttls = tcp_server();
 my $nntps = tcp_server();
-my ($pid, $tail_pid);
-END {
-	foreach ($pid, $tail_pid) {
-		kill 'TERM', $_ if defined $_;
-	}
-};
-
 my $ibx = PublicInbox::Inbox->new({
 	inboxdir => $inboxdir,
 	name => 'nntpd-tls',
@@ -91,6 +83,7 @@ EOF
 my $nntps_addr = $nntps->sockhost . ':' . $nntps->sockport;
 my $starttls_addr = $starttls->sockhost . ':' . $starttls->sockport;
 my $env = { PI_CONFIG => $pi_config };
+my $td;
 
 for my $args (
 	[ "--cert=$cert", "--key=$key",
@@ -100,14 +93,8 @@ for my $args (
 	for ($out, $err) {
 		open my $fh, '>', $_ or die "truncate: $!";
 	}
-	if (my $tail_cmd = $ENV{TAIL}) { # don't assume GNU tail
-		$tail_pid = fork;
-		if (defined $tail_pid && $tail_pid == 0) {
-			exec(split(' ', $tail_cmd), $out, $err);
-		}
-	}
-	my $cmd = [ $nntpd, '-W0', @$args, "--stdout=$out", "--stderr=$err" ];
-	$pid = spawn_listener($env, $cmd, [ $starttls, $nntps ]);
+	my $cmd = [ '-nntpd', '-W0', @$args, "--stdout=$out", "--stderr=$err" ];
+	$td = start_script($cmd, $env, { 3 => $starttls, 4 => $nntps });
 	my %o = (
 		SSL_hostname => 'server.local',
 		SSL_verifycn_name => 'server.local',
@@ -205,21 +192,15 @@ for my $args (
 	};
 
 	$c = undef;
-	kill('TERM', $pid);
-	is($pid, waitpid($pid, 0), 'nntpd exited successfully');
+	$td->kill;
+	$td->join;
 	is($?, 0, 'no error in exited process');
-	$pid = undef;
 	my $eout = eval {
 		open my $fh, '<', $err or die "open $err failed: $!";
 		local $/;
 		<$fh>;
 	};
 	unlike($eout, qr/wide/i, 'no Wide character warnings');
-	if (defined $tail_pid) {
-		kill 'TERM', $tail_pid;
-		waitpid($tail_pid, 0);
-		$tail_pid = undef;
-	}
 }
 done_testing();
 
diff --git a/t/nntpd-validate.t b/t/nntpd-validate.t
index de024394..e3c10d9c 100644
--- a/t/nntpd-validate.t
+++ b/t/nntpd-validate.t
@@ -10,9 +10,15 @@ use Symbol qw(gensym);
 use Time::HiRes qw(clock_gettime CLOCK_MONOTONIC);
 my $inbox_dir = $ENV{GIANT_INBOX_DIR};
 plan skip_all => "GIANT_INBOX_DIR not defined for $0" unless $inbox_dir;
+if (my $m = $ENV{TEST_RUN_MODE}) {
+	plan skip_all => "threads conflict w/ TEST_RUN_MODE=$m";
+}
 my $mid = $ENV{TEST_MID};
 
 # This test is also an excuse for me to experiment with Perl threads :P
+# TODO: get rid of threads, I was reading an old threads(3perl) manpage
+# and missed the WARNING in the newer ones about it being "discouraged"
+# in perlpolicy(1).
 unless (eval 'use threads; 1') {
 	plan skip_all => "$0 requires a threaded perl" if $@;
 }
@@ -37,13 +43,8 @@ if ($test_tls && !-r $key || !-r $cert) {
 require './t/common.perl';
 my $keep_tmp = !!$ENV{TEST_KEEP_TMP};
 my $tmpdir = tempdir('nntpd-validate-XXXXXX',TMPDIR => 1,CLEANUP => $keep_tmp);
-my (%OPT, $pid, $tail_pid, $host_port, $group);
+my (%OPT, $td, $host_port, $group);
 my $batch = 1000;
-END {
-	foreach ($pid, $tail_pid) {
-		kill 'TERM', $_ if defined $_;
-	}
-};
 if (($ENV{NNTP_TEST_URL} // '') =~ m!\Anntp://([^/]+)/([^/]+)\z!) {
 	($host_port, $group) = ($1, $2);
 	$host_port .= ":119" unless index($host_port, ':') > 0;
@@ -149,7 +150,6 @@ sub make_local_server {
 	$group = 'inbox.test.perf.nntpd';
 	my $ibx = { inboxdir => $inbox_dir, newsgroup => $group };
 	$ibx = PublicInbox::Inbox->new($ibx);
-	my $nntpd = 'blib/script/public-inbox-nntpd';
 	my $pi_config = "$tmpdir/config";
 	{
 		open my $fh, '>', $pi_config or die "open($pi_config): $!";
@@ -165,20 +165,13 @@ sub make_local_server {
 	for ($out, $err) {
 		open my $fh, '>', $_ or die "truncate: $!";
 	}
-	if (my $tail_cmd = $ENV{TAIL}) { # don't assume GNU tail
-		$tail_pid = fork;
-		if (defined $tail_pid && $tail_pid == 0) {
-			open STDOUT, '>&STDERR' or die ">&2 failed: $!";
-			exec(split(' ', $tail_cmd), $out, $err);
-		}
-	}
 	my $sock = tcp_server();
 	ok($sock, 'sock created');
 	$host_port = $sock->sockhost . ':' . $sock->sockport;
 
 	# not using multiple workers, here, since we want to increase
 	# the chance of tripping concurrency bugs within PublicInbox/NNTP*.pm
-	my $cmd = [ $nntpd, "--stdout=$out", "--stderr=$err", '-W0' ];
+	my $cmd = [ '-nntpd', "--stdout=$out", "--stderr=$err", '-W0' ];
 	push @$cmd, "-lnntp://$host_port";
 	if ($test_tls) {
 		push @$cmd, "--cert=$cert", "--key=$key";
@@ -190,7 +183,9 @@ sub make_local_server {
 		);
 	}
 	print STDERR "# CMD ". join(' ', @$cmd). "\n";
-	$pid = spawn_listener({ PI_CONFIG => $pi_config }, $cmd, [$sock]);
+	my $env = { PI_CONFIG => $pi_config };
+	# perl threads and run_mode != 0 don't get along
+	$td = start_script($cmd, $env, { run_mode => 0, 3 => $sock });
 }
 
 package DigestPipe;
diff --git a/t/nntpd.t b/t/nntpd.t
index b516ffd1..eb9be9b7 100644
--- a/t/nntpd.t
+++ b/t/nntpd.t
@@ -29,7 +29,6 @@ my $out = "$tmpdir/stdout.log";
 my $inboxdir = "$tmpdir/main.git";
 my $group = 'test-nntpd';
 my $addr = $group . '@example.com';
-my $nntpd = 'blib/script/public-inbox-nntpd';
 SKIP: {
 	skip "git 2.6+ required for V2Writable", 1 if $version == 1;
 	use_ok 'PublicInbox::V2Writable';
@@ -37,9 +36,8 @@ SKIP: {
 
 my %opts;
 my $sock = tcp_server();
-my $pid;
+my $td;
 my $len;
-END { kill 'TERM', $pid if defined $pid };
 
 my $ibx = {
 	inboxdir => $inboxdir,
@@ -90,9 +88,8 @@ EOF
 	}
 
 	ok($sock, 'sock created');
-	my $cmd = [ $nntpd, "--stdout=$out", "--stderr=$err" ];
-	$pid = spawn_listener(undef, $cmd, [ $sock ]);
-	ok(defined $pid, 'forked nntpd process successfully');
+	my $cmd = [ '-nntpd', "--stdout=$out", "--stderr=$err" ];
+	$td = start_script($cmd, undef, { 3 => $sock });
 	my $host_port = $sock->sockhost . ':' . $sock->sockport;
 	my $n = Net::NNTP->new($host_port);
 	my $list = $n->list;
@@ -306,7 +303,7 @@ Date: Fri, 02 Oct 1993 00:00:00 +0000
 		is($? >> 8, 0, 'no errors');
 	}
 	SKIP: {
-		my @of = `lsof -p $pid 2>/dev/null`;
+		my @of = `lsof -p $td->{pid} 2>/dev/null`;
 		skip('lsof broken', 1) if (!scalar(@of) || $?);
 		my @xap = grep m!Search/Xapian!, @of;
 		is_deeply(\@xap, [], 'Xapian not loaded in nntpd');
@@ -315,7 +312,7 @@ Date: Fri, 02 Oct 1993 00:00:00 +0000
 		setsockopt($s, IPPROTO_TCP, TCP_NODELAY, 1);
 		syswrite($s, 'HDR List-id 1-');
 		select(undef, undef, undef, 0.15);
-		ok(kill('TERM', $pid), 'killed nntpd');
+		ok($td->kill, 'killed nntpd');
 		select(undef, undef, undef, 0.15);
 		syswrite($s, "\r\n");
 		$buf = '';
@@ -329,7 +326,7 @@ Date: Fri, 02 Oct 1993 00:00:00 +0000
 	}
 
 	$n = $s = undef;
-	is($pid, waitpid($pid, 0), 'nntpd exited successfully');
+	$td->join;
 	my $eout = eval {
 		local $/;
 		open my $fh, '<', $err or die "open $err failed: $!";
@@ -339,6 +336,8 @@ Date: Fri, 02 Oct 1993 00:00:00 +0000
 	unlike($eout, qr/wide/i, 'no Wide character warnings');
 }
 
+diag "$td";
+$td = undef;
 done_testing();
 
 sub read_til_dot {
diff --git a/t/perf-nntpd.t b/t/perf-nntpd.t
index 7abf2249..c7d2eaff 100644
--- a/t/perf-nntpd.t
+++ b/t/perf-nntpd.t
@@ -10,18 +10,9 @@ use Net::NNTP;
 my $pi_dir = $ENV{GIANT_PI_DIR};
 plan skip_all => "GIANT_PI_DIR not defined for $0" unless $pi_dir;
 eval { require PublicInbox::Search };
-my ($host_port, $group, %opts, $s, $pid);
+my ($host_port, $group, %opts, $s, $td);
 require './t/common.perl';
 
-END {
-	if ($s) {
-		$s->print("QUIT\r\n");
-		$s->getline;
-		$s = undef;
-	}
-	kill 'TERM', $pid if defined $pid;
-};
-
 if (($ENV{NNTP_TEST_URL} || '') =~ m!\Anntp://([^/]+)/([^/]+)\z!) {
 	($host_port, $group) = ($1, $2);
 	$host_port .= ":119" unless index($host_port, ':') > 0;
@@ -29,7 +20,6 @@ if (($ENV{NNTP_TEST_URL} || '') =~ m!\Anntp://([^/]+)/([^/]+)\z!) {
 	$group = 'inbox.test.perf.nntpd';
 	my $ibx = { inboxdir => $pi_dir, newsgroup => $group };
 	$ibx = PublicInbox::Inbox->new($ibx);
-	my $nntpd = 'blib/script/public-inbox-nntpd';
 	my $tmpdir = tempdir('perf-nntpd-XXXXXX', TMPDIR => 1, CLEANUP => 1);
 
 	my $pi_config = "$tmpdir/config";
@@ -46,8 +36,8 @@ if (($ENV{NNTP_TEST_URL} || '') =~ m!\Anntp://([^/]+)/([^/]+)\z!) {
 
 	my $sock = tcp_server();
 	ok($sock, 'sock created');
-	my $cmd = [ $nntpd, '-W0' ];
-	$pid = spawn_listener({ PI_CONFIG => $pi_config }, $cmd, [$sock]);
+	my $cmd = [ '-nntpd', '-W0' ];
+	$td = start_script($cmd, { PI_CONFIG => $pi_config }, { 3 => $sock });
 	$host_port = $sock->sockhost . ':' . $sock->sockport;
 }
 %opts = (
@@ -110,6 +100,12 @@ $t = timeit(1, sub {
 });
 diag 'newnews took: ' . timestr($t) . " for $n";
 
+if ($s) {
+	$s->print("QUIT\r\n");
+	$s->getline;
+}
+
+
 done_testing();
 
 1;
diff --git a/t/v2mirror.t b/t/v2mirror.t
index 2c7f6a84..1a39ce49 100644
--- a/t/v2mirror.t
+++ b/t/v2mirror.t
@@ -21,7 +21,6 @@ use PublicInbox::MIME;
 use PublicInbox::Config;
 # FIXME: too much setup
 my $tmpdir = tempdir('pi-v2mirror-XXXXXX', TMPDIR => 1, CLEANUP => 1);
-my $script = 'blib/script/public-inbox';
 my $pi_config = "$tmpdir/config";
 {
 	open my $fh, '>', $pi_config or die "open($pi_config): $!";
@@ -60,19 +59,10 @@ ok($epoch_max > 0, "multiple epochs");
 $v2w->done;
 $ibx->cleanup;
 
-my ($sock, $pid);
-
-# TODO: replace this with ->DESTROY:
-my $owner_pid = $$;
-END { kill('TERM', $pid) if defined($pid) && $owner_pid == $$ };
-
-$! = 0;
-$sock = tcp_server();
+my $sock = tcp_server();
 ok($sock, 'sock created');
-my $httpd = "$script-httpd";
-my $cmd = [ $httpd, '-W0', "--stdout=$tmpdir/out", "--stderr=$tmpdir/err" ];
-ok(defined($pid = spawn_listener(undef, $cmd, [ $sock ])),
-	'spawned httpd process successfully');
+my $cmd = [ '-httpd', '-W0', "--stdout=$tmpdir/out", "--stderr=$tmpdir/err" ];
+my $td = start_script($cmd, undef, { 3 => $sock });
 my ($host, $port) = ($sock->sockhost, $sock->sockport);
 $sock = undef;
 
@@ -194,9 +184,8 @@ is($mibx->git->check($to_purge), undef, 'unindex+prune successful in mirror');
 	is(scalar($mset->items), 0, '1@example.com no longer visible in mirror');
 }
 
-ok(kill('TERM', $pid), 'killed httpd');
-$pid = undef;
-waitpid(-1, 0);
+ok($td->kill, 'killed httpd');
+$td->join;
 
 done_testing();
 
diff --git a/t/v2writable.t b/t/v2writable.t
index 28420bb9..4bb6d733 100644
--- a/t/v2writable.t
+++ b/t/v2writable.t
@@ -163,12 +163,10 @@ EOF
 	close $fh or die "close: $!\n";
 	my $sock = tcp_server();
 	ok($sock, 'sock created');
-	my $pid;
 	my $len;
-	END { kill 'TERM', $pid if defined $pid };
-	my $nntpd = 'blib/script/public-inbox-nntpd';
-	my $cmd = [ $nntpd, '-W0', "--stdout=$out", "--stderr=$err" ];
-	$pid = spawn_listener({ PI_CONFIG => $pi_config }, $cmd, [ $sock ]);
+	my $cmd = [ '-nntpd', '-W0', "--stdout=$out", "--stderr=$err" ];
+	my $env = { PI_CONFIG => $pi_config };
+	my $td = start_script($cmd, $env, { 3 => $sock });
 	my $host_port = $sock->sockhost . ':' . $sock->sockport;
 	my $n = Net::NNTP->new($host_port);
 	$n->group($group);
diff --git a/t/www_listing.t b/t/www_listing.t
index 61a059e5..9cde3575 100644
--- a/t/www_listing.t
+++ b/t/www_listing.t
@@ -64,15 +64,13 @@ sub tiny_test {
 		'epoch 1 in description');
 }
 
-my $pid;
-END { kill 'TERM', $pid if defined $pid };
+my $td;
 SKIP: {
 	my $err = "$tmpdir/stderr.log";
 	my $out = "$tmpdir/stdout.log";
 	my $alt = "$tmpdir/alt.git";
 	my $cfgfile = "$tmpdir/config";
 	my $v2 = "$tmpdir/v2";
-	my $httpd = 'blib/script/public-inbox-httpd';
 	my $sock = tcp_server();
 	ok($sock, 'sock created');
 	my ($host, $port) = ($sock->sockhost, $sock->sockport);
@@ -106,8 +104,8 @@ SKIP: {
 
 	close $fh or die;
 	my $env = { PI_CONFIG => $cfgfile };
-	my $cmd = [ $httpd, '-W0', "--stdout=$out", "--stderr=$err" ];
-	$pid = spawn_listener($env, $cmd, [$sock]);
+	my $cmd = [ '-httpd', '-W0', "--stdout=$out", "--stderr=$err" ];
+	$td = start_script($cmd, $env, { 3 => $sock });
 	$sock = undef;
 
 	tiny_test($host, $port);

Re: [PATCH 29/29] t/common: start_script replaces spawn_listener

[Eric Wong]

Eric Wong <e@80x24.org> wrote:
> diff --git a/t/nntpd.t b/t/nntpd.t
> index b516ffd1..eb9be9b7 100644
> --- a/t/nntpd.t
> +++ b/t/nntpd.t
> @@ -339,6 +336,8 @@ Date: Fri, 02 Oct 1993 00:00:00 +0000
>  	unlike($eout, qr/wide/i, 'no Wide character warnings');
>  }
>  
> +diag "$td";

Oops, diag was leftover and should not be there.

> diff --git a/t/v2writable.t b/t/v2writable.t
> index 28420bb9..4bb6d733 100644
> --- a/t/v2writable.t
> +++ b/t/v2writable.t
> @@ -163,12 +163,10 @@ EOF
>  	close $fh or die "close: $!\n";
>  	my $sock = tcp_server();
>  	ok($sock, 'sock created');
> -	my $pid;
>  	my $len;
> -	END { kill 'TERM', $pid if defined $pid };
> -	my $nntpd = 'blib/script/public-inbox-nntpd';
> -	my $cmd = [ $nntpd, '-W0', "--stdout=$out", "--stderr=$err" ];
> -	$pid = spawn_listener({ PI_CONFIG => $pi_config }, $cmd, [ $sock ]);
> +	my $cmd = [ '-nntpd', '-W0', "--stdout=$out", "--stderr=$err" ];
> +	my $env = { PI_CONFIG => $pi_config };
> +	my $td = start_script($cmd, $env, { 3 => $sock });
>  	my $host_port = $sock->sockhost . ':' . $sock->sockport;
>  	my $n = Net::NNTP->new($host_port);
>  	$n->group($group);

"$n" (Net::NNTP) and "$td" (TestDaemon) going out of scope at
the same time seems to trigger some strange ->DESTROY
interaction since both classes have a ->DESTROY method.

This causes t/v2writable to be stuck until the 60s
EvCleanup::later timer fires (well, I'm pretty sure
it's the EvCleanup::later timer, since it takes ~60s
to fail and not 180s).

Trying to strace the nntpd kicks it right away, so I'm
trhing to reproduce it and strace the Net::NNTP client
process...

Re: [PATCH 29/29] t/common: start_script replaces spawn_listener

[Eric Wong]

Eric Wong <e@80x24.org> wrote:
> "$n" (Net::NNTP) and "$td" (TestDaemon) going out of scope at
> the same time seems to trigger some strange ->DESTROY
> interaction since both classes have a ->DESTROY method.

Nope, I was wrong about that :x

> This causes t/v2writable to be stuck until the 60s
> EvCleanup::later timer fires (well, I'm pretty sure
> it's the EvCleanup::later timer, since it takes ~60s
> to fail and not 180s).

Actual problem seems to be END {} not firing for EvCleanup.pm
because of POSIX::_exit use in the test.  But NNTP shutdown
seems to have some other problems, too.  Will try to sort out
sometime later.

Anyways, the other 28 patches in this series seem fine and
are in master.

[PATCH 00/17] test fixes and cleanups

[Eric Wong]

There's some fixes for race conditions around daemon
startup and shutdown and resurrects start_script for
slightly improved test performance.

And slowly eliminating all END{} block usages

Eric Wong (17):
  tests: disable daemon workers in a few more places
  tests: use strict everywhere
  t/v1-add-remove-add: quiet down "git init"
  t/xcpdb-reshard: test xcpdb --compact
  t/httpd-corner: wait for worker process death
  t/nntpd-tls: sometimes SSL_connect succeeds quickly
  .gitignore: ignore local prove(1) files
  daemon: use sigprocmask to block signals at startup
  daemon: use sigprocmask when respawning workers
  daemon: avoid race when quitting workers
  t/common: start_script replaces spawn_listener
  t/nntpd-validate: get rid of threads dependency
  xapcmd: replace Xtmpdirs with File::Temp->newdir
  tests: use File::Temp->newdir instead of tempdir()
  tests: quiet down commit graph
  t/perf-*.t: use $ENV{GIANT_INBOX_DIR} consistently
  tests: move giant inbox/git dependent tests to xt/

 .gitignore                   |   2 +
 MANIFEST                     |  11 ++-
 lib/PublicInbox/Daemon.pm    |  35 +++++--
 lib/PublicInbox/Xapcmd.pm    |  73 +++++---------
 t/.gitconfig                 |   4 +
 t/admin.t                    |   4 +-
 t/altid.t                    |   4 +-
 t/altid_v2.t                 |   3 +-
 t/cgi.t                      |   3 +-
 t/common.perl                | 184 ++++++++++++++++++++++++++---------
 t/config.t                   |   4 +-
 t/convert-compact.t          |   3 +-
 t/edit.t                     |   3 +-
 t/emergency.t                |   4 +-
 t/feed.t                     |   3 +-
 t/filter_rubylang.t          |   5 +-
 t/git.t                      |   6 +-
 t/html_index.t               |   4 +-
 t/httpd-corner.psgi          |   2 +-
 t/httpd-corner.t             |  70 +++++++------
 t/httpd-https.t              |  31 ++----
 t/httpd-unix.t               |  51 +++++-----
 t/httpd.t                    |  16 ++-
 t/import.t                   |   4 +-
 t/indexlevels-mirror.t       |   3 +-
 t/init.t                     |   3 +-
 t/mda.t                      |   3 +-
 t/mda_filter_rubylang.t      |   3 +-
 t/mid.t                      |   1 +
 t/msgmap.t                   |   4 +-
 t/nntpd-tls.t                |  42 +++-----
 t/nntpd.t                    |  19 ++--
 t/nulsubject.t               |   4 +-
 t/over.t                     |   4 +-
 t/plack.t                    |   4 +-
 t/psgi_attach.t              |   4 +-
 t/psgi_bad_mids.t            |   4 +-
 t/psgi_mount.t               |   4 +-
 t/psgi_multipart_not.t       |   4 +-
 t/psgi_scan_all.t            |   4 +-
 t/psgi_search.t              |   4 +-
 t/psgi_text.t                |   4 +-
 t/psgi_v2.t                  |   3 +-
 t/purge.t                    |   4 +-
 t/qspawn.t                   |   1 +
 t/replace.t                  |   6 +-
 t/search-thr-index.t         |   4 +-
 t/search.t                   |   4 +-
 t/solver_git.t               |   3 +-
 t/spamcheck_spamc.t          |   4 +-
 t/v1-add-remove-add.t        |   6 +-
 t/v1reindex.t                |   3 +-
 t/v2-add-remove-add.t        |   3 +-
 t/v2mda.t                    |   3 +-
 t/v2mirror.t                 |  26 ++---
 t/v2reindex.t                |   3 +-
 t/v2writable.t               |  13 ++-
 t/watch_filter_rubylang.t    |   3 +-
 t/watch_maildir.t            |   5 +-
 t/watch_maildir_v2.t         |  10 +-
 t/www_listing.t              |  11 +--
 t/xcpdb-reshard.t            |   7 +-
 {t => xt}/git-http-backend.t |  19 ++--
 {t => xt}/nntpd-validate.t   |  57 +++++------
 {t => xt}/perf-msgview.t     |   6 +-
 {t => xt}/perf-nntpd.t       |  34 +++----
 {t => xt}/perf-threading.t   |   8 +-
 67 files changed, 462 insertions(+), 431 deletions(-)
 create mode 100644 t/.gitconfig
 rename {t => xt}/git-http-backend.t (87%)
 rename {t => xt}/nntpd-validate.t (85%)
 rename {t => xt}/perf-msgview.t (85%)
 rename {t => xt}/perf-nntpd.t (79%)
 rename {t => xt}/perf-threading.t (72%)

[PATCH 01/17] tests: disable daemon workers in a few more places

[Eric Wong]

There were still a few places where we used worker processes
unnecessarily in tests, causing a small amount of unnecessary
overhead.

Followup-to: ad221e9b2852f6c5 ("t/*.t: disable nntpd/httpd worker processes in most tests")
---
 t/httpd-unix.t | 1 +
 t/nntpd.t      | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/t/httpd-unix.t b/t/httpd-unix.t
index d0c70a72..2c918281 100644
--- a/t/httpd-unix.t
+++ b/t/httpd-unix.t
@@ -24,6 +24,7 @@ END { kill 'TERM', $pid if defined $pid };
 
 my $spawn_httpd = sub {
 	my (@args) = @_;
+	push @args, '-W0';
 	$pid = fork;
 	if ($pid == 0) {
 		exec $httpd, @args, "--stdout=$out", "--stderr=$err", $psgi;
diff --git a/t/nntpd.t b/t/nntpd.t
index b516ffd1..4795dc00 100644
--- a/t/nntpd.t
+++ b/t/nntpd.t
@@ -90,7 +90,7 @@ EOF
 	}
 
 	ok($sock, 'sock created');
-	my $cmd = [ $nntpd, "--stdout=$out", "--stderr=$err" ];
+	my $cmd = [ $nntpd, '-W0', "--stdout=$out", "--stderr=$err" ];
 	$pid = spawn_listener(undef, $cmd, [ $sock ]);
 	ok(defined $pid, 'forked nntpd process successfully');
 	my $host_port = $sock->sockhost . ':' . $sock->sockport;

[PATCH 02/17] tests: use strict everywhere

[Eric Wong]

The "strict" pragma makes code easier to debug, and we had
undeclared variables as a result in t/watch_maildir_v2.t.
So use it everywhere to be consistent with the rest of our
code.
---
 t/mid.t              | 1 +
 t/qspawn.t           | 1 +
 t/watch_maildir.t    | 1 +
 t/watch_maildir_v2.t | 7 ++++---
 4 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/t/mid.t b/t/mid.t
index 98b0c200..ecac04de 100644
--- a/t/mid.t
+++ b/t/mid.t
@@ -1,5 +1,6 @@
 # Copyright (C) 2016-2019 all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+use strict;
 use Test::More;
 use PublicInbox::MID qw(mid_escape mids references mids_for_index);
 
diff --git a/t/qspawn.t b/t/qspawn.t
index 58c6febb..fc288a2d 100644
--- a/t/qspawn.t
+++ b/t/qspawn.t
@@ -1,5 +1,6 @@
 # Copyright (C) 2016-2019 all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+use strict;
 use Test::More;
 use_ok 'PublicInbox::Qspawn';
 
diff --git a/t/watch_maildir.t b/t/watch_maildir.t
index e6cd599c..41d50329 100644
--- a/t/watch_maildir.t
+++ b/t/watch_maildir.t
@@ -1,5 +1,6 @@
 # Copyright (C) 2016-2019 all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+use strict;
 use Test::More;
 use File::Temp qw/tempdir/;
 use Email::MIME;
diff --git a/t/watch_maildir_v2.t b/t/watch_maildir_v2.t
index 7fe23521..e0e8a13f 100644
--- a/t/watch_maildir_v2.t
+++ b/t/watch_maildir_v2.t
@@ -1,5 +1,6 @@
 # Copyright (C) 2018-2019 all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+use strict;
 use Test::More;
 use File::Temp qw/tempdir/;
 use PublicInbox::MIME;
@@ -106,7 +107,7 @@ More majordomo info at  http://vger.kernel.org/majordomo-info.html\n);
 		local $SIG{__WARN__} = sub {}; # quiet spam check warning
 		PublicInbox::WatchMaildir->new($config)->scan('full');
 	}
-	($nr, $msgs) = $srch->reopen->query('');
+	my ($nr, $msgs) = $srch->reopen->query('');
 	is($nr, 0, 'inbox is still empty');
 	is(unlink(glob("$maildir/new/*")), 1);
 }
@@ -119,7 +120,7 @@ More majordomo info at  http://vger.kernel.org/majordomo-info.html\n);
 	PublicInbox::Emergency->new($maildir)->prepare(\$msg);
 	$config->{'publicinboxwatch.spamcheck'} = 'spamc';
 	PublicInbox::WatchMaildir->new($config)->scan('full');
-	($nr, $msgs) = $srch->reopen->query('');
+	my ($nr, $msgs) = $srch->reopen->query('');
 	is($nr, 1, 'inbox has one mail after spamc OK-ed a message');
 	my $mref = $ibx->msg_by_smsg($msgs->[0]);
 	like($$mref, qr/something\n\z/s, 'message scrubbed on import');
@@ -132,7 +133,7 @@ More majordomo info at  http://vger.kernel.org/majordomo-info.html\n);
 	$msg = eval { local $/; <$fh> };
 	PublicInbox::Emergency->new($maildir)->prepare(\$msg);
 	PublicInbox::WatchMaildir->new($config)->scan('full');
-	($nr, $msgs) = $srch->reopen->query('dfpost:6e006fd7');
+	my ($nr, $msgs) = $srch->reopen->query('dfpost:6e006fd7');
 	is($nr, 1, 'diff postimage found');
 	my $post = $msgs->[0];
 	($nr, $msgs) = $srch->query('dfpre:090d998b6c2c');

[PATCH 03/17] t/v1-add-remove-add: quiet down "git init"

[Eric Wong]

Use the "-q" flag like everywhere else.
---
 t/v1-add-remove-add.t | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/t/v1-add-remove-add.t b/t/v1-add-remove-add.t
index 3facd87e..035fba5c 100644
--- a/t/v1-add-remove-add.t
+++ b/t/v1-add-remove-add.t
@@ -13,7 +13,7 @@ foreach my $mod (qw(DBD::SQLite Search::Xapian)) {
 }
 require PublicInbox::SearchIdx;
 my $inboxdir = tempdir('pi-add-remove-add-XXXXXX', TMPDIR => 1, CLEANUP => 1);
-is(system(qw(git init --bare), $inboxdir), 0);
+is(system(qw(git init -q --bare), $inboxdir), 0);
 my $ibx = {
 	inboxdir => $inboxdir,
 	name => 'test-add-remove-add',

[PATCH 04/17] t/xcpdb-reshard: test xcpdb --compact

[Eric Wong]

We did not have a test for this, and need to guard against
regressions when changing Xapcmd to use File::Temp->newdir
in future commits.
---
 t/xcpdb-reshard.t | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/t/xcpdb-reshard.t b/t/xcpdb-reshard.t
index 88e6c3dc..9335843d 100644
--- a/t/xcpdb-reshard.t
+++ b/t/xcpdb-reshard.t
@@ -50,7 +50,9 @@ my %nums = map {; "$_->{num}" => 1 } @$orig;
 # ensure we can go up or down in shards, or stay the same:
 for my $R (qw(2 4 1 3 3)) {
 	delete $ibx->{search}; # release old handles
-	ok(run_script([@xcpdb, "-R$R", $ibx->{inboxdir}]), "xcpdb -R$R");
+	my $cmd = [@xcpdb, "-R$R", $ibx->{inboxdir}];
+	push @$cmd, '--compact' if $R == 1;
+	ok(run_script($cmd), "xcpdb -R$R");
 	my @new_shards = grep(m!/\d+\z!, glob("$ibx->{inboxdir}/xap*/*"));
 	is(scalar(@new_shards), $R, 'resharded to two shards');
 	my $msgs = $ibx->search->query('s:this');

[PATCH 05/17] t/httpd-corner: wait for worker process death

[Eric Wong]

We need to ensure the worker process is terminated before
starting a new connection, so leave a persistent HTTP/1.1
connection open and wait for the SIGKILL to take effect
and drop the client.
---
 t/httpd-corner.psgi |  2 +-
 t/httpd-corner.t    | 19 ++++++++++++-------
 2 files changed, 13 insertions(+), 8 deletions(-)

diff --git a/t/httpd-corner.psgi b/t/httpd-corner.psgi
index bf38d1ff..18e556be 100644
--- a/t/httpd-corner.psgi
+++ b/t/httpd-corner.psgi
@@ -87,7 +87,7 @@ my $app = sub {
 		});
 	} elsif ($path eq '/pid') {
 		$code = 200;
-		push @$body, $$;
+		push @$body, "$$\n";
 	}
 
 	[ $code, $h, $body ]
diff --git a/t/httpd-corner.t b/t/httpd-corner.t
index b063d9fa..cc36c7e1 100644
--- a/t/httpd-corner.t
+++ b/t/httpd-corner.t
@@ -76,17 +76,22 @@ my $spawn_httpd = sub {
 $spawn_httpd->();
 if ('test worker death') {
 	my $conn = conn_for($sock, 'killed worker');
-	$conn->write("GET /pid HTTP/1.0\r\n\r\n");
-	ok($conn->read(my $buf, 8192), 'read response');
-	my ($head, $body) = split(/\r\n\r\n/, $buf);
-	like($body, qr/\A[0-9]+\z/, '/pid response');
-	my $pid = $body;
+	$conn->write("GET /pid HTTP/1.1\r\nHost:example.com\r\n\r\n");
+	my $pid;
+	while (defined(my $line = $conn->getline)) {
+		next unless $line eq "\r\n";
+		chomp($pid = $conn->getline);
+		last;
+	}
+	like($pid, qr/\A[0-9]+\z/, '/pid response');
 	is(kill('KILL', $pid), 1, 'killed worker');
+	is($conn->getline, undef, 'worker died and EOF-ed client');
 
 	$conn = conn_for($sock, 'respawned worker');
 	$conn->write("GET /pid HTTP/1.0\r\n\r\n");
-	ok($conn->read($buf, 8192), 'read response');
-	($head, $body) = split(/\r\n\r\n/, $buf);
+	ok($conn->read(my $buf, 8192), 'read response');
+	my ($head, $body) = split(/\r\n\r\n/, $buf);
+	chomp($body);
 	like($body, qr/\A[0-9]+\z/, '/pid response');
 	isnt($body, $pid, 'respawned worker');
 }

[PATCH 06/17] t/nntpd-tls: sometimes SSL_connect succeeds quickly

[Eric Wong]

It seems caching can happen within OpenSSL or negotiation
can be delayed in some cases.  In any case, don't barf on
PublicInbox::TLS::epollbit() when connect_SSL succeeds
unexpectedly.

---
 t/nntpd-tls.t | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/t/nntpd-tls.t b/t/nntpd-tls.t
index 0b6afcef..4e71e82d 100644
--- a/t/nntpd-tls.t
+++ b/t/nntpd-tls.t
@@ -120,8 +120,14 @@ for my $args (
 	my $slow = tcp_connect($nntps, Blocking => 0);
 	$slow = IO::Socket::SSL->start_SSL($slow, SSL_startHandshake => 0, %o);
 	my $slow_done = $slow->connect_SSL;
-	diag('W: connect_SSL early OK, slow client test invalid') if $slow_done;
-	my @poll = (fileno($slow), PublicInbox::TLS::epollbit());
+	my @poll;
+	if ($slow_done) {
+		diag('W: connect_SSL early OK, slow client test invalid');
+		use PublicInbox::Syscall qw(EPOLLIN EPOLLOUT);
+		@poll = (fileno($slow), EPOLLIN | EPOLLOUT);
+	} else {
+		@poll = (fileno($slow), PublicInbox::TLS::epollbit());
+	}
 	# we should call connect_SSL much later...
 
 	# NNTPS

[PATCH 07/17] .gitignore: ignore local prove(1) files

[Eric Wong]

As described in prove(1), .prove is storage for --state=save
and .proverc allows per-worktree customizations.
---
 .gitignore | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.gitignore b/.gitignore
index 167d08bf..9eb97751 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,5 @@
+/.prove
+/.proverc
 /config.mak
 /MANIFEST.gen
 /Makefile.old

[PATCH 08/17] daemon: use sigprocmask to block signals at startup

[Eric Wong]

`$SIG{FOO} = "IGNORE"' will cause the daemon to miss signals
entirely.  Instead, we can use sigprocmask to block signal
delivery until we have our signal handlers setup.  This closes a
race where a PID file can be written for an init script and a
signal to be dropped via "IGNORE".
---
 lib/PublicInbox/Daemon.pm | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/lib/PublicInbox/Daemon.pm b/lib/PublicInbox/Daemon.pm
index b3743f5c..e830a98f 100644
--- a/lib/PublicInbox/Daemon.pm
+++ b/lib/PublicInbox/Daemon.pm
@@ -8,7 +8,7 @@ use warnings;
 use Getopt::Long qw/:config gnu_getopt no_ignore_case auto_abbrev/;
 use IO::Handle;
 use IO::Socket;
-use POSIX qw(WNOHANG);
+use POSIX qw(WNOHANG :signal_h);
 use Socket qw(IPPROTO_TCP SOL_SOCKET);
 sub SO_ACCEPTFILTER () { 0x1000 }
 use Cwd qw/abs_path/;
@@ -19,7 +19,7 @@ require PublicInbox::EvCleanup;
 require PublicInbox::Listener;
 require PublicInbox::ParentPipe;
 my @CMD;
-my $set_user;
+my ($set_user, $oldset);
 my (@cfg_listen, $stdout, $stderr, $group, $user, $pid_file, $daemonize);
 my $worker_processes = 1;
 my @listeners;
@@ -76,9 +76,11 @@ sub accept_tls_opt ($) {
 
 sub daemon_prepare ($) {
 	my ($default_listen) = @_;
+	$oldset = POSIX::SigSet->new();
+	my $newset = POSIX::SigSet->new();
+	$newset->fillset or die "fillset: $!";
+	sigprocmask(SIG_SETMASK, $newset, $oldset) or die "sigprocmask: $!";
 	@CMD = ($0, @ARGV);
-	$SIG{HUP} = $SIG{USR1} = $SIG{USR2} = $SIG{PIPE} =
-		$SIG{TTIN} = $SIG{TTOU} = $SIG{WINCH} = 'IGNORE';
 	my %opts = (
 		'l|listen=s' => \@cfg_listen,
 		'1|stdout=s' => \$stdout,
@@ -482,6 +484,7 @@ sub master_loop {
 			syswrite($w, '.');
 		};
 	}
+	sigprocmask(SIG_SETMASK, $oldset) or die "sigprocmask: $!";
 	reopen_logs();
 	# main loop
 	my $quit = 0;
@@ -616,6 +619,7 @@ sub daemon_loop ($$$$) {
 		# this calls epoll_create:
 		PublicInbox::Listener->new($_, $tls_cb || $post_accept)
 	} @listeners;
+	sigprocmask(SIG_SETMASK, $oldset) or die "sigprocmask: $!";
 	PublicInbox::DS->EventLoop;
 	$parent_pipe = undef;
 }

[PATCH 09/17] daemon: use sigprocmask when respawning workers

[Eric Wong]

We need to block signals in workers during respawns
until they're ready to receive signals.
---
 lib/PublicInbox/Daemon.pm | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/lib/PublicInbox/Daemon.pm b/lib/PublicInbox/Daemon.pm
index e830a98f..90f11137 100644
--- a/lib/PublicInbox/Daemon.pm
+++ b/lib/PublicInbox/Daemon.pm
@@ -19,7 +19,7 @@ require PublicInbox::EvCleanup;
 require PublicInbox::Listener;
 require PublicInbox::ParentPipe;
 my @CMD;
-my ($set_user, $oldset);
+my ($set_user, $oldset, $newset);
 my (@cfg_listen, $stdout, $stderr, $group, $user, $pid_file, $daemonize);
 my $worker_processes = 1;
 my @listeners;
@@ -77,7 +77,7 @@ sub accept_tls_opt ($) {
 sub daemon_prepare ($) {
 	my ($default_listen) = @_;
 	$oldset = POSIX::SigSet->new();
-	my $newset = POSIX::SigSet->new();
+	$newset = POSIX::SigSet->new();
 	$newset->fillset or die "fillset: $!";
 	sigprocmask(SIG_SETMASK, $newset, $oldset) or die "sigprocmask: $!";
 	@CMD = ($0, @ARGV);
@@ -536,6 +536,7 @@ sub master_loop {
 			}
 			$n = $worker_processes;
 		}
+		sigprocmask(SIG_SETMASK, $newset) or die "sigprocmask: $!";
 		foreach my $i ($n..($worker_processes - 1)) {
 			my $pid = fork;
 			if (!defined $pid) {
@@ -548,6 +549,7 @@ sub master_loop {
 				$pids{$pid} = $i;
 			}
 		}
+		sigprocmask(SIG_SETMASK, $oldset) or die "sigprocmask: $!";
 		# just wait on signal events here:
 		sysread($r, my $buf, 8);
 	}

[PATCH 10/17] daemon: avoid race when quitting workers

[Eric Wong]

While the master process has a self-pipe to avoid missing
signals, worker processes lack that aside from a pipe to
detect master death.

That pipe doesn't exist when there's no master process,
so it's possible DS::close never finishes because it
never woke up from epoll_wait.  So create a pipe on
the worker_quit signal and force it into epoll/kevent
so it wakes up right away.
---
 lib/PublicInbox/Daemon.pm | 21 +++++++++++++++++----
 1 file changed, 17 insertions(+), 4 deletions(-)

diff --git a/lib/PublicInbox/Daemon.pm b/lib/PublicInbox/Daemon.pm
index 90f11137..0e3b95d2 100644
--- a/lib/PublicInbox/Daemon.pm
+++ b/lib/PublicInbox/Daemon.pm
@@ -252,6 +252,11 @@ sub daemonize () {
 	}
 }
 
+sub shrink_pipes {
+	if ($^O eq 'linux') { # 1031: F_SETPIPE_SZ, 4096: page size
+		fcntl($_, 1031, 4096) for @_;
+	}
+}
 
 sub worker_quit {
 	# killing again terminates immediately:
@@ -260,6 +265,17 @@ sub worker_quit {
 	$_->close foreach @listeners; # call PublicInbox::DS::close
 	@listeners = ();
 
+	# create a lazy self-pipe which kicks us out of the EventLoop
+	# so DS::PostEventLoop can fire
+	if (pipe(my ($r, $w))) {
+		shrink_pipes($w);
+
+		# shrink_pipes == noop
+		PublicInbox::ParentPipe->new($r, *shrink_pipes);
+		close $w; # wake up from the event loop
+	} else {
+		warn "E: pipe failed ($!), quit unreliable\n";
+	}
 	my $proc_name;
 	my $warn = 0;
 	# drop idle connections and try to quit gracefully
@@ -468,10 +484,7 @@ sub unlink_pid_file_safe_ish ($$) {
 sub master_loop {
 	pipe(my ($p0, $p1)) or die "failed to create parent-pipe: $!";
 	pipe(my ($r, $w)) or die "failed to create self-pipe: $!";
-
-	if ($^O eq 'linux') { # 1031: F_SETPIPE_SZ = 1031
-		fcntl($_, 1031, 4096) for ($w, $p1);
-	}
+	shrink_pipes($w, $p1);
 
 	IO::Handle::blocking($w, 0);
 	my $set_workers = $worker_processes;

Re: [PATCH 10/17] daemon: avoid race when quitting workers

[Eric Wong]

Eric Wong <e@80x24.org> wrote:
> While the master process has a self-pipe to avoid missing
> signals, worker processes lack that aside from a pipe to
> detect master death.
> 
> That pipe doesn't exist when there's no master process,
> so it's possible DS::close never finishes because it
> never woke up from epoll_wait.  So create a pipe on
> the worker_quit signal and force it into epoll/kevent
> so it wakes up right away.

Nope.  A self-pipe in any pure-Perl form is insufficient,
since Perl_csighandler() can't wake up processes from any
sleeping syscalls (no-internal self-pipe/eventfd like
other VMs).

Fortunately, Perl exposes sigprocmask, so signalfd should be
usable on Linux.  EVFILT_SIGNAL also exists for kevent...

IO::Poll users might be out-of-luck (anybody try GNU/Hurd?)

[PATCH 0/2] fix kqueue support and missed signal wakeups

[Eric Wong]

signalfd and EVFILT_SIGNAL are pretty nice, actually.  I'm
actually glad Perl5 allows users to call sigprocmask and use
these new APIs effectively, compared to other runtimes which
purport to know better :P

Note: the likelyhood of coalesced signals increases in high
load situations, but I don't think it matters in practice;
since we already account for coalescing in handling SIGCHLD.

Eric Wong (2):
  dskqxs: fix missing EV_DISPATCH define
  httpd|nntpd: avoid missed signal wakeups

 MANIFEST                   |   3 +
 lib/PublicInbox/DS.pm      |   6 +-
 lib/PublicInbox/DSKQXS.pm  | 105 +++++++++++++++++----
 lib/PublicInbox/Daemon.pm  | 183 ++++++++++++++++++-------------------
 lib/PublicInbox/Sigfd.pm   |  63 +++++++++++++
 lib/PublicInbox/Syscall.pm |  42 ++++++++-
 t/ds-kqxs.t                |  42 +++++++++
 t/ds-poll.t                |  16 +---
 t/sigfd.t                  |  65 +++++++++++++
 9 files changed, 397 insertions(+), 128 deletions(-)
 create mode 100644 lib/PublicInbox/Sigfd.pm
 create mode 100644 t/ds-kqxs.t
 create mode 100644 t/sigfd.t

[PATCH 1/2] dskqxs: fix missing EV_DISPATCH define

[Eric Wong]

Oops, IO::KQueue support was broken due to this missing
constant.  Add a new ds-kqxs.t test case to ensure we
test the IO::KQueue path if IO::KQueue is available.
---
 MANIFEST                  |  1 +
 lib/PublicInbox/DSKQXS.pm |  2 ++
 t/ds-kqxs.t               | 14 ++++++++++++++
 t/ds-poll.t               | 16 ++++------------
 4 files changed, 21 insertions(+), 12 deletions(-)
 create mode 100644 t/ds-kqxs.t

diff --git a/MANIFEST b/MANIFEST
index 9fd639f5..a50c1246 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -207,6 +207,7 @@ t/config_limiter.t
 t/content_id.t
 t/convert-compact.t
 t/data/0001.patch
+t/ds-kqxs.t
 t/ds-leak.t
 t/ds-poll.t
 t/edit.t
diff --git a/lib/PublicInbox/DSKQXS.pm b/lib/PublicInbox/DSKQXS.pm
index 1c3b970b..84e146f8 100644
--- a/lib/PublicInbox/DSKQXS.pm
+++ b/lib/PublicInbox/DSKQXS.pm
@@ -21,6 +21,8 @@ use PublicInbox::Syscall qw(EPOLLONESHOT EPOLLIN EPOLLOUT EPOLLET
 our @EXPORT_OK = qw(epoll_ctl epoll_wait);
 my $owner_pid = -1; # kqueue is close-on-fork (yes, fork, not exec)
 
+sub EV_DISPATCH () { 0x0080 }
+
 # map EPOLL* bits to kqueue EV_* flags for EV_SET
 sub kq_flag ($$) {
 	my ($bit, $ev) = @_;
diff --git a/t/ds-kqxs.t b/t/ds-kqxs.t
new file mode 100644
index 00000000..785570c3
--- /dev/null
+++ b/t/ds-kqxs.t
@@ -0,0 +1,14 @@
+# Copyright (C) 2019 all contributors <meta@public-inbox.org>
+# Licensed the same as Danga::Socket (and Perl5)
+# License: GPL-1.0+ or Artistic-1.0-Perl
+#  <https://www.gnu.org/licenses/gpl-1.0.txt>
+#  <https://dev.perl.org/licenses/artistic.html>
+use strict;
+use Test::More;
+unless (eval { require IO::KQueue }) {
+	my $m = $^O !~ /bsd/ ? 'DSKQXS is only for *BSD systems'
+				: "no IO::KQueue, skipping $0: $@";
+	plan skip_all => $m;
+}
+local $ENV{TEST_IOPOLLER} = 'PublicInbox::DSKQXS';
+require './t/ds-poll.t';
diff --git a/t/ds-poll.t b/t/ds-poll.t
index c9dcdd22..21f8b295 100644
--- a/t/ds-poll.t
+++ b/t/ds-poll.t
@@ -7,7 +7,7 @@ use strict;
 use warnings;
 use Test::More;
 use PublicInbox::Syscall qw(:epoll);
-my $cls = 'PublicInbox::DSPoll';
+my $cls = $ENV{TEST_IOPOLLER} // 'PublicInbox::DSPoll';
 use_ok $cls;
 my $p = $cls->new;
 
@@ -43,16 +43,8 @@ my @fds = sort(map { $_->[0] } @$events);
 my @exp = sort((fileno($r), fileno($x)));
 is_deeply(\@fds, \@exp, 'got both ready FDs');
 
-# EPOLL_CTL_DEL doesn't matter for kqueue, we do it in native epoll
-# to avoid a kernel-wide lock; but its not needed for native kqueue
-# paths so DSKQXS makes it a noop (as did Danga::Socket::close).
-SKIP: {
-	if ($cls ne 'PublicInbox::DSPoll') {
-		skip "$cls doesn't handle EPOLL_CTL_DEL", 2;
-	}
-	is($p->epoll_ctl(EPOLL_CTL_DEL, fileno($r), 0), 0, 'EPOLL_CTL_DEL OK');
-	$n = $p->epoll_wait(9, 0, $events);
-	is($n, 0, 'nothing ready after EPOLL_CTL_DEL');
-};
+is($p->epoll_ctl(EPOLL_CTL_DEL, fileno($r), 0), 0, 'EPOLL_CTL_DEL OK');
+$n = $p->epoll_wait(9, 0, $events);
+is($n, 0, 'nothing ready after EPOLL_CTL_DEL');
 
 done_testing;

[PATCH 2/2] httpd|nntpd: avoid missed signal wakeups

[Eric Wong]

Our attempt at using a self-pipe in signal handlers was
ineffective, since pure Perl code execution is deferred
and Perl doesn't use an internal self-pipe/eventfd.  In
retrospect, I actually prefer the simplicity of Perl in
this regard...

We can use sigprocmask() from Perl, so we can introduce
signalfd(2) and EVFILT_SIGNAL support on Linux and *BSD-based
systems, respectively.  These OS primitives allow us to avoid a
race where Perl checks for signals right before epoll_wait() or
kevent() puts the process to sleep.

The (few) systems nowadays without signalfd(2) or IO::KQueue
will now see wakeups every second to avoid missed signals.
---
 MANIFEST                   |   2 +
 lib/PublicInbox/DS.pm      |   6 +-
 lib/PublicInbox/DSKQXS.pm  | 103 +++++++++++++++++----
 lib/PublicInbox/Daemon.pm  | 183 ++++++++++++++++++-------------------
 lib/PublicInbox/Sigfd.pm   |  63 +++++++++++++
 lib/PublicInbox/Syscall.pm |  42 ++++++++-
 t/ds-kqxs.t                |  28 ++++++
 t/sigfd.t                  |  65 +++++++++++++
 8 files changed, 376 insertions(+), 116 deletions(-)
 create mode 100644 lib/PublicInbox/Sigfd.pm
 create mode 100644 t/sigfd.t

diff --git a/MANIFEST b/MANIFEST
index a50c1246..098e656d 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -141,6 +141,7 @@ lib/PublicInbox/SearchIdxShard.pm
 lib/PublicInbox/SearchMsg.pm
 lib/PublicInbox/SearchThread.pm
 lib/PublicInbox/SearchView.pm
+lib/PublicInbox/Sigfd.pm
 lib/PublicInbox/SolverGit.pm
 lib/PublicInbox/Spamcheck.pm
 lib/PublicInbox/Spamcheck/Spamc.pm
@@ -266,6 +267,7 @@ t/replace.t
 t/reply.t
 t/search-thr-index.t
 t/search.t
+t/sigfd.t
 t/solve/0001-simple-mod.patch
 t/solve/0002-rename-with-modifications.patch
 t/solver_git.t
diff --git a/lib/PublicInbox/DS.pm b/lib/PublicInbox/DS.pm
index 7f7cb85d..17c640f4 100644
--- a/lib/PublicInbox/DS.pm
+++ b/lib/PublicInbox/DS.pm
@@ -53,6 +53,7 @@ our (
      $LoopTimeout,               # timeout of event loop in milliseconds
      $DoneInit,                  # if we've done the one-time module init yet
      @Timers,                    # timers
+     $in_loop,
      );
 
 Reset();
@@ -249,7 +250,7 @@ sub reap_pids {
 sub enqueue_reap ($) { push @$nextq, \&reap_pids };
 
 sub EpollEventLoop {
-    local $SIG{CHLD} = \&enqueue_reap;
+    local $in_loop = 1;
     while (1) {
         my @events;
         my $i;
@@ -628,8 +629,7 @@ sub shutdn ($) {
 # must be called with eval, PublicInbox::DS may not be loaded (see t/qspawn.t)
 sub dwaitpid ($$$) {
     my ($pid, $cb, $arg) = @_;
-    my $chld = $SIG{CHLD};
-    if (defined($chld) && $chld eq \&enqueue_reap) {
+    if ($in_loop) {
         push @$WaitPids, [ $pid, $cb, $arg ];
 
         # We could've just missed our SIGCHLD, cover it, here:
diff --git a/lib/PublicInbox/DSKQXS.pm b/lib/PublicInbox/DSKQXS.pm
index 84e146f8..a56079e2 100644
--- a/lib/PublicInbox/DSKQXS.pm
+++ b/lib/PublicInbox/DSKQXS.pm
@@ -8,18 +8,20 @@
 # like epoll to simplify the code in DS.pm.  This is NOT meant to be
 # an all encompassing emulation of epoll via IO::KQueue, but just to
 # support cases public-inbox-nntpd/httpd care about.
-# A pure-Perl version using syscall() is planned, and it should be
-# faster due to the lack of syscall overhead.
+#
+# It also implements signalfd(2) emulation via "tie".
+#
+# A pure-Perl version using syscall() is planned.
 package PublicInbox::DSKQXS;
 use strict;
 use warnings;
-use parent qw(IO::KQueue);
 use parent qw(Exporter);
+use Symbol qw(gensym);
 use IO::KQueue;
+use Errno qw(EAGAIN);
 use PublicInbox::Syscall qw(EPOLLONESHOT EPOLLIN EPOLLOUT EPOLLET
-	EPOLL_CTL_ADD EPOLL_CTL_MOD EPOLL_CTL_DEL);
+	EPOLL_CTL_ADD EPOLL_CTL_MOD EPOLL_CTL_DEL SFD_NONBLOCK);
 our @EXPORT_OK = qw(epoll_ctl epoll_wait);
-my $owner_pid = -1; # kqueue is close-on-fork (yes, fork, not exec)
 
 sub EV_DISPATCH () { 0x0080 }
 
@@ -41,29 +43,90 @@ sub kq_flag ($$) {
 
 sub new {
 	my ($class) = @_;
-	die 'non-singleton use not supported' if $owner_pid == $$;
-	$owner_pid = $$;
-	$class->SUPER::new;
+	bless { kq => IO::KQueue->new, owner_pid => $$ }, $class;
+}
+
+# returns a new instance which behaves like signalfd on Linux.
+# It's wasteful in that it uses another FD, but it simplifies
+# our epoll-oriented code.
+sub signalfd {
+	my ($class, $signo, $flags) = @_;
+	my $sym = gensym;
+	tie *$sym, $class, $signo, $flags; # calls TIEHANDLE
+	$sym
+}
+
+sub TIEHANDLE { # similar to signalfd()
+	my ($class, $signo, $flags) = @_;
+	my $self = $class->new;
+	$self->{timeout} = ($flags & SFD_NONBLOCK) ? 0 : -1;
+	my $kq = $self->{kq};
+	$kq->EV_SET($_, EVFILT_SIGNAL, EV_ADD) for @$signo;
+	$self;
+}
+
+sub READ { # called by sysread() for signalfd compatibility
+	my ($self, undef, $len, $off) = @_; # $_[1] = buf
+	die "bad args for signalfd read" if ($len % 128) // defined($off);
+	my $timeout = $self->{timeout};
+	my $sigbuf = $self->{sigbuf} //= [];
+	my $nr = $len / 128;
+	my $r = 0;
+	$_[1] = '';
+	do {
+		while ($nr--) {
+			my $signo = shift(@$sigbuf) or last;
+			# caller only cares about signalfd_siginfo.ssi_signo:
+			$_[1] .= pack('L', $signo) . ("\0" x 124);
+			$r += 128;
+		}
+		return $r if $r;
+		my @events = eval { $self->{kq}->kevent($timeout) };
+		# workaround https://rt.cpan.org/Ticket/Display.html?id=116615
+		if ($@) {
+			next if $@ =~ /Interrupted system call/;
+			die;
+		}
+		if (!scalar(@events) && $timeout == 0) {
+			$! = EAGAIN;
+			return;
+		}
+
+		# Grab the kevent.ident (signal number).  The kevent.data
+		# field shows coalesced signals, and maybe we'll use it
+		# in the future...
+		@$sigbuf = map { $_->[0] } @events;
+	} while (1);
 }
 
+# for fileno() calls in PublicInbox::DS
+sub FILENO { ${$_[0]->{kq}} }
+
 sub epoll_ctl {
 	my ($self, $op, $fd, $ev) = @_;
+	my $kq = $self->{kq};
 	if ($op == EPOLL_CTL_MOD) {
-		$self->EV_SET($fd, EVFILT_READ, kq_flag(EPOLLIN, $ev));
-		$self->EV_SET($fd, EVFILT_WRITE, kq_flag(EPOLLOUT, $ev));
+		$kq->EV_SET($fd, EVFILT_READ, kq_flag(EPOLLIN, $ev));
+		$kq->EV_SET($fd, EVFILT_WRITE, kq_flag(EPOLLOUT, $ev));
 	} elsif ($op == EPOLL_CTL_DEL) {
-		$self->EV_SET($fd, EVFILT_READ, EV_DISABLE);
-		$self->EV_SET($fd, EVFILT_WRITE, EV_DISABLE);
-	} else {
-		$self->EV_SET($fd, EVFILT_READ, EV_ADD|kq_flag(EPOLLIN, $ev));
-		$self->EV_SET($fd, EVFILT_WRITE, EV_ADD|kq_flag(EPOLLOUT, $ev));
+		$kq->EV_SET($fd, EVFILT_READ, EV_DISABLE);
+		$kq->EV_SET($fd, EVFILT_WRITE, EV_DISABLE);
+	} else { # EPOLL_CTL_ADD
+		$kq->EV_SET($fd, EVFILT_READ, EV_ADD|kq_flag(EPOLLIN, $ev));
+
+		# we call this blindly for read-only FDs such as tied
+		# DSKQXS (signalfd emulation) and Listeners
+		eval {
+			$kq->EV_SET($fd, EVFILT_WRITE, EV_ADD |
+							kq_flag(EPOLLOUT, $ev));
+		};
 	}
 	0;
 }
 
 sub epoll_wait {
 	my ($self, $maxevents, $timeout_msec, $events) = @_;
-	@$events = eval { $self->kevent($timeout_msec) };
+	@$events = eval { $self->{kq}->kevent($timeout_msec) };
 	if (my $err = $@) {
 		# workaround https://rt.cpan.org/Ticket/Display.html?id=116615
 		if ($err =~ /Interrupted system call/) {
@@ -76,11 +139,13 @@ sub epoll_wait {
 	scalar(@$events);
 }
 
+# kqueue is close-on-fork (not exec), so we must not close it
+# in forked processes:
 sub DESTROY {
 	my ($self) = @_;
-	if ($owner_pid == $$) {
-		POSIX::close($$self);
-		$owner_pid = -1;
+	my $kq = delete $self->{kq} or return;
+	if (delete($self->{owner_pid}) == $$) {
+		POSIX::close($$kq);
 	}
 }
 
diff --git a/lib/PublicInbox/Daemon.pm b/lib/PublicInbox/Daemon.pm
index 0e3b95d2..c7a71ba0 100644
--- a/lib/PublicInbox/Daemon.pm
+++ b/lib/PublicInbox/Daemon.pm
@@ -15,9 +15,11 @@ use Cwd qw/abs_path/;
 STDOUT->autoflush(1);
 STDERR->autoflush(1);
 use PublicInbox::DS qw(now);
+use PublicInbox::Syscall qw(SFD_NONBLOCK);
 require PublicInbox::EvCleanup;
 require PublicInbox::Listener;
 require PublicInbox::ParentPipe;
+require PublicInbox::Sigfd;
 my @CMD;
 my ($set_user, $oldset, $newset);
 my (@cfg_listen, $stdout, $stderr, $group, $user, $pid_file, $daemonize);
@@ -74,12 +76,14 @@ sub accept_tls_opt ($) {
 	{ SSL_server => 1, SSL_startHandshake => 0, SSL_reuse_ctx => $ctx };
 }
 
+sub sig_setmask { sigprocmask(SIG_SETMASK, @_) or die "sigprocmask: $!" }
+
 sub daemon_prepare ($) {
 	my ($default_listen) = @_;
 	$oldset = POSIX::SigSet->new();
 	$newset = POSIX::SigSet->new();
 	$newset->fillset or die "fillset: $!";
-	sigprocmask(SIG_SETMASK, $newset, $oldset) or die "sigprocmask: $!";
+	sig_setmask($newset, $oldset);
 	@CMD = ($0, @ARGV);
 	my %opts = (
 		'l|listen=s' => \@cfg_listen,
@@ -252,30 +256,12 @@ sub daemonize () {
 	}
 }
 
-sub shrink_pipes {
-	if ($^O eq 'linux') { # 1031: F_SETPIPE_SZ, 4096: page size
-		fcntl($_, 1031, 4096) for @_;
-	}
-}
-
-sub worker_quit {
+sub worker_quit { # $_[0] = signal name or number (unused)
 	# killing again terminates immediately:
 	exit unless @listeners;
 
 	$_->close foreach @listeners; # call PublicInbox::DS::close
 	@listeners = ();
-
-	# create a lazy self-pipe which kicks us out of the EventLoop
-	# so DS::PostEventLoop can fire
-	if (pipe(my ($r, $w))) {
-		shrink_pipes($w);
-
-		# shrink_pipes == noop
-		PublicInbox::ParentPipe->new($r, *shrink_pipes);
-		close $w; # wake up from the event loop
-	} else {
-		warn "E: pipe failed ($!), quit unreliable\n";
-	}
 	my $proc_name;
 	my $warn = 0;
 	# drop idle connections and try to quit gracefully
@@ -398,7 +384,7 @@ processes when multiple service instances start.
 	@rv
 }
 
-sub upgrade () {
+sub upgrade { # $_[0] = signal name or number (unused)
 	if ($reexec_pid) {
 		warn "upgrade in-progress: $reexec_pid\n";
 		return;
@@ -453,7 +439,7 @@ sub upgrade_aborted ($) {
 	warn $@, "\n" if $@;
 }
 
-sub reap_children () {
+sub reap_children { # $_[0] = 'CHLD' or POSIX::SIGCHLD()
 	while (1) {
 		my $p = waitpid(-1, WNOHANG) or return;
 		if (defined $reexec_pid && $p == $reexec_pid) {
@@ -483,60 +469,50 @@ sub unlink_pid_file_safe_ish ($$) {
 
 sub master_loop {
 	pipe(my ($p0, $p1)) or die "failed to create parent-pipe: $!";
-	pipe(my ($r, $w)) or die "failed to create self-pipe: $!";
-	shrink_pipes($w, $p1);
-
-	IO::Handle::blocking($w, 0);
+	# 1031: F_SETPIPE_SZ, 4096: page size
+	fcntl($p1, 1031, 4096) if $^O eq 'linux';
 	my $set_workers = $worker_processes;
-	my @caught;
-	my $master_pid = $$;
-	foreach my $s (qw(HUP CHLD QUIT INT TERM USR1 USR2 TTIN TTOU WINCH)) {
-		$SIG{$s} = sub {
-			return if $$ != $master_pid;
-			push @caught, $s;
-			syswrite($w, '.');
-		};
-	}
-	sigprocmask(SIG_SETMASK, $oldset) or die "sigprocmask: $!";
 	reopen_logs();
-	# main loop
 	my $quit = 0;
-	while (1) {
-		while (my $s = shift @caught) {
-			if ($s eq 'USR1') {
-				reopen_logs();
-				kill_workers($s);
-			} elsif ($s eq 'USR2') {
-				upgrade();
-			} elsif ($s =~ /\A(?:QUIT|TERM|INT)\z/) {
-				exit if $quit++;
-				kill_workers($s);
-			} elsif ($s eq 'WINCH') {
-				if (-t STDIN || -t STDOUT || -t STDERR) {
-					warn
-"ignoring SIGWINCH since we are not daemonized\n";
-					$SIG{WINCH} = 'IGNORE';
-				} else {
-					$worker_processes = 0;
-				}
-			} elsif ($s eq 'HUP') {
-				$worker_processes = $set_workers;
-				kill_workers($s);
-			} elsif ($s eq 'TTIN') {
-				if ($set_workers > $worker_processes) {
-					++$worker_processes;
-				} else {
-					$worker_processes = ++$set_workers;
-				}
-			} elsif ($s eq 'TTOU') {
-				if ($set_workers > 0) {
-					$worker_processes = --$set_workers;
-				}
-			} elsif ($s eq 'CHLD') {
-				reap_children();
+	my $ignore_winch;
+	my $quit_cb = sub { exit if $quit++; kill_workers($_[0]) };
+	my $sig = {
+		USR1 => sub { reopen_logs(); kill_workers($_[0]); },
+		USR2 => \&upgrade,
+		QUIT => $quit_cb,
+		INT => $quit_cb,
+		TERM => $quit_cb,
+		WINCH => sub {
+			return if $ignore_winch;
+			if (-t STDIN || -t STDOUT || -t STDERR) {
+				$ignore_winch = 1;
+				warn <<EOF;
+ignoring SIGWINCH since we are not daemonized
+EOF
+			} else {
+				$worker_processes = 0;
 			}
-		}
-
+		},
+		HUP => sub {
+			$worker_processes = $set_workers;
+			kill_workers($_[0]);
+		},
+		TTIN => sub {
+			if ($set_workers > $worker_processes) {
+				++$worker_processes;
+			} else {
+				$worker_processes = ++$set_workers;
+			}
+		},
+		TTOU => sub {
+			$worker_processes = --$set_workers if $set_workers > 0;
+		},
+		CHLD => \&reap_children,
+	};
+	my $sigfd = PublicInbox::Sigfd->new($sig, 0);
+	local %SIG = (%SIG, %$sig) if !$sigfd;
+	sig_setmask($oldset) if !$sigfd;
+	while (1) { # main loop
 		my $n = scalar keys %pids;
 		if ($quit) {
 			exit if $n == 0;
@@ -549,22 +525,29 @@ sub master_loop {
 			}
 			$n = $worker_processes;
 		}
-		sigprocmask(SIG_SETMASK, $newset) or die "sigprocmask: $!";
-		foreach my $i ($n..($worker_processes - 1)) {
-			my $pid = fork;
-			if (!defined $pid) {
-				warn "failed to fork worker[$i]: $!\n";
-			} elsif ($pid == 0) {
-				$set_user->() if $set_user;
-				return $p0; # run normal work code
-			} else {
-				warn "PID=$pid is worker[$i]\n";
-				$pids{$pid} = $i;
+		my $want = $worker_processes - 1;
+		if ($n <= $want) {
+			sig_setmask($newset) if !$sigfd;
+			for my $i ($n..$want) {
+				my $pid = fork;
+				if (!defined $pid) {
+					warn "failed to fork worker[$i]: $!\n";
+				} elsif ($pid == 0) {
+					$set_user->() if $set_user;
+					return $p0; # run normal work code
+				} else {
+					warn "PID=$pid is worker[$i]\n";
+					$pids{$pid} = $i;
+				}
 			}
+			sig_setmask($oldset) if !$sigfd;
+		}
+
+		if ($sigfd) { # Linux and IO::KQueue users:
+			$sigfd->wait_once;
+		} else { # wake up every second
+			sleep(1);
 		}
-		sigprocmask(SIG_SETMASK, $oldset) or die "sigprocmask: $!";
-		# just wait on signal events here:
-		sysread($r, my $buf, 8);
 	}
 	exit # never gets here, just for documentation
 }
@@ -606,6 +589,18 @@ sub daemon_loop ($$$$) {
 			$nntpd->{accept_tls} = $v;
 		}
 	}
+	my $sig = {
+		HUP => $refresh,
+		INT => \&worker_quit,
+		QUIT => \&worker_quit,
+		TERM => \&worker_quit,
+		TTIN => 'IGNORE',
+		TTOU => 'IGNORE',
+		USR1 => \&reopen_logs,
+		USR2 => 'IGNORE',
+		WINCH => 'IGNORE',
+		CHLD => \&PublicInbox::DS::enqueue_reap,
+	};
 	my $parent_pipe;
 	if ($worker_processes > 0) {
 		$refresh->(); # preload by default
@@ -614,16 +609,11 @@ sub daemon_loop ($$$$) {
 	} else {
 		reopen_logs();
 		$set_user->() if $set_user;
-		$SIG{USR2} = sub { worker_quit() if upgrade() };
+		$sig->{USR2} = sub { worker_quit() if upgrade() };
 		$refresh->();
 	}
 	$uid = $gid = undef;
 	reopen_logs();
-	$SIG{QUIT} = $SIG{INT} = $SIG{TERM} = *worker_quit;
-	$SIG{USR1} = *reopen_logs;
-	$SIG{HUP} = $refresh;
-	$SIG{CHLD} = 'DEFAULT';
-	$SIG{$_} = 'IGNORE' for qw(USR2 TTIN TTOU WINCH);
 	@listeners = map {;
 		my $tls_cb = $post_accept{sockname($_)};
 
@@ -634,7 +624,14 @@ sub daemon_loop ($$$$) {
 		# this calls epoll_create:
 		PublicInbox::Listener->new($_, $tls_cb || $post_accept)
 	} @listeners;
-	sigprocmask(SIG_SETMASK, $oldset) or die "sigprocmask: $!";
+	my $sigfd = PublicInbox::Sigfd->new($sig, SFD_NONBLOCK);
+	local %SIG = (%SIG, %$sig) if !$sigfd;
+	if (!$sigfd) {
+		# wake up every second to accept signals if we don't
+		# have signalfd or IO::KQueue:
+		sig_setmask($oldset);
+		PublicInbox::DS->SetLoopTimeout(1000);
+	}
 	PublicInbox::DS->EventLoop;
 	$parent_pipe = undef;
 }
diff --git a/lib/PublicInbox/Sigfd.pm b/lib/PublicInbox/Sigfd.pm
new file mode 100644
index 00000000..ec5d7145
--- /dev/null
+++ b/lib/PublicInbox/Sigfd.pm
@@ -0,0 +1,63 @@
+# Copyright (C) 2019 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+package PublicInbox::Sigfd;
+use strict;
+use parent qw(PublicInbox::DS);
+use fields qw(sig); # hashref similar to %SIG, but signal numbers as keys
+use PublicInbox::Syscall qw(signalfd EPOLLIN EPOLLET SFD_NONBLOCK);
+use POSIX ();
+use IO::Handle ();
+
+# returns a coderef to unblock signals if neither signalfd or kqueue
+# are available.
+sub new {
+	my ($class, $sig, $flags) = @_;
+	my $self = fields::new($class);
+	my %signo = map {;
+		my $cb = $sig->{$_};
+		my $num = ($_ eq 'WINCH' && $^O =~ /linux|bsd/i) ? 28 : do {
+			my $m = "SIG$_";
+			POSIX->$m;
+		};
+		$num => $cb;
+	} keys %$sig;
+	my $io;
+	my $fd = signalfd(-1, [keys %signo], $flags);
+	if (defined $fd && $fd >= 0) {
+		$io = IO::Handle->new_from_fd($fd, 'r+');
+	} elsif (eval { require PublicInbox::DSKQXS }) {
+		$io = PublicInbox::DSKQXS->signalfd([keys %signo], $flags);
+	} else {
+		return; # wake up every second to check for signals
+	}
+	if ($flags & SFD_NONBLOCK) { # it can go into the event loop
+		$self->SUPER::new($io, EPOLLIN | EPOLLET);
+	} else { # master main loop
+		$self->{sock} = $io;
+	}
+	$self->{sig} = \%signo;
+	$self;
+}
+
+# PublicInbox::Daemon in master main loop (blocking)
+sub wait_once ($) {
+	my ($self) = @_;
+	my $r = sysread($self->{sock}, my $buf, 128 * 64);
+	if (defined($r)) {
+		while (1) {
+			my $sig = unpack('L', $buf);
+			my $cb = $self->{sig}->{$sig};
+			$cb->($sig) if $cb ne 'IGNORE';
+			return $r if length($buf) == 128;
+			$buf = substr($buf, 128);
+		}
+	}
+	$r;
+}
+
+# called by PublicInbox::DS in epoll_wait loop
+sub event_step {
+	while (wait_once($_[0])) {} # non-blocking
+}
+
+1;
diff --git a/lib/PublicInbox/Syscall.pm b/lib/PublicInbox/Syscall.pm
index da8a6c86..487013d5 100644
--- a/lib/PublicInbox/Syscall.pm
+++ b/lib/PublicInbox/Syscall.pm
@@ -24,7 +24,8 @@ $VERSION     = "0.25";
 @EXPORT_OK   = qw(epoll_ctl epoll_create epoll_wait
                   EPOLLIN EPOLLOUT EPOLLET
                   EPOLL_CTL_ADD EPOLL_CTL_DEL EPOLL_CTL_MOD
-                  EPOLLONESHOT EPOLLEXCLUSIVE);
+                  EPOLLONESHOT EPOLLEXCLUSIVE
+                  signalfd SFD_NONBLOCK);
 %EXPORT_TAGS = (epoll => [qw(epoll_ctl epoll_create epoll_wait
                              EPOLLIN EPOLLOUT
                              EPOLL_CTL_ADD EPOLL_CTL_DEL EPOLL_CTL_MOD
@@ -42,6 +43,11 @@ use constant EPOLLET => (1 << 31);
 use constant EPOLL_CTL_ADD => 1;
 use constant EPOLL_CTL_DEL => 2;
 use constant EPOLL_CTL_MOD => 3;
+use constant {
+	SFD_CLOEXEC => 02000000,
+	SFD_NONBLOCK => 00004000,
+};
+
 
 our $loaded_syscall = 0;
 
@@ -63,6 +69,7 @@ our (
      $SYS_epoll_create,
      $SYS_epoll_ctl,
      $SYS_epoll_wait,
+     $SYS_signalfd4,
      );
 
 our $no_deprecated = 0;
@@ -88,63 +95,75 @@ if ($^O eq "linux") {
         $SYS_epoll_create = 254;
         $SYS_epoll_ctl    = 255;
         $SYS_epoll_wait   = 256;
+        $SYS_signalfd4 = 327;
     } elsif ($machine eq "x86_64") {
         $SYS_epoll_create = 213;
         $SYS_epoll_ctl    = 233;
         $SYS_epoll_wait   = 232;
+        $SYS_signalfd4 = 289;
     } elsif ($machine =~ m/^parisc/) {
         $SYS_epoll_create = 224;
         $SYS_epoll_ctl    = 225;
         $SYS_epoll_wait   = 226;
         $u64_mod_8        = 1;
+        $SYS_signalfd4 = 309;
     } elsif ($machine =~ m/^ppc64/) {
         $SYS_epoll_create = 236;
         $SYS_epoll_ctl    = 237;
         $SYS_epoll_wait   = 238;
         $u64_mod_8        = 1;
+        $SYS_signalfd4 = 313;
     } elsif ($machine eq "ppc") {
         $SYS_epoll_create = 236;
         $SYS_epoll_ctl    = 237;
         $SYS_epoll_wait   = 238;
         $u64_mod_8        = 1;
+        $SYS_signalfd4 = 313;
     } elsif ($machine =~ m/^s390/) {
         $SYS_epoll_create = 249;
         $SYS_epoll_ctl    = 250;
         $SYS_epoll_wait   = 251;
         $u64_mod_8        = 1;
+        $SYS_signalfd4 = 322;
     } elsif ($machine eq "ia64") {
         $SYS_epoll_create = 1243;
         $SYS_epoll_ctl    = 1244;
         $SYS_epoll_wait   = 1245;
         $u64_mod_8        = 1;
+        $SYS_signalfd4 = 289;
     } elsif ($machine eq "alpha") {
         # natural alignment, ints are 32-bits
         $SYS_epoll_create = 407;
         $SYS_epoll_ctl    = 408;
         $SYS_epoll_wait   = 409;
         $u64_mod_8        = 1;
+        $SYS_signalfd4 = 484;
     } elsif ($machine eq "aarch64") {
         $SYS_epoll_create = 20;  # (sys_epoll_create1)
         $SYS_epoll_ctl    = 21;
         $SYS_epoll_wait   = 22;  # (sys_epoll_pwait)
         $u64_mod_8        = 1;
         $no_deprecated    = 1;
+        $SYS_signalfd4 = 74;
     } elsif ($machine =~ m/arm(v\d+)?.*l/) {
         # ARM OABI
         $SYS_epoll_create = 250;
         $SYS_epoll_ctl    = 251;
         $SYS_epoll_wait   = 252;
         $u64_mod_8        = 1;
+        $SYS_signalfd4 = 355;
     } elsif ($machine =~ m/^mips64/) {
         $SYS_epoll_create = 5207;
         $SYS_epoll_ctl    = 5208;
         $SYS_epoll_wait   = 5209;
         $u64_mod_8        = 1;
+        $SYS_signalfd4 = 5283;
     } elsif ($machine =~ m/^mips/) {
         $SYS_epoll_create = 4248;
         $SYS_epoll_ctl    = 4249;
         $SYS_epoll_wait   = 4250;
         $u64_mod_8        = 1;
+        $SYS_signalfd4 = 4324;
     } else {
         # as a last resort, try using the *.ph files which may not
         # exist or may be wrong
@@ -152,6 +171,11 @@ if ($^O eq "linux") {
         $SYS_epoll_create = eval { &SYS_epoll_create; } || 0;
         $SYS_epoll_ctl    = eval { &SYS_epoll_ctl;    } || 0;
         $SYS_epoll_wait   = eval { &SYS_epoll_wait;   } || 0;
+
+	# Note: do NOT add new syscalls to depend on *.ph, here.
+	# Better to miss syscalls (so we can fallback to IO::Poll)
+	# than to use wrong ones, since the names are not stable
+	# (at least not on FreeBSD), if the actual numbers are.
     }
 
     if ($u64_mod_8) {
@@ -228,6 +252,22 @@ sub epoll_wait_mod8 {
     return $ct;
 }
 
+sub signalfd ($$$) {
+	my ($fd, $signos, $flags) = @_;
+	if ($SYS_signalfd4) {
+		# Not sure if there's a way to get pack/unpack to get the
+		# contents of POSIX::SigSet to a buffer, but prepping the
+		# bitmap like one would for select() works:
+		my $buf = "\0" x 8;
+		vec($buf, $_ - 1, 1) = 1 for @$signos;
+
+		syscall($SYS_signalfd4, $fd, $buf, 8, $flags|SFD_CLOEXEC);
+	} else {
+		$! = ENOSYS;
+		undef;
+	}
+}
+
 1;
 
 =head1 WARRANTY
diff --git a/t/ds-kqxs.t b/t/ds-kqxs.t
index 785570c3..43b6333f 100644
--- a/t/ds-kqxs.t
+++ b/t/ds-kqxs.t
@@ -10,5 +10,33 @@ unless (eval { require IO::KQueue }) {
 				: "no IO::KQueue, skipping $0: $@";
 	plan skip_all => $m;
 }
+
+if ('ensure nested kqueue works for signalfd emulation') {
+	require POSIX;
+	my $new = POSIX::SigSet->new(POSIX::SIGHUP());
+	my $old = POSIX::SigSet->new;
+	my $hup = 0;
+	local $SIG{HUP} = sub { $hup++ };
+	POSIX::sigprocmask(POSIX::SIG_SETMASK(), $new, $old) or die;
+	my $kqs = IO::KQueue->new or die;
+	$kqs->EV_SET(POSIX::SIGHUP(), IO::KQueue::EVFILT_SIGNAL(),
+			IO::KQueue::EV_ADD());
+	kill('HUP', $$) or die;
+	my @events = $kqs->kevent(3000);
+	is(scalar(@events), 1, 'got one event');
+	is($events[0]->[0], POSIX::SIGHUP(), 'got SIGHUP');
+	my $parent = IO::KQueue->new or die;
+	my $kqfd = $$kqs;
+	$parent->EV_SET($kqfd, IO::KQueue::EVFILT_READ(), IO::KQueue::EV_ADD());
+	kill('HUP', $$) or die;
+	@events = $parent->kevent(3000);
+	is(scalar(@events), 1, 'got one event');
+	is($events[0]->[0], $kqfd, 'got kqfd');
+	is($hup, 0, '$SIG{HUP} did not fire');
+	POSIX::sigprocmask(POSIX::SIG_SETMASK(), $old) or die;
+	defined(POSIX::close($kqfd)) or die;
+	defined(POSIX::close($$parent)) or die;
+}
+
 local $ENV{TEST_IOPOLLER} = 'PublicInbox::DSKQXS';
 require './t/ds-poll.t';
diff --git a/t/sigfd.t b/t/sigfd.t
new file mode 100644
index 00000000..34f30de8
--- /dev/null
+++ b/t/sigfd.t
@@ -0,0 +1,65 @@
+# Copyright (C) 2019 all contributors <meta@public-inbox.org>
+use strict;
+use Test::More;
+use IO::Handle;
+use POSIX qw(:signal_h);
+use Errno qw(ENOSYS);
+use PublicInbox::Syscall qw(SFD_NONBLOCK);
+require_ok 'PublicInbox::Sigfd';
+
+SKIP: {
+	if ($^O ne 'linux' && !eval { require IO::KQueue }) {
+		skip 'signalfd requires Linux or IO::KQueue to emulate', 10;
+	}
+	my $new = POSIX::SigSet->new;
+	$new->fillset or die "sigfillset: $!";
+	my $old = POSIX::SigSet->new;
+	sigprocmask(SIG_SETMASK, $new, $old) or die "sigprocmask $!";
+	my $hit = {};
+	my $sig = {};
+	local $SIG{HUP} = sub { $hit->{HUP}->{normal}++ };
+	local $SIG{TERM} = sub { $hit->{TERM}->{normal}++ };
+	local $SIG{INT} = sub { $hit->{INT}->{normal}++ };
+	for my $s (qw(HUP TERM INT)) {
+		$sig->{$s} = sub { $hit->{$s}->{sigfd}++ };
+	}
+	my $sigfd = PublicInbox::Sigfd->new($sig, 0);
+	if ($sigfd) {
+		require PublicInbox::DS;
+		ok($sigfd, 'Sigfd->new works');
+		kill('HUP', $$) or die "kill $!";
+		kill('INT', $$) or die "kill $!";
+		my $fd = fileno($sigfd->{sock});
+		ok($fd >= 0, 'fileno(Sigfd->{sock}) works');
+		my $rvec = '';
+		vec($rvec, $fd, 1) = 1;
+		is(select($rvec, undef, undef, undef), 1, 'select() works');
+		ok($sigfd->wait_once, 'wait_once reported success');
+		for my $s (qw(HUP INT)) {
+			is($hit->{$s}->{sigfd}, 1, "sigfd fired $s");
+			is($hit->{$s}->{normal}, undef,
+				'normal $SIG{$s} not fired');
+		}
+		$sigfd = undef;
+
+		my $nbsig = PublicInbox::Sigfd->new($sig, SFD_NONBLOCK);
+		ok($nbsig, 'Sigfd->new SFD_NONBLOCK works');
+		is($nbsig->wait_once, undef, 'nonblocking ->wait_once');
+		ok($! == Errno::EAGAIN, 'got EAGAIN');
+		kill('HUP', $$) or die "kill $!";
+		PublicInbox::DS->SetPostLoopCallback(sub {}); # loop once
+		PublicInbox::DS->EventLoop;
+		is($hit->{HUP}->{sigfd}, 2, 'HUP sigfd fired in event loop');
+		kill('TERM', $$) or die "kill $!";
+		kill('HUP', $$) or die "kill $!";
+		PublicInbox::DS->EventLoop;
+		PublicInbox::DS->Reset;
+		is($hit->{TERM}->{sigfd}, 1, 'TERM sigfd fired in event loop');
+		is($hit->{HUP}->{sigfd}, 3, 'HUP sigfd fired in event loop');
+	} else {
+		skip('signalfd disabled?', 10);
+	}
+	sigprocmask(SIG_SETMASK, $old) or die "sigprocmask $!";
+}
+
+done_testing;

[PATCH 11/17] t/common: start_script replaces spawn_listener

[Eric Wong]

We can shave several hundred milliseconds off tests which spawn
daemons by preloading and avoiding startup time for common
modules which are already loaded in the parent process.

This also gives ENV{TAIL} support to all tests which support
daemons which log to stdout/stderr.
---
 t/common.perl        | 174 +++++++++++++++++++++++++++++++------------
 t/git-http-backend.t |  14 ++--
 t/httpd-corner.t     |  48 +++++++-----
 t/httpd-https.t      |  28 ++-----
 t/httpd-unix.t       |  47 ++++++------
 t/httpd.t            |  13 ++--
 t/nntpd-tls.t        |  29 ++------
 t/nntpd-validate.t   |  27 +++----
 t/nntpd.t            |  16 ++--
 t/perf-nntpd.t       |  22 +++---
 t/v2mirror.t         |  21 ++----
 t/v2writable.t       |   8 +-
 t/www_listing.t      |   8 +-
 13 files changed, 234 insertions(+), 221 deletions(-)

diff --git a/t/common.perl b/t/common.perl
index c5693080..2126a761 100644
--- a/t/common.perl
+++ b/t/common.perl
@@ -30,30 +30,6 @@ sub tcp_connect {
 	$s;
 }
 
-sub spawn_listener {
-	my ($env, $cmd, $socks) = @_;
-	my $pid = fork;
-	defined $pid or die "fork failed: $!\n";
-	if ($pid == 0) {
-		# pretend to be systemd (cf. sd_listen_fds(3))
-		my $fd = 3; # 3 == SD_LISTEN_FDS_START
-		foreach my $s (@$socks) {
-			my $fl = fcntl($s, F_GETFD, 0);
-			if (($fl & FD_CLOEXEC) != FD_CLOEXEC) {
-				warn "got FD:".fileno($s)." w/o CLOEXEC\n";
-			}
-			fcntl($s, F_SETFD, $fl &= ~FD_CLOEXEC);
-			dup2(fileno($s), $fd++) or die "dup2 failed: $!\n";
-		}
-		$ENV{LISTEN_PID} = $$;
-		$ENV{LISTEN_FDS} = scalar @$socks;
-		%ENV = (%ENV, %$env) if $env;
-		exec @$cmd;
-		die "FAIL: ",join(' ', @$cmd), ": $!\n";
-	}
-	$pid;
-}
-
 sub require_git ($;$) {
 	my ($req, $maybe) = @_;
 	my ($req_maj, $req_min) = split(/\./, $req);
@@ -68,7 +44,6 @@ sub require_git ($;$) {
 	1;
 }
 
-my %cached_scripts;
 sub key2script ($) {
 	my ($key) = @_;
 	return $key if $key =~ m!\A/!;
@@ -105,11 +80,10 @@ sub run_script_exit (;$) {
 	die RUN_SCRIPT_EXIT;
 }
 
-sub run_script ($;$$) {
-	my ($cmd, $env, $opt) = @_;
-	my ($key, @argv) = @$cmd;
-	my $run_mode = $ENV{TEST_RUN_MODE} // $opt->{run_mode} // 1;
-	my $sub = $run_mode == 0 ? undef : ($cached_scripts{$key} //= do {
+my %cached_scripts;
+sub key2sub ($) {
+	my ($key) = @_;
+	$cached_scripts{$key} //= do {
 		my $f = key2script($key);
 		open my $fh, '<', $f or die "open $f: $!";
 		my $str = do { local $/; <$fh> };
@@ -129,8 +103,34 @@ $str
 1;
 EOF
 		$pkg->can('main');
-	}); # do
+	}
+}
 
+sub _run_sub ($$$) {
+	my ($sub, $key, $argv) = @_;
+	local @ARGV = @$argv;
+	$run_script_exit_code = undef;
+	my $exit_code = eval { $sub->(@$argv) };
+	if ($@ eq RUN_SCRIPT_EXIT) {
+		$@ = '';
+		$exit_code = $run_script_exit_code;
+		$? = ($exit_code << 8);
+	} elsif (defined($exit_code)) {
+		$? = ($exit_code << 8);
+	} elsif ($@) { # mimic die() behavior when uncaught
+		warn "E: eval-ed $key: $@\n";
+		$? = ($! << 8) if $!;
+		$? = (255 << 8) if $? == 0;
+	} else {
+		die "BUG: eval-ed $key: no exit code or \$@\n";
+	}
+}
+
+sub run_script ($;$$) {
+	my ($cmd, $env, $opt) = @_;
+	my ($key, @argv) = @$cmd;
+	my $run_mode = $ENV{TEST_RUN_MODE} // $opt->{run_mode} // 1;
+	my $sub = $run_mode == 0 ? undef : key2sub($key);
 	my $fhref = [];
 	my $spawn_opt = {};
 	for my $fd (0..2) {
@@ -162,22 +162,7 @@ EOF
 		local %ENV = $env ? (%ENV, %$env) : %ENV;
 		local %SIG = %SIG;
 		_prepare_redirects($fhref);
-		local @ARGV = @argv;
-		$run_script_exit_code = undef;
-		my $exit_code = eval { $sub->(@argv) };
-		if ($@ eq RUN_SCRIPT_EXIT) {
-			$@ = '';
-			$exit_code = $run_script_exit_code;
-			$? = ($exit_code << 8);
-		} elsif (defined($exit_code)) {
-			$? = ($exit_code << 8);
-		} elsif ($@) { # mimic die() behavior when uncaught
-			warn "E: eval-ed $key: $@\n";
-			$? = ($! << 8) if $!;
-			$? = (255 << 8) if $? == 0;
-		} else {
-			die "BUG: eval-ed $key: no exit code or \$@\n";
-		}
+		_run_sub($sub, $key, \@argv);
 	}
 
 	# slurp the redirects back into user-supplied strings
@@ -191,4 +176,99 @@ EOF
 	$? == 0;
 }
 
+sub wait_for_tail () { sleep(2) }
+
+sub start_script {
+	my ($cmd, $env, $opt) = @_;
+	my ($key, @argv) = @$cmd;
+	my $run_mode = $ENV{TEST_RUN_MODE} // $opt->{run_mode} // 1;
+	my $sub = $run_mode == 0 ? undef : key2sub($key);
+	my $tail_pid;
+	if (my $tail_cmd = $ENV{TAIL}) {
+		my @paths;
+		for (@argv) {
+			next unless /\A--std(?:err|out)=(.+)\z/;
+			push @paths, $1;
+		}
+		if (@paths) {
+			defined($tail_pid = fork) or die "fork: $!\n";
+			if ($tail_pid == 0) {
+				# make sure files exist, first
+				open my $fh, '>>', $_ for @paths;
+				open(STDOUT, '>&STDERR') or die "1>&2: $!";
+				exec(split(' ', $tail_cmd), @paths);
+				die "$tail_cmd failed: $!";
+			}
+			wait_for_tail();
+		}
+	}
+	defined(my $pid = fork) or die "fork: $!\n";
+	if ($pid == 0) {
+		# pretend to be systemd (cf. sd_listen_fds(3))
+		# 3 == SD_LISTEN_FDS_START
+		my $fd;
+		for ($fd = 0; 1; $fd++) {
+			my $s = $opt->{$fd};
+			last if $fd >= 3 && !defined($s);
+			next unless $s;
+			my $fl = fcntl($s, F_GETFD, 0);
+			if (($fl & FD_CLOEXEC) != FD_CLOEXEC) {
+				warn "got FD:".fileno($s)." w/o CLOEXEC\n";
+			}
+			fcntl($s, F_SETFD, $fl &= ~FD_CLOEXEC);
+			dup2(fileno($s), $fd) or die "dup2 failed: $!\n";
+		}
+		%ENV = (%ENV, %$env) if $env;
+		my $fds = $fd - 3;
+		if ($fds > 0) {
+			$ENV{LISTEN_PID} = $$;
+			$ENV{LISTEN_FDS} = $fds;
+		}
+		$0 = join(' ', @$cmd);
+		if ($sub) {
+			_run_sub($sub, $key, \@argv);
+			POSIX::_exit($? >> 8);
+		} else {
+			exec(key2script($key), @argv);
+			die "FAIL: ",join(' ', $key, @argv), ": $!\n";
+		}
+	}
+	TestProcess->new($pid, $tail_pid);
+}
+
+package TestProcess;
+use strict;
+
+# prevent new threads from inheriting these objects
+sub CLONE_SKIP { 1 }
+
+sub new {
+	my ($klass, $pid, $tail_pid) = @_;
+	bless { pid => $pid, tail_pid => $tail_pid, owner => $$ }, $klass;
+}
+
+sub kill {
+	my ($self, $sig) = @_;
+	CORE::kill($sig // 'TERM', $self->{pid});
+}
+
+sub join {
+	my ($self) = @_;
+	my $pid = delete $self->{pid} or return;
+	my $ret = waitpid($pid, 0);
+	defined($ret) or die "waitpid($pid): $!";
+	$ret == $pid or die "waitpid($pid) != $ret";
+}
+
+sub DESTROY {
+	my ($self) = @_;
+	return if $self->{owner} != $$;
+	if (my $tail = delete $self->{tail_pid}) {
+		::wait_for_tail();
+		CORE::kill('TERM', $tail);
+	}
+	my $pid = delete $self->{pid} or return;
+	CORE::kill('TERM', $pid);
+}
+
 1;
diff --git a/t/git-http-backend.t b/t/git-http-backend.t
index c2a04653..c4dc09a1 100644
--- a/t/git-http-backend.t
+++ b/t/git-http-backend.t
@@ -22,12 +22,10 @@ my $psgi = "./t/git-http-backend.psgi";
 my $tmpdir = tempdir('pi-git-http-backend-XXXXXX', TMPDIR => 1, CLEANUP => 1);
 my $err = "$tmpdir/stderr.log";
 my $out = "$tmpdir/stdout.log";
-my $httpd = 'blib/script/public-inbox-httpd';
 my $sock = tcp_server();
 my $host = $sock->sockhost;
 my $port = $sock->sockport;
-my $pid;
-END { kill 'TERM', $pid if defined $pid };
+my $td;
 
 my $get_maxrss = sub {
         my $http = Net::HTTP->new(Host => "$host:$port");
@@ -44,9 +42,8 @@ my $get_maxrss = sub {
 
 {
 	ok($sock, 'sock created');
-	my $cmd = [ $httpd, '-W0', "--stdout=$out", "--stderr=$err", $psgi ];
-	ok(defined($pid = spawn_listener(undef, $cmd, [$sock])),
-	   'forked httpd process successfully');
+	my $cmd = [ '-httpd', '-W0', "--stdout=$out", "--stderr=$err", $psgi ];
+	$td = start_script($cmd, undef, { 3 => $sock });
 }
 my $mem_a = $get_maxrss->();
 
@@ -113,9 +110,8 @@ SKIP: {
 }
 
 {
-	ok(kill('TERM', $pid), 'killed httpd');
-	$pid = undef;
-	waitpid(-1, 0);
+	ok($td->kill, 'killed httpd');
+	$td->join;
 }
 
 done_testing();
diff --git a/t/httpd-corner.t b/t/httpd-corner.t
index cc36c7e1..eca77d7f 100644
--- a/t/httpd-corner.t
+++ b/t/httpd-corner.t
@@ -26,7 +26,6 @@ my $fifo = "$tmpdir/fifo";
 ok(defined mkfifo($fifo, 0777), 'created FIFO');
 my $err = "$tmpdir/stderr.log";
 my $out = "$tmpdir/stdout.log";
-my $httpd = 'blib/script/public-inbox-httpd';
 my $psgi = "./t/httpd-corner.psgi";
 my $sock = tcp_server() or die;
 
@@ -64,13 +63,11 @@ sub unix_server ($) {
 my $upath = "$tmpdir/s";
 my $unix = unix_server($upath);
 ok($unix, 'UNIX socket created');
-my $pid;
-END { kill 'TERM', $pid if defined $pid };
+my $td;
 my $spawn_httpd = sub {
 	my (@args) = @_;
-	my $cmd = [ $httpd, @args, "--stdout=$out", "--stderr=$err", $psgi ];
-	$pid = spawn_listener(undef, $cmd, [ $sock, $unix ]);
-	ok(defined $pid, 'forked httpd process successfully');
+	my $cmd = [ '-httpd', @args, "--stdout=$out", "--stderr=$err", $psgi ];
+	$td = start_script($cmd, undef, { 3 => $sock, 4 => $unix });
 };
 
 $spawn_httpd->();
@@ -213,16 +210,14 @@ sub conn_for {
 	open my $f, '>', $fifo or die "open $fifo: $!\n";
 	$f->autoflush(1);
 	ok(print($f "hello\n"), 'wrote something to fifo');
-	my $kpid = $pid;
-	$pid = undef;
-	is(kill('TERM', $kpid), 1, 'started graceful shutdown');
+	is($td->kill, 1, 'started graceful shutdown');
 	ok(print($f "world\n"), 'wrote else to fifo');
 	close $f or die "close fifo: $!\n";
 	$conn->read(my $buf, 8192);
 	my ($head, $body) = split(/\r\n\r\n/, $buf, 2);
 	like($head, qr!\AHTTP/1\.[01] 200 OK!, 'got 200 for slow-header');
 	is($body, "hello\nworld\n", 'read expected body');
-	is(waitpid($kpid, 0), $kpid, 'reaped httpd');
+	$td->join;
 	is($?, 0, 'no error');
 	$spawn_httpd->('-W0');
 }
@@ -244,15 +239,13 @@ sub conn_for {
 		$conn->sysread($buf, 8192);
 		is($buf, $c, 'got trickle for reading');
 	}
-	my $kpid = $pid;
-	$pid = undef;
-	is(kill('TERM', $kpid), 1, 'started graceful shutdown');
+	is($td->kill, 1, 'started graceful shutdown');
 	ok(print($f "world\n"), 'wrote else to fifo');
 	close $f or die "close fifo: $!\n";
 	$conn->sysread($buf, 8192);
 	is($buf, "world\n", 'read expected body');
 	is($conn->sysread($buf, 8192), 0, 'got EOF from server');
-	is(waitpid($kpid, 0), $kpid, 'reaped httpd');
+	$td->join;
 	is($?, 0, 'no error');
 	$spawn_httpd->('-W0');
 }
@@ -346,9 +339,7 @@ SKIP: {
 	$conn->write("Content-Length: $len\r\n");
 	delay();
 	$conn->write("\r\n");
-	my $kpid = $pid;
-	$pid = undef;
-	is(kill('TERM', $kpid), 1, 'started graceful shutdown');
+	is($td->kill, 1, 'started graceful shutdown');
 	delay();
 	my $n = 0;
 	foreach my $c ('a'..'z') {
@@ -356,7 +347,7 @@ SKIP: {
 	}
 	is($n, $len, 'wrote alphabet');
 	$check_self->($conn);
-	is(waitpid($kpid, 0), $kpid, 'reaped httpd');
+	$td->join;
 	is($?, 0, 'no error');
 	$spawn_httpd->('-W0');
 }
@@ -553,12 +544,29 @@ SKIP: {
 	defined(my $x = getsockopt($sock, SOL_SOCKET, $var)) or die;
 	is($x, $accf_arg, 'SO_ACCEPTFILTER unchanged if previously set');
 };
+
 SKIP: {
 	skip 'only testing lsof(8) output on Linux', 1 if $^O ne 'linux';
 	skip 'no lsof in PATH', 1 unless which('lsof');
-	my @lsof = `lsof -p $pid`;
+	my @lsof = `lsof -p $td->{pid}`;
 	is_deeply([grep(/\bdeleted\b/, @lsof)], [], 'no lingering deleted inputs');
-	is_deeply([grep(/\bpipe\b/, @lsof)], [], 'no extra pipes with -W0');
+
+	# filter out pipes inherited from the parent
+	my @this = `lsof -p $$`;
+	my $bad;
+	sub extract_inodes {
+		map {;
+			my @f = split(' ', $_);
+			my $inode = $f[-2];
+			$bad = $_ if $inode !~ /\A[0-9]+\z/;
+			$inode => 1;
+		} grep (/\bpipe\b/, @_);
+	}
+	my %child = extract_inodes(@lsof);
+	my %parent = extract_inodes(@this);
+	skip("inode not in expected format: $bad", 1) if defined($bad);
+	delete @child{(keys %parent)};
+	is_deeply([], [keys %child], 'no extra pipes with -W0');
 };
 
 done_testing();
diff --git a/t/httpd-https.t b/t/httpd-https.t
index 22c62bf4..81a11108 100644
--- a/t/httpd-https.t
+++ b/t/httpd-https.t
@@ -23,14 +23,8 @@ my $psgi = "./t/httpd-corner.psgi";
 my $tmpdir = tempdir('pi-httpd-https-XXXXXX', TMPDIR => 1, CLEANUP => 1);
 my $err = "$tmpdir/stderr.log";
 my $out = "$tmpdir/stdout.log";
-my $httpd = 'blib/script/public-inbox-httpd';
 my $https = tcp_server();
-my ($pid, $tail_pid);
-END {
-	foreach ($pid, $tail_pid) {
-		kill 'TERM', $_ if defined $_;
-	}
-};
+my $td;
 my $https_addr = $https->sockhost . ':' . $https->sockport;
 
 for my $args (
@@ -39,15 +33,9 @@ for my $args (
 	for ($out, $err) {
 		open my $fh, '>', $_ or die "truncate: $!";
 	}
-	if (my $tail_cmd = $ENV{TAIL}) { # don't assume GNU tail
-		$tail_pid = fork;
-		if (defined $tail_pid && $tail_pid == 0) {
-			exec(split(' ', $tail_cmd), $out, $err);
-		}
-	}
-	my $cmd = [ $httpd, '-W0', @$args,
+	my $cmd = [ '-httpd', '-W0', @$args,
 			"--stdout=$out", "--stderr=$err", $psgi ];
-	$pid = spawn_listener(undef, $cmd, [ $https ]);
+	$td = start_script($cmd, undef, { 3 => $https });
 	my %o = (
 		SSL_hostname => 'server.local',
 		SSL_verifycn_name => 'server.local',
@@ -119,15 +107,9 @@ for my $args (
 	};
 
 	$c = undef;
-	kill('TERM', $pid);
-	is($pid, waitpid($pid, 0), 'httpd exited successfully');
+	$td->kill;
+	$td->join;
 	is($?, 0, 'no error in exited process');
-	$pid = undef;
-	if (defined $tail_pid) {
-		kill 'TERM', $tail_pid;
-		waitpid($tail_pid, 0);
-		$tail_pid = undef;
-	}
 }
 done_testing();
 1;
diff --git a/t/httpd-unix.t b/t/httpd-unix.t
index 2c918281..5ec70fd8 100644
--- a/t/httpd-unix.t
+++ b/t/httpd-unix.t
@@ -4,6 +4,8 @@
 use strict;
 use warnings;
 use Test::More;
+require './t/common.perl';
+use Errno qw(EADDRINUSE);
 
 foreach my $mod (qw(Plack::Util Plack::Builder HTTP::Date HTTP::Status)) {
 	eval "require $mod";
@@ -14,23 +16,16 @@ use File::Temp qw/tempdir/;
 use IO::Socket::UNIX;
 my $tmpdir = tempdir('httpd-unix-XXXXXX', TMPDIR => 1, CLEANUP => 1);
 my $unix = "$tmpdir/unix.sock";
-my $httpd = 'blib/script/public-inbox-httpd';
 my $psgi = './t/httpd-corner.psgi';
 my $out = "$tmpdir/out.log";
 my $err = "$tmpdir/err.log";
-
-my $pid;
-END { kill 'TERM', $pid if defined $pid };
+my $td;
 
 my $spawn_httpd = sub {
 	my (@args) = @_;
 	push @args, '-W0';
-	$pid = fork;
-	if ($pid == 0) {
-		exec $httpd, @args, "--stdout=$out", "--stderr=$err", $psgi;
-		die "FAIL: $!\n";
-	}
-	ok(defined $pid, 'forked httpd process successfully');
+	my $cmd = [ '-httpd', @args, "--stdout=$out", "--stderr=$err", $psgi ];
+	$td = start_script($cmd);
 };
 
 {
@@ -65,15 +60,18 @@ sub check_sock ($) {
 check_sock($unix);
 
 { # do not clobber existing socket
-	my $fpid = fork;
-	if ($fpid == 0) {
-		open STDOUT, '>>', "$tmpdir/1" or die "redirect failed: $!";
-		open STDERR, '>>', "$tmpdir/2" or die "redirect failed: $!";
-		exec $httpd, '-l', $unix, '-W0', $psgi;
-		die "FAIL: $!\n";
-	}
-	is($fpid, waitpid($fpid, 0), 'second httpd exits');
-	isnt($?, 0, 'httpd failed with failure to bind');
+	my %err = ( 'linux' => EADDRINUSE );
+	open my $out, '>>', "$tmpdir/1" or die "redirect failed: $!";
+	open my $err, '>>', "$tmpdir/2" or die "redirect failed: $!";
+	my $cmd = ['-httpd', '-l', $unix, '-W0', $psgi];
+	my $ftd = start_script($cmd, undef, { 1 => $out, 2 => $err });
+	$ftd->join;
+	isnt($?, 0, 'httpd failure set $?');
+	SKIP: {
+		my $ec = $err{$^O} or
+			skip("not sure if $^O fails with EADDRINUSE", 1);
+		is($? >> 8, $ec, 'httpd failed with EADDRINUSE');
+	};
 	open my $fh, "$tmpdir/2" or die "failed to open $tmpdir/2: $!";
 	local $/;
 	my $e = <$fh>;
@@ -82,10 +80,8 @@ check_sock($unix);
 }
 
 {
-	my $kpid = $pid;
-	$pid = undef;
-	is(kill('TERM', $kpid), 1, 'terminate existing process');
-	is(waitpid($kpid, 0), $kpid, 'existing httpd terminated');
+	is($td->kill, 1, 'terminate existing process');
+	$td->join;
 	is($?, 0, 'existing httpd exited successfully');
 	ok(-S $unix, 'unix socket still exists');
 }
@@ -96,9 +92,8 @@ SKIP: {
 
 	# wait for daemonization
 	$spawn_httpd->("-l$unix", '-D', '-P', "$tmpdir/pid");
-	my $kpid = $pid;
-	$pid = undef;
-	is(waitpid($kpid, 0), $kpid, 'existing httpd terminated');
+	$td->join;
+	is($?, 0, 'daemonized process OK');
 	check_sock($unix);
 
 	ok(-f "$tmpdir/pid", 'pid file written');
diff --git a/t/httpd.t b/t/httpd.t
index e7527ed6..ce8063b2 100644
--- a/t/httpd.t
+++ b/t/httpd.t
@@ -21,13 +21,11 @@ my $maindir = "$tmpdir/main.git";
 my $group = 'test-httpd';
 my $addr = $group . '@example.com';
 my $cfgpfx = "publicinbox.$group";
-my $httpd = 'blib/script/public-inbox-httpd';
 my $sock = tcp_server();
-my $pid;
+my $td;
 use_ok 'PublicInbox::Git';
 use_ok 'PublicInbox::Import';
 use_ok 'Email::MIME';
-END { kill 'TERM', $pid if defined $pid };
 {
 	local $ENV{HOME} = $home;
 	my $cmd = [ '-init', $group, $maindir, 'http://example.com/', $addr ];
@@ -52,8 +50,8 @@ EOF
 		$im->done($mime);
 	}
 	ok($sock, 'sock created');
-	$cmd = [ $httpd, '-W0', "--stdout=$out", "--stderr=$err" ];
-	$pid = spawn_listener(undef, $cmd, [$sock]);
+	$cmd = [ '-httpd', '-W0', "--stdout=$out", "--stderr=$err" ];
+	$td = start_script($cmd, undef, { 3 => $sock });
 	my $host = $sock->sockhost;
 	my $port = $sock->sockport;
 	my $conn = tcp_connect($sock);
@@ -78,9 +76,8 @@ EOF
 			"http://$host:$port/$group", "$tmpdir/dumb.git"),
 		0, 'clone successful');
 
-	ok(kill('TERM', $pid), 'killed httpd');
-	$pid = undef;
-	waitpid(-1, 0);
+	ok($td->kill, 'killed httpd');
+	$td->join;
 
 	is(system('git', "--git-dir=$tmpdir/clone.git",
 		  qw(fsck --no-verbose)), 0,
diff --git a/t/nntpd-tls.t b/t/nntpd-tls.t
index 4e71e82d..5d170b78 100644
--- a/t/nntpd-tls.t
+++ b/t/nntpd-tls.t
@@ -41,16 +41,8 @@ my $inboxdir = "$tmpdir";
 my $pi_config = "$tmpdir/pi_config";
 my $group = 'test-nntpd-tls';
 my $addr = $group . '@example.com';
-my $nntpd = 'blib/script/public-inbox-nntpd';
 my $starttls = tcp_server();
 my $nntps = tcp_server();
-my ($pid, $tail_pid);
-END {
-	foreach ($pid, $tail_pid) {
-		kill 'TERM', $_ if defined $_;
-	}
-};
-
 my $ibx = PublicInbox::Inbox->new({
 	inboxdir => $inboxdir,
 	name => 'nntpd-tls',
@@ -91,6 +83,7 @@ EOF
 my $nntps_addr = $nntps->sockhost . ':' . $nntps->sockport;
 my $starttls_addr = $starttls->sockhost . ':' . $starttls->sockport;
 my $env = { PI_CONFIG => $pi_config };
+my $td;
 
 for my $args (
 	[ "--cert=$cert", "--key=$key",
@@ -100,14 +93,8 @@ for my $args (
 	for ($out, $err) {
 		open my $fh, '>', $_ or die "truncate: $!";
 	}
-	if (my $tail_cmd = $ENV{TAIL}) { # don't assume GNU tail
-		$tail_pid = fork;
-		if (defined $tail_pid && $tail_pid == 0) {
-			exec(split(' ', $tail_cmd), $out, $err);
-		}
-	}
-	my $cmd = [ $nntpd, '-W0', @$args, "--stdout=$out", "--stderr=$err" ];
-	$pid = spawn_listener($env, $cmd, [ $starttls, $nntps ]);
+	my $cmd = [ '-nntpd', '-W0', @$args, "--stdout=$out", "--stderr=$err" ];
+	$td = start_script($cmd, $env, { 3 => $starttls, 4 => $nntps });
 	my %o = (
 		SSL_hostname => 'server.local',
 		SSL_verifycn_name => 'server.local',
@@ -211,21 +198,15 @@ for my $args (
 	};
 
 	$c = undef;
-	kill('TERM', $pid);
-	is($pid, waitpid($pid, 0), 'nntpd exited successfully');
+	$td->kill;
+	$td->join;
 	is($?, 0, 'no error in exited process');
-	$pid = undef;
 	my $eout = eval {
 		open my $fh, '<', $err or die "open $err failed: $!";
 		local $/;
 		<$fh>;
 	};
 	unlike($eout, qr/wide/i, 'no Wide character warnings');
-	if (defined $tail_pid) {
-		kill 'TERM', $tail_pid;
-		waitpid($tail_pid, 0);
-		$tail_pid = undef;
-	}
 }
 done_testing();
 
diff --git a/t/nntpd-validate.t b/t/nntpd-validate.t
index de024394..e3c10d9c 100644
--- a/t/nntpd-validate.t
+++ b/t/nntpd-validate.t
@@ -10,9 +10,15 @@ use Symbol qw(gensym);
 use Time::HiRes qw(clock_gettime CLOCK_MONOTONIC);
 my $inbox_dir = $ENV{GIANT_INBOX_DIR};
 plan skip_all => "GIANT_INBOX_DIR not defined for $0" unless $inbox_dir;
+if (my $m = $ENV{TEST_RUN_MODE}) {
+	plan skip_all => "threads conflict w/ TEST_RUN_MODE=$m";
+}
 my $mid = $ENV{TEST_MID};
 
 # This test is also an excuse for me to experiment with Perl threads :P
+# TODO: get rid of threads, I was reading an old threads(3perl) manpage
+# and missed the WARNING in the newer ones about it being "discouraged"
+# in perlpolicy(1).
 unless (eval 'use threads; 1') {
 	plan skip_all => "$0 requires a threaded perl" if $@;
 }
@@ -37,13 +43,8 @@ if ($test_tls && !-r $key || !-r $cert) {
 require './t/common.perl';
 my $keep_tmp = !!$ENV{TEST_KEEP_TMP};
 my $tmpdir = tempdir('nntpd-validate-XXXXXX',TMPDIR => 1,CLEANUP => $keep_tmp);
-my (%OPT, $pid, $tail_pid, $host_port, $group);
+my (%OPT, $td, $host_port, $group);
 my $batch = 1000;
-END {
-	foreach ($pid, $tail_pid) {
-		kill 'TERM', $_ if defined $_;
-	}
-};
 if (($ENV{NNTP_TEST_URL} // '') =~ m!\Anntp://([^/]+)/([^/]+)\z!) {
 	($host_port, $group) = ($1, $2);
 	$host_port .= ":119" unless index($host_port, ':') > 0;
@@ -149,7 +150,6 @@ sub make_local_server {
 	$group = 'inbox.test.perf.nntpd';
 	my $ibx = { inboxdir => $inbox_dir, newsgroup => $group };
 	$ibx = PublicInbox::Inbox->new($ibx);
-	my $nntpd = 'blib/script/public-inbox-nntpd';
 	my $pi_config = "$tmpdir/config";
 	{
 		open my $fh, '>', $pi_config or die "open($pi_config): $!";
@@ -165,20 +165,13 @@ sub make_local_server {
 	for ($out, $err) {
 		open my $fh, '>', $_ or die "truncate: $!";
 	}
-	if (my $tail_cmd = $ENV{TAIL}) { # don't assume GNU tail
-		$tail_pid = fork;
-		if (defined $tail_pid && $tail_pid == 0) {
-			open STDOUT, '>&STDERR' or die ">&2 failed: $!";
-			exec(split(' ', $tail_cmd), $out, $err);
-		}
-	}
 	my $sock = tcp_server();
 	ok($sock, 'sock created');
 	$host_port = $sock->sockhost . ':' . $sock->sockport;
 
 	# not using multiple workers, here, since we want to increase
 	# the chance of tripping concurrency bugs within PublicInbox/NNTP*.pm
-	my $cmd = [ $nntpd, "--stdout=$out", "--stderr=$err", '-W0' ];
+	my $cmd = [ '-nntpd', "--stdout=$out", "--stderr=$err", '-W0' ];
 	push @$cmd, "-lnntp://$host_port";
 	if ($test_tls) {
 		push @$cmd, "--cert=$cert", "--key=$key";
@@ -190,7 +183,9 @@ sub make_local_server {
 		);
 	}
 	print STDERR "# CMD ". join(' ', @$cmd). "\n";
-	$pid = spawn_listener({ PI_CONFIG => $pi_config }, $cmd, [$sock]);
+	my $env = { PI_CONFIG => $pi_config };
+	# perl threads and run_mode != 0 don't get along
+	$td = start_script($cmd, $env, { run_mode => 0, 3 => $sock });
 }
 
 package DigestPipe;
diff --git a/t/nntpd.t b/t/nntpd.t
index 4795dc00..3c928610 100644
--- a/t/nntpd.t
+++ b/t/nntpd.t
@@ -29,7 +29,6 @@ my $out = "$tmpdir/stdout.log";
 my $inboxdir = "$tmpdir/main.git";
 my $group = 'test-nntpd';
 my $addr = $group . '@example.com';
-my $nntpd = 'blib/script/public-inbox-nntpd';
 SKIP: {
 	skip "git 2.6+ required for V2Writable", 1 if $version == 1;
 	use_ok 'PublicInbox::V2Writable';
@@ -37,9 +36,8 @@ SKIP: {
 
 my %opts;
 my $sock = tcp_server();
-my $pid;
+my $td;
 my $len;
-END { kill 'TERM', $pid if defined $pid };
 
 my $ibx = {
 	inboxdir => $inboxdir,
@@ -90,9 +88,8 @@ EOF
 	}
 
 	ok($sock, 'sock created');
-	my $cmd = [ $nntpd, '-W0', "--stdout=$out", "--stderr=$err" ];
-	$pid = spawn_listener(undef, $cmd, [ $sock ]);
-	ok(defined $pid, 'forked nntpd process successfully');
+	my $cmd = [ '-nntpd', '-W0', "--stdout=$out", "--stderr=$err" ];
+	$td = start_script($cmd, undef, { 3 => $sock });
 	my $host_port = $sock->sockhost . ':' . $sock->sockport;
 	my $n = Net::NNTP->new($host_port);
 	my $list = $n->list;
@@ -306,7 +303,7 @@ Date: Fri, 02 Oct 1993 00:00:00 +0000
 		is($? >> 8, 0, 'no errors');
 	}
 	SKIP: {
-		my @of = `lsof -p $pid 2>/dev/null`;
+		my @of = `lsof -p $td->{pid} 2>/dev/null`;
 		skip('lsof broken', 1) if (!scalar(@of) || $?);
 		my @xap = grep m!Search/Xapian!, @of;
 		is_deeply(\@xap, [], 'Xapian not loaded in nntpd');
@@ -315,7 +312,7 @@ Date: Fri, 02 Oct 1993 00:00:00 +0000
 		setsockopt($s, IPPROTO_TCP, TCP_NODELAY, 1);
 		syswrite($s, 'HDR List-id 1-');
 		select(undef, undef, undef, 0.15);
-		ok(kill('TERM', $pid), 'killed nntpd');
+		ok($td->kill, 'killed nntpd');
 		select(undef, undef, undef, 0.15);
 		syswrite($s, "\r\n");
 		$buf = '';
@@ -329,7 +326,7 @@ Date: Fri, 02 Oct 1993 00:00:00 +0000
 	}
 
 	$n = $s = undef;
-	is($pid, waitpid($pid, 0), 'nntpd exited successfully');
+	$td->join;
 	my $eout = eval {
 		local $/;
 		open my $fh, '<', $err or die "open $err failed: $!";
@@ -339,6 +336,7 @@ Date: Fri, 02 Oct 1993 00:00:00 +0000
 	unlike($eout, qr/wide/i, 'no Wide character warnings');
 }
 
+$td = undef;
 done_testing();
 
 sub read_til_dot {
diff --git a/t/perf-nntpd.t b/t/perf-nntpd.t
index 7abf2249..c7d2eaff 100644
--- a/t/perf-nntpd.t
+++ b/t/perf-nntpd.t
@@ -10,18 +10,9 @@ use Net::NNTP;
 my $pi_dir = $ENV{GIANT_PI_DIR};
 plan skip_all => "GIANT_PI_DIR not defined for $0" unless $pi_dir;
 eval { require PublicInbox::Search };
-my ($host_port, $group, %opts, $s, $pid);
+my ($host_port, $group, %opts, $s, $td);
 require './t/common.perl';
 
-END {
-	if ($s) {
-		$s->print("QUIT\r\n");
-		$s->getline;
-		$s = undef;
-	}
-	kill 'TERM', $pid if defined $pid;
-};
-
 if (($ENV{NNTP_TEST_URL} || '') =~ m!\Anntp://([^/]+)/([^/]+)\z!) {
 	($host_port, $group) = ($1, $2);
 	$host_port .= ":119" unless index($host_port, ':') > 0;
@@ -29,7 +20,6 @@ if (($ENV{NNTP_TEST_URL} || '') =~ m!\Anntp://([^/]+)/([^/]+)\z!) {
 	$group = 'inbox.test.perf.nntpd';
 	my $ibx = { inboxdir => $pi_dir, newsgroup => $group };
 	$ibx = PublicInbox::Inbox->new($ibx);
-	my $nntpd = 'blib/script/public-inbox-nntpd';
 	my $tmpdir = tempdir('perf-nntpd-XXXXXX', TMPDIR => 1, CLEANUP => 1);
 
 	my $pi_config = "$tmpdir/config";
@@ -46,8 +36,8 @@ if (($ENV{NNTP_TEST_URL} || '') =~ m!\Anntp://([^/]+)/([^/]+)\z!) {
 
 	my $sock = tcp_server();
 	ok($sock, 'sock created');
-	my $cmd = [ $nntpd, '-W0' ];
-	$pid = spawn_listener({ PI_CONFIG => $pi_config }, $cmd, [$sock]);
+	my $cmd = [ '-nntpd', '-W0' ];
+	$td = start_script($cmd, { PI_CONFIG => $pi_config }, { 3 => $sock });
 	$host_port = $sock->sockhost . ':' . $sock->sockport;
 }
 %opts = (
@@ -110,6 +100,12 @@ $t = timeit(1, sub {
 });
 diag 'newnews took: ' . timestr($t) . " for $n";
 
+if ($s) {
+	$s->print("QUIT\r\n");
+	$s->getline;
+}
+
+
 done_testing();
 
 1;
diff --git a/t/v2mirror.t b/t/v2mirror.t
index 2c7f6a84..1a39ce49 100644
--- a/t/v2mirror.t
+++ b/t/v2mirror.t
@@ -21,7 +21,6 @@ use PublicInbox::MIME;
 use PublicInbox::Config;
 # FIXME: too much setup
 my $tmpdir = tempdir('pi-v2mirror-XXXXXX', TMPDIR => 1, CLEANUP => 1);
-my $script = 'blib/script/public-inbox';
 my $pi_config = "$tmpdir/config";
 {
 	open my $fh, '>', $pi_config or die "open($pi_config): $!";
@@ -60,19 +59,10 @@ ok($epoch_max > 0, "multiple epochs");
 $v2w->done;
 $ibx->cleanup;
 
-my ($sock, $pid);
-
-# TODO: replace this with ->DESTROY:
-my $owner_pid = $$;
-END { kill('TERM', $pid) if defined($pid) && $owner_pid == $$ };
-
-$! = 0;
-$sock = tcp_server();
+my $sock = tcp_server();
 ok($sock, 'sock created');
-my $httpd = "$script-httpd";
-my $cmd = [ $httpd, '-W0', "--stdout=$tmpdir/out", "--stderr=$tmpdir/err" ];
-ok(defined($pid = spawn_listener(undef, $cmd, [ $sock ])),
-	'spawned httpd process successfully');
+my $cmd = [ '-httpd', '-W0', "--stdout=$tmpdir/out", "--stderr=$tmpdir/err" ];
+my $td = start_script($cmd, undef, { 3 => $sock });
 my ($host, $port) = ($sock->sockhost, $sock->sockport);
 $sock = undef;
 
@@ -194,9 +184,8 @@ is($mibx->git->check($to_purge), undef, 'unindex+prune successful in mirror');
 	is(scalar($mset->items), 0, '1@example.com no longer visible in mirror');
 }
 
-ok(kill('TERM', $pid), 'killed httpd');
-$pid = undef;
-waitpid(-1, 0);
+ok($td->kill, 'killed httpd');
+$td->join;
 
 done_testing();
 
diff --git a/t/v2writable.t b/t/v2writable.t
index 28420bb9..4bb6d733 100644
--- a/t/v2writable.t
+++ b/t/v2writable.t
@@ -163,12 +163,10 @@ EOF
 	close $fh or die "close: $!\n";
 	my $sock = tcp_server();
 	ok($sock, 'sock created');
-	my $pid;
 	my $len;
-	END { kill 'TERM', $pid if defined $pid };
-	my $nntpd = 'blib/script/public-inbox-nntpd';
-	my $cmd = [ $nntpd, '-W0', "--stdout=$out", "--stderr=$err" ];
-	$pid = spawn_listener({ PI_CONFIG => $pi_config }, $cmd, [ $sock ]);
+	my $cmd = [ '-nntpd', '-W0', "--stdout=$out", "--stderr=$err" ];
+	my $env = { PI_CONFIG => $pi_config };
+	my $td = start_script($cmd, $env, { 3 => $sock });
 	my $host_port = $sock->sockhost . ':' . $sock->sockport;
 	my $n = Net::NNTP->new($host_port);
 	$n->group($group);
diff --git a/t/www_listing.t b/t/www_listing.t
index 61a059e5..9cde3575 100644
--- a/t/www_listing.t
+++ b/t/www_listing.t
@@ -64,15 +64,13 @@ sub tiny_test {
 		'epoch 1 in description');
 }
 
-my $pid;
-END { kill 'TERM', $pid if defined $pid };
+my $td;
 SKIP: {
 	my $err = "$tmpdir/stderr.log";
 	my $out = "$tmpdir/stdout.log";
 	my $alt = "$tmpdir/alt.git";
 	my $cfgfile = "$tmpdir/config";
 	my $v2 = "$tmpdir/v2";
-	my $httpd = 'blib/script/public-inbox-httpd';
 	my $sock = tcp_server();
 	ok($sock, 'sock created');
 	my ($host, $port) = ($sock->sockhost, $sock->sockport);
@@ -106,8 +104,8 @@ SKIP: {
 
 	close $fh or die;
 	my $env = { PI_CONFIG => $cfgfile };
-	my $cmd = [ $httpd, '-W0', "--stdout=$out", "--stderr=$err" ];
-	$pid = spawn_listener($env, $cmd, [$sock]);
+	my $cmd = [ '-httpd', '-W0', "--stdout=$out", "--stderr=$err" ];
+	$td = start_script($cmd, $env, { 3 => $sock });
 	$sock = undef;
 
 	tiny_test($host, $port);

[PATCH 12/17] t/nntpd-validate: get rid of threads dependency

[Eric Wong]

Threads are officially discouraged by perl5-porters and proves
problematic with my Perl installation when using run_mode=1
to speed up tests.  So just use fork() and pipes to share
results from Net::NNTP.
---
 t/nntpd-validate.t | 39 +++++++++++++++++++++++----------------
 1 file changed, 23 insertions(+), 16 deletions(-)

diff --git a/t/nntpd-validate.t b/t/nntpd-validate.t
index e3c10d9c..da6985be 100644
--- a/t/nntpd-validate.t
+++ b/t/nntpd-validate.t
@@ -8,21 +8,11 @@ use File::Temp qw(tempdir);
 use Test::More;
 use Symbol qw(gensym);
 use Time::HiRes qw(clock_gettime CLOCK_MONOTONIC);
+use POSIX qw(_exit);
 my $inbox_dir = $ENV{GIANT_INBOX_DIR};
 plan skip_all => "GIANT_INBOX_DIR not defined for $0" unless $inbox_dir;
-if (my $m = $ENV{TEST_RUN_MODE}) {
-	plan skip_all => "threads conflict w/ TEST_RUN_MODE=$m";
-}
 my $mid = $ENV{TEST_MID};
 
-# This test is also an excuse for me to experiment with Perl threads :P
-# TODO: get rid of threads, I was reading an old threads(3perl) manpage
-# and missed the WARNING in the newer ones about it being "discouraged"
-# in perlpolicy(1).
-unless (eval 'use threads; 1') {
-	plan skip_all => "$0 requires a threaded perl" if $@;
-}
-
 # Net::NNTP is part of the standard library, but distros may split it off...
 foreach my $mod (qw(DBD::SQLite Net::NNTP Compress::Raw::Zlib)) {
 	eval "require $mod";
@@ -134,11 +124,29 @@ my (@keys, %thr, %res);
 for my $m (@tests) {
 	my $key = join(',', @$m);
 	push @keys, $key;
-	diag "$key start";
-	$thr{$key} = threads->create(\&do_get_all, $m);
+	pipe(my ($r, $w)) or die;
+	my $pid = fork;
+	if ($pid == 0) {
+		close $r or die;
+		my $res = do_get_all($m);
+		print $w $res or die;
+		$w->flush;
+		_exit(0);
+	}
+	close $w or die;
+	$thr{$key} = [ $pid, $r ];
+}
+for my $key (@keys) {
+	my ($pid, $r) = @{delete $thr{$key}};
+	local $/;
+	$res{$key} = <$r>;
+	defined $res{$key} or die "nothing for $key";
+	my $w = waitpid($pid, 0);
+	defined($w) or die;
+	$w == $pid or die "waitpid($pid) != $w)";
+	is($?, 0, "`$key' exited successfully")
 }
 
-$res{$_} = $thr{$_}->join for @keys;
 my $plain = $res{''};
 ok($plain, "plain got $plain");
 is($res{$_}, $plain, "$_ matches '' result") for @keys;
@@ -184,8 +192,7 @@ sub make_local_server {
 	}
 	print STDERR "# CMD ". join(' ', @$cmd). "\n";
 	my $env = { PI_CONFIG => $pi_config };
-	# perl threads and run_mode != 0 don't get along
-	$td = start_script($cmd, $env, { run_mode => 0, 3 => $sock });
+	$td = start_script($cmd, $env, { 3 => $sock });
 }
 
 package DigestPipe;

[PATCH 13/17] xapcmd: replace Xtmpdirs with File::Temp->newdir

[Eric Wong]

Since we're using Perl 5.10.1 and File::Temp 0.19+, we don't
need Xtmpdirs at all for cleaning up tempdirs on failure and
can just rely on the DESTROY handler provided by File::Temp.
---
 lib/PublicInbox/Xapcmd.pm | 73 +++++++++++++++------------------------
 1 file changed, 27 insertions(+), 46 deletions(-)

diff --git a/lib/PublicInbox/Xapcmd.pm b/lib/PublicInbox/Xapcmd.pm
index 28285898..78b3a9ac 100644
--- a/lib/PublicInbox/Xapcmd.pm
+++ b/lib/PublicInbox/Xapcmd.pm
@@ -6,7 +6,7 @@ use warnings;
 use PublicInbox::Spawn qw(which spawn);
 use PublicInbox::Over;
 use PublicInbox::Search;
-use File::Temp qw(tempdir);
+use File::Temp ();
 use File::Path qw(remove_tree);
 use File::Basename qw(dirname);
 use POSIX ();
@@ -24,13 +24,14 @@ sub commit_changes ($$$$) {
 	$SIG{INT} or die 'BUG: $SIG{INT} not handled';
 	my @old_shard;
 
-	while (my ($old, $new) = each %$tmp) {
+	while (my ($old, $newdir) = each %$tmp) {
 		next if $old eq ''; # no invalid paths
 		my @st = stat($old);
 		if (!@st && !defined($opt->{reshard})) {
 			die "failed to stat($old): $!";
 		}
 
+		my $new = $newdir->dirname if defined($newdir);
 		my $over = "$old/over.sqlite3";
 		if (-f $over) { # only for v1, v2 over is untouched
 			defined $new or die "BUG: $over exists when culling v2";
@@ -145,6 +146,12 @@ sub process_queue {
 	}
 }
 
+sub setup_signals () {
+	# http://www.tldp.org/LDP/abs/html/exitcodes.html
+	$SIG{INT} = sub { exit(130) };
+	$SIG{HUP} = $SIG{PIPE} = $SIG{TERM} = sub { exit(1) };
+}
+
 sub run {
 	my ($ibx, $task, $opt) = @_; # task = 'cpdb' or 'compact'
 	my $cb = \&${\"PublicInbox::Xapcmd::$task"};
@@ -164,7 +171,7 @@ sub run {
 	my $old = $ibx->search->xdir(1);
 	-d $old or die "$old does not exist\n";
 
-	my $tmp = PublicInbox::Xtmpdirs->new;
+	my $tmp = {};
 	my $v = $ibx->{version} ||= 1;
 	my @q;
 	my $reshard = $opt->{reshard};
@@ -173,7 +180,7 @@ sub run {
 	}
 
 	local %SIG = %SIG;
-	$tmp->setup_signals;
+	setup_signals();
 
 	# we want temporary directories to be as deep as possible,
 	# so v2 shards can keep "xap$SCHEMA_VERSION" on a separate FS.
@@ -182,10 +189,10 @@ sub run {
 			warn
 "--reshard=$reshard ignored for v1 $ibx->{inboxdir}\n";
 		}
-		my $old_parent = dirname($old);
-		same_fs_or_die($old_parent, $old);
+		my $dir = dirname($old);
+		same_fs_or_die($dir, $old);
 		my $v = PublicInbox::Search::SCHEMA_VERSION();
-		my $wip = tempdir("xapian$v-XXXXXXXX", DIR => $old_parent);
+		my $wip = File::Temp->newdir("xapian$v-XXXXXXXX", DIR => $dir);
 		$tmp->{$old} = $wip;
 		push @q, [ $old, $wip ];
 	} else {
@@ -213,8 +220,8 @@ sub run {
 		}
 		foreach my $dn (0..$max_shard) {
 			my $tmpl = "$dn-XXXXXXXX";
-			my $wip = tempdir($tmpl, DIR => $old);
-			same_fs_or_die($old, $wip);
+			my $wip = File::Temp->newdir($tmpl, DIR => $old);
+			same_fs_or_die($old, $wip->dirname);
 			my $cur = "$old/$dn";
 			push @q, [ $src // $cur , $wip ];
 			$tmp->{$cur} = $wip;
@@ -267,7 +274,8 @@ sub progress_pfx ($) {
 # xapian-compact wrapper
 sub compact ($$) {
 	my ($args, $opt) = @_;
-	my ($src, $dst) = @$args;
+	my ($src, $newdir) = @$args;
+	my $dst = ref($newdir) ? $newdir->dirname : $newdir;
 	my ($r, $w);
 	my $pfx = $opt->{-progress_pfx} ||= progress_pfx($src);
 	my $pr = $opt->{-progress};
@@ -349,7 +357,8 @@ sub cpdb_loop ($$$;$$) {
 # to the overhead of Perl.
 sub cpdb ($$) {
 	my ($args, $opt) = @_;
-	my ($old, $new) = @$args;
+	my ($old, $newdir) = @$args;
+	my $new = $newdir->dirname;
 	my ($src, $cur_shard);
 	my $reshard;
 	if (ref($old) eq 'ARRAY') {
@@ -372,15 +381,14 @@ sub cpdb ($$) {
 		$src = Search::Xapian::Database->new($old);
 	}
 
-	my ($xtmp, $tmp);
+	my ($tmp, $ft);
 	local %SIG = %SIG;
 	if ($opt->{compact}) {
-		my $newdir = dirname($new);
-		same_fs_or_die($newdir, $new);
-		$tmp = tempdir("$new.compact-XXXXXX", DIR => $newdir);
-		$xtmp = PublicInbox::Xtmpdirs->new;
-		$xtmp->setup_signals;
-		$xtmp->{$new} = $tmp;
+		my $dir = dirname($new);
+		same_fs_or_die($dir, $new);
+		$ft = File::Temp->newdir("$new.compact-XXXXXX", DIR => $dir);
+		setup_signals();
+		$tmp = $ft->dirname;
 	} else {
 		$tmp = $new;
 	}
@@ -439,7 +447,7 @@ sub cpdb ($$) {
 	}
 
 	$pr->(sprintf($pr_data->{fmt}, $pr_data->{nr})) if $pr;
-	return unless $xtmp;
+	return unless $opt->{compact};
 
 	$src = $dst = undef; # flushes and closes
 
@@ -447,33 +455,6 @@ sub cpdb ($$) {
 	# since $dst isn't readable by HTTP or NNTP clients, yet:
 	compact([ $tmp, $new ], $opt);
 	remove_tree($tmp) or die "failed to remove $tmp: $!\n";
-	$xtmp = undef;
-}
-
-# slightly easier-to-manage manage than END{} blocks
-package PublicInbox::Xtmpdirs;
-use strict;
-use warnings;
-use File::Path qw(remove_tree);
-
-sub setup_signals () {
-	# http://www.tldp.org/LDP/abs/html/exitcodes.html
-	$SIG{INT} = sub { exit(130) };
-	$SIG{HUP} = $SIG{PIPE} = $SIG{TERM} = sub { exit(1) };
-}
-
-sub new {
-	bless { '' => $$ }, $_[0]; # old shard => new (WIP) shard
-}
-
-sub DESTROY {
-	my ($self) = @_;
-	my $owner_pid = delete($self->{''}) or return;
-	return if $owner_pid != $$;
-	foreach my $new (values %$self) {
-		defined $new or next; # may be undef if resharding
-		remove_tree($new) unless -d "$new/old";
-	}
 }
 
 1;

[PATCH 14/17] tests: use File::Temp->newdir instead of tempdir()

[Eric Wong]

We'll also introduce a tmpdir() API to give tempdirs
consistent names.
---
 t/admin.t                 |  4 ++--
 t/altid.t                 |  4 ++--
 t/altid_v2.t              |  3 +--
 t/cgi.t                   |  3 +--
 t/common.perl             | 10 ++++++++++
 t/config.t                |  4 ++--
 t/convert-compact.t       |  3 +--
 t/edit.t                  |  3 +--
 t/emergency.t             |  4 ++--
 t/feed.t                  |  3 +--
 t/filter_rubylang.t       |  5 ++---
 t/git-http-backend.t      |  5 ++---
 t/git.t                   |  6 +++---
 t/html_index.t            |  4 ++--
 t/httpd-corner.t          |  3 +--
 t/httpd-https.t           |  3 +--
 t/httpd-unix.t            |  3 +--
 t/httpd.t                 |  3 +--
 t/import.t                |  4 ++--
 t/indexlevels-mirror.t    |  3 +--
 t/init.t                  |  3 +--
 t/mda.t                   |  3 +--
 t/mda_filter_rubylang.t   |  3 +--
 t/msgmap.t                |  4 ++--
 t/nntpd-tls.t             |  3 +--
 t/nntpd-validate.t        |  7 +++----
 t/nntpd.t                 |  3 +--
 t/nulsubject.t            |  4 ++--
 t/over.t                  |  4 ++--
 t/perf-nntpd.t            |  6 +++---
 t/plack.t                 |  4 ++--
 t/psgi_attach.t           |  4 ++--
 t/psgi_bad_mids.t         |  4 ++--
 t/psgi_mount.t            |  4 ++--
 t/psgi_multipart_not.t    |  4 ++--
 t/psgi_scan_all.t         |  4 ++--
 t/psgi_search.t           |  4 ++--
 t/psgi_text.t             |  4 ++--
 t/psgi_v2.t               |  3 +--
 t/purge.t                 |  3 +--
 t/replace.t               |  3 +--
 t/search-thr-index.t      |  4 ++--
 t/search.t                |  4 ++--
 t/solver_git.t            |  3 +--
 t/spamcheck_spamc.t       |  4 ++--
 t/v1-add-remove-add.t     |  6 +++---
 t/v1reindex.t             |  3 +--
 t/v2-add-remove-add.t     |  3 +--
 t/v2mda.t                 |  3 +--
 t/v2mirror.t              |  3 +--
 t/v2reindex.t             |  3 +--
 t/v2writable.t            |  3 +--
 t/watch_filter_rubylang.t |  3 +--
 t/watch_maildir.t         |  4 ++--
 t/watch_maildir_v2.t      |  3 +--
 t/www_listing.t           |  3 +--
 t/xcpdb-reshard.t         |  3 +--
 57 files changed, 97 insertions(+), 119 deletions(-)

diff --git a/t/admin.t b/t/admin.t
index 0024df15..6458982b 100644
--- a/t/admin.t
+++ b/t/admin.t
@@ -3,9 +3,9 @@
 use strict;
 use warnings;
 use Test::More;
-use File::Temp qw(tempdir);
+require './t/common.perl';
 use_ok 'PublicInbox::Admin', qw(resolve_repo_dir);
-my $tmpdir = tempdir('pi-admin.XXXXXX', TMPDIR => 1, CLEANUP => 1);
+my ($tmpdir, $for_destroy) = tmpdir();
 my $git_dir = "$tmpdir/v1";
 my $v2_dir = "$tmpdir/v2";
 my ($res, $err, $v);
diff --git a/t/altid.t b/t/altid.t
index 4ab004c4..86e7f9de 100644
--- a/t/altid.t
+++ b/t/altid.t
@@ -3,7 +3,7 @@
 use strict;
 use warnings;
 use Test::More;
-use File::Temp qw/tempdir/;
+require './t/common.perl';
 foreach my $mod (qw(DBD::SQLite Search::Xapian)) {
 	eval "require $mod";
 	plan skip_all => "$mod missing for altid.t" if $@;
@@ -13,7 +13,7 @@ use_ok 'PublicInbox::Msgmap';
 use_ok 'PublicInbox::SearchIdx';
 use_ok 'PublicInbox::Import';
 use_ok 'PublicInbox::Inbox';
-my $tmpdir = tempdir('pi-altid-XXXXXX', TMPDIR => 1, CLEANUP => 1);
+my ($tmpdir, $for_destroy) = tmpdir();
 my $git_dir = "$tmpdir/a.git";
 my $alt_file = "$tmpdir/another-nntp.sqlite3";
 my $altid = [ "serial:gmane:file=$alt_file" ];
diff --git a/t/altid_v2.t b/t/altid_v2.t
index 2c1d8616..9e152fc4 100644
--- a/t/altid_v2.t
+++ b/t/altid_v2.t
@@ -3,7 +3,6 @@
 use strict;
 use warnings;
 use Test::More;
-use File::Temp qw/tempdir/;
 require './t/common.perl';
 require_git(2.6);
 foreach my $mod (qw(DBD::SQLite Search::Xapian)) {
@@ -13,7 +12,7 @@ foreach my $mod (qw(DBD::SQLite Search::Xapian)) {
 
 use_ok 'PublicInbox::V2Writable';
 use_ok 'PublicInbox::Inbox';
-my $tmpdir = tempdir('pi-altidv2-XXXXXX', TMPDIR => 1, CLEANUP => 1);
+my ($tmpdir, $for_destroy) = tmpdir();
 my $inboxdir = "$tmpdir/inbox";
 my $full = "$tmpdir/inbox/another-nntp.sqlite3";
 my $altid = [ 'serial:gmane:file=another-nntp.sqlite3' ];
diff --git a/t/cgi.t b/t/cgi.t
index 3c09ecd6..62cea499 100644
--- a/t/cgi.t
+++ b/t/cgi.t
@@ -6,9 +6,8 @@ use strict;
 use warnings;
 use Test::More;
 use Email::MIME;
-use File::Temp qw/tempdir/;
 require './t/common.perl';
-my $tmpdir = tempdir('pi-cgi-XXXXXX', TMPDIR => 1, CLEANUP => 1);
+my ($tmpdir, $for_destroy) = tmpdir();
 my $home = "$tmpdir/pi-home";
 my $pi_home = "$home/.public-inbox";
 my $pi_config = "$pi_home/config";
diff --git a/t/common.perl b/t/common.perl
index 2126a761..0ff5de4a 100644
--- a/t/common.perl
+++ b/t/common.perl
@@ -7,6 +7,16 @@ use strict;
 use warnings;
 use IO::Socket::INET;
 
+sub tmpdir (;$) {
+	my ($base) = @_;
+	require File::Temp;
+	unless (defined $base) {
+		($base) = ($0 =~ m!\b([^/]+)\.[^\.]+\z!);
+	}
+	my $tmpdir = File::Temp->newdir("pi-$base-$$-XXXXXX", TMPDIR => 1);
+	($tmpdir->dirname, $tmpdir);
+}
+
 sub tcp_server () {
 	IO::Socket::INET->new(
 		LocalAddr => '127.0.0.1',
diff --git a/t/config.t b/t/config.t
index 0866f264..ade2e796 100644
--- a/t/config.t
+++ b/t/config.t
@@ -4,8 +4,8 @@ use strict;
 use warnings;
 use Test::More;
 use PublicInbox::Config;
-use File::Temp qw/tempdir/;
-my $tmpdir = tempdir('pi-config-XXXXXX', TMPDIR => 1, CLEANUP => 1);
+require './t/common.perl';
+my ($tmpdir, $for_destroy) = tmpdir();
 
 {
 	is(system(qw(git init -q --bare), $tmpdir), 0, "git init successful");
diff --git a/t/convert-compact.t b/t/convert-compact.t
index 0661ed14..b8dc5ed5 100644
--- a/t/convert-compact.t
+++ b/t/convert-compact.t
@@ -3,7 +3,6 @@
 use strict;
 use warnings;
 use Test::More;
-use File::Temp qw/tempdir/;
 use PublicInbox::MIME;
 use PublicInbox::Spawn qw(which);
 require './t/common.perl';
@@ -18,7 +17,7 @@ which('xapian-compact') or
 
 use_ok 'PublicInbox::V2Writable';
 use PublicInbox::Import;
-my $tmpdir = tempdir('convert-compact-XXXXXX', TMPDIR => 1, CLEANUP => 1);
+my ($tmpdir, $for_destroy) = tmpdir();
 my $ibx = {
 	inboxdir => "$tmpdir/v1",
 	name => 'test-v1',
diff --git a/t/edit.t b/t/edit.t
index 09e0cddd..02df6cda 100644
--- a/t/edit.t
+++ b/t/edit.t
@@ -4,7 +4,6 @@
 use strict;
 use warnings;
 use Test::More;
-use File::Temp qw/tempdir/;
 require './t/common.perl';
 require_git(2.6);
 require PublicInbox::Inbox;
@@ -18,7 +17,7 @@ foreach my $mod (@mods) {
 	plan skip_all => "missing $mod for $0" if $@;
 };
 
-my $tmpdir = tempdir('pi-edit-XXXXXX', TMPDIR => 1, CLEANUP => 1);
+my ($tmpdir, $for_destroy) = tmpdir();
 my $inboxdir = "$tmpdir/v2";
 my $ibx = PublicInbox::Inbox->new({
 	inboxdir => $inboxdir,
diff --git a/t/emergency.t b/t/emergency.t
index c28826a0..d6c7b6d5 100644
--- a/t/emergency.t
+++ b/t/emergency.t
@@ -3,8 +3,8 @@
 use strict;
 use warnings;
 use Test::More;
-use File::Temp qw/tempdir/;
-my $tmpdir = tempdir('emergency-XXXXXX', TMPDIR => 1, CLEANUP => 1);
+require './t/common.perl';
+my ($tmpdir, $for_destroy) = tmpdir();
 use_ok 'PublicInbox::Emergency';
 
 {
diff --git a/t/feed.t b/t/feed.t
index 93da3717..daf97a72 100644
--- a/t/feed.t
+++ b/t/feed.t
@@ -9,7 +9,6 @@ use PublicInbox::Git;
 use PublicInbox::Import;
 use PublicInbox::Config;
 use PublicInbox::Inbox;
-use File::Temp qw/tempdir/;
 my $have_xml_feed = eval { require XML::Feed; 1 };
 require './t/common.perl';
 
@@ -24,7 +23,7 @@ sub string_feed {
 	$str;
 }
 
-my $tmpdir = tempdir('pi-feed-XXXXXX', TMPDIR => 1, CLEANUP => 1);
+my ($tmpdir, $for_destroy) = tmpdir();
 my $git_dir = "$tmpdir/gittest";
 my $ibx = PublicInbox::Inbox->new({
 	address => 'test@example',
diff --git a/t/filter_rubylang.t b/t/filter_rubylang.t
index 7b1da11c..33753925 100644
--- a/t/filter_rubylang.t
+++ b/t/filter_rubylang.t
@@ -4,7 +4,7 @@ use strict;
 use warnings;
 use Test::More;
 use Email::MIME;
-use File::Temp qw/tempdir/;
+require './t/common.perl';
 use_ok 'PublicInbox::Filter::RubyLang';
 
 my $f = PublicInbox::Filter::RubyLang->new;
@@ -26,8 +26,7 @@ SKIP: {
 	eval 'require DBD::SQLite';
 	skip 'DBD::SQLite missing for altid mapping', 4 if $@;
 	use_ok 'PublicInbox::Inbox';
-	my $git_dir = tempdir('pi-filter_rubylang-XXXXXX',
-				TMPDIR => 1, CLEANUP => 1);
+	my ($git_dir, $for_destroy) = tmpdir();
 	is(mkdir("$git_dir/public-inbox"), 1, "created public-inbox dir");
 	my $altid = [ "serial:ruby-core:file=msgmap.sqlite3" ];
 	my $ibx = PublicInbox::Inbox->new({ inboxdir => $git_dir,
diff --git a/t/git-http-backend.t b/t/git-http-backend.t
index c4dc09a1..a927d89e 100644
--- a/t/git-http-backend.t
+++ b/t/git-http-backend.t
@@ -6,8 +6,8 @@
 use strict;
 use warnings;
 use Test::More;
-use File::Temp qw/tempdir/;
 use POSIX qw(setsid);
+require './t/common.perl';
 
 my $git_dir = $ENV{GIANT_GIT_DIR};
 plan 'skip_all' => 'GIANT_GIT_DIR not defined' unless $git_dir;
@@ -17,9 +17,8 @@ foreach my $mod (qw(BSD::Resource
 	eval "require $mod";
 	plan skip_all => "$mod missing for git-http-backend.t" if $@;
 }
-require './t/common.perl';
 my $psgi = "./t/git-http-backend.psgi";
-my $tmpdir = tempdir('pi-git-http-backend-XXXXXX', TMPDIR => 1, CLEANUP => 1);
+my ($tmpdir, $for_destroy) = tmpdir();
 my $err = "$tmpdir/stderr.log";
 my $out = "$tmpdir/stdout.log";
 my $sock = tcp_server();
diff --git a/t/git.t b/t/git.t
index a496f851..cc4fc591 100644
--- a/t/git.t
+++ b/t/git.t
@@ -3,8 +3,8 @@
 use strict;
 use warnings;
 use Test::More;
-use File::Temp qw/tempdir/;
-my $dir = tempdir('pi-git-XXXXXX', TMPDIR => 1, CLEANUP => 1);
+require './t/common.perl';
+my ($dir, $for_destroy) = tmpdir();
 use PublicInbox::Spawn qw(popen_rd);
 
 use_ok 'PublicInbox::Git';
@@ -67,7 +67,7 @@ if (1) {
 }
 
 if ('alternates reloaded') {
-	my $alt = tempdir('pi-git-XXXXXX', TMPDIR => 1, CLEANUP => 1);
+	my ($alt, $alt_obj) = tmpdir();
 	my @cmd = ('git', "--git-dir=$alt", qw(hash-object -w --stdin));
 	is(system(qw(git init -q --bare), $alt), 0, 'create alt directory');
 	open my $fh, '<', "$alt/config" or die "open failed: $!\n";
diff --git a/t/html_index.t b/t/html_index.t
index 2f4b4d1b..51ea9a25 100644
--- a/t/html_index.t
+++ b/t/html_index.t
@@ -8,8 +8,8 @@ use PublicInbox::Feed;
 use PublicInbox::Git;
 use PublicInbox::Import;
 use PublicInbox::Inbox;
-use File::Temp qw/tempdir/;
-my $tmpdir = tempdir('pi-http-XXXXXX', TMPDIR => 1, CLEANUP => 1);
+require './t/common.perl';
+my ($tmpdir, $for_destroy) = tmpdir();
 my $git_dir = "$tmpdir/gittest";
 my $ibx = PublicInbox::Inbox->new({
 	address => 'test@example',
diff --git a/t/httpd-corner.t b/t/httpd-corner.t
index eca77d7f..551af2b2 100644
--- a/t/httpd-corner.t
+++ b/t/httpd-corner.t
@@ -14,14 +14,13 @@ foreach my $mod (qw(Plack::Util Plack::Builder HTTP::Date HTTP::Status)) {
 }
 
 use Digest::SHA qw(sha1_hex);
-use File::Temp qw/tempdir/;
 use IO::Socket;
 use IO::Socket::UNIX;
 use Fcntl qw(:seek);
 use Socket qw(IPPROTO_TCP TCP_NODELAY SOL_SOCKET);
 use POSIX qw(mkfifo);
 require './t/common.perl';
-my $tmpdir = tempdir('httpd-corner-XXXXXX', TMPDIR => 1, CLEANUP => 1);
+my ($tmpdir, $for_destroy) = tmpdir();
 my $fifo = "$tmpdir/fifo";
 ok(defined mkfifo($fifo, 0777), 'created FIFO');
 my $err = "$tmpdir/stderr.log";
diff --git a/t/httpd-https.t b/t/httpd-https.t
index 81a11108..de74c20e 100644
--- a/t/httpd-https.t
+++ b/t/httpd-https.t
@@ -3,7 +3,6 @@
 use strict;
 use warnings;
 use Test::More;
-use File::Temp qw(tempdir);
 use Socket qw(SOCK_STREAM IPPROTO_TCP SOL_SOCKET);
 # IO::Poll is part of the standard library, but distros may split them off...
 foreach my $mod (qw(IO::Socket::SSL IO::Poll)) {
@@ -20,7 +19,7 @@ use_ok 'PublicInbox::TLS';
 use_ok 'IO::Socket::SSL';
 require './t/common.perl';
 my $psgi = "./t/httpd-corner.psgi";
-my $tmpdir = tempdir('pi-httpd-https-XXXXXX', TMPDIR => 1, CLEANUP => 1);
+my ($tmpdir, $for_destroy) = tmpdir();
 my $err = "$tmpdir/stderr.log";
 my $out = "$tmpdir/stdout.log";
 my $https = tcp_server();
diff --git a/t/httpd-unix.t b/t/httpd-unix.t
index 5ec70fd8..f7881cfa 100644
--- a/t/httpd-unix.t
+++ b/t/httpd-unix.t
@@ -12,9 +12,8 @@ foreach my $mod (qw(Plack::Util Plack::Builder HTTP::Date HTTP::Status)) {
 	plan skip_all => "$mod missing for httpd-unix.t" if $@;
 }
 
-use File::Temp qw/tempdir/;
 use IO::Socket::UNIX;
-my $tmpdir = tempdir('httpd-unix-XXXXXX', TMPDIR => 1, CLEANUP => 1);
+my ($tmpdir, $for_destroy) = tmpdir();
 my $unix = "$tmpdir/unix.sock";
 my $psgi = './t/httpd-corner.psgi';
 my $out = "$tmpdir/out.log";
diff --git a/t/httpd.t b/t/httpd.t
index ce8063b2..f0b4efb4 100644
--- a/t/httpd.t
+++ b/t/httpd.t
@@ -8,12 +8,11 @@ foreach my $mod (qw(Plack::Util Plack::Builder HTTP::Date HTTP::Status)) {
 	eval "require $mod";
 	plan skip_all => "$mod missing for httpd.t" if $@;
 }
-use File::Temp qw/tempdir/;
 use Socket qw(IPPROTO_TCP SOL_SOCKET);
 require './t/common.perl';
 
 # FIXME: too much setup
-my $tmpdir = tempdir('pi-httpd-XXXXXX', TMPDIR => 1, CLEANUP => 1);
+my ($tmpdir, $for_destroy) = tmpdir();
 my $home = "$tmpdir/pi-home";
 my $err = "$tmpdir/stderr.log";
 my $out = "$tmpdir/stdout.log";
diff --git a/t/import.t b/t/import.t
index d309eec5..2f5b08a5 100644
--- a/t/import.t
+++ b/t/import.t
@@ -9,9 +9,9 @@ use PublicInbox::Import;
 use PublicInbox::Spawn qw(spawn);
 use IO::File;
 use Fcntl qw(:DEFAULT);
-use File::Temp qw/tempdir tempfile/;
-my $dir = tempdir('pi-import-XXXXXX', TMPDIR => 1, CLEANUP => 1);
+use File::Temp qw/tempfile/;
 require './t/common.perl';
+my ($dir, $for_destroy) = tmpdir();
 
 is(system(qw(git init -q --bare), $dir), 0, 'git init successful');
 my $git = PublicInbox::Git->new($dir);
diff --git a/t/indexlevels-mirror.t b/t/indexlevels-mirror.t
index d129237e..f1c338e1 100644
--- a/t/indexlevels-mirror.t
+++ b/t/indexlevels-mirror.t
@@ -6,7 +6,6 @@ use Test::More;
 use PublicInbox::MIME;
 use PublicInbox::Inbox;
 use PublicInbox::InboxWritable;
-use File::Temp qw/tempdir/;
 require PublicInbox::Admin;
 require './t/common.perl';
 my $PI_TEST_VERSION = $ENV{PI_TEST_VERSION} || 2;
@@ -32,7 +31,7 @@ my $mime = PublicInbox::MIME->create(
 sub import_index_incremental {
 	my ($v, $level) = @_;
 	my $this = "pi-$v-$level-indexlevels";
-	my $tmpdir = tempdir("$this-tmp-XXXXXX", TMPDIR => 1, CLEANUP => 1);
+	my ($tmpdir, $for_destroy) = tmpdir();
 	my $ibx = PublicInbox::Inbox->new({
 		inboxdir => "$tmpdir/testbox",
 		name => $this,
diff --git a/t/init.t b/t/init.t
index 2442eeec..16550868 100644
--- a/t/init.t
+++ b/t/init.t
@@ -4,10 +4,9 @@ use strict;
 use warnings;
 use Test::More;
 use PublicInbox::Config;
-use File::Temp qw/tempdir/;
 require './t/common.perl';
-my $tmpdir = tempdir('pi-init-XXXXXX', TMPDIR => 1, CLEANUP => 1);
 use File::Basename;
+my ($tmpdir, $for_destroy) = tmpdir();
 sub quiet_fail {
 	my ($cmd, $msg) = @_;
 	my $err = '';
diff --git a/t/mda.t b/t/mda.t
index 89dedd4a..47d06132 100644
--- a/t/mda.t
+++ b/t/mda.t
@@ -4,12 +4,11 @@ use strict;
 use warnings;
 use Test::More;
 use Email::MIME;
-use File::Temp qw/tempdir/;
 use Cwd qw(getcwd);
 use PublicInbox::MID qw(mid2path);
 use PublicInbox::Git;
 require './t/common.perl';
-my $tmpdir = tempdir('pi-mda-XXXXXX', TMPDIR => 1, CLEANUP => 1);
+my ($tmpdir, $for_destroy) = tmpdir();
 my $home = "$tmpdir/pi-home";
 my $pi_home = "$home/.public-inbox";
 my $pi_config = "$pi_home/config";
diff --git a/t/mda_filter_rubylang.t b/t/mda_filter_rubylang.t
index e971b440..ce17d5a9 100644
--- a/t/mda_filter_rubylang.t
+++ b/t/mda_filter_rubylang.t
@@ -3,7 +3,6 @@
 use strict;
 use warnings;
 use Test::More;
-use File::Temp qw/tempdir/;
 use PublicInbox::MIME;
 use PublicInbox::Config;
 require './t/common.perl';
@@ -15,7 +14,7 @@ foreach my $mod (@mods) {
 }
 
 use_ok 'PublicInbox::V2Writable';
-my $tmpdir = tempdir('mda-XXXXXX', TMPDIR => 1, CLEANUP => 1);
+my ($tmpdir, $for_destroy) = tmpdir();
 my $pi_config = "$tmpdir/pi_config";
 local $ENV{PI_CONFIG} = $pi_config;
 local $ENV{PI_EMERGENCY} = "$tmpdir/emergency";
diff --git a/t/msgmap.t b/t/msgmap.t
index f1250bca..7fcd131a 100644
--- a/t/msgmap.t
+++ b/t/msgmap.t
@@ -3,7 +3,7 @@
 use strict;
 use warnings;
 use Test::More;
-use File::Temp qw/tempdir/;
+require './t/common.perl';
 
 foreach my $mod (qw(DBD::SQLite)) {
 	eval "require $mod";
@@ -11,7 +11,7 @@ foreach my $mod (qw(DBD::SQLite)) {
 }
 
 use_ok 'PublicInbox::Msgmap';
-my $tmpdir = tempdir('pi-msgmap-XXXXXX', TMPDIR => 1, CLEANUP => 1);
+my ($tmpdir, $for_destroy) = tmpdir();
 my $d = PublicInbox::Msgmap->new($tmpdir, 1);
 
 my %mid2num;
diff --git a/t/nntpd-tls.t b/t/nntpd-tls.t
index 5d170b78..bbcc04c0 100644
--- a/t/nntpd-tls.t
+++ b/t/nntpd-tls.t
@@ -3,7 +3,6 @@
 use strict;
 use warnings;
 use Test::More;
-use File::Temp qw(tempdir);
 use Socket qw(SOCK_STREAM IPPROTO_TCP SOL_SOCKET);
 # IO::Poll and Net::NNTP are part of the standard library, but
 # distros may split them off...
@@ -34,7 +33,7 @@ eval { require Compress::Raw::Zlib } or
 	$need_zlib = 'Compress::Raw::Zlib missing';
 my $version = 2; # v2 needs newer git
 require_git('2.6') if $version >= 2;
-my $tmpdir = tempdir('pi-nntpd-tls-XXXXXX', TMPDIR => 1, CLEANUP => 1);
+my ($tmpdir, $for_destroy) = tmpdir();
 my $err = "$tmpdir/stderr.log";
 my $out = "$tmpdir/stdout.log";
 my $inboxdir = "$tmpdir";
diff --git a/t/nntpd-validate.t b/t/nntpd-validate.t
index da6985be..39108639 100644
--- a/t/nntpd-validate.t
+++ b/t/nntpd-validate.t
@@ -4,7 +4,6 @@
 # Integration test to validate compression.
 use strict;
 use warnings;
-use File::Temp qw(tempdir);
 use Test::More;
 use Symbol qw(gensym);
 use Time::HiRes qw(clock_gettime CLOCK_MONOTONIC);
@@ -31,8 +30,8 @@ if ($test_tls && !-r $key || !-r $cert) {
 	plan skip_all => "certs/ missing for $0, run $^X ./certs/create-certs.perl";
 }
 require './t/common.perl';
-my $keep_tmp = !!$ENV{TEST_KEEP_TMP};
-my $tmpdir = tempdir('nntpd-validate-XXXXXX',TMPDIR => 1,CLEANUP => $keep_tmp);
+my ($tmpdir, $ftd) = tmpdir();
+$File::Temp::KEEP_ALL = !!$ENV{TEST_KEEP_TMP};
 my (%OPT, $td, $host_port, $group);
 my $batch = 1000;
 if (($ENV{NNTP_TEST_URL} // '') =~ m!\Anntp://([^/]+)/([^/]+)\z!) {
@@ -63,7 +62,7 @@ sub do_get_all {
 	my $dig = Digest::SHA->new(1);
 	my $digfh = gensym;
 	my $tmpfh;
-	if ($keep_tmp) {
+	if ($File::Temp::KEEP_ALL) {
 		open $tmpfh, '>', "$tmpdir/$desc.raw" or die $!;
 	}
 	my $tmp = { dig => $dig, tmpfh => $tmpfh };
diff --git a/t/nntpd.t b/t/nntpd.t
index 3c928610..5b697344 100644
--- a/t/nntpd.t
+++ b/t/nntpd.t
@@ -13,7 +13,6 @@ require PublicInbox::InboxWritable;
 use Email::Simple;
 use IO::Socket;
 use Socket qw(IPPROTO_TCP TCP_NODELAY);
-use File::Temp qw/tempdir/;
 use Net::NNTP;
 use Sys::Hostname;
 require './t/common.perl';
@@ -22,7 +21,7 @@ require './t/common.perl';
 my $version = $ENV{PI_TEST_VERSION} || 2;
 require_git('2.6') if $version == 2;
 
-my $tmpdir = tempdir('pi-nntpd-XXXXXX', TMPDIR => 1, CLEANUP => 1);
+my ($tmpdir, $for_destroy) = tmpdir();
 my $home = "$tmpdir/pi-home";
 my $err = "$tmpdir/stderr.log";
 my $out = "$tmpdir/stdout.log";
diff --git a/t/nulsubject.t b/t/nulsubject.t
index 4c07f509..617997c0 100644
--- a/t/nulsubject.t
+++ b/t/nulsubject.t
@@ -3,11 +3,11 @@
 use strict;
 use warnings;
 use Test::More;
-use File::Temp qw/tempdir/;
+require './t/common.perl';
 
 use_ok 'PublicInbox::Import';
 use_ok 'PublicInbox::Git';
-my $tmpdir = tempdir('pi-nulsubject-XXXXXX', TMPDIR => 1, CLEANUP => 1);
+my ($tmpdir, $for_destroy) = tmpdir();
 my $git_dir = "$tmpdir/a.git";
 
 {
diff --git a/t/over.t b/t/over.t
index 48c835f8..27168a33 100644
--- a/t/over.t
+++ b/t/over.t
@@ -3,15 +3,15 @@
 use strict;
 use warnings;
 use Test::More;
-use File::Temp qw/tempdir/;
 use Compress::Zlib qw(compress);
+require './t/common.perl';
 foreach my $mod (qw(DBD::SQLite)) {
 	eval "require $mod";
 	plan skip_all => "$mod missing for over.t" if $@;
 }
 
 use_ok 'PublicInbox::OverIdx';
-my $tmpdir = tempdir('pi-over-XXXXXX', TMPDIR => 1, CLEANUP => 1);
+my ($tmpdir, $for_destroy) = tmpdir();
 my $over = PublicInbox::OverIdx->new("$tmpdir/over.sqlite3");
 $over->connect;
 my $x = $over->next_tid;
diff --git a/t/perf-nntpd.t b/t/perf-nntpd.t
index c7d2eaff..6f891ddb 100644
--- a/t/perf-nntpd.t
+++ b/t/perf-nntpd.t
@@ -5,12 +5,11 @@ use warnings;
 use Test::More;
 use Benchmark qw(:all :hireswallclock);
 use PublicInbox::Inbox;
-use File::Temp qw/tempdir/;
 use Net::NNTP;
 my $pi_dir = $ENV{GIANT_PI_DIR};
 plan skip_all => "GIANT_PI_DIR not defined for $0" unless $pi_dir;
 eval { require PublicInbox::Search };
-my ($host_port, $group, %opts, $s, $td);
+my ($host_port, $group, %opts, $s, $td, $tmp_obj);
 require './t/common.perl';
 
 if (($ENV{NNTP_TEST_URL} || '') =~ m!\Anntp://([^/]+)/([^/]+)\z!) {
@@ -20,7 +19,8 @@ if (($ENV{NNTP_TEST_URL} || '') =~ m!\Anntp://([^/]+)/([^/]+)\z!) {
 	$group = 'inbox.test.perf.nntpd';
 	my $ibx = { inboxdir => $pi_dir, newsgroup => $group };
 	$ibx = PublicInbox::Inbox->new($ibx);
-	my $tmpdir = tempdir('perf-nntpd-XXXXXX', TMPDIR => 1, CLEANUP => 1);
+	my $tmpdir;
+	($tmpdir, $tmp_obj) = tmpdir();
 
 	my $pi_config = "$tmpdir/config";
 	{
diff --git a/t/plack.t b/t/plack.t
index 9308813f..6023a419 100644
--- a/t/plack.t
+++ b/t/plack.t
@@ -4,9 +4,9 @@ use strict;
 use warnings;
 use Test::More;
 use Email::MIME;
-use File::Temp qw/tempdir/;
+require './t/common.perl';
 my $psgi = "./examples/public-inbox.psgi";
-my $tmpdir = tempdir('pi-plack-XXXXXX', TMPDIR => 1, CLEANUP => 1);
+my ($tmpdir, $for_destroy) = tmpdir();
 my $pi_config = "$tmpdir/config";
 my $maindir = "$tmpdir/main.git";
 my $addr = 'test-public@example.com';
diff --git a/t/psgi_attach.t b/t/psgi_attach.t
index 96f0cb47..45f05bac 100644
--- a/t/psgi_attach.t
+++ b/t/psgi_attach.t
@@ -4,8 +4,8 @@ use strict;
 use warnings;
 use Test::More;
 use Email::MIME;
-use File::Temp qw/tempdir/;
-my $tmpdir = tempdir('psgi-attach-XXXXXX', TMPDIR => 1, CLEANUP => 1);
+require './t/common.perl';
+my ($tmpdir, $for_destroy) = tmpdir();
 my $maindir = "$tmpdir/main.git";
 my $addr = 'test-public@example.com';
 my $cfgpfx = "publicinbox.test";
diff --git a/t/psgi_bad_mids.t b/t/psgi_bad_mids.t
index c7c94718..0e8fa114 100644
--- a/t/psgi_bad_mids.t
+++ b/t/psgi_bad_mids.t
@@ -3,10 +3,10 @@
 use strict;
 use warnings;
 use Test::More;
-use File::Temp qw/tempdir/;
 use PublicInbox::MIME;
 use PublicInbox::Config;
 use PublicInbox::WWW;
+require './t/common.perl';
 my @mods = qw(DBD::SQLite HTTP::Request::Common Plack::Test
 		URI::Escape Plack::Builder);
 foreach my $mod (@mods) {
@@ -15,7 +15,7 @@ foreach my $mod (@mods) {
 }
 use_ok($_) for @mods;
 use_ok 'PublicInbox::V2Writable';
-my $inboxdir = tempdir('pi-bad-mids-XXXXXX', TMPDIR => 1, CLEANUP => 1);
+my ($inboxdir, $for_destroy) = tmpdir();
 my $cfgpfx = "publicinbox.bad-mids";
 my $ibx = {
 	inboxdir => $inboxdir,
diff --git a/t/psgi_mount.t b/t/psgi_mount.t
index 7de2bc0e..ca573e1e 100644
--- a/t/psgi_mount.t
+++ b/t/psgi_mount.t
@@ -4,8 +4,8 @@ use strict;
 use warnings;
 use Test::More;
 use Email::MIME;
-use File::Temp qw/tempdir/;
-my $tmpdir = tempdir('psgi-path-XXXXXX', TMPDIR => 1, CLEANUP => 1);
+require './t/common.perl';
+my ($tmpdir, $for_destroy) = tmpdir();
 my $maindir = "$tmpdir/main.git";
 my $addr = 'test-public@example.com';
 my $cfgpfx = "publicinbox.test";
diff --git a/t/psgi_multipart_not.t b/t/psgi_multipart_not.t
index 40bc3c18..d3489f2d 100644
--- a/t/psgi_multipart_not.t
+++ b/t/psgi_multipart_not.t
@@ -3,10 +3,10 @@
 use strict;
 use warnings;
 use Test::More;
-use File::Temp qw/tempdir/;
 use Email::MIME;
 use PublicInbox::Config;
 use PublicInbox::WWW;
+require './t/common.perl';
 my @mods = qw(DBD::SQLite Search::Xapian HTTP::Request::Common
               Plack::Test URI::Escape Plack::Builder Plack::Test);
 foreach my $mod (@mods) {
@@ -15,7 +15,7 @@ foreach my $mod (@mods) {
 }
 use_ok($_) for @mods;
 use_ok 'PublicInbox::V2Writable';
-my $repo = tempdir('pi-psgi-multipart-not.XXXXXX', TMPDIR => 1, CLEANUP => 1);
+my ($repo, $for_destroy) = tmpdir();
 my $ibx = PublicInbox::Inbox->new({
 	inboxdir => $repo,
 	name => 'multipart-not',
diff --git a/t/psgi_scan_all.t b/t/psgi_scan_all.t
index 707807a7..5d4cc263 100644
--- a/t/psgi_scan_all.t
+++ b/t/psgi_scan_all.t
@@ -4,8 +4,8 @@ use strict;
 use warnings;
 use Test::More;
 use Email::MIME;
-use File::Temp qw/tempdir/;
 use PublicInbox::Config;
+require './t/common.perl';
 my @mods = qw(HTTP::Request::Common Plack::Test URI::Escape DBD::SQLite);
 foreach my $mod (@mods) {
 	eval "require $mod";
@@ -13,7 +13,7 @@ foreach my $mod (@mods) {
 }
 use_ok 'PublicInbox::V2Writable';
 foreach my $mod (@mods) { use_ok $mod; }
-my $tmp = tempdir('pi-scan_all-XXXXXX', TMPDIR => 1, CLEANUP => 1);
+my ($tmp, $for_destroy) = tmpdir();
 my $cfg = '';
 
 foreach my $i (1..2) {
diff --git a/t/psgi_search.t b/t/psgi_search.t
index 4cd0e499..0c430aea 100644
--- a/t/psgi_search.t
+++ b/t/psgi_search.t
@@ -3,13 +3,13 @@
 use strict;
 use warnings;
 use Test::More;
-use File::Temp qw/tempdir/;
 use Email::MIME;
 use PublicInbox::Config;
 use PublicInbox::Inbox;
 use PublicInbox::InboxWritable;
 use PublicInbox::WWW;
 use bytes (); # only for bytes::length
+require './t/common.perl';
 my @mods = qw(DBD::SQLite Search::Xapian HTTP::Request::Common Plack::Test
 		URI::Escape Plack::Builder);
 foreach my $mod (@mods) {
@@ -18,7 +18,7 @@ foreach my $mod (@mods) {
 }
 
 use_ok $_ foreach (@mods, qw(PublicInbox::SearchIdx));
-my $tmpdir = tempdir('pi-psgi-search.XXXXXX', TMPDIR => 1, CLEANUP => 1);
+my ($tmpdir, $for_destroy) = tmpdir();
 
 my $ibx = PublicInbox::Inbox->new({
 	inboxdir => $tmpdir,
diff --git a/t/psgi_text.t b/t/psgi_text.t
index da7c6f57..b9564181 100644
--- a/t/psgi_text.t
+++ b/t/psgi_text.t
@@ -4,8 +4,8 @@ use strict;
 use warnings;
 use Test::More;
 use Email::MIME;
-use File::Temp qw/tempdir/;
-my $tmpdir = tempdir('psgi-text-XXXXXX', TMPDIR => 1, CLEANUP => 1);
+require './t/common.perl';
+my ($tmpdir, $for_destroy) = tmpdir();
 my $maindir = "$tmpdir/main.git";
 my $addr = 'test-public@example.com';
 my $cfgpfx = "publicinbox.test";
diff --git a/t/psgi_v2.t b/t/psgi_v2.t
index c7550e2d..1163e2bf 100644
--- a/t/psgi_v2.t
+++ b/t/psgi_v2.t
@@ -5,7 +5,6 @@ use warnings;
 use Test::More;
 require './t/common.perl';
 require_git(2.6);
-use File::Temp qw/tempdir/;
 use PublicInbox::MIME;
 use PublicInbox::Config;
 use PublicInbox::WWW;
@@ -18,7 +17,7 @@ foreach my $mod (@mods) {
 }
 use_ok($_) for @mods;
 use_ok 'PublicInbox::V2Writable';
-my $inboxdir = tempdir('pi-v2_dupes-XXXXXX', TMPDIR => 1, CLEANUP => 1);
+my ($inboxdir, $for_destroy) = tmpdir();
 my $ibx = {
 	inboxdir => $inboxdir,
 	name => 'test-v2writable',
diff --git a/t/purge.t b/t/purge.t
index bcdbad52..12644d69 100644
--- a/t/purge.t
+++ b/t/purge.t
@@ -3,7 +3,6 @@
 use strict;
 use warnings;
 use Test::More;
-use File::Temp qw/tempdir/;
 require './t/common.perl';
 require_git(2.6);
 my @mods = qw(DBI DBD::SQLite);
@@ -13,7 +12,7 @@ foreach my $mod (@mods) {
 };
 use Cwd qw(abs_path); # we need this since we chdir below
 my $purge = abs_path('blib/script/public-inbox-purge');
-my $tmpdir = tempdir('pi-purge-XXXXXX', TMPDIR => 1, CLEANUP => 1);
+my ($tmpdir, $for_destroy) = tmpdir();
 use_ok 'PublicInbox::V2Writable';
 my $inboxdir = "$tmpdir/v2";
 my $ibx = PublicInbox::Inbox->new({
diff --git a/t/replace.t b/t/replace.t
index 24f7537c..039c6bc6 100644
--- a/t/replace.t
+++ b/t/replace.t
@@ -5,7 +5,6 @@ use warnings;
 use Test::More;
 use PublicInbox::MIME;
 use PublicInbox::InboxWritable;
-use File::Temp qw/tempdir/;
 require './t/common.perl';
 require_git(2.6); # replace is v2 only, for now...
 foreach my $mod (qw(DBD::SQLite)) {
@@ -17,7 +16,7 @@ sub test_replace ($$$) {
 	my ($v, $level, $opt) = @_;
 	diag "v$v $level replace";
 	my $this = "pi-$v-$level-replace";
-	my $tmpdir = tempdir("$this-tmp-XXXXXX", TMPDIR => 1, CLEANUP => 1);
+	my ($tmpdir, $for_destroy) = tmpdir($this);
 	my $ibx = PublicInbox::Inbox->new({
 		inboxdir => "$tmpdir/testbox",
 		name => $this,
diff --git a/t/search-thr-index.t b/t/search-thr-index.t
index 26339989..4f793657 100644
--- a/t/search-thr-index.t
+++ b/t/search-thr-index.t
@@ -4,7 +4,6 @@ use strict;
 use warnings;
 use bytes (); # only for bytes::length
 use Test::More;
-use File::Temp qw/tempdir/;
 use PublicInbox::MID qw(mids);
 use Email::MIME;
 my @mods = qw(DBI DBD::SQLite Search::Xapian);
@@ -14,7 +13,8 @@ foreach my $mod (@mods) {
 }
 require PublicInbox::SearchIdx;
 require PublicInbox::Inbox;
-my $tmpdir = tempdir('pi-search-thr-index.XXXXXX', TMPDIR => 1, CLEANUP => 1);
+require './t/common.perl';
+my ($tmpdir, $for_destroy) = tmpdir();
 my $git_dir = "$tmpdir/a.git";
 
 is(0, system(qw(git init -q --bare), $git_dir), "git init (main)");
diff --git a/t/search.t b/t/search.t
index b6531ab3..58684138 100644
--- a/t/search.t
+++ b/t/search.t
@@ -10,9 +10,9 @@ foreach my $mod (@mods) {
 };
 require PublicInbox::SearchIdx;
 require PublicInbox::Inbox;
-use File::Temp qw/tempdir/;
+require './t/common.perl';
 use Email::MIME;
-my $tmpdir = tempdir('pi-search-XXXXXX', TMPDIR => 1, CLEANUP => 1);
+my ($tmpdir, $for_destroy) = tmpdir();
 my $git_dir = "$tmpdir/a.git";
 my $ibx = PublicInbox::Inbox->new({ inboxdir => $git_dir });
 my ($root_id, $last_id);
diff --git a/t/solver_git.t b/t/solver_git.t
index baab40a4..9bda157d 100644
--- a/t/solver_git.t
+++ b/t/solver_git.t
@@ -3,7 +3,6 @@
 use strict;
 use warnings;
 use Test::More;
-use File::Temp qw(tempdir);
 use Cwd qw(abs_path);
 require './t/common.perl';
 require_git(2.6);
@@ -22,7 +21,7 @@ $git_dir = abs_path($git_dir);
 
 use_ok "PublicInbox::$_" for (qw(Inbox V2Writable MIME Git SolverGit));
 
-my $inboxdir = tempdir('pi-solver-XXXXXX', TMPDIR => 1, CLEANUP => 1);
+my ($inboxdir, $for_destroy) = tmpdir();
 my $opts = {
 	inboxdir => $inboxdir,
 	name => 'test-v2writable',
diff --git a/t/spamcheck_spamc.t b/t/spamcheck_spamc.t
index c13108f4..a4a01a8b 100644
--- a/t/spamcheck_spamc.t
+++ b/t/spamcheck_spamc.t
@@ -5,9 +5,9 @@ use warnings;
 use Test::More;
 use Email::Simple;
 use IO::File;
-use File::Temp qw/tempdir/;
 use Fcntl qw(:DEFAULT SEEK_SET);
-my $tmpdir = tempdir('spamcheck_spamc-XXXXXX', TMPDIR => 1, CLEANUP => 1);
+require './t/common.perl';
+my ($tmpdir, $for_destroy) = tmpdir();
 
 use_ok 'PublicInbox::Spamcheck::Spamc';
 my $spamc = PublicInbox::Spamcheck::Spamc->new;
diff --git a/t/v1-add-remove-add.t b/t/v1-add-remove-add.t
index 035fba5c..13e9f29c 100644
--- a/t/v1-add-remove-add.t
+++ b/t/v1-add-remove-add.t
@@ -5,15 +5,15 @@ use warnings;
 use Test::More;
 use PublicInbox::MIME;
 use PublicInbox::Import;
-use File::Temp qw/tempdir/;
+require './t/common.perl';
 
 foreach my $mod (qw(DBD::SQLite Search::Xapian)) {
 	eval "require $mod";
 	plan skip_all => "$mod missing for v1-add-remove-add.t" if $@;
 }
 require PublicInbox::SearchIdx;
-my $inboxdir = tempdir('pi-add-remove-add-XXXXXX', TMPDIR => 1, CLEANUP => 1);
-is(system(qw(git init -q --bare), $inboxdir), 0);
+my ($inboxdir, $for_destroy) = tmpdir();
+is(system(qw(git init --bare -q), $inboxdir), 0);
 my $ibx = {
 	inboxdir => $inboxdir,
 	name => 'test-add-remove-add',
diff --git a/t/v1reindex.t b/t/v1reindex.t
index e3547753..c0e21a56 100644
--- a/t/v1reindex.t
+++ b/t/v1reindex.t
@@ -5,7 +5,6 @@ use warnings;
 use Test::More;
 use PublicInbox::MIME;
 use PublicInbox::ContentId qw(content_digest);
-use File::Temp qw/tempdir/;
 use File::Path qw(remove_tree);
 require './t/common.perl';
 require_git(2.6);
@@ -16,7 +15,7 @@ foreach my $mod (qw(DBD::SQLite Search::Xapian)) {
 }
 use_ok 'PublicInbox::SearchIdx';
 use_ok 'PublicInbox::Import';
-my $inboxdir = tempdir('pi-v1reindex-XXXXXX', TMPDIR => 1, CLEANUP => 1);
+my ($inboxdir, $for_destroy) = tmpdir();
 is(system(qw(git init -q --bare), $inboxdir), 0);
 my $ibx_config = {
 	inboxdir => $inboxdir,
diff --git a/t/v2-add-remove-add.t b/t/v2-add-remove-add.t
index 438fe3db..c0dec300 100644
--- a/t/v2-add-remove-add.t
+++ b/t/v2-add-remove-add.t
@@ -4,7 +4,6 @@ use strict;
 use warnings;
 use Test::More;
 use PublicInbox::MIME;
-use File::Temp qw/tempdir/;
 require './t/common.perl';
 require_git(2.6);
 
@@ -13,7 +12,7 @@ foreach my $mod (qw(DBD::SQLite Search::Xapian)) {
 	plan skip_all => "$mod missing for v2-add-remove-add.t" if $@;
 }
 use_ok 'PublicInbox::V2Writable';
-my $inboxdir = tempdir('pi-add-remove-add-XXXXXX', TMPDIR => 1, CLEANUP => 1);
+my ($inboxdir, $for_destroy) = tmpdir();
 my $ibx = {
 	inboxdir => "$inboxdir/v2",
 	name => 'test-v2writable',
diff --git a/t/v2mda.t b/t/v2mda.t
index 0cd852b1..11a517e4 100644
--- a/t/v2mda.t
+++ b/t/v2mda.t
@@ -4,7 +4,6 @@ use strict;
 use warnings;
 use Test::More;
 use PublicInbox::MIME;
-use File::Temp qw/tempdir/;
 use Fcntl qw(SEEK_SET);
 use Cwd;
 require './t/common.perl';
@@ -16,7 +15,7 @@ foreach my $mod (qw(DBD::SQLite Search::Xapian)) {
 	plan skip_all => "$mod missing for v2mda.t" if $@;
 }
 use_ok 'PublicInbox::V2Writable';
-my $tmpdir = tempdir('pi-v2mda-XXXXXX', TMPDIR => 1, CLEANUP => 1);
+my ($tmpdir, $for_destroy) = tmpdir();
 my $ibx = {
 	inboxdir => "$tmpdir/inbox",
 	name => 'test-v2writable',
diff --git a/t/v2mirror.t b/t/v2mirror.t
index 1a39ce49..96657fdc 100644
--- a/t/v2mirror.t
+++ b/t/v2mirror.t
@@ -12,7 +12,6 @@ foreach my $mod (qw(Plack::Util Plack::Builder
 	eval "require $mod";
 	plan skip_all => "$mod missing for v2mirror.t" if $@;
 }
-use File::Temp qw/tempdir/;
 use IO::Socket;
 use POSIX qw(dup2);
 use_ok 'PublicInbox::V2Writable';
@@ -20,7 +19,7 @@ use PublicInbox::InboxWritable;
 use PublicInbox::MIME;
 use PublicInbox::Config;
 # FIXME: too much setup
-my $tmpdir = tempdir('pi-v2mirror-XXXXXX', TMPDIR => 1, CLEANUP => 1);
+my ($tmpdir, $for_destroy) = tmpdir();
 my $pi_config = "$tmpdir/config";
 {
 	open my $fh, '>', $pi_config or die "open($pi_config): $!";
diff --git a/t/v2reindex.t b/t/v2reindex.t
index 3e56ddfa..e222d0f1 100644
--- a/t/v2reindex.t
+++ b/t/v2reindex.t
@@ -5,7 +5,6 @@ use warnings;
 use Test::More;
 use PublicInbox::MIME;
 use PublicInbox::ContentId qw(content_digest);
-use File::Temp qw/tempdir/;
 use File::Path qw(remove_tree);
 require './t/common.perl';
 require_git(2.6);
@@ -15,7 +14,7 @@ foreach my $mod (qw(DBD::SQLite Search::Xapian)) {
 	plan skip_all => "$mod missing for v2reindex.t" if $@;
 }
 use_ok 'PublicInbox::V2Writable';
-my $inboxdir = tempdir('pi-v2reindex-XXXXXX', TMPDIR => 1, CLEANUP => 1);
+my ($inboxdir, $for_destroy) = tmpdir();
 my $ibx_config = {
 	inboxdir => $inboxdir,
 	name => 'test-v2writable',
diff --git a/t/v2writable.t b/t/v2writable.t
index 4bb6d733..7519b487 100644
--- a/t/v2writable.t
+++ b/t/v2writable.t
@@ -5,7 +5,6 @@ use warnings;
 use Test::More;
 use PublicInbox::MIME;
 use PublicInbox::ContentId qw(content_digest);
-use File::Temp qw/tempdir/;
 require './t/common.perl';
 require_git(2.6);
 foreach my $mod (qw(DBD::SQLite Search::Xapian)) {
@@ -14,7 +13,7 @@ foreach my $mod (qw(DBD::SQLite Search::Xapian)) {
 }
 use_ok 'PublicInbox::V2Writable';
 umask 007;
-my $inboxdir = tempdir('pi-v2writable-XXXXXX', TMPDIR => 1, CLEANUP => 1);
+my ($inboxdir, $for_destroy) = tmpdir();
 my $ibx = {
 	inboxdir => $inboxdir,
 	name => 'test-v2writable',
diff --git a/t/watch_filter_rubylang.t b/t/watch_filter_rubylang.t
index 57ab3b91..c4078879 100644
--- a/t/watch_filter_rubylang.t
+++ b/t/watch_filter_rubylang.t
@@ -4,7 +4,6 @@ use strict;
 use warnings;
 require './t/common.perl';
 use Test::More;
-use File::Temp qw/tempdir/;
 use PublicInbox::MIME;
 use PublicInbox::Config;
 my @mods = qw(Filesys::Notify::Simple DBD::SQLite Search::Xapian);
@@ -15,7 +14,7 @@ foreach my $mod (@mods) {
 
 use_ok 'PublicInbox::WatchMaildir';
 use_ok 'PublicInbox::Emergency';
-my $tmpdir = tempdir('watch-XXXXXX', TMPDIR => 1, CLEANUP => 1);
+my ($tmpdir, $for_destroy) = tmpdir();
 local $ENV{PI_CONFIG} = "$tmpdir/pi_config";
 
 my @v = qw(V1);
diff --git a/t/watch_maildir.t b/t/watch_maildir.t
index 41d50329..d2e6fecd 100644
--- a/t/watch_maildir.t
+++ b/t/watch_maildir.t
@@ -2,17 +2,17 @@
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 use strict;
 use Test::More;
-use File::Temp qw/tempdir/;
 use Email::MIME;
 use Cwd;
 use PublicInbox::Config;
+require './t/common.perl';
 my @mods = qw(Filesys::Notify::Simple);
 foreach my $mod (@mods) {
 	eval "require $mod";
 	plan skip_all => "$mod missing for watch_maildir.t" if $@;
 }
 
-my $tmpdir = tempdir('watch_maildir-XXXXXX', TMPDIR => 1, CLEANUP => 1);
+my ($tmpdir, $for_destroy) = tmpdir();
 my $git_dir = "$tmpdir/test.git";
 my $maildir = "$tmpdir/md";
 my $spamdir = "$tmpdir/spam";
diff --git a/t/watch_maildir_v2.t b/t/watch_maildir_v2.t
index e0e8a13f..53f1bdfc 100644
--- a/t/watch_maildir_v2.t
+++ b/t/watch_maildir_v2.t
@@ -2,7 +2,6 @@
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 use strict;
 use Test::More;
-use File::Temp qw/tempdir/;
 use PublicInbox::MIME;
 use Cwd;
 use PublicInbox::Config;
@@ -14,7 +13,7 @@ foreach my $mod (@mods) {
 	plan skip_all => "$mod missing for watch_maildir_v2.t" if $@;
 }
 require PublicInbox::V2Writable;
-my $tmpdir = tempdir('watch_maildir-v2-XXXXXX', TMPDIR => 1, CLEANUP => 1);
+my ($tmpdir, $for_destroy) = tmpdir();
 my $inboxdir = "$tmpdir/v2";
 my $maildir = "$tmpdir/md";
 my $spamdir = "$tmpdir/spam";
diff --git a/t/www_listing.t b/t/www_listing.t
index 9cde3575..c9201213 100644
--- a/t/www_listing.t
+++ b/t/www_listing.t
@@ -5,7 +5,6 @@ use strict;
 use warnings;
 use Test::More;
 use PublicInbox::Spawn qw(which);
-use File::Temp qw/tempdir/;
 require './t/common.perl';
 my @mods = qw(URI::Escape Plack::Builder Digest::SHA
 		IO::Compress::Gzip IO::Uncompress::Gunzip HTTP::Tiny);
@@ -19,7 +18,7 @@ plan skip_all => "JSON module missing: $@" if $@;
 
 use_ok 'PublicInbox::Git';
 
-my $tmpdir = tempdir('www_listing-tmp-XXXXXX', TMPDIR => 1, CLEANUP => 1);
+my ($tmpdir, $for_destroy) = tmpdir();
 my $bare = PublicInbox::Git->new("$tmpdir/bare.git");
 is(system(qw(git init -q --bare), $bare->{git_dir}), 0, 'git init --bare');
 is(PublicInbox::WwwListing::fingerprint($bare), undef,
diff --git a/t/xcpdb-reshard.t b/t/xcpdb-reshard.t
index 9335843d..ebf156a3 100644
--- a/t/xcpdb-reshard.t
+++ b/t/xcpdb-reshard.t
@@ -10,7 +10,6 @@ foreach my $mod (@mods) {
 };
 require './t/common.perl';
 require_git('2.6');
-use File::Temp qw/tempdir/;
 use PublicInbox::MIME;
 use PublicInbox::InboxWritable;
 
@@ -25,7 +24,7 @@ my $mime = PublicInbox::MIME->create(
 );
 
 my ($this) = (split('/', $0))[-1];
-my $tmpdir = tempdir($this.'-XXXXXX', TMPDIR => 1, CLEANUP => 1);
+my ($tmpdir, $for_destroy) = tmpdir();
 my $ibx = PublicInbox::Inbox->new({
 	inboxdir => "$tmpdir/testbox",
 	name => $this,

[PATCH 15/17] tests: quiet down commit graph

[Eric Wong]

Newer versions of git enable the commit graph by default.
Since we blow away our temporary directories every test,
generating graphis is a waste and clutters stderr with
"Computing commit graph generation numbers" messages.
---
 MANIFEST       | 1 +
 t/.gitconfig   | 4 ++++
 t/purge.t      | 1 +
 t/replace.t    | 3 +++
 t/v2mirror.t   | 2 ++
 t/v2writable.t | 2 ++
 6 files changed, 13 insertions(+)
 create mode 100644 t/.gitconfig

diff --git a/MANIFEST b/MANIFEST
index 689d3d4e..9475667b 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -194,6 +194,7 @@ scripts/report-spam
 scripts/slrnspool2maildir
 scripts/ssoma-replay
 scripts/xhdr-num2mid
+t/.gitconfig
 t/address.t
 t/admin.t
 t/altid.t
diff --git a/t/.gitconfig b/t/.gitconfig
new file mode 100644
index 00000000..645a3041
--- /dev/null
+++ b/t/.gitconfig
@@ -0,0 +1,4 @@
+; this becomes ~/.gitconfig for tests where we use
+; "$ENV{HOME} = '/path/to/worktree/t'" in tests
+[gc]
+	writeCommitGraph = false
diff --git a/t/purge.t b/t/purge.t
index 12644d69..db09b731 100644
--- a/t/purge.t
+++ b/t/purge.t
@@ -11,6 +11,7 @@ foreach my $mod (@mods) {
 	plan skip_all => "missing $mod for t/purge.t" if $@;
 };
 use Cwd qw(abs_path); # we need this since we chdir below
+local $ENV{HOME} = abs_path('t');
 my $purge = abs_path('blib/script/public-inbox-purge');
 my ($tmpdir, $for_destroy) = tmpdir();
 use_ok 'PublicInbox::V2Writable';
diff --git a/t/replace.t b/t/replace.t
index 039c6bc6..e9361856 100644
--- a/t/replace.t
+++ b/t/replace.t
@@ -6,12 +6,15 @@ use Test::More;
 use PublicInbox::MIME;
 use PublicInbox::InboxWritable;
 require './t/common.perl';
+use Cwd qw(abs_path);
 require_git(2.6); # replace is v2 only, for now...
 foreach my $mod (qw(DBD::SQLite)) {
 	eval "require $mod";
 	plan skip_all => "$mod missing for $0" if $@;
 }
 
+local $ENV{HOME} = abs_path('t');
+
 sub test_replace ($$$) {
 	my ($v, $level, $opt) = @_;
 	diag "v$v $level replace";
diff --git a/t/v2mirror.t b/t/v2mirror.t
index 96657fdc..a45a262e 100644
--- a/t/v2mirror.t
+++ b/t/v2mirror.t
@@ -4,7 +4,9 @@ use strict;
 use warnings;
 use Test::More;
 require './t/common.perl';
+use Cwd qw(abs_path);
 require_git(2.6);
+local $ENV{HOME} = abs_path('t');
 
 # Integration tests for HTTP cloning + mirroring
 foreach my $mod (qw(Plack::Util Plack::Builder
diff --git a/t/v2writable.t b/t/v2writable.t
index 7519b487..8bbcd45a 100644
--- a/t/v2writable.t
+++ b/t/v2writable.t
@@ -6,11 +6,13 @@ use Test::More;
 use PublicInbox::MIME;
 use PublicInbox::ContentId qw(content_digest);
 require './t/common.perl';
+use Cwd qw(abs_path);
 require_git(2.6);
 foreach my $mod (qw(DBD::SQLite Search::Xapian)) {
 	eval "require $mod";
 	plan skip_all => "$mod missing for nntpd.t" if $@;
 }
+local $ENV{HOME} = abs_path('t');
 use_ok 'PublicInbox::V2Writable';
 umask 007;
 my ($inboxdir, $for_destroy) = tmpdir();

[PATCH 16/17] t/perf-*.t: use $ENV{GIANT_INBOX_DIR} consistently

[Eric Wong]

It's more consistent with our current terminology and
"PI_DIR" is already used to override ~/.public-inbox/
(which holds "config" and possibly other files which affect
all inboxes for a particular user, but is not an inbox itself);
so stop advertising GIANT_PI_DIR in skip messages.
---
 t/perf-msgview.t   | 6 +++---
 t/perf-nntpd.t     | 8 ++++----
 t/perf-threading.t | 8 ++++----
 3 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/t/perf-msgview.t b/t/perf-msgview.t
index 492ed487..22d8ce20 100644
--- a/t/perf-msgview.t
+++ b/t/perf-msgview.t
@@ -8,8 +8,8 @@ use PublicInbox::Inbox;
 use PublicInbox::View;
 require './t/common.perl';
 
-my $pi_dir = $ENV{GIANT_PI_DIR};
-plan skip_all => "GIANT_PI_DIR not defined for $0" unless $pi_dir;
+my $inboxdir = $ENV{GIANT_INBOX_DIR} // $ENV{GIANT_PI_DIR};
+plan skip_all => "GIANT_INBOX_DIR not defined for $0" unless $inboxdir;
 
 my @cat = qw(cat-file --buffer --batch-check --batch-all-objects);
 if (require_git(2.19, 1)) {
@@ -20,7 +20,7 @@ if (require_git(2.19, 1)) {
 }
 
 use_ok 'Plack::Util';
-my $ibx = PublicInbox::Inbox->new({ inboxdir => $pi_dir, name => 'name' });
+my $ibx = PublicInbox::Inbox->new({ inboxdir => $inboxdir, name => 'name' });
 my $git = $ibx->git;
 my $fh = $git->popen(@cat);
 my $vec = '';
diff --git a/t/perf-nntpd.t b/t/perf-nntpd.t
index 6f891ddb..5a176e08 100644
--- a/t/perf-nntpd.t
+++ b/t/perf-nntpd.t
@@ -6,8 +6,8 @@ use Test::More;
 use Benchmark qw(:all :hireswallclock);
 use PublicInbox::Inbox;
 use Net::NNTP;
-my $pi_dir = $ENV{GIANT_PI_DIR};
-plan skip_all => "GIANT_PI_DIR not defined for $0" unless $pi_dir;
+my $inboxdir = $ENV{GIANT_INBOX_DIR} // $ENV{GIANT_PI_DIR};
+plan skip_all => "GIANT_INBOX_DIR not defined for $0" unless defined($inboxdir);
 eval { require PublicInbox::Search };
 my ($host_port, $group, %opts, $s, $td, $tmp_obj);
 require './t/common.perl';
@@ -17,7 +17,7 @@ if (($ENV{NNTP_TEST_URL} || '') =~ m!\Anntp://([^/]+)/([^/]+)\z!) {
 	$host_port .= ":119" unless index($host_port, ':') > 0;
 } else {
 	$group = 'inbox.test.perf.nntpd';
-	my $ibx = { inboxdir => $pi_dir, newsgroup => $group };
+	my $ibx = { inboxdir => $inboxdir, newsgroup => $group };
 	$ibx = PublicInbox::Inbox->new($ibx);
 	my $tmpdir;
 	($tmpdir, $tmp_obj) = tmpdir();
@@ -28,7 +28,7 @@ if (($ENV{NNTP_TEST_URL} || '') =~ m!\Anntp://([^/]+)/([^/]+)\z!) {
 		print $fh <<"" or die "print $pi_config: $!";
 [publicinbox "test"]
 	newsgroup = $group
-	inboxdir = $pi_dir
+	inboxdir = $inboxdir
 	address = test\@example.com
 
 		close $fh or die "close($pi_config): $!";
diff --git a/t/perf-threading.t b/t/perf-threading.t
index 8d28b3a0..1038bda5 100644
--- a/t/perf-threading.t
+++ b/t/perf-threading.t
@@ -7,12 +7,12 @@ use warnings;
 use Test::More;
 use Benchmark qw(:all);
 use PublicInbox::Inbox;
-my $pi_dir = $ENV{GIANT_PI_DIR};
-plan skip_all => "GIANT_PI_DIR not defined for $0" unless $pi_dir;
-my $ibx = PublicInbox::Inbox->new({ inboxdir => $pi_dir });
+my $inboxdir = $ENV{GIANT_INBOX_DIR} // $ENV{GIANT_PI_DIR};
+plan skip_all => "GIANT_INBOX_DIR not defined for $0" unless $inboxdir;
+my $ibx = PublicInbox::Inbox->new({ inboxdir => $inboxdir });
 eval { require PublicInbox::Search };
 my $srch = $ibx->search;
-plan skip_all => "$pi_dir not configured for search $0 $@" unless $srch;
+plan skip_all => "$inboxdir not configured for search $0 $@" unless $srch;
 
 require PublicInbox::View;
 

[PATCH 17/17] tests: move giant inbox/git dependent tests to xt/

[Eric Wong]

xt/ is typically reserved for "eXtended tests" intended for
the maintainers and not ordinary users.  Since these require
special configuration and do nothing by waste cycles
during startup, they qualify.
---
 MANIFEST                     | 10 +++++-----
 {t => xt}/git-http-backend.t |  0
 {t => xt}/nntpd-validate.t   |  0
 {t => xt}/perf-msgview.t     |  0
 {t => xt}/perf-nntpd.t       |  0
 {t => xt}/perf-threading.t   |  0
 6 files changed, 5 insertions(+), 5 deletions(-)
 rename {t => xt}/git-http-backend.t (100%)
 rename {t => xt}/nntpd-validate.t (100%)
 rename {t => xt}/perf-msgview.t (100%)
 rename {t => xt}/perf-nntpd.t (100%)
 rename {t => xt}/perf-threading.t (100%)

diff --git a/MANIFEST b/MANIFEST
index 9475667b..9fd639f5 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -219,7 +219,6 @@ t/filter_rubylang.t
 t/filter_subjecttag.t
 t/filter_vger.t
 t/git-http-backend.psgi
-t/git-http-backend.t
 t/git.fast-import-data
 t/git.t
 t/hl_mod.t
@@ -247,13 +246,9 @@ t/msgmap.t
 t/msgtime.t
 t/nntp.t
 t/nntpd-tls.t
-t/nntpd-validate.t
 t/nntpd.t
 t/nulsubject.t
 t/over.t
-t/perf-msgview.t
-t/perf-nntpd.t
-t/perf-threading.t
 t/plack.t
 t/precheck.t
 t/psgi_attach.t
@@ -291,3 +286,8 @@ t/watch_maildir.t
 t/watch_maildir_v2.t
 t/www_listing.t
 t/xcpdb-reshard.t
+xt/git-http-backend.t
+xt/nntpd-validate.t
+xt/perf-msgview.t
+xt/perf-nntpd.t
+xt/perf-threading.t
diff --git a/t/git-http-backend.t b/xt/git-http-backend.t
similarity index 100%
rename from t/git-http-backend.t
rename to xt/git-http-backend.t
diff --git a/t/nntpd-validate.t b/xt/nntpd-validate.t
similarity index 100%
rename from t/nntpd-validate.t
rename to xt/nntpd-validate.t
diff --git a/t/perf-msgview.t b/xt/perf-msgview.t
similarity index 100%
rename from t/perf-msgview.t
rename to xt/perf-msgview.t
diff --git a/t/perf-nntpd.t b/xt/perf-nntpd.t
similarity index 100%
rename from t/perf-nntpd.t
rename to xt/perf-nntpd.t
diff --git a/t/perf-threading.t b/xt/perf-threading.t
similarity index 100%
rename from t/perf-threading.t
rename to xt/perf-threading.t