[PATCH 0/5] miscellaneous minor changes

[PATCH 0/5] miscellaneous minor changes

[Eric Wong]

Subject threading is now disabled completely to hopefully
expose misuse (or broken) email clients.  NNTP messages
now include HTTP/HTTPS archive headers, and there's some
minor test improvements with more to come.

Eric Wong (5):
      http: remove needless binmode call
      view: disable subject threading
      nntp: append Archived-At and List-Archive headers
      t/nntpd.t: stop hard coding message :bytes into test
      t/*.t: reduce -mda calls

 lib/PublicInbox/HTTP.pm       |  1 -
 lib/PublicInbox/NNTP.pm       | 27 +++++++++++++++++++++++++--
 lib/PublicInbox/NNTPD.pm      |  3 ++-
 lib/PublicInbox/NewsGroup.pm  | 11 ++++++++++-
 lib/PublicInbox/SearchView.pm | 17 +++++++----------
 lib/PublicInbox/Thread.pm     | 14 ++++++++++++++
 lib/PublicInbox/View.pm       | 16 +++-------------
 t/cgi.t                       | 41 +++++++++++++++++------------------------
 t/httpd.t                     | 22 +++++++++-------------
 t/nntp.t                      | 34 ++++++++++++++++++++++++++++++++++
 t/nntpd.t                     | 31 ++++++++++++++-----------------
 t/plack.t                     | 43 +++++++++++++------------------------------
 12 files changed, 148 insertions(+), 112 deletions(-)

[PATCH 1/5] http: remove needless binmode call

[Eric Wong]

Unnecessary on *nix, and we won't support systems
which do insane things.
---
 lib/PublicInbox/HTTP.pm | 1 -
 1 file changed, 1 deletion(-)

diff --git a/lib/PublicInbox/HTTP.pm b/lib/PublicInbox/HTTP.pm
index 88b6fb8..3286e79 100644
--- a/lib/PublicInbox/HTTP.pm
+++ b/lib/PublicInbox/HTTP.pm
@@ -264,7 +264,6 @@ sub input_prepare {
 	# TODO: expire idle clients on ENFILE / EMFILE
 	return unless $input;
 
-	binmode $input;
 	$env->{'psgi.input'} = $input;
 	$self->{env} = $env;
 	$self->{input_left} = $len || 0;

[PATCH 2/5] view: disable subject threading

[Eric Wong]

Broken threads should be exposed to hopefully encourage people to
use proper mail clients which set In-Reply-To headers.
---
 lib/PublicInbox/SearchView.pm | 17 +++++++----------
 lib/PublicInbox/Thread.pm     | 14 ++++++++++++++
 lib/PublicInbox/View.pm       | 16 +++-------------
 3 files changed, 24 insertions(+), 23 deletions(-)

diff --git a/lib/PublicInbox/SearchView.pm b/lib/PublicInbox/SearchView.pm
index 41d3200..c0cd1ff 100644
--- a/lib/PublicInbox/SearchView.pm
+++ b/lib/PublicInbox/SearchView.pm
@@ -11,6 +11,7 @@ use PublicInbox::View;
 use PublicInbox::MID qw(mid2path mid_clean mid_mime);
 use Email::MIME;
 require PublicInbox::Git;
+require PublicInbox::Thread;
 our $LIM = 50;
 
 sub sres_top_html {
@@ -151,23 +152,19 @@ sub tdump {
 	} ($mset->items);
 
 	my @rootset;
-	my $th = PublicInbox::View::thread_results(\@m, 0, $q->{r});
-	if ($q->{r}) {
+	my $th = PublicInbox::Thread->new(@m);
+	$th->thread;
+	if ($q->{r}) { # order by relevance
 		$th->order(sub {
 			sort { (eval { $pct{$b->topmost->messageid} } || 0)
 					<=>
 				(eval { $pct{$a->topmost->messageid} } || 0)
 			} @_;
 		});
-		@rootset = $th->rootset;
-	} else {
-		@rootset = sort {
-			(eval { $b->topmost->message->header('X-PI-TS') } || 0)
-				<=>
-			(eval { $a->topmost->message->header('X-PI-TS') } || 0)
-		} $th->rootset;
+	} else { # order by time (default for threaded view)
+		$th->order(*PublicInbox::View::sort_ts);
 	}
-
+	@rootset = $th->rootset;
 	my $git = $ctx->{git} ||= PublicInbox::Git->new($ctx->{git_dir});
 	my $state = {
 		ctx => $ctx,
diff --git a/lib/PublicInbox/Thread.pm b/lib/PublicInbox/Thread.pm
index 781ffff..44a565a 100644
--- a/lib/PublicInbox/Thread.pm
+++ b/lib/PublicInbox/Thread.pm
@@ -12,6 +12,20 @@ package PublicInbox::Thread;
 use strict;
 use warnings;
 use base qw(Mail::Thread);
+# WARNING! both these Mail::Thread knobs were found by inspecting
+# the Mail::Thread 2.55 source code, and we have some monkey patches
+# in PublicInbox::Thread to fix memory leaks.  Since Mail::Thread
+# appears unmaintained, I suppose it's safe to depend on these
+# variables for now:
+{
+	no warnings 'once';
+	# we want strict threads to expose (and hopefully discourage)
+	# use of broken email clients
+	$Mail::Thread::nosubject = 1;
+	# Keep ghosts with only a single direct child,
+	# don't hide that there may be missing messages.
+	$Mail::Thread::noprune = 1;
+}
 
 if ($Mail::Thread::VERSION <= 2.55) {
 	eval q(sub _container_class { 'PublicInbox::Thread::Container' });
diff --git a/lib/PublicInbox/View.pm b/lib/PublicInbox/View.pm
index 63810dc..70eb44e 100644
--- a/lib/PublicInbox/View.pm
+++ b/lib/PublicInbox/View.pm
@@ -670,21 +670,11 @@ sub msg_timestamp {
 }
 
 sub thread_results {
-	my ($msgs, $nosubject, $nosort) = @_;
+	my ($msgs) = @_;
 	require PublicInbox::Thread;
 	my $th = PublicInbox::Thread->new(@$msgs);
-
-	# WARNING! both these Mail::Thread knobs were found by inspecting
-	# the Mail::Thread 2.55 source code, and we have some monkey patches
-	# in PublicInbox::Thread to fix memory leaks.  Since Mail::Thread
-	# appears unmaintained, I suppose it's safe to depend on these
-	# variables for now:
-	no warnings 'once';
-	$Mail::Thread::nosubject = $nosubject;
-	# Keep ghosts with only a single direct child:
-	$Mail::Thread::noprune = 1;
 	$th->thread;
-	$th->order(*sort_ts) unless $nosort;
+	$th->order(*sort_ts);
 	$th
 }
 
@@ -879,7 +869,7 @@ sub emit_index_topics {
 		my $sres = $state->{srch}->query('', \%opts);
 		my $nr = scalar @{$sres->{msgs}} or last;
 
-		for (thread_results(load_results($sres), 1)->rootset) {
+		for (thread_results(load_results($sres))->rootset) {
 			add_topic($state, $_, 0);
 		}
 		$opts{offset} += $nr;

[PATCH 3/5] nntp: append Archived-At and List-Archive headers

[Eric Wong]

For readers using NNTP, we should do our best to advertise the
clonable HTTP/HTTPS URLs and the message permalink URL for
ease-of-referencing messages, since we don't want the NNTP server
and it's sequential article numbers to be relied on.
---
 lib/PublicInbox/NNTP.pm      | 27 +++++++++++++++++++++++++--
 lib/PublicInbox/NNTPD.pm     |  3 ++-
 lib/PublicInbox/NewsGroup.pm | 11 ++++++++++-
 t/nntp.t                     | 34 ++++++++++++++++++++++++++++++++++
 4 files changed, 71 insertions(+), 4 deletions(-)

diff --git a/lib/PublicInbox/NNTP.pm b/lib/PublicInbox/NNTP.pm
index a4cf25e..3e0faaf 100644
--- a/lib/PublicInbox/NNTP.pm
+++ b/lib/PublicInbox/NNTP.pm
@@ -15,6 +15,7 @@ use Email::MIME;
 use Data::Dumper qw(Dumper);
 use POSIX qw(strftime);
 use Time::HiRes qw(clock_gettime CLOCK_MONOTONIC);
+use URI::Escape qw(uri_escape_utf8);
 use constant {
 	r501 => '501 command syntax error',
 	r221 => '221 Header follows',
@@ -426,6 +427,29 @@ sub cmd_quit ($) {
 	undef;
 }
 
+sub header_append ($$$) {
+	my ($hdr, $k, $v) = @_;
+	my @v = $hdr->header($k);
+	foreach (@v) {
+		return if $v eq $_;
+	}
+	$hdr->header_set($k, @v, $v);
+}
+
+sub set_nntp_headers {
+	my ($hdr, $ng, $n, $mid) = @_;
+
+	# clobber some
+	$hdr->header_set('Newsgroups', $ng->{name});
+	$hdr->header_set('Xref', xref($ng, $n));
+	header_append($hdr, 'List-Post', "<mailto:$ng->{address}>");
+	if (my $url = $ng->{url}) {
+		$mid = uri_escape_utf8($mid);
+		header_append($hdr, 'Archived-At', "<$url$mid/>");
+		header_append($hdr, 'List-Archive', "<$url>");
+	}
+}
+
 sub art_lookup ($$$) {
 	my ($self, $art, $set_headers) = @_;
 	my $ng = $self->{ng};
@@ -468,8 +492,7 @@ found:
 	return $err unless $s;
 	my $lines;
 	if ($set_headers) {
-		$s->header_set('Newsgroups', $ng->{name});
-		$s->header_set('Xref', xref($ng, $n));
+		set_nntp_headers($s->header_obj, $ng, $n, $mid);
 		$lines = $s->body =~ tr!\n!\n!;
 
 		# must be last
diff --git a/lib/PublicInbox/NNTPD.pm b/lib/PublicInbox/NNTPD.pm
index 85109ea..2c84fb3 100644
--- a/lib/PublicInbox/NNTPD.pm
+++ b/lib/PublicInbox/NNTPD.pm
@@ -30,11 +30,12 @@ sub refresh_groups () {
 		my $git_dir = $pi_config->{$k};
 		my $addr = $pi_config->{"publicinbox.$g.address"};
 		my $ngname = $pi_config->{"publicinbox.$g.newsgroup"};
+		my $url = $pi_config->{"publicinbox.$g.url"};
 		if (defined $ngname) {
 			next if ($ngname eq ''); # disabled
 			$g = $ngname;
 		}
-		my $ng = PublicInbox::NewsGroup->new($g, $git_dir, $addr);
+		my $ng = PublicInbox::NewsGroup->new($g, $git_dir, $addr, $url);
 		my $old_ng = $self->{groups}->{$g};
 
 		# Reuse the old one if possible since it can hold
diff --git a/lib/PublicInbox/NewsGroup.pm b/lib/PublicInbox/NewsGroup.pm
index adac919..98a3595 100644
--- a/lib/PublicInbox/NewsGroup.pm
+++ b/lib/PublicInbox/NewsGroup.pm
@@ -13,12 +13,21 @@ require PublicInbox::Search;
 require PublicInbox::Git;
 
 sub new {
-	my ($class, $name, $git_dir, $address) = @_;
+	my ($class, $name, $git_dir, $address, $url) = @_;
+
+	# first email address is preferred
 	$address = $address->[0] if ref($address);
+	if ($url) {
+		# assume protocol-relative URLs which start with '//' means
+		# the server supports both HTTP and HTTPS, favor HTTPS.
+		$url = "https:$url" if $url =~ m!\A//!;
+		$url .= '/' if $url !~ m!/\z!;
+	}
 	my $self = bless {
 		name => $name,
 		git_dir => $git_dir,
 		address => $address,
+		url => $url,
 	}, $class;
 	$self->{domain} = ($address =~ /\@(\S+)\z/) ? $1 : 'localhost';
 	$self;
diff --git a/t/nntp.t b/t/nntp.t
index 388620e..5513c7b 100644
--- a/t/nntp.t
+++ b/t/nntp.t
@@ -11,6 +11,7 @@ foreach my $mod (qw(DBD::SQLite Search::Xapian Danga::Socket)) {
 }
 
 use_ok 'PublicInbox::NNTP';
+use_ok 'PublicInbox::NewsGroup';
 
 {
 	sub quote_str {
@@ -95,4 +96,37 @@ use_ok 'PublicInbox::NNTP';
 	}
 }
 
+{ # test setting NNTP headers in HEAD and ARTICLE requests
+	require Email::MIME;
+	my $u = 'https://example.com/a/';
+	my $ng = PublicInbox::NewsGroup->new('test', 'test.git',
+				'a@example.com', '//example.com/a');
+	is($ng->{url}, $u, 'URL expanded');
+	my $mid = 'a@b';
+	my $mime = Email::MIME->new("Message-ID: <$mid>\r\n\r\n");
+	PublicInbox::NNTP::set_nntp_headers($mime->header_obj, $ng, 1, $mid);
+	is_deeply([ $mime->header('Message-ID') ], [ "<$mid>" ],
+		'Message-ID unchanged');
+	is_deeply([ $mime->header('Archived-At') ], [ "<${u}a%40b/>" ],
+		'Archived-At: set');
+	is_deeply([ $mime->header('List-Archive') ], [ "<$u>" ],
+		'List-Archive: set');
+	is_deeply([ $mime->header('List-Post') ], [ '<mailto:a@example.com>' ],
+		'List-Post: set');
+	is_deeply([ $mime->header('Newsgroups') ], [ 'test' ],
+		'Newsgroups: set');
+	is_deeply([ $mime->header('Xref') ], [ 'example.com test:1' ],
+		'Xref: set');
+
+	$ng->{url} = 'http://mirror.example.com/m/';
+	PublicInbox::NNTP::set_nntp_headers($mime->header_obj, $ng, 2, $mid);
+	is_deeply([ $mime->header('Message-ID') ], [ "<$mid>" ],
+		'Message-ID unchanged');
+	is_deeply([ $mime->header('Archived-At') ],
+		[ "<${u}a%40b/>", '<http://mirror.example.com/m/a%40b/>' ],
+		'Archived-At: appended');
+	is_deeply([ $mime->header('Xref') ], [ 'example.com test:2' ],
+		'Old Xref: clobbered');
+}
+
 done_testing();

[PATCH 4/5] t/nntpd.t: stop hard coding message :bytes into test

[Eric Wong]

It limits flexibility and makes it harder to switch
to use PublicImport::Import.
---
 t/nntpd.t | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/t/nntpd.t b/t/nntpd.t
index d033221..b026bd5 100644
--- a/t/nntpd.t
+++ b/t/nntpd.t
@@ -46,6 +46,7 @@ my %opts = (
 );
 my $sock = IO::Socket::INET->new(%opts);
 my $pid;
+my $len;
 END { kill 'TERM', $pid if defined $pid };
 {
 	local $ENV{HOME} = $home;
@@ -69,6 +70,8 @@ EOF
 		IPC::Run::run([$mda], \$in);
 		is(0, $?, 'ran MDA correctly');
 		is(0, system($index, $maindir), 'indexed git dir');
+		$simple->header_set('List-Id', "<$addr>");
+		$len = length($simple->as_string);
 	}
 
 	ok($sock, 'sock created');
@@ -151,7 +154,7 @@ EOF
 			'Thu, 01 Jan 1970 06:06:06 +0000',
 			'<nntp@example.com>',
 			'',
-			'202',
+			$len,
 			'1' ] }, "XOVER range works");
 
 	is_deeply($n->xover('1'), {
@@ -160,7 +163,7 @@ EOF
 			'Thu, 01 Jan 1970 06:06:06 +0000',
 			'<nntp@example.com>',
 			'',
-			'202',
+			$len,
 			'1' ] }, "XOVER by article works");
 
 	{
@@ -173,7 +176,7 @@ EOF
 		like($r[0], qr/^224 /, 'got 224 response for OVER');
 		is($r[1], "0\thihi\tMe <me\@example.com>\t" .
 			"Thu, 01 Jan 1970 06:06:06 +0000\t" .
-			"$mid\t\t202\t1", 'OVER by Message-ID works');
+			"$mid\t\t$len\t1", 'OVER by Message-ID works');
 		is($r[2], '.', 'correctly terminated response');
 	}
 

[PATCH 5/5] t/*.t: reduce -mda calls

[Eric Wong]

Process startup times are atrocious for fast tests and there's far
too much setup involved.  Rely on git-fast-import instead; but
more work is needed in this area.
---
 t/cgi.t   | 41 +++++++++++++++++------------------------
 t/httpd.t | 22 +++++++++-------------
 t/nntpd.t | 26 ++++++++++----------------
 t/plack.t | 43 +++++++++++++------------------------------
 4 files changed, 49 insertions(+), 83 deletions(-)

diff --git a/t/cgi.t b/t/cgi.t
index dcbfb4c..fbdbfbf 100644
--- a/t/cgi.t
+++ b/t/cgi.t
@@ -1,5 +1,7 @@
 # Copyright (C) 2014-2015 all contributors <meta@public-inbox.org>
 # License: AGPLv3 or later (https://www.gnu.org/licenses/agpl-3.0.txt)
+# FIXME: this test is too slow and most non-CGI-requirements
+# should be moved over to things which use test_psgi
 use strict;
 use warnings;
 use Test::More;
@@ -9,22 +11,16 @@ use Cwd;
 use IPC::Run qw/run/;
 
 use constant CGI => "blib/script/public-inbox.cgi";
-my $mda = "blib/script/public-inbox-mda";
 my $index = "blib/script/public-inbox-index";
 my $tmpdir = tempdir('pi-cgi-XXXXXX', TMPDIR => 1, CLEANUP => 1);
 my $home = "$tmpdir/pi-home";
 my $pi_home = "$home/.public-inbox";
 my $pi_config = "$pi_home/config";
 my $maindir = "$tmpdir/main.git";
-my $main_bin = getcwd()."/t/main-bin";
-my $main_path = "$main_bin:$ENV{PATH}"; # for spamc ham mock
 my $addr = 'test-public@example.com';
 my $cfgpfx = "publicinbox.test";
 
 {
-	ok(-x "$main_bin/spamc",
-		"spamc ham mock found (run in top of source tree");
-	ok(-x $mda, "$mda is executable");
 	is(1, mkdir($home, 0755), "setup ~/ for testing");
 	is(1, mkdir($pi_home, 0755), "setup ~/.public-inbox");
 	is(0, system(qw(git init -q --bare), $maindir), "git init (main)");
@@ -42,15 +38,18 @@ my $cfgpfx = "publicinbox.test";
 	}
 }
 
-my $failbox = "$home/fail.mbox";
-local $ENV{PI_EMERGENCY} = $failbox;
+use_ok 'PublicInbox::Git';
+use_ok 'PublicInbox::Import';
+use_ok 'Email::MIME';
+my $git = PublicInbox::Git->new($maindir);
+my $im = PublicInbox::Import->new($git, 'test', $addr);
+
 {
 	local $ENV{HOME} = $home;
-	local $ENV{ORIGINAL_RECIPIENT} = $addr;
 
 	# ensure successful message delivery
 	{
-		my $simple = Email::Simple->new(<<EOF);
+		my $mime = Email::MIME->new(<<EOF);
 From: Me <me\@example.com>
 To: You <you\@example.com>
 Cc: $addr
@@ -60,15 +59,15 @@ Date: Thu, 01 Jan 1970 00:00:00 +0000
 
 zzzzzz
 EOF
-		my $in = $simple->as_string;
-		run_with_env({PATH => $main_path}, [$mda], \$in);
+		$im->add($mime);
+		$im->done;
 		my $rev = `git --git-dir=$maindir rev-list HEAD`;
 		like($rev, qr/\A[a-f0-9]{40}/, "good revision committed");
 	}
 
 	# deliver a reply, too
 	{
-		my $reply = Email::Simple->new(<<EOF);
+		my $reply = Email::MIME->new(<<EOF);
 From: You <you\@example.com>
 To: Me <me\@example.com>
 Cc: $addr
@@ -82,8 +81,8 @@ Me wrote:
 
 what?
 EOF
-		my $in = $reply->as_string;
-		run_with_env({PATH => $main_path}, [$mda], \$in);
+		$im->add($reply);
+		$im->done;
 		my $rev = `git --git-dir=$maindir rev-list HEAD`;
 		like($rev, qr/\A[a-f0-9]{40}/, "good revision committed");
 	}
@@ -150,7 +149,7 @@ EOF
 {
 	local $ENV{HOME} = $home;
 	my $slashy_mid = 'slashy/asdf@example.com';
-	my $reply = Email::Simple->new(<<EOF);
+	my $reply = Email::MIME->new(<<EOF);
 From: You <you\@example.com>
 To: Me <me\@example.com>
 Cc: $addr
@@ -160,13 +159,8 @@ Date: Thu, 01 Jan 1970 00:00:01 +0000
 
 slashy
 EOF
-	my $in = $reply->as_string;
-
-	{
-		local $ENV{HOME} = $home;
-		local $ENV{ORIGINAL_RECIPIENT} = $addr;
-		run_with_env({PATH => $main_path}, [$mda], \$in);
-	}
+	$im->add($reply);
+	$im->done;
 
 	my $res = cgi_run("/test/slashy%2fasdf%40example.com/raw");
 	like($res->{body}, qr/Message-Id: <\Q$slashy_mid\E>/,
@@ -200,7 +194,6 @@ EOF
 # retrieve thread as an mbox
 {
 	local $ENV{HOME} = $home;
-	local $ENV{PATH} = $main_path;
 	my $path = "/test/blahblah%40example.com/t.mbox.gz";
 	my $res = cgi_run($path);
 	like($res->{head}, qr/^Status: 501 /, "search not-yet-enabled");
diff --git a/t/httpd.t b/t/httpd.t
index 781fe03..0e19b56 100644
--- a/t/httpd.t
+++ b/t/httpd.t
@@ -14,24 +14,16 @@ use Cwd qw/getcwd/;
 use IO::Socket;
 use Fcntl qw(FD_CLOEXEC F_SETFD F_GETFD);
 use Socket qw(SO_KEEPALIVE IPPROTO_TCP TCP_NODELAY);
-use IPC::Run;
 
 # FIXME: too much setup
 my $tmpdir = tempdir('pi-httpd-XXXXXX', TMPDIR => 1, CLEANUP => 1);
 my $home = "$tmpdir/pi-home";
 my $err = "$tmpdir/stderr.log";
 my $out = "$tmpdir/stdout.log";
-my $pi_home = "$home/.public-inbox";
-my $pi_config = "$pi_home/config";
 my $maindir = "$tmpdir/main.git";
-my $main_bin = getcwd()."/t/main-bin";
-my $main_path = "$main_bin:$ENV{PATH}"; # for spamc ham mock
 my $group = 'test-httpd';
 my $addr = $group . '@example.com';
 my $cfgpfx = "publicinbox.$group";
-my $failbox = "$home/fail.mbox";
-local $ENV{PI_EMERGENCY} = $failbox;
-my $mda = 'blib/script/public-inbox-mda';
 my $httpd = 'blib/script/public-inbox-httpd';
 my $init = 'blib/script/public-inbox-init';
 
@@ -44,6 +36,9 @@ my %opts = (
 );
 my $sock = IO::Socket::INET->new(%opts);
 my $pid;
+use_ok 'PublicInbox::Git';
+use_ok 'PublicInbox::Import';
+use_ok 'Email::MIME';
 END { kill 'TERM', $pid if defined $pid };
 {
 	local $ENV{HOME} = $home;
@@ -52,8 +47,7 @@ END { kill 'TERM', $pid if defined $pid };
 
 	# ensure successful message delivery
 	{
-		local $ENV{ORIGINAL_RECIPIENT} = $addr;
-		my $in = <<EOF;
+		my $mime = Email::MIME->new(<<EOF);
 From: Me <me\@example.com>
 To: You <you\@example.com>
 Cc: $addr
@@ -63,9 +57,11 @@ Date: Thu, 01 Jan 1970 06:06:06 +0000
 
 nntp
 EOF
-		local $ENV{PATH} = $main_path;
-		IPC::Run::run([$mda], \$in);
-		is(0, $?, 'ran MDA correctly');
+		$mime->header_set('List-Id', "<$addr>");
+		my $git = PublicInbox::Git->new($maindir);
+		my $im = PublicInbox::Import->new($git, 'test', $addr);
+		$im->add($mime);
+		$im->done($mime);
 	}
 	ok($sock, 'sock created');
 	$! = 0;
diff --git a/t/nntpd.t b/t/nntpd.t
index b026bd5..a389a3e 100644
--- a/t/nntpd.t
+++ b/t/nntpd.t
@@ -16,26 +16,20 @@ use Fcntl qw(FD_CLOEXEC F_SETFD F_GETFD);
 use Socket qw(SO_KEEPALIVE IPPROTO_TCP TCP_NODELAY);
 use File::Temp qw/tempdir/;
 use Net::NNTP;
-use IPC::Run qw(run);
 
 my $tmpdir = tempdir('pi-nntpd-XXXXXX', TMPDIR => 1, CLEANUP => 1);
 my $home = "$tmpdir/pi-home";
 my $err = "$tmpdir/stderr.log";
 my $out = "$tmpdir/stdout.log";
-my $pi_home = "$home/.public-inbox";
-my $pi_config = "$pi_home/config";
 my $maindir = "$tmpdir/main.git";
-my $main_bin = getcwd()."/t/main-bin";
-my $main_path = "$main_bin:$ENV{PATH}"; # for spamc ham mock
 my $group = 'test-nntpd';
 my $addr = $group . '@example.com';
 my $cfgpfx = "publicinbox.$group";
-my $failbox = "$home/fail.mbox";
-local $ENV{PI_EMERGENCY} = $failbox;
-my $mda = 'blib/script/public-inbox-mda';
 my $nntpd = 'blib/script/public-inbox-nntpd';
 my $init = 'blib/script/public-inbox-init';
 my $index = 'blib/script/public-inbox-index';
+use_ok 'PublicInbox::Import';
+use_ok 'PublicInbox::Git';
 
 my %opts = (
 	LocalAddr => '127.0.0.1',
@@ -51,11 +45,11 @@ END { kill 'TERM', $pid if defined $pid };
 {
 	local $ENV{HOME} = $home;
 	system($init, $group, $maindir, 'http://example.com/', $addr);
+	my $len;
 
 	# ensure successful message delivery
 	{
-		local $ENV{ORIGINAL_RECIPIENT} = $addr;
-		my $simple = Email::Simple->new(<<EOF);
+		my $mime = Email::MIME->new(<<EOF);
 From: Me <me\@example.com>
 To: You <you\@example.com>
 Cc: $addr
@@ -65,13 +59,13 @@ Date: Thu, 01 Jan 1970 06:06:06 +0000
 
 nntp
 EOF
-		my $in = $simple->as_string;
-		local $ENV{PATH} = $main_path;
-		IPC::Run::run([$mda], \$in);
-		is(0, $?, 'ran MDA correctly');
+		$mime->header_set('List-Id', "<$addr>");
+		$len = length($mime->as_string);
+		my $git = PublicInbox::Git->new($maindir);
+		my $im = PublicInbox::Import->new($git, 'test', $addr);
+		$im->add($mime);
+		$im->done;
 		is(0, system($index, $maindir), 'indexed git dir');
-		$simple->header_set('List-Id', "<$addr>");
-		$len = length($simple->as_string);
 	}
 
 	ok($sock, 'sock created');
diff --git a/t/plack.t b/t/plack.t
index 1ae5873..c8dd7bf 100644
--- a/t/plack.t
+++ b/t/plack.t
@@ -5,36 +5,24 @@ use warnings;
 use Test::More;
 use Email::MIME;
 use File::Temp qw/tempdir/;
-use Cwd;
-use IPC::Run qw/run/;
 my $psgi = "examples/public-inbox.psgi";
-my $mda = "blib/script/public-inbox-mda";
 my $tmpdir = tempdir('pi-plack-XXXXXX', TMPDIR => 1, CLEANUP => 1);
-my $home = "$tmpdir/pi-home";
-my $pi_home = "$home/.public-inbox";
-my $pi_config = "$pi_home/config";
+my $pi_config = "$tmpdir/config";
 my $maindir = "$tmpdir/main.git";
-my $main_bin = getcwd()."/t/main-bin";
-my $main_path = "$main_bin:$ENV{PATH}"; # for spamc ham mock
 my $addr = 'test-public@example.com';
 my $cfgpfx = "publicinbox.test";
-my $failbox = "$home/fail.mbox";
-local $ENV{PI_EMERGENCY} = $failbox;
 my @mods = qw(HTTP::Request::Common Plack::Request Plack::Test
 	Mail::Thread URI::Escape);
 foreach my $mod (@mods) {
 	eval "require $mod";
 	plan skip_all => "$mod missing for plack.t" if $@;
 }
+use_ok 'PublicInbox::Import';
+use_ok 'PublicInbox::Git';
 
 foreach my $mod (@mods) { use_ok $mod; }
 {
 	ok(-f $psgi, "psgi example file found");
-	ok(-x "$main_bin/spamc",
-		"spamc ham mock found (run in top of source tree");
-	ok(-x $mda, "$mda is executable");
-	is(1, mkdir($home, 0755), "setup ~/ for testing");
-	is(1, mkdir($pi_home, 0755), "setup ~/.public-inbox");
 	is(0, system(qw(git init -q --bare), $maindir), "git init (main)");
 	open my $fh, '>', "$maindir/description" or die "open: $!\n";
 	print $fh "test for public-inbox\n";
@@ -48,12 +36,9 @@ foreach my $mod (@mods) { use_ok $mod; }
 			"setup $k");
 	}
 
-	local $ENV{HOME} = $home;
-	local $ENV{ORIGINAL_RECIPIENT} = $addr;
-
 	# ensure successful message delivery
 	{
-		my $simple = Email::Simple->new(<<EOF);
+		my $mime = Email::MIME->new(<<EOF);
 From: Me <me\@example.com>
 To: You <you\@example.com>
 Cc: $addr
@@ -63,13 +48,17 @@ Date: Thu, 01 Jan 1970 00:00:00 +0000
 
 zzzzzz
 EOF
-		my $in = $simple->as_string;
-		run_with_env({PATH => $main_path}, [$mda], \$in);
-		local $ENV{GIT_DIR} = $maindir;
-		my $rev = `git rev-list HEAD`;
+		my $git = PublicInbox::Git->new($maindir);
+		my $im = PublicInbox::Import->new($git, 'test', $addr);
+		$im->add($mime);
+		$im->done;
+		my $rev = `git --git-dir="$maindir" rev-list HEAD`;
 		like($rev, qr/\A[a-f0-9]{40}/, "good revision committed");
 	}
-	my $app = require $psgi;
+	my $app = eval {
+		local $ENV{PI_CONFIG} = $pi_config;
+		require $psgi;
+	};
 
 	# redirect with trailing /
 	test_psgi($app, sub {
@@ -190,9 +179,3 @@ EOF
 }
 
 done_testing();
-
-sub run_with_env {
-	my ($env, @args) = @_;
-	my $init = sub { foreach my $k (keys %$env) { $ENV{$k} = $env->{$k} } };
-	run(@args, init => $init);
-}