diff options
Diffstat (limited to 't')
| -rw-r--r-- | t/address.t | 5 | ||||
| -rw-r--r-- | t/altid.t | 20 | ||||
| -rw-r--r-- | t/altid_v2.t | 55 | ||||
| -rw-r--r-- | t/content_id.t | 35 | ||||
| -rw-r--r-- | t/convert-compact.t | 104 | ||||
| -rw-r--r-- | t/git.t | 24 | ||||
| -rw-r--r-- | t/import.t | 32 | ||||
| -rw-r--r-- | t/init.t | 45 | ||||
| -rw-r--r-- | t/mid.t | 22 | ||||
| -rw-r--r-- | t/msgmap.t | 4 | ||||
| -rw-r--r-- | t/nntp.t | 6 | ||||
| -rw-r--r-- | t/nntpd.t | 37 | ||||
| -rw-r--r-- | t/over.t | 63 | ||||
| -rw-r--r-- | t/perf-nntpd.t | 135 | ||||
| -rw-r--r-- | t/perf-threading.t | 32 | ||||
| -rw-r--r-- | t/plack.t | 18 | ||||
| -rw-r--r-- | t/psgi_search.t | 6 | ||||
| -rw-r--r-- | t/psgi_v2.t | 245 | ||||
| -rw-r--r-- | t/search-thr-index.t | 14 | ||||
| -rw-r--r-- | t/search.t | 162 | ||||
| -rw-r--r-- | t/thread-all.t | 38 | ||||
| -rw-r--r-- | t/time.t | 28 | ||||
| -rw-r--r-- | t/v1-add-remove-add.t | 45 | ||||
| -rw-r--r-- | t/v2-add-remove-add.t | 42 | ||||
| -rw-r--r-- | t/v2mda.t | 59 | ||||
| -rw-r--r-- | t/v2mirror.t | 176 | ||||
| -rw-r--r-- | t/v2reindex.t | 97 | ||||
| -rw-r--r-- | t/v2writable.t | 280 | ||||
| -rw-r--r-- | t/view.t | 1 | ||||
| -rw-r--r-- | t/watch_maildir.t | 3 | ||||
| -rw-r--r-- | t/watch_maildir_v2.t | 125 |
31 files changed, 1818 insertions, 140 deletions
diff --git a/t/address.t b/t/address.t index e35e4f8b..eced5c46 100644 --- a/t/address.t +++ b/t/address.t @@ -9,8 +9,9 @@ is_deeply([qw(e@example.com e@example.org)], [PublicInbox::Address::emails('User <e@example.com>, e@example.org')], 'address extraction works as expected'); -is_deeply([PublicInbox::Address::emails('"ex@example.com" <ex@example.com>')], - [qw(ex@example.com)]); +is_deeply(['user@example.com'], + [PublicInbox::Address::emails('<user@example.com (Comment)>')], + 'comment after domain accepted before >'); my @names = PublicInbox::Address::names( 'User <e@e>, e@e, "John A. Doe" <j@d>, <x@x>'); @@ -20,7 +20,9 @@ my $altid = [ "serial:gmane:file=$alt_file" ]; { my $mm = PublicInbox::Msgmap->new_file($alt_file, 1); - $mm->mid_set(1234, 'a@example.com'); + is($mm->mid_set(1234, 'a@example.com'), 1, 'mid_set once OK'); + ok(0 == $mm->mid_set(1234, 'a@example.com'), 'mid_set not idempotent'); + ok(0 == $mm->mid_set(1, 'a@example.com'), 'mid_set fails with dup MID'); } { @@ -48,14 +50,20 @@ my $altid = [ "serial:gmane:file=$alt_file" ]; { my $ro = PublicInbox::Search->new($git_dir, $altid); - my $res = $ro->query("gmane:1234"); - is($res->{total}, 1, 'got one match'); - is($res->{msgs}->[0]->mid, 'a@example.com'); + my $msgs = $ro->query("gmane:1234"); + is_deeply([map { $_->mid } @$msgs], ['a@example.com'], 'got one match'); - $res = $ro->query("gmane:666"); - is($res->{total}, 0, 'body did NOT match'); + $msgs = $ro->query("gmane:666"); + is_deeply([], $msgs, 'body did NOT match'); }; +{ + my $mm = PublicInbox::Msgmap->new_file($alt_file, 1); + my ($min, $max) = $mm->minmax; + my $num = $mm->mid_insert('b@example.com'); + ok($num > $max, 'auto-increment goes beyond mid_set'); +} + done_testing(); 1; diff --git a/t/altid_v2.t b/t/altid_v2.t new file mode 100644 index 00000000..87f1452b --- /dev/null +++ b/t/altid_v2.t @@ -0,0 +1,55 @@ +# Copyright (C) 2016-2018 all contributors <meta@public-inbox.org> +# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt> +use strict; +use warnings; +use Test::More; +use File::Temp qw/tempdir/; +foreach my $mod (qw(DBD::SQLite Search::Xapian)) { + eval "require $mod"; + plan skip_all => "$mod missing for altid_v2.t" if $@; +} + +use_ok 'PublicInbox::V2Writable'; +use_ok 'PublicInbox::Inbox'; +my $tmpdir = tempdir('pi-altidv2-XXXXXX', TMPDIR => 1, CLEANUP => 1); +my $mainrepo = "$tmpdir/inbox"; +my $full = "$tmpdir/inbox/another-nntp.sqlite3"; +my $altid = [ 'serial:gmane:file=another-nntp.sqlite3' ]; + +{ + ok(mkdir($mainrepo), 'created repo for msgmap'); + my $mm = PublicInbox::Msgmap->new_file($full, 1); + is($mm->mid_set(1234, 'a@example.com'), 1, 'mid_set once OK'); + ok(0 == $mm->mid_set(1234, 'a@example.com'), 'mid_set not idempotent'); + ok(0 == $mm->mid_set(1, 'a@example.com'), 'mid_set fails with dup MID'); +} + +my $ibx = { + mainrepo => $mainrepo, + name => 'test-v2writable', + version => 2, + -primary_address => 'test@example.com', + altid => $altid, +}; +$ibx = PublicInbox::Inbox->new($ibx); +my $v2w = PublicInbox::V2Writable->new($ibx, 1); +$v2w->add(Email::MIME->create( + header => [ + From => 'a@example.com', + To => 'b@example.com', + 'Content-Type' => 'text/plain', + Subject => 'boo!', + 'Message-ID' => '<a@example.com>', + ], + body => "hello world gmane:666\n", + )); +$v2w->done; + +my $msgs = $ibx->search->reopen->query("gmane:1234"); +is_deeply([map { $_->mid } @$msgs], ['a@example.com'], 'got one match'); +$msgs = $ibx->search->query("gmane:666"); +is_deeply([], $msgs, 'body did NOT match'); + +done_testing(); + +1; diff --git a/t/content_id.t b/t/content_id.t new file mode 100644 index 00000000..01ce65e5 --- /dev/null +++ b/t/content_id.t @@ -0,0 +1,35 @@ +# Copyright (C) 2018 all contributors <meta@public-inbox.org> +# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt> +use strict; +use warnings; +use Test::More; +use PublicInbox::ContentId qw(content_id); +use Email::MIME; + +my $mime = Email::MIME->create( + header => [ + From => 'a@example.com', + To => 'b@example.com', + 'Content-Type' => 'text/plain', + Subject => 'this is a subject', + 'Message-ID' => '<a@example.com>', + Date => 'Fri, 02 Oct 1993 00:00:00 +0000', + ], + body => "hello world\n", +); + +my $orig = content_id($mime); +my $reload = content_id(Email::MIME->new($mime->as_string)); +is($orig, $reload, 'content_id matches after serialization'); + +foreach my $h (qw(From To Cc)) { + my $n = '"Quoted N\'Ame" <foo@EXAMPLE.com>'; + $mime->header_str_set($h, "$n"); + my $q = content_id($mime); + is($n, $mime->header($h), "content_id does not mutate $h:"); + $mime->header_str_set($h, 'Quoted N\'Ame <foo@example.com>'); + my $nq = content_id($mime); + is($nq, $q, "quotes ignored in $h:"); +} + +done_testing(); diff --git a/t/convert-compact.t b/t/convert-compact.t new file mode 100644 index 00000000..ced45415 --- /dev/null +++ b/t/convert-compact.t @@ -0,0 +1,104 @@ +# Copyright (C) 2018 all contributors <meta@public-inbox.org> +# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt> +use strict; +use warnings; +use Test::More; +use File::Temp qw/tempdir/; +use PublicInbox::MIME; +my @mods = qw(DBD::SQLite Search::Xapian); +foreach my $mod (@mods) { + eval "require $mod"; + plan skip_all => "$mod missing for convert-compact.t" if $@; +} +use_ok 'PublicInbox::V2Writable'; +use PublicInbox::Import; +my $tmpdir = tempdir('convert-compact-XXXXXX', TMPDIR => 1, CLEANUP => 1); +my $ibx = { + mainrepo => "$tmpdir/v1", + name => 'test-v1', + -primary_address => 'test@example.com', +}; + +ok(PublicInbox::Import::run_die([qw(git init --bare -q), $ibx->{mainrepo}]), + 'initialized v1 repo'); +ok(umask(077), 'set restrictive umask'); +ok(PublicInbox::Import::run_die([qw(git) , "--git-dir=$ibx->{mainrepo}", + qw(config core.sharedRepository 0644)]), 'set sharedRepository'); +$ibx = PublicInbox::Inbox->new($ibx); +my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx); +my $mime = PublicInbox::MIME->create( + header => [ + From => 'a@example.com', + To => 'test@example.com', + Subject => 'this is a subject', + 'Message-ID' => '<a-mid@b>', + Date => 'Fri, 02 Oct 1993 00:00:00 +0000', + ], + body => "hello world\n", +); +ok($im->add($mime), 'added one message'); +ok($im->remove($mime), 'remove message'); +ok($im->add($mime), 'added message again'); +$im->done; +PublicInbox::SearchIdx->new($ibx, 1)->index_sync; + +is(((stat("$ibx->{mainrepo}/public-inbox"))[2]) & 07777, 0755, + 'sharedRepository respected for v1'); +is(((stat("$ibx->{mainrepo}/public-inbox/msgmap.sqlite3"))[2]) & 07777, 0644, + 'sharedRepository respected for v1 msgmap'); +my @xdir = glob("$ibx->{mainrepo}/public-inbox/xap*/*"); +foreach (@xdir) { + my @st = stat($_); + is($st[2] & 07777, -f _ ? 0644 : 0755, + 'sharedRepository respected on file after convert'); +} + +local $ENV{PATH} = "blib/script:$ENV{PATH}"; +open my $err, '>>', "$tmpdir/err.log" or die "open: err.log $!\n"; +open my $out, '>>', "$tmpdir/out.log" or die "open: out.log $!\n"; +my $rdr = { 1 => fileno($out), 2 => fileno($err) }; + +my $cmd = [ 'public-inbox-compact', $ibx->{mainrepo} ]; +ok(PublicInbox::Import::run_die($cmd, undef, $rdr), 'v1 compact works'); + +@xdir = glob("$ibx->{mainrepo}/public-inbox/xap*"); +is(scalar(@xdir), 1, 'got one xapian directory after compact'); +is(((stat($xdir[0]))[2]) & 07777, 0755, + 'sharedRepository respected on v1 compact'); + +$cmd = [ 'public-inbox-convert', $ibx->{mainrepo}, "$tmpdir/v2" ]; +ok(PublicInbox::Import::run_die($cmd, undef, $rdr), 'convert works'); +@xdir = glob("$tmpdir/v2/xap*/*"); +foreach (@xdir) { + my @st = stat($_); + is($st[2] & 07777, -f _ ? 0644 : 0755, + 'sharedRepository respected after convert'); +} + +$cmd = [ 'public-inbox-compact', "$tmpdir/v2" ]; +my $env = { NPROC => 2 }; +ok(PublicInbox::Import::run_die($cmd, $env, $rdr), 'v2 compact works'); +$ibx->{mainrepo} = "$tmpdir/v2"; +$ibx->{version} = 2; + +@xdir = glob("$tmpdir/v2/xap*/*"); +foreach (@xdir) { + my @st = stat($_); + is($st[2] & 07777, -f _ ? 0644 : 0755, + 'sharedRepository respected after v2 compact'); +} +is(((stat("$tmpdir/v2/msgmap.sqlite3"))[2]) & 07777, 0644, + 'sharedRepository respected for v2 msgmap'); + +@xdir = (glob("$tmpdir/v2/git/*.git/objects/*/*"), + glob("$tmpdir/v2/git/*.git/objects/pack/*")); +foreach (@xdir) { + my @st = stat($_); + is($st[2] & 07777, -f _ ? 0444 : 0755, + 'sharedRepository respected after v2 compact'); +} +my $msgs = $ibx->recent({limit => 1000}); +is($msgs->[0]->{mid}, 'a-mid@b', 'message exists in history'); +is(scalar @$msgs, 1, 'only one message in history'); + +done_testing(); @@ -6,6 +6,7 @@ use Test::More; use File::Temp qw/tempdir/; my $dir = tempdir('pi-git-XXXXXX', TMPDIR => 1, CLEANUP => 1); use Cwd qw/getcwd/; +use PublicInbox::Spawn qw(popen_rd); use_ok 'PublicInbox::Git'; { @@ -137,6 +138,29 @@ if (1) { is($all, join('', @ref), 'qx returned array when wanted'); my $nl = scalar @ref; ok($nl > 1, "qx returned array length of $nl"); + + $gcf->qx(qw(repack -adbq)); + ok($gcf->packed_bytes > 0, 'packed size is positive'); +} + +if ('alternates reloaded') { + my $alt = tempdir('pi-git-XXXXXX', TMPDIR => 1, CLEANUP => 1); + my @cmd = ('git', "--git-dir=$alt", qw(hash-object -w --stdin)); + is(system(qw(git init -q --bare), $alt), 0, 'create alt directory'); + open my $fh, '<', "$alt/config" or die "open failed: $!\n"; + my $rd = popen_rd(\@cmd, {}, { 0 => fileno($fh) } ); + close $fh or die "close failed: $!"; + chomp(my $remote = <$rd>); + my $gcf = PublicInbox::Git->new($dir); + is($gcf->cat_file($remote), undef, "remote file not found"); + open $fh, '>>', "$dir/objects/info/alternates" or + die "open failed: $!\n"; + print $fh "$alt/objects" or die "print failed: $!\n"; + close $fh or die "close failed: $!"; + my $found = $gcf->cat_file($remote); + open $fh, '<', "$alt/config" or die "open failed: $!\n"; + my $config = eval { local $/; <$fh> }; + is($$found, $config, 'alternates reloaded'); } done_testing(); @@ -6,7 +6,10 @@ use Test::More; use PublicInbox::MIME; use PublicInbox::Git; use PublicInbox::Import; -use File::Temp qw/tempdir/; +use PublicInbox::Spawn qw(spawn); +use IO::File; +use Fcntl qw(:DEFAULT); +use File::Temp qw/tempdir tempfile/; my $dir = tempdir('pi-import-XXXXXX', TMPDIR => 1, CLEANUP => 1); is(system(qw(git init -q --bare), $dir), 0, 'git init successful'); @@ -20,10 +23,33 @@ my $mime = PublicInbox::MIME->create( 'Content-Type' => 'text/plain', Subject => 'this is a subject', 'Message-ID' => '<a@example.com>', + Date => 'Fri, 02 Oct 1993 00:00:00 +0000', ], body => "hello world\n", ); + +$im->{want_object_info} = 1 if 'v2'; like($im->add($mime), qr/\A:\d+\z/, 'added one message'); + +if ('v2') { + my $info = $im->{last_object}; + like($info->[0], qr/\A[a-f0-9]{40}\z/, 'got last object_id'); + is($mime->as_string, ${$info->[2]}, 'string matches'); + is($info->[1], length(${$info->[2]}), 'length matches'); + my @cmd = ('git', "--git-dir=$git->{git_dir}", qw(hash-object --stdin)); + my $in = tempfile(); + print $in $mime->as_string or die "write failed: $!"; + $in->flush or die "flush failed: $!"; + $in->seek(0, SEEK_SET); + my $out = tempfile(); + my $pid = spawn(\@cmd, {}, { 0 => fileno($in), 1 => fileno($out)}); + is(waitpid($pid, 0), $pid, 'waitpid succeeds on hash-object'); + is($?, 0, 'hash-object'); + $out->seek(0, SEEK_SET); + chomp(my $hashed_obj = <$out>); + is($hashed_obj, $info->[0], "last object_id matches exp"); +} + $im->done; my @revs = $git->qx(qw(rev-list HEAD)); is(scalar @revs, 1, 'one revision created'); @@ -64,6 +90,8 @@ isnt($msg->header('Subject'), $mime->header('Subject'), 'subject mismatch'); $mime->header_set('Message-Id', '<failcheck@example.com>'); is($im->add($mime, sub { undef }), undef, 'check callback fails'); is($im->remove($mime), undef, 'message not added, so not removed'); - +is(undef, $im->checkpoint, 'checkpoint works before ->done'); $im->done; +is(undef, $im->checkpoint, 'checkpoint works after ->done'); +$im->checkpoint; done_testing(); @@ -7,6 +7,23 @@ use PublicInbox::Config; use File::Temp qw/tempdir/; my $tmpdir = tempdir('pi-init-XXXXXX', TMPDIR => 1, CLEANUP => 1); use constant pi_init => 'blib/script/public-inbox-init'; +use PublicInbox::Import; +use File::Basename; +open my $null, '>>', '/dev/null'; +my $rdr = { 2 => fileno($null) }; +sub quiet_fail { + my ($cmd, $msg) = @_; + # run_die doesn't take absolute paths: + my $path = $ENV{PATH}; + if (index($cmd->[0], '/') >= 0) { + my ($dir, $base) = ($cmd->[0] =~ m!\A(.+)/([^/]+)\z!); + $path = "$dir:$path"; + $cmd->[0] = $base; + } + local $ENV{PATH} = $path; + eval { PublicInbox::Import::run_die($cmd, undef, $rdr) }; + isnt($@, '', $msg); +} { local $ENV{PI_DIR} = "$tmpdir/.public-inbox/"; @@ -23,6 +40,34 @@ use constant pi_init => 'blib/script/public-inbox-init'; qw(http://example.com/clist clist@example.com)); is(system(@cmd), 0, 'public-inbox-init clist OK'); is((stat($cfgfile))[2] & 07777, 0666, "permissions preserved"); + + @cmd = (pi_init, 'clist', '-V2', "$tmpdir/clist", + qw(http://example.com/clist clist@example.com)); + quiet_fail(\@cmd, 'attempting to init V2 from V1 fails'); +} + +SKIP: { + foreach my $mod (qw(DBD::SQLite Search::Xapian::WritableDatabase)) { + eval "require $mod"; + skip "$mod missing for v2", 2 if $@; + } + local $ENV{PI_DIR} = "$tmpdir/.public-inbox/"; + my $cfgfile = "$ENV{PI_DIR}/config"; + my @cmd = (pi_init, '-V2', 'v2list', "$tmpdir/v2list", + qw(http://example.com/v2list v2list@example.com)); + is(system(@cmd), 0, 'public-inbox-init -V2 OK'); + ok(-d "$tmpdir/v2list", 'v2list directory exists'); + ok(-f "$tmpdir/v2list/msgmap.sqlite3", 'msgmap exists'); + ok(-d "$tmpdir/v2list/all.git", 'catch-all.git directory exists'); + @cmd = (pi_init, 'v2list', "$tmpdir/v2list", + qw(http://example.com/v2list v2list@example.com)); + is(system(@cmd), 0, 'public-inbox-init is idempotent'); + ok(! -d "$tmpdir/public-inbox" && !-d "$tmpdir/objects", + 'idempotent invocation w/o -V2 does not make inbox v1'); + + @cmd = (pi_init, 'v2list', "-V1", "$tmpdir/v2list", + qw(http://example.com/v2list v2list@example.com)); + quiet_fail(\@cmd, 'initializing V2 as V1 fails'); } done_testing(); @@ -1,11 +1,31 @@ # Copyright (C) 2016-2018 all contributors <meta@public-inbox.org> # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt> use Test::More; -use PublicInbox::MID qw(mid_escape); +use PublicInbox::MID qw(mid_escape mids references); is(mid_escape('foo!@(bar)'), 'foo!@(bar)'); is(mid_escape('foo%!@(bar)'), 'foo%25!@(bar)'); is(mid_escape('foo%!@(bar)'), 'foo%25!@(bar)'); +{ + use Email::MIME; + my $mime = Email::MIME->create; + $mime->header_set('Message-Id', '<mid-1@a>'); + is_deeply(['mid-1@a'], mids($mime->header_obj), 'mids in common case'); + $mime->header_set('Message-Id', '<mid-1@a>', '<mid-2@b>'); + is_deeply(['mid-1@a', 'mid-2@b'], mids($mime->header_obj), '2 mids'); + $mime->header_set('Message-Id', '<mid-1@a>', '<mid-1@a>'); + is_deeply(['mid-1@a'], mids($mime->header_obj), 'dup mids'); + $mime->header_set('Message-Id', '<mid-1@a> comment'); + is_deeply(['mid-1@a'], mids($mime->header_obj), 'comment ignored'); + $mime->header_set('Message-Id', 'bare-mid'); + is_deeply(['bare-mid'], mids($mime->header_obj), 'bare mid OK'); + + $mime->header_set('References', '<hello> <world>'); + $mime->header_set('In-Reply-To', '<weld>'); + is_deeply(['hello', 'world', 'weld'], references($mime->header_obj), + 'references combines with In-Reply-To'); +} + done_testing(); 1; @@ -65,4 +65,8 @@ my $orig = $d->mid_insert('spam@1'); $d->mid_delete('spam@1'); is($d->mid_insert('spam@2'), 1 + $orig, "last number not recycled"); +my $tmp = $d->tmp_clone; +is_deeply([$d->minmax], [$tmp->minmax], 'Cloned temporary DB matches'); +ok($tmp->mid_delete('spam@2'), 'temporary DB is writable'); + done_testing(); @@ -109,7 +109,9 @@ use_ok 'PublicInbox::Inbox'; is($ng->base_url, $u, 'URL expanded'); my $mid = 'a@b'; my $mime = Email::MIME->new("Message-ID: <$mid>\r\n\r\n"); - PublicInbox::NNTP::set_nntp_headers($mime->header_obj, $ng, 1, $mid); + my $hdr = $mime->header_obj; + my $mock_self = { nntpd => { grouplist => [] } }; + PublicInbox::NNTP::set_nntp_headers($mock_self, $hdr, $ng, 1, $mid); is_deeply([ $mime->header('Message-ID') ], [ "<$mid>" ], 'Message-ID unchanged'); is_deeply([ $mime->header('Archived-At') ], [ "<${u}a\@b/>" ], @@ -124,7 +126,7 @@ use_ok 'PublicInbox::Inbox'; 'Xref: set'); $ng->{-base_url} = 'http://mirror.example.com/m/'; - PublicInbox::NNTP::set_nntp_headers($mime->header_obj, $ng, 2, $mid); + PublicInbox::NNTP::set_nntp_headers($mock_self, $hdr, $ng, 2, $mid); is_deeply([ $mime->header('Message-ID') ], [ "<$mid>" ], 'Message-ID unchanged'); is_deeply([ $mime->header('Archived-At') ], @@ -21,14 +21,18 @@ my $tmpdir = tempdir('pi-nntpd-XXXXXX', TMPDIR => 1, CLEANUP => 1); my $home = "$tmpdir/pi-home"; my $err = "$tmpdir/stderr.log"; my $out = "$tmpdir/stdout.log"; -my $maindir = "$tmpdir/main.git"; +my $mainrepo = "$tmpdir/main.git"; my $group = 'test-nntpd'; my $addr = $group . '@example.com'; my $nntpd = 'blib/script/public-inbox-nntpd'; my $init = 'blib/script/public-inbox-init'; use_ok 'PublicInbox::Import'; +use_ok 'PublicInbox::Inbox'; use_ok 'PublicInbox::Git'; +use_ok 'PublicInbox::V2Writable'; +# XXX FIXME: make it easier to test both versions +my $version = int($ENV{PI_VERSION} || 1); my %opts = ( LocalAddr => '127.0.0.1', ReuseAddr => 1, @@ -40,14 +44,34 @@ my $sock = IO::Socket::INET->new(%opts); my $pid; my $len; END { kill 'TERM', $pid if defined $pid }; + +my $ibx = { + mainrepo => $mainrepo, + name => $group, + version => $version, + -primary_address => $addr, +}; +$ibx = PublicInbox::Inbox->new($ibx); { local $ENV{HOME} = $home; - system($init, $group, $maindir, 'http://example.com/', $addr); + my @cmd = ($init, $group, $mainrepo, 'http://example.com/', $addr); + push @cmd, "-V$version"; + is(system(@cmd), 0, 'init OK'); is(system(qw(git config), "--file=$home/.public-inbox/config", "publicinbox.$group.newsgroup", $group), 0, 'enabled newsgroup'); my $len; + my $im; + if ($version == 2) { + $im = PublicInbox::V2Writable->new($ibx); + } elsif ($version == 1) { + my $git = PublicInbox::Git->new($mainrepo); + $im = PublicInbox::Import->new($git, 'test', $addr); + } else { + die "unsupported version: $version"; + } + # ensure successful message delivery { my $mime = Email::MIME->new(<<EOF); @@ -66,12 +90,12 @@ EOF $list_id =~ s/@/./; $mime->header_set('List-Id', "<$list_id>"); $len = length($mime->as_string); - my $git = PublicInbox::Git->new($maindir); - my $im = PublicInbox::Import->new($git, 'test', $addr); $im->add($mime); $im->done; - my $s = PublicInbox::SearchIdx->new($maindir, 1); - $s->index_sync; + if ($version == 1) { + my $s = PublicInbox::SearchIdx->new($mainrepo, 1); + $s->index_sync; + } } ok($sock, 'sock created'); @@ -99,6 +123,7 @@ EOF my $list = $n->list; is_deeply($list, { $group => [ qw(1 1 n) ] }, 'LIST works'); is_deeply([$n->group($group)], [ qw(0 1 1), $group ], 'GROUP works'); + is_deeply($n->listgroup($group), [1], 'listgroup OK'); %opts = ( PeerAddr => $host_port, diff --git a/t/over.t b/t/over.t new file mode 100644 index 00000000..c0d9d5e5 --- /dev/null +++ b/t/over.t @@ -0,0 +1,63 @@ +# Copyright (C) 2018 all contributors <meta@public-inbox.org> +# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt> +use strict; +use warnings; +use Test::More; +use File::Temp qw/tempdir/; +use Compress::Zlib qw(compress); +foreach my $mod (qw(DBD::SQLite)) { + eval "require $mod"; + plan skip_all => "$mod missing for over.t" if $@; +} + +use_ok 'PublicInbox::OverIdx'; +my $tmpdir = tempdir('pi-over-XXXXXX', TMPDIR => 1, CLEANUP => 1); +my $over = PublicInbox::OverIdx->new("$tmpdir/over.sqlite3"); +$over->connect; +my $x = $over->next_tid; +is(int($x), $x, 'integer tid'); +my $y = $over->next_tid; +is($y, $x+1, 'tid increases'); + +$x = $over->sid('hello-world'); +is(int($x), $x, 'integer sid'); +$y = $over->sid('hello-WORLD'); +is($y, $x+1, 'sid ncreases'); +is($over->sid('hello-world'), $x, 'idempotent'); +$over->disconnect; + +$over = PublicInbox::OverIdx->new("$tmpdir/over.sqlite3"); +$over->connect; +is($over->sid('hello-world'), $x, 'idempotent across reopen'); +$over->each_by_mid('never', sub { fail('should not be called') }); + +$x = $over->create_ghost('never'); +is(int($x), $x, 'integer tid for ghost'); +$y = $over->create_ghost('NEVAR'); +is($y, $x + 1, 'integer tid for ghost increases'); + +my $ddd = compress(''); +foreach my $s ('', undef) { + $over->add_over([0, 0, 98, [ 'a' ], [], $s, $ddd]); + $over->add_over([0, 0, 99, [ 'b' ], [], $s, $ddd]); + my $msgs = [ map { $_->{num} } @{$over->get_thread('a')} ]; + is_deeply([98], $msgs, + 'messages not linked by empty subject'); +} + +$over->add_over([0, 0, 98, [ 'a' ], [], 's', $ddd]); +$over->add_over([0, 0, 99, [ 'b' ], [], 's', $ddd]); +foreach my $mid (qw(a b)) { + my $msgs = [ map { $_->{num} } @{$over->get_thread('a')} ]; + is_deeply([98, 99], $msgs, 'linked messages by subject'); +} +$over->add_over([0, 0, 98, [ 'a' ], [], 's', $ddd]); +$over->add_over([0, 0, 99, [ 'b' ], ['a'], 'diff', $ddd]); +foreach my $mid (qw(a b)) { + my $msgs = [ map { $_->{num} } @{$over->get_thread($mid)} ]; + is_deeply([98, 99], $msgs, "linked messages by Message-ID: <$mid>"); +} + +$over->rollback_lazy; + +done_testing(); diff --git a/t/perf-nntpd.t b/t/perf-nntpd.t new file mode 100644 index 00000000..e5021532 --- /dev/null +++ b/t/perf-nntpd.t @@ -0,0 +1,135 @@ +# Copyright (C) 2018 all contributors <meta@public-inbox.org> +# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt> +use strict; +use warnings; +use Test::More; +use Benchmark qw(:all :hireswallclock); +use PublicInbox::Inbox; +use File::Temp qw/tempdir/; +use POSIX qw(dup2); +use Fcntl qw(FD_CLOEXEC F_SETFD F_GETFD); +use Net::NNTP; +my $pi_dir = $ENV{GIANT_PI_DIR}; +plan skip_all => "GIANT_PI_DIR not defined for $0" unless $pi_dir; +eval { require PublicInbox::Search }; +my ($host_port, $group, %opts, $s, $pid); +END { + if ($s) { + $s->print("QUIT\r\n"); + $s->getline; + $s = undef; + } + kill 'TERM', $pid if defined $pid; +}; + +if (($ENV{NNTP_TEST_URL} || '') =~ m!\Anntp://([^/]+)/([^/]+)\z!) { + ($host_port, $group) = ($1, $2); + $host_port .= ":119" unless index($host_port, ':') > 0; +} else { + $group = 'inbox.test.perf.nntpd'; + my $ibx = { mainrepo => $pi_dir, newsgroup => $group }; + $ibx = PublicInbox::Inbox->new($ibx); + my $nntpd = 'blib/script/public-inbox-nntpd'; + my $tmpdir = tempdir('perf-nntpd-XXXXXX', TMPDIR => 1, CLEANUP => 1); + + my $pi_config = "$tmpdir/config"; + { + open my $fh, '>', $pi_config or die "open($pi_config): $!"; + print $fh <<"" or die "print $pi_config: $!"; +[publicinbox "test"] + newsgroup = $group + mainrepo = $pi_dir + address = test\@example.com + + close $fh or die "close($pi_config): $!"; + } + + %opts = ( + LocalAddr => '127.0.0.1', + ReuseAddr => 1, + Proto => 'tcp', + Listen => 1024, + ); + my $sock = IO::Socket::INET->new(%opts); + + ok($sock, 'sock created'); + $! = 0; + $pid = fork; + if ($pid == 0) { + # pretend to be systemd + my $fl = fcntl($sock, F_GETFD, 0); + dup2(fileno($sock), 3) or die "dup2 failed: $!\n"; + dup2(1, 2) or die "dup2 failed: $!\n"; + fcntl($sock, F_SETFD, $fl &= ~FD_CLOEXEC); + $ENV{LISTEN_PID} = $$; + $ENV{LISTEN_FDS} = 1; + $ENV{PI_CONFIG} = $pi_config; + exec $nntpd, '-W0'; + die "FAIL: $!\n"; + } + ok(defined $pid, 'forked nntpd process successfully'); + $host_port = $sock->sockhost . ':' . $sock->sockport; +} +%opts = ( + PeerAddr => $host_port, + Proto => 'tcp', + Timeout => 1, +); +$s = IO::Socket::INET->new(%opts); +$s->autoflush(1); +my $buf = $s->getline; +is($buf, "201 server ready - post via email\r\n", 'got greeting'); + +my $t = timeit(10, sub { + ok($s->print("GROUP $group\r\n"), 'changed group'); + $buf = $s->getline; +}); +diag 'GROUP took: ' . timestr($t); + +my ($tot, $min, $max) = ($buf =~ /\A211 (\d+) (\d+) (\d+) /); +ok($tot && $min && $max, 'got GROUP response'); +my $nr = $max - $min; +my $nmax = 50000; +my $nmin = $max - $nmax; +$nmin = $min if $nmin < $min; +my $res; +my $spec = "$nmin-$max"; +my $n; + +sub read_until_dot ($) { + my $n = 0; + do { + $buf = $s->getline; + ++$n + } until $buf eq ".\r\n"; + $n; +} + +$t = timeit(1, sub { + $s->print("XOVER $spec\r\n"); + $n = read_until_dot($s); +}); +diag 'xover took: ' . timestr($t) . " for $n"; + +$t = timeit(1, sub { + $s->print("HDR From $spec\r\n"); + $n = read_until_dot($s); + +}); +diag "XHDR From ". timestr($t) . " for $n"; + +my $date = $ENV{NEWNEWS_DATE}; +unless ($date) { + my (undef, undef, undef, $d, $m, $y) = gmtime(time - 30 * 86400); + $date = sprintf('%04u%02u%02u', $y + 1900, $m, $d); + diag "NEWNEWS_DATE undefined, using $date"; +} +$t = timeit(1, sub { + $s->print("NEWNEWS * $date 000000 GMT\r\n"); + $n = read_until_dot($s); +}); +diag 'newnews took: ' . timestr($t) . " for $n"; + +done_testing(); + +1; diff --git a/t/perf-threading.t b/t/perf-threading.t new file mode 100644 index 00000000..15779c93 --- /dev/null +++ b/t/perf-threading.t @@ -0,0 +1,32 @@ +# Copyright (C) 2016-2018 all contributors <meta@public-inbox.org> +# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt> +# +# real-world testing of search threading +use strict; +use warnings; +use Test::More; +use Benchmark qw(:all); +use PublicInbox::Inbox; +my $pi_dir = $ENV{GIANT_PI_DIR}; +plan skip_all => "GIANT_PI_DIR not defined for $0" unless $pi_dir; +my $ibx = PublicInbox::Inbox->new({ mainrepo => $pi_dir }); +eval { require PublicInbox::Search }; +my $srch = $ibx->search; +plan skip_all => "$pi_dir not configured for search $0 $@" unless $srch; + +require PublicInbox::View; + +my $msgs; +my $elapsed = timeit(1, sub { + $msgs = $srch->{over_ro}->recent({limit => 200000}); +}); +my $n = scalar(@$msgs); +ok($n, 'got some messages'); +diag "enquire: ".timestr($elapsed)." for $n"; + +$elapsed = timeit(1, sub { + PublicInbox::View::thread_results({-inbox => $ibx}, $msgs); +}); +diag "thread_results ".timestr($elapsed); + +done_testing(); @@ -18,6 +18,7 @@ foreach my $mod (@mods) { } use_ok 'PublicInbox::Import'; use_ok 'PublicInbox::Git'; +my @ls; foreach my $mod (@mods) { use_ok $mod; } { @@ -55,6 +56,8 @@ EOF $im->done; my $rev = `git --git-dir="$maindir" rev-list HEAD`; like($rev, qr/\A[a-f0-9]{40}/, "good revision committed"); + @ls = `git --git-dir="$maindir" ls-tree -r --name-only HEAD`; + chomp @ls; } my $app = eval { local $ENV{PI_CONFIG} = $pi_config; @@ -198,6 +201,21 @@ EOF "$sfx redirected to /mbox.gz"); }); } + test_psgi($app, sub { + my ($cb) = @_; + # for a while, we used to support /$INBOX/$X40/ + # when we "compressed" long Message-IDs to SHA-1 + # Now we're stuck supporting them forever :< + foreach my $path (@ls) { + $path =~ tr!/!!d; + my $from = "http://example.com/test/$path/"; + my $res = $cb->(GET($from)); + is(301, $res->code, 'is permanent redirect'); + like($res->header('Location'), + qr!/test/blah\@example\.com/!, + 'redirect from x40 MIDs works'); + } + }); } done_testing(); diff --git a/t/psgi_search.t b/t/psgi_search.t index cf5a7e91..2f033016 100644 --- a/t/psgi_search.t +++ b/t/psgi_search.t @@ -30,8 +30,7 @@ EOF my $num = 0; # nb. using internal API, fragile! -my $xdb = $rw->_xdb_acquire; -$xdb->begin_transaction; +$rw->begin_txn_lazy; foreach (reverse split(/\n\n/, $data)) { $_ .= "\n"; @@ -42,8 +41,7 @@ foreach (reverse split(/\n\n/, $data)) { ok($doc_id, 'message added: '. $mid); } -$xdb->commit_transaction; -$rw = undef; +$rw->commit_txn_lazy; my $cfgpfx = "publicinbox.test"; my $config = PublicInbox::Config->new({ diff --git a/t/psgi_v2.t b/t/psgi_v2.t new file mode 100644 index 00000000..65448dc4 --- /dev/null +++ b/t/psgi_v2.t @@ -0,0 +1,245 @@ +# Copyright (C) 2018 all contributors <meta@public-inbox.org> +# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt> +use strict; +use warnings; +use Test::More; +use File::Temp qw/tempdir/; +use PublicInbox::MIME; +use PublicInbox::Config; +use PublicInbox::WWW; +use PublicInbox::MID qw(mids); +my @mods = qw(DBD::SQLite Search::Xapian HTTP::Request::Common Plack::Test + URI::Escape Plack::Builder); +foreach my $mod (@mods) { + eval "require $mod"; + plan skip_all => "$mod missing for psgi_v2_dupes.t" if $@; +} +use_ok($_) for @mods; +use_ok 'PublicInbox::V2Writable'; +my $mainrepo = tempdir('pi-v2_dupes-XXXXXX', TMPDIR => 1, CLEANUP => 1); +my $ibx = { + mainrepo => $mainrepo, + name => 'test-v2writable', + version => 2, + -primary_address => 'test@example.com', +}; +$ibx = PublicInbox::Inbox->new($ibx); +my $new_mid; + +my $im = PublicInbox::V2Writable->new($ibx, 1); +$im->{parallel} = 0; + +my $mime = PublicInbox::MIME->create( + header => [ + From => 'a@example.com', + To => 'test@example.com', + Subject => 'this is a subject', + 'Message-ID' => '<a-mid@b>', + Date => 'Fri, 02 Oct 1993 00:00:00 +0000', + ], + body => "hello world\n", +); +ok($im->add($mime), 'added one message'); +$mime->body_set("hello world!\n"); + +my @warn; +local $SIG{__WARN__} = sub { push @warn, @_ }; +$mime->header_set(Date => 'Fri, 02 Oct 1993 00:01:00 +0000'); +ok($im->add($mime), 'added duplicate-but-different message'); +is(scalar(@warn), 1, 'got one warning'); +my $mids = mids($mime->header_obj); +$new_mid = $mids->[1]; +$im->done; + +my $cfgpfx = "publicinbox.v2test"; +my $cfg = { + "$cfgpfx.address" => $ibx->{-primary_address}, + "$cfgpfx.mainrepo" => $mainrepo, +}; +my $config = PublicInbox::Config->new($cfg); +my $www = PublicInbox::WWW->new($config); +my ($res, $raw, @from_); +test_psgi(sub { $www->call(@_) }, sub { + my ($cb) = @_; + $res = $cb->(GET('/v2test/a-mid@b/raw')); + $raw = $res->content; + like($raw, qr/^hello world$/m, 'got first message'); + like($raw, qr/^hello world!$/m, 'got second message'); + @from_ = ($raw =~ m/^From /mg); + is(scalar(@from_), 2, 'two From_ lines'); + + $res = $cb->(GET("/v2test/$new_mid/raw")); + $raw = $res->content; + like($raw, qr/^hello world!$/m, 'second message with new Message-Id'); + @from_ = ($raw =~ m/^From /mg); + is(scalar(@from_), 1, 'only one From_ line'); + + # Atom feed should sort by Date: (if Received is missing) + $res = $cb->(GET('/v2test/new.atom')); + my @bodies = ($res->content =~ />(hello [^<]+)</mg); + is_deeply(\@bodies, [ "hello world!\n", "hello world\n" ], + 'Atom ordering is chronological'); + + # new.html should sort by Date:, too (if Received is missing) + $res = $cb->(GET('/v2test/new.html')); + @bodies = ($res->content =~ /^(hello [^<]+)$/mg); + is_deeply(\@bodies, [ "hello world!\n", "hello world\n" ], + 'new.html ordering is chronological'); +}); + +$mime->header_set('Message-Id', 'a-mid@b'); +$mime->body_set("hello ghosts\n"); +ok($im->add($mime), 'added 3rd duplicate-but-different message'); +is(scalar(@warn), 2, 'got another warning'); +like($warn[0], qr/mismatched/, 'warned about mismatched messages'); +is($warn[0], $warn[1], 'both warnings are the same'); + +$mids = mids($mime->header_obj); +my $third = $mids->[-1]; +$im->done; + +test_psgi(sub { $www->call(@_) }, sub { + my ($cb) = @_; + $res = $cb->(GET("/v2test/$third/raw")); + $raw = $res->content; + like($raw, qr/^hello ghosts$/m, 'got third message'); + @from_ = ($raw =~ m/^From /mg); + is(scalar(@from_), 1, 'one From_ line'); + + $res = $cb->(GET('/v2test/a-mid@b/raw')); + $raw = $res->content; + like($raw, qr/^hello world$/m, 'got first message'); + like($raw, qr/^hello world!$/m, 'got second message'); + like($raw, qr/^hello ghosts$/m, 'got third message'); + @from_ = ($raw =~ m/^From /mg); + is(scalar(@from_), 3, 'three From_ lines'); + $config->each_inbox(sub { $_[0]->search->reopen }); + + SKIP: { + eval { require IO::Uncompress::Gunzip }; + skip 'IO::Uncompress::Gunzip missing', 4 if $@; + + $res = $cb->(GET('/v2test/a-mid@b/t.mbox.gz')); + my $out; + my $in = $res->content; + my $status = IO::Uncompress::Gunzip::gunzip(\$in => \$out); + like($out, qr/^hello world$/m, 'got first in t.mbox.gz'); + like($out, qr/^hello world!$/m, 'got second in t.mbox.gz'); + like($out, qr/^hello ghosts$/m, 'got third in t.mbox.gz'); + @from_ = ($out =~ m/^From /mg); + is(scalar(@from_), 3, 'three From_ lines in t.mbox.gz'); + + # search interface + $res = $cb->(POST('/v2test/?q=m:a-mid@b&x=m')); + $in = $res->content; + $status = IO::Uncompress::Gunzip::gunzip(\$in => \$out); + like($out, qr/^hello world$/m, 'got first in mbox POST'); + like($out, qr/^hello world!$/m, 'got second in mbox POST'); + like($out, qr/^hello ghosts$/m, 'got third in mbox POST'); + @from_ = ($out =~ m/^From /mg); + is(scalar(@from_), 3, 'three From_ lines in mbox POST'); + + # all.mbox.gz interface + $res = $cb->(GET('/v2test/all.mbox.gz')); + $in = $res->content; + $status = IO::Uncompress::Gunzip::gunzip(\$in => \$out); + like($out, qr/^hello world$/m, 'got first in all.mbox'); + like($out, qr/^hello world!$/m, 'got second in all.mbox'); + like($out, qr/^hello ghosts$/m, 'got third in all.mbox'); + @from_ = ($out =~ m/^From /mg); + is(scalar(@from_), 3, 'three From_ lines in all.mbox'); + }; + + $res = $cb->(GET('/v2test/?q=m:a-mid@b&x=t')); + is($res->code, 200, 'success with threaded search'); + my $raw = $res->content; + ok($raw =~ s/\A.*>Results 1-3 of 3<//s, 'got all results'); + my @over = ($raw =~ m/\d{4}-\d+-\d+\s+\d+:\d+ (.+)$/gm); + is_deeply(\@over, [ '<a', '` <a', '` <a' ], 'threaded messages show up'); + + local $SIG{__WARN__} = 'DEFAULT'; + $res = $cb->(GET('/v2test/a-mid@b/')); + $raw = $res->content; + like($raw, qr/^hello world$/m, 'got first message'); + like($raw, qr/^hello world!$/m, 'got second message'); + like($raw, qr/^hello ghosts$/m, 'got third message'); + @from_ = ($raw =~ m/>From: /mg); + is(scalar(@from_), 3, 'three From: lines'); + foreach my $mid ('a-mid@b', $new_mid, $third) { + like($raw, qr/<\Q$mid\E>/s, "Message-ID $mid shown"); + } + like($raw, qr/\b3\+ messages\b/, 'thread overview shown'); + + my $exp = [ qw(<a-mid@b> <reuse@mid>) ]; + $mime->header_set('Message-Id', @$exp); + $mime->header_set('Subject', '4th dupe'); + local $SIG{__WARN__} = sub {}; + ok($im->add($mime), 'added one message'); + $im->done; + my @h = $mime->header('Message-ID'); + is_deeply($exp, \@h, 'reused existing Message-ID'); + + $config->each_inbox(sub { $_[0]->search->reopen }); + + $res = $cb->(GET('/v2test/new.atom')); + my @ids = ($res->content =~ m!<id>urn:uuid:([^<]+)</id>!sg); + my %ids; + $ids{$_}++ for @ids; + is_deeply([qw(1 1 1 1)], [values %ids], 'feed ids unique'); + + $res = $cb->(GET('/v2test/reuse@mid/T/')); + $raw = $res->content; + like($raw, qr/\b4\+ messages\b/, 'thread overview shown with /T/'); + @over = ($raw =~ m/^\d{4}-\d+-\d+\s+\d+:\d+ (.+)$/gm); + is_deeply(\@over, [ '<a', '` <a', '` <a', '` <a' ], + 'duplicate messages share the same root'); + + $res = $cb->(GET('/v2test/reuse@mid/t/')); + $raw = $res->content; + like($raw, qr/\b4\+ messages\b/, 'thread overview shown with /t/'); + + $res = $cb->(GET('/v2test/0/info/refs')); + is($res->code, 200, 'got info refs for dumb clones'); + $res = $cb->(GET('/v2test/info/refs')); + is($res->code, 404, 'unpartitioned git URL fails'); + + # ensure conflicted attachments can be resolved + foreach my $body (qw(old new)) { + my $parts = [ + PublicInbox::MIME->create( + attributes => { content_type => 'text/plain' }, + body => 'blah', + ), + PublicInbox::MIME->create( + attributes => { + filename => 'attach.txt', + content_type => 'text/plain', + }, + body => $body + ) + ]; + $mime = PublicInbox::MIME->create( + parts => $parts, + header_str => [ From => 'root@z', + 'Message-ID' => '<a@dup>', + Subject => 'hi'] + ); + ok($im->add($mime), "added attachment $body"); + } + $im->done; + $config->each_inbox(sub { $_[0]->search->reopen }); + $res = $cb->(GET('/v2test/a@dup/')); + my @links = ($res->content =~ m!"\.\./([^/]+/2-attach\.txt)\"!g); + is(scalar(@links), 2, 'both attachment links exist'); + isnt($links[0], $links[1], 'attachment links are different'); + { + my $old = $cb->(GET('/v2test/' . $links[0])); + my $new = $cb->(GET('/v2test/' . $links[1])); + is($old->content, 'old', 'got expected old content'); + is($new->content, 'new', 'got expected new content'); + } +}); + +done_testing(); + +1; diff --git a/t/search-thr-index.t b/t/search-thr-index.t index c3534f6b..2aa97bff 100644 --- a/t/search-thr-index.t +++ b/t/search-thr-index.t @@ -4,6 +4,7 @@ use strict; use warnings; use Test::More; use File::Temp qw/tempdir/; +use PublicInbox::MID qw(mids); use Email::MIME; eval { require PublicInbox::SearchIdx; }; plan skip_all => "Xapian missing for search" if $@; @@ -31,8 +32,7 @@ EOF my $num = 0; # nb. using internal API, fragile! -my $xdb = $rw->_xdb_acquire; -$xdb->begin_transaction; +my $xdb = $rw->begin_txn_lazy; my @mids; foreach (reverse split(/\n\n/, $data)) { @@ -41,18 +41,20 @@ foreach (reverse split(/\n\n/, $data)) { $mime->header_set('From' => 'bw@g'); $mime->header_set('To' => 'git@vger.kernel.org'); my $bytes = bytes::length($mime->as_string); - my $doc_id = $rw->add_message($mime, $bytes, ++$num, 'ignored'); - my $mid = $mime->header('Message-Id'); + my $mid = mids($mime->header_obj)->[0]; + my $doc_id = $rw->add_message($mime, $bytes, ++$num, 'ignored', $mid); push @mids, $mid; ok($doc_id, 'message added: '. $mid); } my $prev; foreach my $mid (@mids) { - my $res = $rw->get_thread($mid); - is(3, $res->{total}, "got all messages from $mid"); + my $msgs = $rw->{over}->get_thread($mid); + is(3, scalar(@$msgs), "got all messages from $mid"); } +$rw->commit_txn_lazy; + done_testing(); 1; @@ -18,36 +18,36 @@ ok($@, "exception raised on non-existent DB"); my $rw = PublicInbox::SearchIdx->new($git_dir, 1); $rw->_xdb_acquire; $rw->_xdb_release; +my $ibx = $rw->{-inbox}; $rw = undef; my $ro = PublicInbox::Search->new($git_dir); my $rw_commit = sub { - $rw->{xdb}->commit_transaction if $rw && $rw->{xdb}; + $rw->commit_txn_lazy if $rw; $rw = PublicInbox::SearchIdx->new($git_dir, 1); - $rw->_xdb_acquire->begin_transaction; + $rw->begin_txn_lazy; }; { # git repository perms - is(PublicInbox::SearchIdx->_git_config_perm(undef), - &PublicInbox::SearchIdx::PERM_GROUP, + is($ibx->_git_config_perm(), &PublicInbox::InboxWritable::PERM_GROUP, "undefined permission is group"); - is(PublicInbox::SearchIdx::_umask_for( - PublicInbox::SearchIdx->_git_config_perm('0644')), + is(PublicInbox::InboxWritable::_umask_for( + PublicInbox::InboxWritable->_git_config_perm('0644')), 0022, "644 => umask(0022)"); - is(PublicInbox::SearchIdx::_umask_for( - PublicInbox::SearchIdx->_git_config_perm('0600')), + is(PublicInbox::InboxWritable::_umask_for( + PublicInbox::InboxWritable->_git_config_perm('0600')), 0077, "600 => umask(0077)"); - is(PublicInbox::SearchIdx::_umask_for( - PublicInbox::SearchIdx->_git_config_perm('0640')), + is(PublicInbox::InboxWritable::_umask_for( + PublicInbox::InboxWritable->_git_config_perm('0640')), 0027, "640 => umask(0027)"); - is(PublicInbox::SearchIdx::_umask_for( - PublicInbox::SearchIdx->_git_config_perm('group')), + is(PublicInbox::InboxWritable::_umask_for( + PublicInbox::InboxWritable->_git_config_perm('group')), 0007, 'group => umask(0007)'); - is(PublicInbox::SearchIdx::_umask_for( - PublicInbox::SearchIdx->_git_config_perm('everybody')), + is(PublicInbox::InboxWritable::_umask_for( + PublicInbox::InboxWritable->_git_config_perm('everybody')), 0002, 'everybody => umask(0002)'); - is(PublicInbox::SearchIdx::_umask_for( - PublicInbox::SearchIdx->_git_config_perm('umask')), + is(PublicInbox::InboxWritable::_umask_for( + PublicInbox::InboxWritable->_git_config_perm('umask')), umask, 'umask => existing umask'); } @@ -82,18 +82,16 @@ my $rw_commit = sub { } sub filter_mids { - my ($res) = @_; - sort(map { $_->mid } @{$res->{msgs}}); + my ($msgs) = @_; + sort(map { $_->mid } @$msgs); } { $rw_commit->(); $ro->reopen; - my $found = $ro->lookup_message('<root@s>'); - ok($found, "message found"); - is($root_id, $found->{doc_id}, 'doc_id set correctly'); - is($found->mid, 'root@s', 'mid set correctly'); - ok(int($found->thread_id) > 0, 'thread_id is an integer'); + my $found = $ro->query('m:root@s'); + is(scalar(@$found), 1, "message found"); + is($found->[0]->mid, 'root@s', 'mid set correctly'); my ($res, @res); my @exp = sort qw(root@s last@s); @@ -107,12 +105,12 @@ sub filter_mids { is_deeply(\@res, \@exp, 'got expected results for s:"" match'); $res = $ro->query('s:"Hello world"', {limit => 1}); - is(scalar @{$res->{msgs}}, 1, "limit works"); - my $first = $res->{msgs}->[0]; + is(scalar @$res, 1, "limit works"); + my $first = $res->[0]; $res = $ro->query('s:"Hello world"', {offset => 1}); - is(scalar @{$res->{msgs}}, 1, "offset works"); - my $second = $res->{msgs}->[0]; + is(scalar @$res, 1, "offset works"); + my $second = $res->[0]; isnt($first, $second, "offset returned different result from limit"); } @@ -148,7 +146,13 @@ sub filter_mids { my $ghost_id = $rw->add_message($was_ghost); is($ghost_id, int($ghost_id), "ghost_id is an integer: $ghost_id"); - ok($ghost_id < $reply_id, "ghost vivified from earlier message"); + my $msgs = $rw->{over}->get_thread('ghost-message@s'); + is(scalar(@$msgs), 2, 'got both messages in ghost thread'); + foreach (qw(sid tid)) { + is($msgs->[0]->{$_}, $msgs->[1]->{$_}, "{$_} match"); + } + isnt($msgs->[0]->{num}, $msgs->[1]->{num}, "num do not match"); + ok($_->{num} > 0, 'positive art num') foreach @$msgs } # search thread on ghost @@ -164,7 +168,14 @@ sub filter_mids { # body $res = $ro->query('goodbye'); - is($res->{msgs}->[0]->mid, 'last@s', 'got goodbye message body'); + is($res->[0]->mid, 'last@s', 'got goodbye message body'); + + # datestamp + $res = $ro->query('dt:20101002000001..20101002000001'); + @res = filter_mids($res); + is_deeply(\@res, ['ghost-message@s'], 'exact Date: match works'); + $res = $ro->query('dt:20101002000002..20101002000002'); + is_deeply($res, [], 'exact Date: match down to the second'); } # long message-id @@ -210,7 +221,7 @@ sub filter_mids { $rw_commit->(); $ro->reopen; my $t = $ro->get_thread('root@s'); - is($t->{total}, 4, "got all 4 mesages in thread"); + is(scalar(@$t), 4, "got all 4 mesages in thread"); my @exp = sort($long_reply_mid, 'root@s', 'last@s', $long_mid); @res = filter_mids($t); is_deeply(\@res, \@exp, "get_thread works"); @@ -239,13 +250,13 @@ sub filter_mids { ], body => "theatre\nfade\n")); my $res = $rw->query("theatre"); - is($res->{total}, 2, "got both matches"); - is($res->{msgs}->[0]->mid, 'nquote@a', "non-quoted scores higher"); - is($res->{msgs}->[1]->mid, 'quote@a', "quoted result still returned"); + is(scalar(@$res), 2, "got both matches"); + is($res->[0]->mid, 'nquote@a', "non-quoted scores higher"); + is($res->[1]->mid, 'quote@a', "quoted result still returned"); $res = $rw->query("illusions"); - is($res->{total}, 1, "got a match for quoted text"); - is($res->{msgs}->[0]->mid, 'quote@a', + is(scalar(@$res), 1, "got a match for quoted text"); + is($res->[0]->mid, 'quote@a', "quoted result returned if nothing else"); } @@ -264,10 +275,9 @@ sub filter_mids { ], body => "LOOP!\n")); ok($doc_id > 0, "doc_id defined with circular reference"); - my $smsg = $rw->lookup_message('circle@a'); + my $smsg = $rw->query('m:circle@a', {limit=>1})->[0]; is($smsg->references, '', "no references created"); - my $msg = PublicInbox::SearchMsg->load_doc($smsg->{doc}); - is($s, $msg->subject, 'long subject not rewritten'); + is($s, $smsg->subject, 'long subject not rewritten'); } { @@ -281,51 +291,51 @@ sub filter_mids { my $mime = Email::MIME->new($str); my $doc_id = $rw->add_message($mime); ok($doc_id > 0, 'message indexed doc_id with UTF-8'); - my $smsg = $rw->lookup_message('testmessage@example.com'); - my $msg = PublicInbox::SearchMsg->load_doc($smsg->{doc}); - + my $msg = $rw->query('m:testmessage@example.com', {limit => 1})->[0]; is($mime->header('Subject'), $msg->subject, 'UTF-8 subject preserved'); } { - my $res = $ro->query('d:19931002..20101002'); - ok(scalar @{$res->{msgs}} > 0, 'got results within range'); - $res = $ro->query('d:20101003..'); - is(scalar @{$res->{msgs}}, 0, 'nothing after 20101003'); - $res = $ro->query('d:..19931001'); - is(scalar @{$res->{msgs}}, 0, 'nothing before 19931001'); + my $msgs = $ro->query('d:19931002..20101002'); + ok(scalar(@$msgs) > 0, 'got results within range'); + $msgs = $ro->query('d:20101003..'); + is(scalar(@$msgs), 0, 'nothing after 20101003'); + $msgs = $ro->query('d:..19931001'); + is(scalar(@$msgs), 0, 'nothing before 19931001'); } # names and addresses { - my $res = $ro->query('t:list@example.com'); - is(scalar @{$res->{msgs}}, 6, 'searched To: successfully'); - foreach my $smsg (@{$res->{msgs}}) { + my $mset = $ro->query('t:list@example.com', {mset => 1}); + is($mset->size, 6, 'searched To: successfully'); + foreach my $m ($mset->items) { + my $smsg = $ro->lookup_article($m->get_docid); like($smsg->to, qr/\blist\@example\.com\b/, 'to appears'); } - $res = $ro->query('tc:list@example.com'); - is(scalar @{$res->{msgs}}, 6, 'searched To+Cc: successfully'); - foreach my $smsg (@{$res->{msgs}}) { + $mset = $ro->query('tc:list@example.com', {mset => 1}); + is($mset->size, 6, 'searched To+Cc: successfully'); + foreach my $m ($mset->items) { + my $smsg = $ro->lookup_article($m->get_docid); my $tocc = join("\n", $smsg->to, $smsg->cc); like($tocc, qr/\blist\@example\.com\b/, 'tocc appears'); } foreach my $pfx ('tcf:', 'c:') { - $res = $ro->query($pfx . 'foo@example.com'); - is(scalar @{$res->{msgs}}, 1, - "searched $pfx successfully for Cc:"); - foreach my $smsg (@{$res->{msgs}}) { + my $mset = $ro->query($pfx . 'foo@example.com', { mset => 1 }); + is($mset->items, 1, "searched $pfx successfully for Cc:"); + foreach my $m ($mset->items) { + my $smsg = $ro->lookup_article($m->get_docid); like($smsg->cc, qr/\bfoo\@example\.com\b/, 'cc appears'); } } foreach my $pfx ('', 'tcf:', 'f:') { - $res = $ro->query($pfx . 'Laggy'); - is(scalar @{$res->{msgs}}, 1, + my $res = $ro->query($pfx . 'Laggy'); + is(scalar(@$res), 1, "searched $pfx successfully for From:"); - foreach my $smsg (@{$res->{msgs}}) { + foreach my $smsg (@$res) { like($smsg->from, qr/Laggy Sender/, "From appears with $pfx"); } @@ -336,23 +346,23 @@ sub filter_mids { $rw_commit->(); $ro->reopen; my $res = $ro->query('b:hello'); - is(scalar @{$res->{msgs}}, 0, 'no match on body search only'); + is(scalar(@$res), 0, 'no match on body search only'); $res = $ro->query('bs:smith'); - is(scalar @{$res->{msgs}}, 0, + is(scalar(@$res), 0, 'no match on body+subject search for From'); $res = $ro->query('q:theatre'); - is(scalar @{$res->{msgs}}, 1, 'only one quoted body'); - like($res->{msgs}->[0]->from, qr/\AQuoter/, 'got quoted body'); + is(scalar(@$res), 1, 'only one quoted body'); + like($res->[0]->from, qr/\AQuoter/, 'got quoted body'); $res = $ro->query('nq:theatre'); - is(scalar @{$res->{msgs}}, 1, 'only one non-quoted body'); - like($res->{msgs}->[0]->from, qr/\ANon-Quoter/, 'got non-quoted body'); + is(scalar @$res, 1, 'only one non-quoted body'); + like($res->[0]->from, qr/\ANon-Quoter/, 'got non-quoted body'); foreach my $pfx (qw(b: bs:)) { $res = $ro->query($pfx . 'theatre'); - is(scalar @{$res->{msgs}}, 2, "searched both bodies for $pfx"); - like($res->{msgs}->[0]->from, qr/\ANon-Quoter/, + is(scalar @$res, 2, "searched both bodies for $pfx"); + like($res->[0]->from, qr/\ANon-Quoter/, "non-quoter first for $pfx"); } } @@ -391,14 +401,22 @@ sub filter_mids { $rw_commit->(); $ro->reopen; my $n = $ro->query('n:attached_fart.txt'); - is(scalar @{$n->{msgs}}, 1, 'got result for n:'); + is(scalar @$n, 1, 'got result for n:'); my $res = $ro->query('part_deux.txt'); - is(scalar @{$res->{msgs}}, 1, 'got result without n:'); - is($n->{msgs}->[0]->mid, $res->{msgs}->[0]->mid, + is(scalar @$res, 1, 'got result without n:'); + is($n->[0]->mid, $res->[0]->mid, 'same result with and without'); my $txt = $ro->query('"inside another"'); - is($txt->{msgs}->[0]->mid, $res->{msgs}->[0]->mid, + is($txt->[0]->mid, $res->[0]->mid, 'search inside text attachments works'); + + my $mid = $n->[0]->mid; + my ($id, $prev); + my $art = $ro->next_by_mid($mid, \$id, \$prev); + ok($art, 'article exists in OVER DB'); + $rw->unindex_blob($amsg); + $rw->commit_txn_lazy; + is($ro->lookup_article($art->{num}), undef, 'gone from OVER DB'); } done_testing(); diff --git a/t/thread-all.t b/t/thread-all.t deleted file mode 100644 index d4e8c1fc..00000000 --- a/t/thread-all.t +++ /dev/null @@ -1,38 +0,0 @@ -# Copyright (C) 2016-2018 all contributors <meta@public-inbox.org> -# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt> -# -# real-world testing of search threading -use strict; -use warnings; -use Test::More; -use Time::HiRes qw(clock_gettime CLOCK_MONOTONIC); -my $pi_dir = $ENV{GIANT_PI_DIR}; -plan skip_all => "GIANT_PI_DIR not defined for $0" unless $pi_dir; -eval { require PublicInbox::Search; }; -plan skip_all => "Xapian missing for $0" if $@; -my $srch = eval { PublicInbox::Search->new($pi_dir) }; -plan skip_all => "$pi_dir not initialized for $0" if $@; - -require PublicInbox::View; -require PublicInbox::SearchThread; - -my $pfx = PublicInbox::Search::xpfx('thread'); -my $opts = { limit => 1000000, asc => 1 }; -my $t0 = clock_gettime(CLOCK_MONOTONIC); -my $elapsed; - -my $sres = $srch->_do_enquire(undef, $opts); -$elapsed = clock_gettime(CLOCK_MONOTONIC) - $t0; -diag "enquire: $elapsed"; - -$t0 = clock_gettime(CLOCK_MONOTONIC); -my $msgs = PublicInbox::View::load_results($srch, $sres); -$elapsed = clock_gettime(CLOCK_MONOTONIC) - $t0; -diag "load_results $elapsed"; - -$t0 = clock_gettime(CLOCK_MONOTONIC); -PublicInbox::View::thread_results($msgs); -$elapsed = clock_gettime(CLOCK_MONOTONIC) - $t0; -diag "thread_results $elapsed"; - -done_testing(); diff --git a/t/time.t b/t/time.t new file mode 100644 index 00000000..370a0bd3 --- /dev/null +++ b/t/time.t @@ -0,0 +1,28 @@ +# Copyright (C) 2018 all contributors <meta@public-inbox.org> +# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt> +use strict; +use warnings; +use Test::More; +use_ok 'PublicInbox::MIME'; +use PublicInbox::MsgTime qw(msg_datestamp); +my $mime = PublicInbox::MIME->create( + header => [ + From => 'a@example.com', + To => 'test@example.com', + Subject => 'this is a subject', + 'Message-ID' => '<a-mid@b>', + Date => 'Fri, 02 Oct 93 00:00:00 +0000', + ], + body => "hello world\n", +); + +my $ts = msg_datestamp($mime->header_obj); +use POSIX qw(strftime); +is(strftime('%Y-%m-%d %H:%M:%S', gmtime($ts)), '1993-10-02 00:00:00', + 'got expected date with 2 digit year'); +$mime->header_set(Date => 'Fri, 02 Oct 101 01:02:03 +0000'); +$ts = msg_datestamp($mime->header_obj); +is(strftime('%Y-%m-%d %H:%M:%S', gmtime($ts)), '2001-10-02 01:02:03', + 'got expected date with 3 digit year'); + +done_testing(); diff --git a/t/v1-add-remove-add.t b/t/v1-add-remove-add.t new file mode 100644 index 00000000..cd6e2811 --- /dev/null +++ b/t/v1-add-remove-add.t @@ -0,0 +1,45 @@ +# Copyright (C) 2018 all contributors <meta@public-inbox.org> +# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt> +use strict; +use warnings; +use Test::More; +use PublicInbox::MIME; +use PublicInbox::Import; +use PublicInbox::SearchIdx; +use File::Temp qw/tempdir/; + +foreach my $mod (qw(DBD::SQLite Search::Xapian)) { + eval "require $mod"; + plan skip_all => "$mod missing for v1-add-remove-add.t" if $@; +} +my $mainrepo = tempdir('pi-add-remove-add-XXXXXX', TMPDIR => 1, CLEANUP => 1); +is(system(qw(git init --bare), $mainrepo), 0); +my $ibx = { + mainrepo => $mainrepo, + name => 'test-add-remove-add', + -primary_address => 'test@example.com', +}; +$ibx = PublicInbox::Inbox->new($ibx); +my $mime = PublicInbox::MIME->create( + header => [ + From => 'a@example.com', + To => 'test@example.com', + Subject => 'this is a subject', + Date => 'Fri, 02 Oct 1993 00:00:00 +0000', + 'Message-ID' => '<a-mid@b>', + ], + body => "hello world\n", +); +my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx); +ok($im->add($mime), 'message added'); +ok($im->remove($mime), 'message added'); +ok($im->add($mime), 'message added again'); +$im->done; +my $rw = PublicInbox::SearchIdx->new($ibx, 1); +$rw->index_sync; +my $msgs = $ibx->recent({limit => 10}); +is($msgs->[0]->{mid}, 'a-mid@b', 'message exists in history'); +is(scalar @$msgs, 1, 'only one message in history'); +is($ibx->mm->num_for('a-mid@b'), 2, 'exists with second article number'); + +done_testing(); diff --git a/t/v2-add-remove-add.t b/t/v2-add-remove-add.t new file mode 100644 index 00000000..c8d12d34 --- /dev/null +++ b/t/v2-add-remove-add.t @@ -0,0 +1,42 @@ +# Copyright (C) 2018 all contributors <meta@public-inbox.org> +# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt> +use strict; +use warnings; +use Test::More; +use PublicInbox::MIME; +use File::Temp qw/tempdir/; + +foreach my $mod (qw(DBD::SQLite Search::Xapian)) { + eval "require $mod"; + plan skip_all => "$mod missing for v2-add-remove-add.t" if $@; +} +use_ok 'PublicInbox::V2Writable'; +my $mainrepo = tempdir('pi-add-remove-add-XXXXXX', TMPDIR => 1, CLEANUP => 1); +my $ibx = { + mainrepo => "$mainrepo/v2", + name => 'test-v2writable', + version => 2, + -primary_address => 'test@example.com', +}; +$ibx = PublicInbox::Inbox->new($ibx); +my $mime = PublicInbox::MIME->create( + header => [ + From => 'a@example.com', + To => 'test@example.com', + Subject => 'this is a subject', + Date => 'Fri, 02 Oct 1993 00:00:00 +0000', + 'Message-ID' => '<a-mid@b>', + ], + body => "hello world\n", +); +my $im = PublicInbox::V2Writable->new($ibx, 1); +$im->{parallel} = 0; +ok($im->add($mime), 'message added'); +ok($im->remove($mime), 'message added'); +ok($im->add($mime), 'message added again'); +$im->done; +my $msgs = $ibx->recent({limit => 1000}); +is($msgs->[0]->{mid}, 'a-mid@b', 'message exists in history'); +is(scalar @$msgs, 1, 'only one message in history'); + +done_testing(); diff --git a/t/v2mda.t b/t/v2mda.t new file mode 100644 index 00000000..ca1bb09c --- /dev/null +++ b/t/v2mda.t @@ -0,0 +1,59 @@ +# Copyright (C) 2018 all contributors <meta@public-inbox.org> +# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt> +use strict; +use warnings; +use Test::More; +use PublicInbox::MIME; +use File::Temp qw/tempdir/; +use Fcntl qw(SEEK_SET); +use Cwd; + +foreach my $mod (qw(DBD::SQLite Search::Xapian)) { + eval "require $mod"; + plan skip_all => "$mod missing for v2mda.t" if $@; +} +use_ok 'PublicInbox::V2Writable'; +my $tmpdir = tempdir('pi-v2mda-XXXXXX', TMPDIR => 1, CLEANUP => 1); +my $ibx = { + mainrepo => "$tmpdir/inbox", + name => 'test-v2writable', + address => [ 'test@example.com' ], +}; +my $mime = PublicInbox::MIME->create( + header => [ + From => 'a@example.com', + To => 'test@example.com', + Subject => 'this is a subject', + Date => 'Fri, 02 Oct 1993 00:00:00 +0000', + 'Message-ID' => '<foo@bar>', + 'List-ID' => '<test.example.com>', + ], + body => "hello world\n", +); + +my $mda = "blib/script/public-inbox-mda"; +ok(-f "blib/script/public-inbox-mda", '-mda exists'); +my $main_bin = getcwd()."/t/main-bin"; +local $ENV{PI_DIR} = "$tmpdir/foo"; +local $ENV{PATH} = "$main_bin:blib/script:$ENV{PATH}"; +my @cmd = (qw(public-inbox-init -V2), $ibx->{name}, + $ibx->{mainrepo}, 'http://localhost/test', + $ibx->{address}->[0]); +ok(PublicInbox::Import::run_die(\@cmd), 'initialized v2 inbox'); + +open my $tmp, '+>', undef or die "failed to open anonymous tempfile: $!"; +ok($tmp->print($mime->as_string), 'wrote to temporary file'); +ok($tmp->flush, 'flushed temporary file'); +ok($tmp->sysseek(0, SEEK_SET), 'seeked'); + +my $rdr = { 0 => fileno($tmp) }; +local $ENV{ORIGINAL_RECIPIENT} = 'test@example.com'; +ok(PublicInbox::Import::run_die(['public-inbox-mda'], undef, $rdr), + 'mda delivered a message'); + +$ibx = PublicInbox::Inbox->new($ibx); +my $msgs = $ibx->search->query(''); +my $saved = $ibx->smsg_mime($msgs->[0]); +is($saved->{mime}->as_string, $mime->as_string, 'injected message'); + +done_testing(); diff --git a/t/v2mirror.t b/t/v2mirror.t new file mode 100644 index 00000000..9e0c9e1d --- /dev/null +++ b/t/v2mirror.t @@ -0,0 +1,176 @@ +# Copyright (C) 2018 all contributors <meta@public-inbox.org> +# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt> +use strict; +use warnings; +use Test::More; + +# Integration tests for HTTP cloning + mirroring +foreach my $mod (qw(Plack::Util Plack::Builder Danga::Socket + HTTP::Date HTTP::Status Search::Xapian DBD::SQLite)) { + eval "require $mod"; + plan skip_all => "$mod missing for v2mirror.t" if $@; +} +use File::Temp qw/tempdir/; +use IO::Socket; +use POSIX qw(dup2); +use_ok 'PublicInbox::V2Writable'; +use PublicInbox::MIME; +use PublicInbox::Config; +use Fcntl qw(FD_CLOEXEC F_SETFD F_GETFD); +# FIXME: too much setup +my $tmpdir = tempdir('pi-v2mirror-XXXXXX', TMPDIR => 1, CLEANUP => 1); +my $script = 'blib/script/public-inbox'; +my $pi_config = "$tmpdir/config"; +{ + open my $fh, '>', $pi_config or die "open($pi_config): $!"; + print $fh <<"" or die "print $pi_config: $!"; +[publicinbox "v2"] + mainrepo = $tmpdir/in + address = test\@example.com + + close $fh or die "close($pi_config): $!"; +} +local $ENV{PI_CONFIG} = $pi_config; + +my $cfg = PublicInbox::Config->new($pi_config); +my $ibx = $cfg->lookup('test@example.com'); +ok($ibx, 'inbox found'); +$ibx->{version} = 2; +my $v2w = PublicInbox::V2Writable->new($ibx, 1); +ok $v2w, 'v2w loaded'; +$v2w->{parallel} = 0; +my $mime = PublicInbox::MIME->new(<<''); +From: Me <me@example.com> +To: You <you@example.com> +Subject: a +Date: Thu, 01 Jan 1970 00:00:00 +0000 + +for my $i (1..9) { + $mime->header_set('Message-ID', "<$i\@example.com>"); + $mime->header_set('Subject', "subject = $i"); + ok($v2w->add($mime), "add msg $i OK"); +} +$v2w->barrier; + +my %opts = ( + LocalAddr => '127.0.0.1', + ReuseAddr => 1, + Proto => 'tcp', + Listen => 1024, +); +my ($sock, $pid); +END { kill 'TERM', $pid if defined $pid }; + +$! = 0; +$sock = IO::Socket::INET->new(%opts); +ok($sock, 'sock created'); +my $fl = fcntl($sock, F_GETFD, 0); +$pid = fork; +if ($pid == 0) { + # pretend to be systemd + fcntl($sock, F_SETFD, $fl &= ~FD_CLOEXEC); + dup2(fileno($sock), 3) or die "dup2 failed: $!\n"; + $ENV{LISTEN_PID} = $$; + $ENV{LISTEN_FDS} = 1; + exec "$script-httpd", "--stdout=$tmpdir/out", "--stderr=$tmpdir/err"; + die "FAIL: $!\n"; +} +ok(defined $pid, 'forked httpd process successfully'); +my ($host, $port) = ($sock->sockhost, $sock->sockport); +$sock = undef; + +my @cmd = (qw(git clone --mirror -q), "http://$host:$port/v2/0", + "$tmpdir/m/git/0.git"); + +is(system(@cmd), 0, 'cloned OK'); +ok(-d "$tmpdir/m/git/0.git", 'mirror OK');; + +@cmd = ("$script-init", '-V2', 'm', "$tmpdir/m", 'http://example.com/m', + 'alt@example.com'); +is(system(@cmd), 0, 'initialized public-inbox -V2'); +is(system("$script-index", "$tmpdir/m"), 0, 'indexed'); + +my $mibx = { mainrepo => "$tmpdir/m", address => 'alt@example.com' }; +$mibx = PublicInbox::Inbox->new($mibx); +is_deeply([$mibx->mm->minmax], [$ibx->mm->minmax], 'index synched minmax'); + +for my $i (10..15) { + $mime->header_set('Message-ID', "<$i\@example.com>"); + $mime->header_set('Subject', "subject = $i"); + ok($v2w->add($mime), "add msg $i OK"); +} +$v2w->barrier; +is(system('git', "--git-dir=$tmpdir/m/git/0.git", 'fetch', '-q'), 0, + 'fetch successful'); + +my $mset = $mibx->search->reopen->query('m:15@example.com', {mset => 1}); +is(scalar($mset->items), 0, 'new message not found in mirror, yet'); +is(system("$script-index", "$tmpdir/m"), 0, 'index updated'); +is_deeply([$mibx->mm->minmax], [$ibx->mm->minmax], 'index synched minmax'); +$mset = $mibx->search->reopen->query('m:15@example.com', {mset => 1}); +is(scalar($mset->items), 1, 'found message in mirror'); + +# purge: +$mime->header_set('Message-ID', '<10@example.com>'); +$mime->header_set('Subject', 'subject = 10'); +{ + my @warn; + local $SIG{__WARN__} = sub { push @warn, @_ }; + ok($v2w->purge($mime), 'purge a message'); + my $warn = join('', @warn); + like($warn, qr/purge rewriting/); + my @subj = ($warn =~ m/^# subject .*$/mg); + is_deeply(\@subj, ["# subject = 10"], "only rewrote one"); +} + +$v2w->barrier; + +my $msgs = $mibx->search->{over_ro}->get_thread('10@example.com'); +my $to_purge = $msgs->[0]->{blob}; +like($to_purge, qr/\A[a-f0-9]{40,}\z/, 'read blob to be purged'); +$mset = $ibx->search->reopen->query('m:10@example.com', {mset => 1}); +is(scalar($mset->items), 0, 'purged message gone from origin'); + +is(system('git', "--git-dir=$tmpdir/m/git/0.git", 'fetch', '-q'), 0, + 'fetch successful'); +{ + open my $err, '+>', "$tmpdir/index-err" or die "open: $!"; + my $ipid = fork; + if ($ipid == 0) { + dup2(fileno($err), 2) or die "dup2 failed: $!"; + exec("$script-index", '--prune', "$tmpdir/m"); + die "exec fail: $!"; + } + ok($ipid, 'running index..'); + is(waitpid($ipid, 0), $ipid, 'index --prune done'); + is($?, 0, 'no error from index'); + ok(seek($err, 0, 0), 'rewound stderr'); + $err = eval { local $/; <$err> }; + like($err, qr/discontiguous range/, 'warned about discontiguous range'); + unlike($err, qr/fatal/, 'no scary fatal error shown'); +} + +$mset = $mibx->search->reopen->query('m:10@example.com', {mset => 1}); +is(scalar($mset->items), 0, 'purged message not found in mirror'); +is_deeply([$mibx->mm->minmax], [$ibx->mm->minmax], 'minmax still synced'); +for my $i ((1..9),(11..15)) { + $mset = $mibx->search->query("m:$i\@example.com", {mset => 1}); + is(scalar($mset->items), 1, "$i\@example.com remains visible"); +} +is($mibx->git->check($to_purge), undef, 'unindex+prune successful in mirror'); + +{ + my @warn; + local $SIG{__WARN__} = sub { push @warn, @_ }; + $v2w->index_sync; + is_deeply(\@warn, [], 'no warnings from index_sync after purge'); +} + +$v2w->done; +ok(kill('TERM', $pid), 'killed httpd'); +$pid = undef; +waitpid(-1, 0); + +done_testing(); + +1; diff --git a/t/v2reindex.t b/t/v2reindex.t new file mode 100644 index 00000000..9bc271fc --- /dev/null +++ b/t/v2reindex.t @@ -0,0 +1,97 @@ +# Copyright (C) 2018 all contributors <meta@public-inbox.org> +# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt> +use strict; +use warnings; +use Test::More; +use PublicInbox::MIME; +use PublicInbox::ContentId qw(content_digest); +use File::Temp qw/tempdir/; +use File::Path qw(remove_tree); + +foreach my $mod (qw(DBD::SQLite Search::Xapian)) { + eval "require $mod"; + plan skip_all => "$mod missing for v2reindex.t" if $@; +} +use_ok 'PublicInbox::V2Writable'; +my $mainrepo = tempdir('pi-v2reindex-XXXXXX', TMPDIR => 1, CLEANUP => 1); +my $ibx = { + mainrepo => $mainrepo, + name => 'test-v2writable', + version => 2, + -primary_address => 'test@example.com', +}; +$ibx = PublicInbox::Inbox->new($ibx); +my $mime = PublicInbox::MIME->create( + header => [ + From => 'a@example.com', + To => 'test@example.com', + Subject => 'this is a subject', + Date => 'Fri, 02 Oct 1993 00:00:00 +0000', + ], + body => "hello world\n", +); +local $ENV{NPROC} = 2; +my $im = PublicInbox::V2Writable->new($ibx, 1); +foreach my $i (1..10) { + $mime->header_set('Message-Id', "<$i\@example.com>"); + ok($im->add($mime), "message $i added"); + if ($i == 4) { + $im->remove($mime); + } +} + +if ('test remove later') { + $mime->header_set('Message-Id', "<5\@example.com>"); + $im->remove($mime); +} + +$im->done; +my $minmax = [ $ibx->mm->minmax ]; +ok(defined $minmax->[0] && defined $minmax->[1], 'minmax defined'); + +eval { $im->index_sync({reindex => 1}) }; +is($@, '', 'no error from reindexing'); +$im->done; + +my $xap = "$mainrepo/xap".PublicInbox::Search::SCHEMA_VERSION(); +remove_tree($xap); +ok(!-d $xap, 'Xapian directories removed'); +eval { $im->index_sync({reindex => 1}) }; +is($@, '', 'no error from reindexing'); +$im->done; +ok(-d $xap, 'Xapian directories recreated'); + +delete $ibx->{mm}; +is_deeply($minmax, [ $ibx->mm->minmax ], 'minmax unchanged'); + +ok(unlink "$mainrepo/msgmap.sqlite3", 'remove msgmap'); +remove_tree($xap); +ok(!-d $xap, 'Xapian directories removed again'); +{ + my @warn; + local $SIG{__WARN__} = sub { push @warn, @_ }; + eval { $im->index_sync({reindex => 1}) }; + is($@, '', 'no error from reindexing without msgmap'); + is(scalar(@warn), 0, 'no warnings from reindexing'); + $im->done; + ok(-d $xap, 'Xapian directories recreated'); + delete $ibx->{mm}; + is_deeply($minmax, [ $ibx->mm->minmax ], 'minmax unchanged'); +} + +ok(unlink "$mainrepo/msgmap.sqlite3", 'remove msgmap'); +remove_tree($xap); +ok(!-d $xap, 'Xapian directories removed again'); +{ + my @warn; + local $SIG{__WARN__} = sub { push @warn, @_ }; + eval { $im->index_sync({reindex => 1}) }; + is($@, '', 'no error from reindexing without msgmap'); + is_deeply(\@warn, [], 'no warnings'); + $im->done; + ok(-d $xap, 'Xapian directories recreated'); + delete $ibx->{mm}; + is_deeply($minmax, [ $ibx->mm->minmax ], 'minmax unchanged'); +} + +done_testing(); diff --git a/t/v2writable.t b/t/v2writable.t new file mode 100644 index 00000000..d37fb06e --- /dev/null +++ b/t/v2writable.t @@ -0,0 +1,280 @@ +# Copyright (C) 2018 all contributors <meta@public-inbox.org> +# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt> +use strict; +use warnings; +use Test::More; +use PublicInbox::MIME; +use PublicInbox::ContentId qw(content_digest); +use File::Temp qw/tempdir/; +foreach my $mod (qw(DBD::SQLite Search::Xapian)) { + eval "require $mod"; + plan skip_all => "$mod missing for nntpd.t" if $@; +} +use_ok 'PublicInbox::V2Writable'; +my $mainrepo = tempdir('pi-v2writable-XXXXXX', TMPDIR => 1, CLEANUP => 1); +my $ibx = { + mainrepo => $mainrepo, + name => 'test-v2writable', + version => 2, + -primary_address => 'test@example.com', +}; +$ibx = PublicInbox::Inbox->new($ibx); +my $mime = PublicInbox::MIME->create( + header => [ + From => 'a@example.com', + To => 'test@example.com', + Subject => 'this is a subject', + 'Message-ID' => '<a-mid@b>', + Date => 'Fri, 02 Oct 1993 00:00:00 +0000', + ], + body => "hello world\n", +); + +my $im = eval { + local $ENV{NPROC} = '1'; + PublicInbox::V2Writable->new($ibx, 1); +}; +is($im->{partitions}, 1, 'one partition when forced'); +ok($im->add($mime), 'ordinary message added'); +my $git0; + +if ('ensure git configs are correct') { + my @cmd = (qw(git config), "--file=$mainrepo/all.git/config", + qw(core.sharedRepository 0644)); + is(system(@cmd), 0, "set sharedRepository in all.git"); + $git0 = PublicInbox::Git->new("$mainrepo/git/0.git"); + chomp(my $v = $git0->qx(qw(config core.sharedRepository))); + is($v, '0644', 'child repo inherited core.sharedRepository'); + chomp($v = $git0->qx(qw(config --bool repack.writeBitmaps))); + is($v, 'true', 'child repo inherited repack.writeBitmaps'); +} + +{ + my @warn; + local $SIG{__WARN__} = sub { push @warn, @_ }; + is($im->add($mime), undef, 'obvious duplicate rejected'); + is(scalar(@warn), 0, 'no warning about resent message'); + + @warn = (); + $mime->header_set('Message-Id', '<a-mid@b>', '<c@d>'); + is($im->add($mime), undef, 'secondary MID ignored if first matches'); + my $sec = PublicInbox::MIME->new($mime->as_string); + $sec->header_set('Date'); + $sec->header_set('Message-Id', '<a-mid@b>', '<c@d>'); + ok($im->add($sec), 'secondary MID used if data is different'); + like(join(' ', @warn), qr/mismatched/, 'warned about mismatch'); + like(join(' ', @warn), qr/alternative/, 'warned about alternative'); + is_deeply([ '<a-mid@b>', '<c@d>' ], + [ $sec->header_obj->header_raw('Message-Id') ], + 'no new Message-Id added'); + + my $sane_mid = qr/\A<[\w\-\.]+\@\w+>\z/; + @warn = (); + $mime->header_set('Message-Id', '<a-mid@b>'); + $mime->body_set('different'); + ok($im->add($mime), 'reused mid ok'); + like(join(' ', @warn), qr/reused/, 'warned about reused MID'); + my @mids = $mime->header_obj->header_raw('Message-Id'); + is($mids[0], '<a-mid@b>', 'original mid not changed'); + like($mids[1], $sane_mid, 'new MID added'); + is(scalar(@mids), 2, 'only one new MID added'); + + @warn = (); + $mime->header_set('Message-Id', '<a-mid@b>'); + $mime->body_set('this one needs a random mid'); + my $hdr = $mime->header_obj; + my $gen = PublicInbox::Import::digest2mid(content_digest($mime), $hdr); + unlike($gen, qr![\+/=]!, 'no URL-unfriendly chars in Message-Id'); + my $fake = PublicInbox::MIME->new($mime->as_string); + $fake->header_set('Message-Id', "<$gen>"); + ok($im->add($fake), 'fake added easily'); + is_deeply(\@warn, [], 'no warnings from a faker'); + ok($im->add($mime), 'random MID made'); + like(join(' ', @warn), qr/using random/, 'warned about using random'); + @mids = $mime->header_obj->header_raw('Message-Id'); + is($mids[0], '<a-mid@b>', 'original mid not changed'); + like($mids[1], $sane_mid, 'new MID added'); + is(scalar(@mids), 2, 'only one new MID added'); + + @warn = (); + $mime->header_set('Message-Id'); + ok($im->add($mime), 'random MID made for MID free message'); + @mids = $mime->header_obj->header_raw('Message-Id'); + like($mids[0], $sane_mid, 'mid was generated'); + is(scalar(@mids), 1, 'new generated'); +} + +{ + $mime->header_set('Message-Id', '<abcde@1>', '<abcde@2>'); + $mime->header_set('References', '<zz-mid@b>'); + ok($im->add($mime), 'message with multiple Message-ID'); + $im->done; + my $srch = $ibx->search; + my $mset1 = $srch->reopen->query('m:abcde@1', { mset => 1 }); + is($mset1->size, 1, 'message found by first MID'); + my $mset2 = $srch->reopen->query('m:abcde@2', { mset => 1 }); + is($mset2->size, 1, 'message found by second MID'); + is((($mset1->items)[0])->get_docid, (($mset2->items)[0])->get_docid, + 'same document'); +} + +SKIP: { + use Fcntl qw(FD_CLOEXEC F_SETFD F_GETFD); + use Net::NNTP; + use IO::Socket; + use Socket qw(SO_KEEPALIVE IPPROTO_TCP TCP_NODELAY); + eval { require Danga::Socket }; + skip "Danga::Socket missing $@", 2 if $@; + my $err = "$mainrepo/stderr.log"; + my $out = "$mainrepo/stdout.log"; + my %opts = ( + LocalAddr => '127.0.0.1', + ReuseAddr => 1, + Proto => 'tcp', + Type => SOCK_STREAM, + Listen => 1024, + ); + my $group = 'inbox.comp.test.v2writable'; + my $pi_config = "$mainrepo/pi_config"; + open my $fh, '>', $pi_config or die "open: $!\n"; + print $fh <<EOF +[publicinbox "test-v2writable"] + mainrepo = $mainrepo + version = 2 + address = test\@example.com + newsgroup = $group +EOF + ; + close $fh or die "close: $!\n"; + my $sock = IO::Socket::INET->new(%opts); + ok($sock, 'sock created'); + my $pid; + my $len; + END { kill 'TERM', $pid if defined $pid }; + $! = 0; + my $fl = fcntl($sock, F_GETFD, 0); + ok(! $!, 'no error from fcntl(F_GETFD)'); + is($fl, FD_CLOEXEC, 'cloexec set by default (Perl behavior)'); + $pid = fork; + if ($pid == 0) { + use POSIX qw(dup2); + $ENV{PI_CONFIG} = $pi_config; + # pretend to be systemd + fcntl($sock, F_SETFD, $fl &= ~FD_CLOEXEC); + dup2(fileno($sock), 3) or die "dup2 failed: $!\n"; + $ENV{LISTEN_PID} = $$; + $ENV{LISTEN_FDS} = 1; + my $nntpd = 'blib/script/public-inbox-nntpd'; + exec $nntpd, "--stdout=$out", "--stderr=$err"; + die "FAIL: $!\n"; + } + ok(defined $pid, 'forked nntpd process successfully'); + $! = 0; + fcntl($sock, F_SETFD, $fl |= FD_CLOEXEC); + ok(! $!, 'no error from fcntl(F_SETFD)'); + my $host_port = $sock->sockhost . ':' . $sock->sockport; + my $n = Net::NNTP->new($host_port); + $n->group($group); + my $x = $n->xover('1-'); + my %uniq; + foreach my $num (sort { $a <=> $b } keys %$x) { + my $mid = $x->{$num}->[3]; + is($uniq{$mid}++, 0, "MID for $num is unique in XOVER"); + is_deeply($n->xhdr('Message-ID', $num), + { $num => $mid }, "XHDR lookup OK on num $num"); + is_deeply($n->xhdr('Message-ID', $mid), + { $mid => $mid }, "XHDR lookup OK on MID $num"); + } + my %nn; + foreach my $mid (@{$n->newnews(0, $group)}) { + is($nn{$mid}++, 0, "MID is unique in NEWNEWS"); + } + is_deeply([sort keys %nn], [sort keys %uniq]); + + my %lg; + foreach my $num (@{$n->listgroup($group)}) { + is($lg{$num}++, 0, "num is unique in LISTGROUP"); + } + is_deeply([sort keys %lg], [sort keys %$x], + 'XOVER and LISTGROUPS return the same article numbers'); + + my $xref = $n->xhdr('Xref', '1-'); + is_deeply([sort keys %lg], [sort keys %$xref], 'Xref range OK'); + + my $mids = $n->xhdr('Message-ID', '1-'); + is_deeply([sort keys %lg], [sort keys %$xref], 'Message-ID range OK'); + + my $rover = $n->xrover('1-'); + is_deeply([sort keys %lg], [sort keys %$rover], 'XROVER range OK'); +}; +{ + local $ENV{NPROC} = 2; + my @before = $git0->qx(qw(log --pretty=oneline)); + my $before = $git0->qx(qw(log --pretty=raw --raw -r --no-abbrev)); + $im = PublicInbox::V2Writable->new($ibx, 1); + is($im->{partitions}, 1, 'detected single partition from previous'); + my $smsg = $im->remove($mime, 'test removal'); + $im->done; + my @after = $git0->qx(qw(log --pretty=oneline)); + my $tip = shift @after; + like($tip, qr/\A[a-f0-9]+ test removal\n\z/s, + 'commit message propagated to git'); + is_deeply(\@after, \@before, 'only one commit written to git'); + is($ibx->mm->num_for($smsg->mid), undef, 'no longer in Msgmap by mid'); + my $num = $smsg->{num}; + like($num, qr/\A\d+\z/, 'numeric number in return message'); + is($ibx->mm->mid_for($num), undef, 'no longer in Msgmap by num'); + my $srch = $ibx->search->reopen; + my $mset = $srch->query('m:'.$smsg->mid, { mset => 1}); + is($mset->size, 0, 'no longer found in Xapian'); + my @log1 = qw(log -1 --pretty=raw --raw -r --no-abbrev --no-renames); + is($srch->{over_ro}->get_art($num), undef, + 'removal propagated to Over DB'); + + my $after = $git0->qx(@log1); + if ($after =~ m!( [a-f0-9]+ )A\td$!m) { + my $oid = $1; + ok(index($before, $oid) > 0, 'no new blob introduced'); + } else { + fail('failed to extract blob from log output'); + } + is($im->remove($mime, 'test removal'), undef, + 'remove is idempotent'); + $im->done; + is($git0->qx(@log1), + $after, 'no git history made with idempotent remove'); + eval { $im->done }; + ok(!$@, '->done is idempotent'); +} + +{ + ok($im->add($mime), 'add message to be purged'); + local $SIG{__WARN__} = sub {}; + ok(my $cmts = $im->purge($mime), 'purged message'); + like($cmts->[0], qr/\A[a-f0-9]{40}\z/, 'purge returned current commit'); + $im->done; +} + +{ + my @warn; + my $x = 'x'x250; + my $y = 'y'x250; + local $SIG{__WARN__} = sub { push @warn, @_ }; + $mime->header_set('Subject', 'long mid'); + $mime->header_set('Message-ID', "<$x>"); + ok($im->add($mime), 'add excessively long Message-ID'); + + $mime->header_set('Message-ID', "<$y>"); + $mime->header_set('References', "<$x>"); + ok($im->add($mime), 'add excessively long References'); + $im->barrier; + + my $msgs = $ibx->search->reopen->get_thread('x'x244); + is(2, scalar(@$msgs), 'got both messages'); + is($msgs->[0]->{mid}, 'x'x244, 'stored truncated mid'); + is($msgs->[1]->{references}, '<'.('x'x244).'>', 'stored truncated ref'); + is($msgs->[1]->{mid}, 'y'x244, 'stored truncated mid(2)'); + $im->done; +} + +done_testing(); @@ -16,6 +16,7 @@ my $ctx = { base_url => sub { 'http://example.com/' }, cloneurl => sub {[]}, nntp_url => sub {[]}, + max_git_part => sub { undef }, description => sub { '' }), }; $ctx->{-inbox}->{-primary_address} = 'test@example.com'; diff --git a/t/watch_maildir.t b/t/watch_maildir.t index 30e94c1e..7178f29e 100644 --- a/t/watch_maildir.t +++ b/t/watch_maildir.t @@ -31,7 +31,8 @@ Date: Sat, 18 Jun 2016 00:00:00 +0000 something EOF PublicInbox::Emergency->new($maildir)->prepare(\$msg); -ok(POSIX::mkfifo("$maildir/cur/fifo", 0777)); +ok(POSIX::mkfifo("$maildir/cur/fifo", 0777), + 'create FIFO to ensure we do not get stuck on it :P'); my $sem = PublicInbox::Emergency->new($spamdir); # create dirs my $config = PublicInbox::Config->new({ diff --git a/t/watch_maildir_v2.t b/t/watch_maildir_v2.t new file mode 100644 index 00000000..a76e413f --- /dev/null +++ b/t/watch_maildir_v2.t @@ -0,0 +1,125 @@ +# Copyright (C) 2018 all contributors <meta@public-inbox.org> +# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt> +use Test::More; +use File::Temp qw/tempdir/; +use PublicInbox::MIME; +use Cwd; +use PublicInbox::Config; +my @mods = qw(Filesys::Notify::Simple PublicInbox::V2Writable); +foreach my $mod (@mods) { + eval "require $mod"; + plan skip_all => "$mod missing for watch_maildir_v2.t" if $@; +} + +my $tmpdir = tempdir('watch_maildir-v2-XXXXXX', TMPDIR => 1, CLEANUP => 1); +my $mainrepo = "$tmpdir/v2"; +my $maildir = "$tmpdir/md"; +my $spamdir = "$tmpdir/spam"; +use_ok 'PublicInbox::WatchMaildir'; +use_ok 'PublicInbox::Emergency'; +my $cfgpfx = "publicinbox.test"; +my $addr = 'test-public@example.com'; +my @cmd = ('blib/script/public-inbox-init', '-V2', 'test', $mainrepo, + 'http://example.com/v2list', $addr); +local $ENV{PI_CONFIG} = "$tmpdir/pi_config"; +is(system(@cmd), 0, 'public-inbox init OK'); + +my $msg = <<EOF; +From: user\@example.com +To: $addr +Subject: spam +Message-Id: <a\@b.com> +Date: Sat, 18 Jun 2016 00:00:00 +0000 + +something +EOF +PublicInbox::Emergency->new($maildir)->prepare(\$msg); +ok(POSIX::mkfifo("$maildir/cur/fifo", 0777), + 'create FIFO to ensure we do not get stuck on it :P'); +my $sem = PublicInbox::Emergency->new($spamdir); # create dirs + +my $config = PublicInbox::Config->new({ + "$cfgpfx.address" => $addr, + "$cfgpfx.mainrepo" => $mainrepo, + "$cfgpfx.watch" => "maildir:$maildir", + "$cfgpfx.filter" => 'PublicInbox::Filter::Vger', + "publicinboxlearn.watchspam" => "maildir:$spamdir", +}); +my $ibx = $config->lookup_name('test'); +ok($ibx, 'found inbox by name'); +my $srch = $ibx->search; + +PublicInbox::WatchMaildir->new($config)->scan('full'); +my ($total, undef) = $srch->reopen->query(''); +is($total, 1, 'got one revision'); + +# my $git = PublicInbox::Git->new("$mainrepo/git/0.git"); +# my @list = $git->qx(qw(rev-list refs/heads/master)); +# is(scalar @list, 1, 'one revision in rev-list'); + +my $write_spam = sub { + is(scalar glob("$spamdir/new/*"), undef, 'no spam existing'); + $sem->prepare(\$msg); + $sem->commit; + my @new = glob("$spamdir/new/*"); + is(scalar @new, 1); + my @p = split(m!/+!, $new[0]); + ok(link($new[0], "$spamdir/cur/".$p[-1].":2,S")); + is(unlink($new[0]), 1); +}; +$write_spam->(); +is(unlink(glob("$maildir/new/*")), 1, 'unlinked old spam'); +PublicInbox::WatchMaildir->new($config)->scan('full'); +is(($srch->reopen->query(''))[0], 0, 'deleted file'); + +# check with scrubbing +{ + $msg .= qq(-- +To unsubscribe from this list: send the line "unsubscribe git" in +the body of a message to majordomo\@vger.kernel.org +More majordomo info at http://vger.kernel.org/majordomo-info.html\n); + PublicInbox::Emergency->new($maildir)->prepare(\$msg); + PublicInbox::WatchMaildir->new($config)->scan('full'); + my ($nr, $msgs) = $srch->reopen->query(''); + is($nr, 1, 'got one file back'); + my $mref = $ibx->msg_by_smsg($msgs->[0]); + like($$mref, qr/something\n\z/s, 'message scrubbed on import'); + + is(unlink(glob("$maildir/new/*")), 1, 'unlinked spam'); + $write_spam->(); + PublicInbox::WatchMaildir->new($config)->scan('full'); + ($nr, $msgs) = $srch->reopen->query(''); + is($nr, 0, 'inbox is empty again'); +} + +{ + my $fail_bin = getcwd()."/t/fail-bin"; + ok(-x "$fail_bin/spamc", "mock spamc exists"); + my $fail_path = "$fail_bin:$ENV{PATH}"; # for spamc ham mock + local $ENV{PATH} = $fail_path; + PublicInbox::Emergency->new($maildir)->prepare(\$msg); + $config->{'publicinboxwatch.spamcheck'} = 'spamc'; + { + local $SIG{__WARN__} = sub {}; # quiet spam check warning + PublicInbox::WatchMaildir->new($config)->scan('full'); + } + ($nr, $msgs) = $srch->reopen->query(''); + is($nr, 0, 'inbox is still empty'); + is(unlink(glob("$maildir/new/*")), 1); +} + +{ + my $main_bin = getcwd()."/t/main-bin"; + ok(-x "$main_bin/spamc", "mock spamc exists"); + my $main_path = "$main_bin:$ENV{PATH}"; # for spamc ham mock + local $ENV{PATH} = $main_path; + PublicInbox::Emergency->new($maildir)->prepare(\$msg); + $config->{'publicinboxwatch.spamcheck'} = 'spamc'; + PublicInbox::WatchMaildir->new($config)->scan('full'); + ($nr, $msgs) = $srch->reopen->query(''); + is($nr, 1, 'inbox has one mail after spamc OK-ed a message'); + my $mref = $ibx->msg_by_smsg($msgs->[0]); + like($$mref, qr/something\n\z/s, 'message scrubbed on import'); +} + +done_testing; |