diff options
Diffstat (limited to 't')
| -rw-r--r-- | t/common.perl | 4 | ||||
| -rw-r--r-- | t/edit.t | 196 | ||||
| -rw-r--r-- | t/git.t | 58 | ||||
| -rw-r--r-- | t/nntpd.t | 37 | ||||
| -rw-r--r-- | t/psgi_v2.t | 2 | ||||
| -rw-r--r-- | t/purge.t | 4 | ||||
| -rw-r--r-- | t/replace.t | 199 | ||||
| -rw-r--r-- | t/v2mirror.t | 8 | ||||
| -rw-r--r-- | t/www_listing.t | 159 |
9 files changed, 603 insertions, 64 deletions
diff --git a/t/common.perl b/t/common.perl index e49a5965..5a898e32 100644 --- a/t/common.perl +++ b/t/common.perl @@ -3,6 +3,8 @@ use Fcntl qw(FD_CLOEXEC F_SETFD F_GETFD); use POSIX qw(dup2); +use strict; +use warnings; sub stream_to_string { my ($res) = @_; @@ -48,7 +50,7 @@ sub require_git ($;$) { my $cur_int = ($cur_maj << 24) | ($cur_min << 16); if ($cur_int < $req_int) { return 0 if $maybe; - plan skip_all => "git $req+ required, have $git_ver"; + plan skip_all => "git $req+ required, have $cur_maj.$cur_min"; } 1; } diff --git a/t/edit.t b/t/edit.t new file mode 100644 index 00000000..6b4e35c3 --- /dev/null +++ b/t/edit.t @@ -0,0 +1,196 @@ +# Copyright (C) 2019 all contributors <meta@public-inbox.org> +# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt> +# edit frontend behavior test (t/replace.t for backend) +use strict; +use warnings; +use Test::More; +use File::Temp qw/tempdir/; +require './t/common.perl'; +require_git(2.6); +require PublicInbox::Inbox; +require PublicInbox::InboxWritable; +require PublicInbox::Config; +use PublicInbox::MID qw(mid_clean); + +my @mods = qw(IPC::Run DBI DBD::SQLite); +foreach my $mod (@mods) { + eval "require $mod"; + plan skip_all => "missing $mod for $0" if $@; +}; +IPC::Run->import(qw(run)); + +my $cmd_pfx = 'blib/script/public-inbox'; +my $tmpdir = tempdir('pi-edit-XXXXXX', TMPDIR => 1, CLEANUP => 1); +my $mainrepo = "$tmpdir/v2"; +my $ibx = PublicInbox::Inbox->new({ + mainrepo => $mainrepo, + name => 'test-v2edit', + version => 2, + -primary_address => 'test@example.com', + indexlevel => 'basic', +}); +$ibx = PublicInbox::InboxWritable->new($ibx, {nproc=>1}); +my $cfgfile = "$tmpdir/config"; +local $ENV{PI_CONFIG} = $cfgfile; +my $file = 't/data/0001.patch'; +open my $fh, '<', $file or die "open: $!"; +my $raw = do { local $/; <$fh> }; +my $im = $ibx->importer(0); +my $mime = PublicInbox::MIME->new($raw); +my $mid = mid_clean($mime->header('Message-Id')); +ok($im->add($mime), 'add message to be edited'); +$im->done; +my ($in, $out, $err, $cmd, $cur, $t); +my $__git_dir = "--git-dir=$ibx->{mainrepo}/git/0.git"; + +$t = '-F FILE'; { + $in = $out = $err = ''; + local $ENV{MAIL_EDITOR} = "$^X -i -p -e 's/boolean prefix/bool pfx/'"; + $cmd = [ "$cmd_pfx-edit", "-F$file", $mainrepo ]; + ok(run($cmd, \$in, \$out, \$err), "$t edit OK"); + $cur = PublicInbox::MIME->new($ibx->msg_by_mid($mid)); + like($cur->header('Subject'), qr/bool pfx/, "$t message edited"); + like($out, qr/[a-f0-9]{40}/, "$t shows commit on success"); +} + +$t = '-m MESSAGE_ID'; { + $in = $out = $err = ''; + local $ENV{MAIL_EDITOR} = "$^X -i -p -e 's/bool pfx/boolean prefix/'"; + $cmd = [ "$cmd_pfx-edit", "-m$mid", $mainrepo ]; + ok(run($cmd, \$in, \$out, \$err), "$t edit OK"); + $cur = PublicInbox::MIME->new($ibx->msg_by_mid($mid)); + like($cur->header('Subject'), qr/boolean prefix/, "$t message edited"); + like($out, qr/[a-f0-9]{40}/, "$t shows commit on success"); +} + +$t = 'no-op -m MESSAGE_ID'; { + $in = $out = $err = ''; + my $before = `git $__git_dir rev-parse HEAD`; + local $ENV{MAIL_EDITOR} = "$^X -i -p -e 's/bool pfx/boolean prefix/'"; + $cmd = [ "$cmd_pfx-edit", "-m$mid", $mainrepo ]; + ok(run($cmd, \$in, \$out, \$err), "$t succeeds"); + my $prev = $cur; + $cur = PublicInbox::MIME->new($ibx->msg_by_mid($mid)); + is_deeply($cur, $prev, "$t makes no change"); + like($cur->header('Subject'), qr/boolean prefix/, + "$t does not change message"); + like($out, qr/NONE/, 'noop shows NONE'); + my $after = `git $__git_dir rev-parse HEAD`; + is($after, $before, 'git head unchanged'); +} + +$t = 'no-op -m MESSAGE_ID w/Status: header'; { # because mutt does it + $in = $out = $err = ''; + my $before = `git $__git_dir rev-parse HEAD`; + local $ENV{MAIL_EDITOR} = + "$^X -i -p -e 's/^Subject:.*/Status: RO\\n\$&/'"; + $cmd = [ "$cmd_pfx-edit", "-m$mid", $mainrepo ]; + ok(run($cmd, \$in, \$out, \$err), "$t succeeds"); + my $prev = $cur; + $cur = PublicInbox::MIME->new($ibx->msg_by_mid($mid)); + is_deeply($cur, $prev, "$t makes no change"); + like($cur->header('Subject'), qr/boolean prefix/, + "$t does not change message"); + is($cur->header('Status'), undef, 'Status header not added'); + like($out, qr/NONE/, 'noop shows NONE'); + my $after = `git $__git_dir rev-parse HEAD`; + is($after, $before, 'git head unchanged'); +} + +$t = '-m MESSAGE_ID can change Received: headers'; { + $in = $out = $err = ''; + my $before = `git $__git_dir rev-parse HEAD`; + local $ENV{MAIL_EDITOR} = + "$^X -i -p -e 's/^Subject:.*/Received: x\\n\$&/'"; + $cmd = [ "$cmd_pfx-edit", "-m$mid", $mainrepo ]; + ok(run($cmd, \$in, \$out, \$err), "$t succeeds"); + $cur = PublicInbox::MIME->new($ibx->msg_by_mid($mid)); + like($cur->header('Subject'), qr/boolean prefix/, + "$t does not change Subject"); + is($cur->header('Received'), 'x', 'added Received header'); +} + +$t = '-m miss'; { + $in = $out = $err = ''; + local $ENV{MAIL_EDITOR} = "$^X -i -p -e 's/boolean/FAIL/'"; + $cmd = [ "$cmd_pfx-edit", "-m$mid-miss", $mainrepo ]; + ok(!run($cmd, \$in, \$out, \$err), "$t fails on invalid MID"); + like($err, qr/No message found/, "$t shows error"); +} + +$t = 'non-interactive editor failure'; { + $in = $out = $err = ''; + local $ENV{MAIL_EDITOR} = "$^X -i -p -e 'END { exit 1 }'"; + $cmd = [ "$cmd_pfx-edit", "-m$mid", $mainrepo ]; + ok(!run($cmd, \$in, \$out, \$err), "$t detected"); + like($err, qr/END \{ exit 1 \}' failed:/, "$t shows error"); +} + +$t = 'mailEditor set in config'; { + $in = $out = $err = ''; + my $rc = system(qw(git config), "--file=$cfgfile", + 'publicinbox.maileditor', + "$^X -i -p -e 's/boolean prefix/bool pfx/'"); + is($rc, 0, 'set publicinbox.mailEditor'); + local $ENV{MAIL_EDITOR}; + local $ENV{GIT_EDITOR} = 'echo should not run'; + $cmd = [ "$cmd_pfx-edit", "-m$mid", $mainrepo ]; + ok(run($cmd, \$in, \$out, \$err), "$t edited message"); + $cur = PublicInbox::MIME->new($ibx->msg_by_mid($mid)); + like($cur->header('Subject'), qr/bool pfx/, "$t message edited"); + unlike($out, qr/should not run/, 'did not run GIT_EDITOR'); +} + +$t = '--raw and mbox escaping'; { + $in = $out = $err = ''; + local $ENV{MAIL_EDITOR} = "$^X -i -p -e 's/^\$/\\nFrom not mbox\\n/'"; + $cmd = [ "$cmd_pfx-edit", "-m$mid", '--raw', $mainrepo ]; + ok(run($cmd, \$in, \$out, \$err), "$t succeeds"); + $cur = PublicInbox::MIME->new($ibx->msg_by_mid($mid)); + like($cur->body, qr/^From not mbox/sm, 'put "From " line into body'); + + local $ENV{MAIL_EDITOR} = "$^X -i -p -e 's/^>From not/\$& an/'"; + $cmd = [ "$cmd_pfx-edit", "-m$mid", $mainrepo ]; + ok(run($cmd, \$in, \$out, \$err), "$t succeeds with mbox escaping"); + $cur = PublicInbox::MIME->new($ibx->msg_by_mid($mid)); + like($cur->body, qr/^From not an mbox/sm, + 'changed "From " line unescaped'); + + local $ENV{MAIL_EDITOR} = "$^X -i -p -e 's/^From not an mbox\\n//s'"; + $cmd = [ "$cmd_pfx-edit", "-m$mid", '--raw', $mainrepo ]; + ok(run($cmd, \$in, \$out, \$err), "$t succeeds again"); + $cur = PublicInbox::MIME->new($ibx->msg_by_mid($mid)); + unlike($cur->body, qr/^From not an mbox/sm, "$t restored body"); +} + +$t = 'reuse Message-ID'; { + my @warn; + local $SIG{__WARN__} = sub { push @warn, @_ }; + ok($im->add($mime), "$t and re-add"); + $im->done; + like($warn[0], qr/reused for mismatched content/, "$t got warning"); +} + +$t = 'edit ambiguous Message-ID with -m'; { + $in = $out = $err = ''; + local $ENV{MAIL_EDITOR} = "$^X -i -p -e 's/bool pfx/boolean prefix/'"; + $cmd = [ "$cmd_pfx-edit", "-m$mid", $mainrepo ]; + ok(!run($cmd, \$in, \$out, \$err), "$t fails w/o --force"); + like($err, qr/Multiple messages with different content found matching/, + "$t shows matches"); + like($err, qr/GIT_DIR=.*git show/is, "$t shows git commands"); +} + +$t .= ' and --force'; { + $in = $out = $err = ''; + local $ENV{MAIL_EDITOR} = "$^X -i -p -e 's/^Subject:.*/Subject:x/i'"; + $cmd = [ "$cmd_pfx-edit", "-m$mid", '--force', $mainrepo ]; + ok(run($cmd, \$in, \$out, \$err), "$t succeeds"); + like($err, qr/Will edit all of them/, "$t notes all will be edited"); + my @dump = `git $__git_dir cat-file --batch --batch-all-objects`; + chomp @dump; + is_deeply([grep(/^Subject:/i, @dump)], [qw(Subject:x Subject:x)], + "$t edited both messages"); +} + +done_testing(); @@ -33,33 +33,7 @@ use_ok 'PublicInbox::Git'; my $raw = $gcf->cat_file($f); is($x[2], length($$raw), 'length matches'); - { - my $size; - my $rv = $gcf->cat_file($f, sub { - my ($in, $left) = @_; - $size = $$left; - 'nothing' - }); - is($rv, 'nothing', 'returned from callback without reading'); - is($size, $x[2], 'set size for callback correctly'); - } - - eval { $gcf->cat_file($f, sub { die 'OMG' }) }; - like($@, qr/\bOMG\b/, 'died in callback propagated'); is(${$gcf->cat_file($f)}, $$raw, 'not broken after failures'); - - { - my ($buf, $r); - my $rv = $gcf->cat_file($f, sub { - my ($in, $left) = @_; - $r = read($in, $buf, 2); - $$left -= $r; - 'blah' - }); - is($r, 2, 'only read 2 bytes'); - is($buf, '--', 'partial read succeeded'); - is($rv, 'blah', 'return value propagated'); - } is(${$gcf->cat_file($f)}, $$raw, 'not broken after partial read'); } @@ -79,44 +53,12 @@ if (1) { my $gcf = PublicInbox::Git->new($dir); my $rsize; - is($gcf->cat_file($buf, sub { - $rsize = ${$_[1]}; - 'x'; - }), 'x', 'checked input'); - is($rsize, $size, 'got correct size on big file'); - my $x = $gcf->cat_file($buf, \$rsize); is($rsize, $size, 'got correct size ref on big file'); is(length($$x), $size, 'read correct number of bytes'); - my $rline; - $gcf->cat_file($buf, sub { - my ($in, $left) = @_; - $rline = <$in>; - $$left -= length($rline); - }); - { - open my $fh, '<', $big_data or die "open failed: $!\n"; - is($rline, <$fh>, 'first line matches'); - }; - - my $all; - $gcf->cat_file($buf, sub { - my ($in, $left) = @_; - my $x = read($in, $all, $$left); - $$left -= $x; - }); - { - open my $fh, '<', $big_data or die "open failed: $!\n"; - local $/; - is($all, <$fh>, 'entire read matches'); - }; - my $ref = $gcf->qx(qw(cat-file blob), $buf); - is($all, $ref, 'qx read giant single string'); - my @ref = $gcf->qx(qw(cat-file blob), $buf); - is($all, join('', @ref), 'qx returned array when wanted'); my $nl = scalar @ref; ok($nl > 1, "qx returned array length of $nl"); @@ -231,6 +231,43 @@ EOF ok($date >= $t0, 'valid date after start'); ok($date <= $t1, 'valid date before stop'); } + if ('leafnode interop') { + my $for_leafnode = PublicInbox::MIME->new(<<""); +From: longheader\@example.com +To: $addr +Subject: none +Date: Fri, 02 Oct 1993 00:00:00 +0000 + + my $long_hdr = 'for-leafnode-'.('y'x200).'@example.com'; + $for_leafnode->header_set('Message-ID', "<$long_hdr>"); + $im->add($for_leafnode); + $im->done; + if ($version == 1) { + my $s = PublicInbox::SearchIdx->new($mainrepo, 1); + $s->index_sync; + } + my $hdr = $n->head("<$long_hdr>"); + my $expect = qr/\AMessage-ID: /i . qr/\Q<$long_hdr>\E/; + ok(scalar(grep(/$expect/, @$hdr)), 'Message-ID not folded'); + ok(scalar(grep(/^Path:/, @$hdr)), 'Path: header found'); + + # it's possible for v2 messages to have 2+ Message-IDs, + # but leafnode can't handle it + if ($version != 1) { + my @mids = ("<$long_hdr>", '<2mid@wtf>'); + $for_leafnode->header_set('Message-ID', @mids); + $for_leafnode->body_set('not-a-dupe'); + my $warn = ''; + $SIG{__WARN__} = sub { $warn .= join('', @_) }; + $im->add($for_leafnode); + $im->done; + like($warn, qr/reused/, 'warned for reused MID'); + $hdr = $n->head('<2mid@wtf>'); + my @hmids = grep(/\AMessage-ID: /i, @$hdr); + is(scalar(@hmids), 1, 'Single Message-ID in header'); + like($hmids[0], qr/: <2mid\@wtf>/, 'got expected mid'); + } + } # pipelined requests: { diff --git a/t/psgi_v2.t b/t/psgi_v2.t index 98112494..5c358cde 100644 --- a/t/psgi_v2.t +++ b/t/psgi_v2.t @@ -202,6 +202,8 @@ test_psgi(sub { $www->call(@_) }, sub { $res = $cb->(GET('/v2test/0/info/refs')); is($res->code, 200, 'got info refs for dumb clones'); + $res = $cb->(GET('/v2test/0.git/info/refs')); + is($res->code, 200, 'got info refs for dumb clones w/ .git suffix'); $res = $cb->(GET('/v2test/info/refs')); is($res->code, 404, 'unpartitioned git URL fails'); @@ -57,7 +57,7 @@ is($? >> 8, 1, 'missed purge exits with 1'); # a successful case: ok(IPC::Run::run([$purge, $mainrepo], \$raw, \$out, \$err), 'match OK'); -like($out, qr/^\t[a-f0-9]{40,}/m, 'removed commit noted'); +like($out, qr/\b[a-f0-9]{40,}/m, 'removed commit noted'); # add (old) vger filter to config file print $cfg_fh <<EOF or die "print $!"; @@ -85,7 +85,7 @@ $out = $err = ''; ok(chdir('/'), "chdir / OK for --all test"); ok(IPC::Run::run([$purge, '--all'], \$pre_scrub, \$out, \$err), 'scrub purge OK'); -like($out, qr/^\t[a-f0-9]{40,}/m, 'removed commit noted'); +like($out, qr/\b[a-f0-9]{40,}/m, 'removed commit noted'); # diag "out: $out"; diag "err: $err"; $out = $err = ''; diff --git a/t/replace.t b/t/replace.t new file mode 100644 index 00000000..6fae5511 --- /dev/null +++ b/t/replace.t @@ -0,0 +1,199 @@ +# Copyright (C) 2019 all contributors <meta@public-inbox.org> +# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt> +use strict; +use warnings; +use Test::More; +use PublicInbox::MIME; +use PublicInbox::InboxWritable; +use File::Temp qw/tempdir/; +require './t/common.perl'; +require_git(2.6); # replace is v2 only, for now... +foreach my $mod (qw(DBD::SQLite)) { + eval "require $mod"; + plan skip_all => "$mod missing for $0" if $@; +} + +sub test_replace ($$$) { + my ($v, $level, $opt) = @_; + diag "v$v $level replace"; + my $this = "pi-$v-$level-replace"; + my $tmpdir = tempdir("$this-tmp-XXXXXX", TMPDIR => 1, CLEANUP => 1); + my $ibx = PublicInbox::Inbox->new({ + mainrepo => "$tmpdir/testbox", + name => $this, + version => $v, + -primary_address => 'test@example.com', + indexlevel => $level, + }); + + my $orig = PublicInbox::MIME->new(<<'EOF'); +From: Barbra Streisand <effect@example.com> +To: test@example.com +Subject: confidential +Message-ID: <replace@example.com> +Date: Fri, 02 Oct 1993 00:00:00 +0000 + +Top secret info about my house in Malibu... +EOF + my $im = PublicInbox::InboxWritable->new($ibx, {nproc=>1})->importer; + # fake a bunch of epochs + $im->{rotate_bytes} = $opt->{rotate_bytes} if $opt->{rotate_bytes}; + + if ($opt->{pre}) { + $opt->{pre}->($im, 1, 2); + $orig->header_set('References', '<1@example.com>'); + } + ok($im->add($orig), 'add message to be replaced'); + if ($opt->{post}) { + $opt->{post}->($im, 3, { 4 => 'replace@example.com' }); + } + $im->done; + my $thread_a = $ibx->over->get_thread('replace@example.com'); + + my %before = map {; delete($_->{blob}) => $_ } @{$ibx->recent}; + my $reject = PublicInbox::MIME->new($orig->as_string); + foreach my $mid (['<replace@example.com>', '<extra@example.com>'], + [], ['<replaced@example.com>']) { + $reject->header_set('Message-ID', @$mid); + my $ok = eval { $im->replace($orig, $reject) }; + like($@, qr/Message-ID.*may not be changed/, + '->replace died on Message-ID change'); + ok(!$ok, 'no replacement happened'); + } + + # prepare the replacement + my $expect = "Move along, nothing to see here\n"; + my $repl = PublicInbox::MIME->new($orig->as_string); + $repl->header_set('From', '<redactor@example.com>'); + $repl->header_set('Subject', 'redacted'); + $repl->header_set('Date', 'Sat, 02 Oct 2010 00:00:00 +0000'); + $repl->body_str_set($expect); + + my @warn; + local $SIG{__WARN__} = sub { push @warn, @_ }; + ok(my $cmts = $im->replace($orig, $repl), 'replaced message'); + my $changed_epochs = 0; + for my $tip (@$cmts) { + next if !defined $tip; + $changed_epochs++; + like($tip, qr/\A[a-f0-9]{40}\z/, + 'replace returned current commit'); + } + is($changed_epochs, 1, 'only one epoch changed'); + + $im->done; + my $m = PublicInbox::MIME->new($ibx->msg_by_mid('replace@example.com')); + is($m->body, $expect, 'replaced message'); + is_deeply(\@warn, [], 'no warnings on noop'); + + my @cat = qw(cat-file --buffer --batch --batch-all-objects); + my $git = $ibx->git; + my @all = $git->qx(@cat); + is_deeply([grep(/confidential/, @all)], [], 'nothing confidential'); + is_deeply([grep(/Streisand/, @all)], [], 'Streisand who?'); + is_deeply([grep(/\bOct 1993\b/, @all)], [], 'nothing from Oct 1993'); + my $t19931002 = qr/ 749520000 /; + is_deeply([grep(/$t19931002/, @all)], [], "nothing matches $t19931002"); + + for my $dir (glob("$ibx->{mainrepo}/git/*.git")) { + my ($bn) = ($dir =~ m!([^/]+)\z!); + is(system(qw(git --git-dir), $dir, qw(fsck --strict)), 0, + "git fsck is clean in epoch $bn"); + } + + my $thread_b = $ibx->over->get_thread('replace@example.com'); + is_deeply([sort map { $_->{mid} } @$thread_b], + [sort map { $_->{mid} } @$thread_a], 'threading preserved'); + + if (my $srch = $ibx->search) { + for my $q ('f:streisand', 's:confidential', 'malibu') { + my $msgs = $srch->query($q); + is_deeply($msgs, [], "no match for $q"); + } + my @ok = ('f:redactor', 's:redacted', 'nothing to see'); + if ($opt->{pre}) { + push @ok, 'm:1@example.com', 'm:2@example.com', + 's:message2', 's:message1'; + } + if ($opt->{post}) { + push @ok, 'm:3@example.com', 'm:4@example.com', + 's:message3', 's:message4'; + } + for my $q (@ok) { + my $msgs = $srch->query($q); + ok($msgs->[0], "got match for $q"); + } + } + + # check overview matches: + my %after = map {; delete($_->{blob}) => $_ } @{$ibx->recent}; + my @before_blobs = keys %before; + foreach my $blob (@before_blobs) { + delete $before{$blob} if delete $after{$blob}; + } + + is(scalar keys %before, 1, 'one unique blob from before left'); + is(scalar keys %after, 1, 'one unique blob from after left'); + foreach my $blob (keys %before) { + is($git->check($blob), undef, 'old blob not found'); + my $smsg = $before{$blob}; + is($smsg->{subject}, 'confidential', 'before subject'); + is($smsg->{mid}, 'replace@example.com', 'before MID'); + } + foreach my $blob (keys %after) { + ok($git->check($blob), 'new blob found'); + my $smsg = $after{$blob}; + is($smsg->{subject}, 'redacted', 'after subject'); + is($smsg->{mid}, 'replace@example.com', 'before MID'); + } + @warn = (); + is($im->replace($orig, $repl), undef, 'no-op replace returns undef'); + is($im->purge($orig), undef, 'no-op purge returns undef'); + is_deeply(\@warn, [], 'no warnings on noop'); +} + +sub pad_msgs { + my ($im, @range) = @_; + for my $i (@range) { + my $irt; + if (ref($i) eq 'HASH') { + ($i, $irt) = each %$i; + } + my $sec = sprintf('%0d', $i); + my $mime = PublicInbox::MIME->new(<<EOF); +From: foo\@example.com +To: test\@example.com +Message-ID: <$i\@example.com> +Date: Fri, 02, Jan 1970 00:00:$sec +0000 +Subject: message$i + +message number$i +EOF + + if (defined($irt)) { + $mime->header_set('References', "<$irt>"); + } + + $im->add($mime); + } +} + +my $opt = { pre => *pad_msgs }; +test_replace(2, 'basic', {}); +test_replace(2, 'basic', $opt); +test_replace(2, 'basic', $opt = { %$opt, post => *pad_msgs }); +test_replace(2, 'basic', $opt = { %$opt, rotate_bytes => 1 }); + +SKIP: if ('test xapian') { + require PublicInbox::Search; + PublicInbox::Search::load_xapian() or skip 'Search::Xapian missing', 8; + for my $l (qw(medium)) { + test_replace(2, $l, {}); + $opt = { pre => *pad_msgs }; + test_replace(2, $l, $opt); + test_replace(2, $l, $opt = { %$opt, post => *pad_msgs }); + test_replace(2, $l, $opt = { %$opt, rotate_bytes => 1 }); + } +}; + +done_testing(); diff --git a/t/v2mirror.t b/t/v2mirror.t index fe05ec4d..c31dcd5b 100644 --- a/t/v2mirror.t +++ b/t/v2mirror.t @@ -80,11 +80,13 @@ $sock = undef; my @cmd; foreach my $i (0..$epoch_max) { - @cmd = (qw(git clone --mirror -q), "http://$host:$port/v2/$i", + my $sfx = $i == 0 ? '.git' : ''; + @cmd = (qw(git clone --mirror -q), + "http://$host:$port/v2/$i$sfx", "$tmpdir/m/git/$i.git"); - is(system(@cmd), 0, 'cloned OK'); - ok(-d "$tmpdir/m/git/$i.git", 'mirror OK'); + is(system(@cmd), 0, "cloned $i.git"); + ok(-d "$tmpdir/m/git/$i.git", "mirror $i OK"); } @cmd = ("$script-init", '-V2', 'm', "$tmpdir/m", 'http://example.com/m', diff --git a/t/www_listing.t b/t/www_listing.t new file mode 100644 index 00000000..1f292980 --- /dev/null +++ b/t/www_listing.t @@ -0,0 +1,159 @@ +# Copyright (C) 2019 all contributors <meta@public-inbox.org> +# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt> +# manifest.js.gz generation and grok-pull integration test +use strict; +use warnings; +use Test::More; +use PublicInbox::Spawn qw(which); +use File::Temp qw/tempdir/; +require './t/common.perl'; +my @mods = qw(URI::Escape Plack::Builder IPC::Run Digest::SHA HTTP::Tiny + IO::Compress::Gzip IO::Uncompress::Gunzip Net::HTTP); +foreach my $mod (@mods) { + eval("require $mod") or plan skip_all => "$mod missing for $0"; +} +use_ok 'PublicInbox::WwwListing'; +use_ok 'PublicInbox::Git'; + +my $fi_data = './t/git.fast-import-data'; +my $tmpdir = tempdir('www_listing-tmp-XXXXXX', TMPDIR => 1, CLEANUP => 1); +my $bare = PublicInbox::Git->new("$tmpdir/bare.git"); +is(system(qw(git init -q --bare), $bare->{git_dir}), 0, 'git init --bare'); +is(PublicInbox::WwwListing::fingerprint($bare), undef, + 'empty repo has no fingerprint'); + +my $cmd = [ 'git', "--git-dir=$bare->{git_dir}", qw(fast-import --quiet) ]; +ok(IPC::Run::run($cmd, '<', $fi_data), 'fast-import'); + +like(PublicInbox::WwwListing::fingerprint($bare), qr/\A[a-f0-9]{40}\z/, + 'got fingerprint with non-empty repo'); + +my $pid; +END { kill 'TERM', $pid if defined $pid }; +SKIP: { + my $json = eval { PublicInbox::WwwListing::_json() }; + skip "JSON module missing: $@", 1 if $@; + my $err = "$tmpdir/stderr.log"; + my $out = "$tmpdir/stdout.log"; + my $alt = "$tmpdir/alt.git"; + my $cfgfile = "$tmpdir/config"; + my $v2 = "$tmpdir/v2"; + my $httpd = 'blib/script/public-inbox-httpd'; + use IO::Socket::INET; + my %opts = ( + LocalAddr => '127.0.0.1', + ReuseAddr => 1, + Proto => 'tcp', + Type => SOCK_STREAM, + Listen => 1024, + ); + my $sock = IO::Socket::INET->new(%opts); + ok($sock, 'sock created'); + my ($host, $port) = ($sock->sockhost, $sock->sockport); + my @clone = qw(git clone -q -s --bare); + is(system(@clone, $bare->{git_dir}, $alt), 0, 'clone shared repo'); + + for my $i (0..2) { + is(system(@clone, $alt, "$v2/git/$i.git"), 0, "clone epoch $i"); + } + ok(open(my $fh, '>', "$v2/inbox.lock"), 'mock a v2 inbox'); + open $fh, '>', "$alt/description" or die; + print $fh "we're all clones\n" or die; + close $fh or die; + is(system('git', "--git-dir=$alt", qw(config gitweb.owner lorelei)), 0, + 'set gitweb user'); + ok(unlink("$bare->{git_dir}/description"), 'removed bare/description'); + open $fh, '>', $cfgfile or die; + print $fh <<"" or die; +[publicinbox "bare"] + mainrepo = $bare->{git_dir} + url = http://$host/bare + address = bare\@example.com +[publicinbox "alt"] + mainrepo = $alt + url = http://$host/alt + address = alt\@example.com +[publicinbox "v2"] + mainrepo = $v2 + url = http://$host/v2 + address = v2\@example.com + + close $fh or die; + my $env = { PI_CONFIG => $cfgfile }; + my $cmd = [ $httpd, "--stdout=$out", "--stderr=$err" ]; + $pid = spawn_listener($env, $cmd, [$sock]); + $sock = undef; + my $http = Net::HTTP->new(Host => "$host:$port"); + $http->write_request(GET => '/manifest.js.gz'); + my ($code, undef, %h) = $http->read_response_headers; + is($code, 200, 'got manifest'); + my $tmp; + my $body = ''; + while (1) { + my $n = $http->read_entity_body(my $buf, 65536); + die unless defined $n; + last if $n == 0; + $body .= $buf; + } + IO::Uncompress::Gunzip::gunzip(\$body => \$tmp); + unlike($tmp, qr/"modified":\s*"/, 'modified is an integer'); + my $manifest = $json->decode($tmp); + ok(my $clone = $manifest->{'/alt'}, '/alt in manifest'); + is($clone->{owner}, 'lorelei', 'owner set'); + is($clone->{reference}, '/bare', 'reference detected'); + is($clone->{description}, "we're all clones", 'description read'); + ok(my $bare = $manifest->{'/bare'}, '/bare in manifest'); + is($bare->{description}, 'Unnamed repository', + 'missing $GIT_DIR/description fallback'); + + like($bare->{fingerprint}, qr/\A[a-f0-9]{40}\z/, 'fingerprint'); + is($clone->{fingerprint}, $bare->{fingerprint}, 'fingerprint matches'); + + is(HTTP::Date::time2str($bare->{modified}), $h{'Last-Modified'}, + 'modified field and Last-Modified header match'); + + ok($manifest->{'/v2/git/0.git'}, 'v2 epoch appeared'); + + skip 'skipping grok-pull integration test', 2 if !which('grok-pull'); + + ok(mkdir("$tmpdir/mirror"), 'prepare grok mirror dest'); + open $fh, '>', "$tmpdir/repos.conf" or die; + print $fh <<"" or die; +# You can pull from multiple grok mirrors, just create +# a separate section for each mirror. The name can be anything. +[test] +site = http://$host:$port +manifest = http://$host:$port/manifest.js.gz +toplevel = $tmpdir/mirror +mymanifest = $tmpdir/local-manifest.js.gz + + close $fh or die; + + system(qw(grok-pull -c), "$tmpdir/repos.conf"); + is($? >> 8, 127, 'grok-pull exit code as expected'); + for (qw(alt bare v2/git/0.git v2/git/1.git v2/git/2.git)) { + ok(-d "$tmpdir/mirror/$_", "grok-pull created $_"); + } + + # support per-inbox manifests, handy for v2: + # /$INBOX/v2/manifest.js.gz + open $fh, '>', "$tmpdir/per-inbox.conf" or die; + print $fh <<"" or die; +# You can pull from multiple grok mirrors, just create +# a separate section for each mirror. The name can be anything. +[v2] +site = http://$host:$port +manifest = http://$host:$port/v2/manifest.js.gz +toplevel = $tmpdir/per-inbox +mymanifest = $tmpdir/per-inbox-manifest.js.gz + + close $fh or die; + ok(mkdir("$tmpdir/per-inbox"), 'prepare single-v2-inbox mirror'); + system(qw(grok-pull -c), "$tmpdir/per-inbox.conf"); + is($? >> 8, 127, 'grok-pull exit code as expected'); + for (qw(v2/git/0.git v2/git/1.git v2/git/2.git)) { + ok(-d "$tmpdir/per-inbox/$_", "grok-pull created $_"); + } +} + +done_testing(); |