diff options
Diffstat (limited to 't')
| -rw-r--r-- | t/check-www-inbox.perl | 58 | ||||
| -rw-r--r-- | t/config.t | 19 | ||||
| -rw-r--r-- | t/git.t | 7 | ||||
| -rw-r--r-- | t/hl_mod.t | 55 | ||||
| -rw-r--r-- | t/hval.t | 3 | ||||
| -rw-r--r-- | t/perf-msgview.t | 50 | ||||
| -rw-r--r-- | t/qspawn.t | 12 | ||||
| -rw-r--r-- | t/solve/0001-simple-mod.patch | 20 | ||||
| -rw-r--r-- | t/solve/0002-rename-with-modifications.patch | 37 | ||||
| -rw-r--r-- | t/solver_git.t | 99 | ||||
| -rw-r--r-- | t/view.t | 2 |
11 files changed, 356 insertions, 6 deletions
diff --git a/t/check-www-inbox.perl b/t/check-www-inbox.perl index 08e62471..db292c50 100644 --- a/t/check-www-inbox.perl +++ b/t/check-www-inbox.perl @@ -1,5 +1,5 @@ #!/usr/bin/perl -w -# Copyright (C) 2016-2018 all contributors <meta@public-inbox.org> +# Copyright (C) 2016-2019 all contributors <meta@public-inbox.org> # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt> # Parallel WWW checker my $usage = "$0 [-j JOBS] [-s SLOW_THRESHOLD] URL_OF_INBOX\n"; @@ -14,6 +14,13 @@ use POSIX qw(:sys_wait_h); use Time::HiRes qw(gettimeofday tv_interval); use WWW::Mechanize; use Data::Dumper; + +# we want to use vfork+exec with spawn, WWW::Mechanize can use too much +# memory and fork(2) fails +use PublicInbox::Spawn qw(spawn which); +$ENV{PERL_INLINE_DIRECTORY} or warn "PERL_INLINE_DIRECTORY unset, may OOM\n"; + +our $tmp_owner = $$; my $nproc = 4; my $slow = 0.5; my %opts = ( @@ -23,7 +30,40 @@ my %opts = ( GetOptions(%opts) or die "bad command-line args\n$usage"; my $root_url = shift or die $usage; +chomp(my $xmlstarlet = which('xmlstarlet')); +my $atom_check = eval { + my $cmd = [ qw(xmlstarlet val -e -) ]; + sub { + my ($in, $out, $err) = @_; + use autodie; + open my $in_fh, '+>', undef; + open my $out_fh, '+>', undef; + open my $err_fh, '+>', undef; + print $in_fh $$in; + $in_fh->flush; + sysseek($in_fh, 0, 0); + my $rdr = { + 0 => fileno($in_fh), + 1 => fileno($out_fh), + 2 => fileno($err_fh), + }; + my $pid = spawn($cmd, undef, $rdr); + defined $pid or die "spawn failure: $!"; + while (waitpid($pid, 0) != $pid) { + next if $!{EINTR}; + warn "waitpid(xmlstarlet, $pid) $!"; + return $!; + } + sysseek($out_fh, 0, 0); + sysread($out_fh, $$out, -s $out_fh); + sysseek($err_fh, 0, 0); + sysread($err_fh, $$err, -s $err_fh); + $? + } +} if $xmlstarlet; + my %workers; +$SIG{INT} = sub { exit 130 }; $SIG{TERM} = sub { exit 0 }; $SIG{CHLD} = sub { while (1) { @@ -108,8 +148,10 @@ while (keys %workers) { # reacts to SIGCHLD sub worker_loop { my ($todo_rd, $done_wr) = @_; + $SIG{CHLD} = 'DEFAULT'; my $m = WWW::Mechanize->new(autocheck => 0); my $cc = LWP::ConnCache->new; + $m->stack_depth(0); # no history $m->conn_cache($cc); while (1) { $todo_rd->recv(my $u, 65535, 0); @@ -134,7 +176,7 @@ sub worker_loop { my $s; # blocking foreach my $l (@links, "DONE\t$u") { - next if $l eq ''; + next if $l eq '' || $l =~ /\.mbox(?:\.gz)\z/; do { $s = $done_wr->send($l, MSG_EOR); } while (!defined $s && $!{EINTR}); @@ -146,7 +188,17 @@ sub worker_loop { # make sure the HTML source doesn't screw up terminals # when people curl the source (not remotely an expert # on languages or encodings, here). - next if $r->header('Content-Type') !~ m!\btext/html\b!; + my $ct = $r->header('Content-Type') || ''; + warn "no Content-Type: $u\n" if $ct eq ''; + + if ($atom_check && $ct =~ m!\bapplication/atom\+xml\b!) { + my $raw = $r->decoded_content; + my ($out, $err) = ('', ''); + my $fail = $atom_check->(\$raw, \$out, \$err); + warn "Atom ($fail) - $u - <1:$out> <2:$err>\n" if $fail; + } + + next if $ct !~ m!\btext/html\b!; my $dc = $r->decoded_content; if ($dc =~ /([\x00-\x08\x0d-\x1f\x7f-\x{99999999}]+)/s) { my $o = $1; @@ -150,4 +150,23 @@ for my $s (@valid) { ok(PublicInbox::Config::valid_inbox_name($s), "$d name accepted"); } +{ + my $f = "$tmpdir/ordered"; + open my $fh, '>', $f or die "open: $!"; + my @expect; + foreach my $i (0..3) { + push @expect, "$i"; + print $fh <<"" or die "print: $!"; +[publicinbox "$i"] + mainrepo = /path/to/$i.git + address = $i\@example.com + + } + close $fh or die "close: $!"; + my $cfg = PublicInbox::Config->new($f); + my @result; + $cfg->each_inbox(sub { push @result, $_[0]->{name} }); + is_deeply(\@result, \@expect); +} + done_testing(); @@ -144,11 +144,16 @@ if ('alternates reloaded') { is($$found, $config, 'alternates reloaded'); } -use_ok 'PublicInbox::Git', qw(git_unquote); +use_ok 'PublicInbox::Git', qw(git_unquote git_quote); my $s; is("foo\nbar", git_unquote($s = '"foo\\nbar"'), 'unquoted newline'); is("Eléanor", git_unquote($s = '"El\\303\\251anor"'), 'unquoted octal'); is(git_unquote($s = '"I\"m"'), 'I"m', 'unquoted dq'); is(git_unquote($s = '"I\\m"'), 'I\\m', 'unquoted backslash'); +is(git_quote($s = "Eléanor"), '"El\\303\\251anor"', 'quoted octal'); +is(git_quote($s = "hello\"world"), '"hello\"world"', 'quoted dq'); +is(git_quote($s = "hello\\world"), '"hello\\\\world"', 'quoted backslash'); +is(git_quote($s = "hello\nworld"), '"hello\\nworld"', 'quoted LF'); + done_testing(); diff --git a/t/hl_mod.t b/t/hl_mod.t new file mode 100644 index 00000000..80f88907 --- /dev/null +++ b/t/hl_mod.t @@ -0,0 +1,55 @@ +#!/usr/bin/perl -w +# Copyright (C) 2019 all contributors <meta@public-inbox.org> +# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt> +use strict; +use warnings; +use Test::More; +eval { require highlight } or + plan skip_all => 'failed to load highlight.pm'; +use_ok 'PublicInbox::HlMod'; +my $hls = PublicInbox::HlMod->new; +ok($hls, 'initialized OK'); +is($hls->_shebang2lang(\"#!/usr/bin/perl -w\n"), 'perl', 'perl shebang OK'); +is($hls->{-ext2lang}->{'pm'}, 'perl', '.pm suffix OK'); +is($hls->{-ext2lang}->{'pl'}, 'perl', '.pl suffix OK'); +is($hls->_path2lang('Makefile'), 'make', 'Makefile OK'); +my $str = do { local $/; open(my $fh, __FILE__); <$fh> }; +my $orig = $str; + +{ + my $ref = $hls->do_hl(\$str, 'foo.perl'); + is(ref($ref), 'SCALAR', 'got a scalar reference back'); + like($$ref, qr/I can see you!/, 'we can see ourselves in output'); + like($$ref, qr/&&/, 'escaped'); + + use PublicInbox::Spawn qw(which); + if (eval { require IPC::Run } && which('w3m')) { + require File::Temp; + my $cmd = [ qw(w3m -T text/html -dump -config /dev/null) ]; + my ($out, $err) = ('', ''); + IPC::Run::run($cmd, \('<pre>'.$$ref.'</pre>'), \$out, \$err); + # expand tabs and normalize whitespace, + # w3m doesn't preserve tabs + $orig =~ s/\t/ /gs; + $out =~ s/\s*\z//sg; + $orig =~ s/\s*\z//sg; + is($out, $orig, 'w3m output matches'); + } +} + +my $nr = $ENV{TEST_MEMLEAK}; +if ($nr && -r "/proc/$$/status") { + my $fh; + open $fh, '<', "/proc/$$/status"; + diag "starting at memtest at ".join('', grep(/VmRSS:/, <$fh>)); + PublicInbox::HlMod->new->do_hl(\$orig) for (1..$nr); + open $fh, '<', "/proc/$$/status"; + diag "creating $nr instances: ".join('', grep(/VmRSS:/, <$fh>)); + my $hls = PublicInbox::HlMod->new; + $hls->do_hl(\$orig) for (1..$nr); + $hls = undef; + open $fh, '<', "/proc/$$/status"; + diag "reused instance $nr times: ".join('', grep(/VmRSS:/, <$fh>)); +} + +done_testing; @@ -43,5 +43,8 @@ is('foo-bar', PublicInbox::Hval::to_filename("foo bar\nanother line\n"), is('foo.bar', PublicInbox::Hval::to_filename("foo....bar"), 'to_filename squeezes -'); +my $s = "\0\x07\n"; +PublicInbox::Hval::src_escape($s); +is($s, "\\0\\a\n", 'src_escape works as intended'); done_testing(); diff --git a/t/perf-msgview.t b/t/perf-msgview.t new file mode 100644 index 00000000..adeb7aac --- /dev/null +++ b/t/perf-msgview.t @@ -0,0 +1,50 @@ +# Copyright (C) 2019 all contributors <meta@public-inbox.org> +# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt> +use strict; +use warnings; +use Test::More; +use Benchmark qw(:all); +use PublicInbox::Inbox; +use PublicInbox::View; +require './t/common.perl'; + +my @cat = qw(cat-file --buffer --batch-check --batch-all-objects); +if (require_git(2.19, 1)) { + push @cat, '--unordered'; +} else { + warn +"git <2.19, cat-file lacks --unordered, locality suffers\n"; +} + +my $pi_dir = $ENV{GIANT_PI_DIR}; +plan skip_all => "GIANT_PI_DIR not defined for $0" unless $pi_dir; + +my $ibx = PublicInbox::Inbox->new({ mainrepo => $pi_dir, name => 'name' }); +my $git = $ibx->git; +my $fh = $git->popen(@cat); +my $vec = ''; +vec($vec, fileno($fh), 1) = 1; +select($vec, undef, undef, 60) or die "timed out waiting for --batch-check"; + +my $ctx = { + env => { HTTP_HOST => 'example.com', 'psgi.url_scheme' => 'https' }, + -inbox => $ibx, +}; +my ($str, $mime, $res, $cmt, $type); +my $n = 0; +my $t = timeit(1, sub { + while (<$fh>) { + ($cmt, $type) = split / /; + next if $type ne 'blob'; + ++$n; + $str = $git->cat_file($cmt); + $mime = PublicInbox::MIME->new($str); + $res = PublicInbox::View::msg_html($ctx, $mime); + $res = $res->[2]; + while (defined($res->getline)) {} + $res->close; + } +}); +diag 'msg_html took '.timestr($t)." for $n messages"; +ok 1; +done_testing(); @@ -1,8 +1,16 @@ -# Copyright (C) 2016-2018 all contributors <meta@public-inbox.org> +# Copyright (C) 2016-2019 all contributors <meta@public-inbox.org> # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt> use Test::More; use_ok 'PublicInbox::Qspawn'; +{ + my $cmd = [qw(sh -c), 'echo >&2 err; echo out']; + my $qsp = PublicInbox::Qspawn->new($cmd, {}, { 2 => 1 }); + my $res; + $qsp->psgi_qx({}, undef, sub { $res = ${$_[0]} }); + is($res, "err\nout\n", 'captured stderr and stdout'); +} + my $limiter = PublicInbox::Qspawn::Limiter->new(1); { my $x = PublicInbox::Qspawn->new([qw(true)]); @@ -23,7 +31,7 @@ my $limiter = PublicInbox::Qspawn::Limiter->new(1); my ($rpipe) = @_; is(0, sysread($rpipe, my $buf, 1), 'read zero bytes from false'); my $err = $x->finish; - is($err, 256, 'error on finish'); + ok($err, 'error on finish'); $run = 1; }); is($run, 1, 'callback ran alright'); diff --git a/t/solve/0001-simple-mod.patch b/t/solve/0001-simple-mod.patch new file mode 100644 index 00000000..c6bb1575 --- /dev/null +++ b/t/solve/0001-simple-mod.patch @@ -0,0 +1,20 @@ +From: WEB DESIGN EXPERT <BOFH@YHBT.net> +To: meta@public-inbox.org +Subject: [PATCH] TODO: take expert web design advice +Date: Mon, 1 Apr 2019 08:15:20 +0000 +Message-Id: <20190401081523.16213-1-BOFH@YHBT.net> + +--- + TODO | 2 ++ + 1 file changed, 2 insertions(+) + +diff --git a/TODO b/TODO +index 605013e..69df7d5 100644 +--- a/TODO ++++ b/TODO +@@ -109,3 +109,5 @@ all need to be considered for everything we introduce) + + * Optional history squashing to reduce commit and intermediate + tree objects ++ ++ * Make use of <blink> and <marquee> tags diff --git a/t/solve/0002-rename-with-modifications.patch b/t/solve/0002-rename-with-modifications.patch new file mode 100644 index 00000000..aa415e01 --- /dev/null +++ b/t/solve/0002-rename-with-modifications.patch @@ -0,0 +1,37 @@ +From: POLITICAL CORRECTNESS EXPERT <BOFH@YHBT.net> +To: meta@public-inbox.org +Subject: [PATCH] POLITICALLY CORRECT FILE NAMING +Date: Mon, 1 Apr 2019 08:15:20 +0000 +Message-Id: <20190401081523.16213-2-BOFH@YHBT.net> + +HACKING MIGHT GET US REPORTED TO EFF-BEE-EYE +AND USE MARKDOWN CUZ MOAR FLAVORS == BETTER +--- + HACKING => CONTRIBUTING.md | 6 +++--- + 1 file changed, 3 insertions(+), 3 deletions(-) + rename HACKING => CONTRIBUTING.md (94%) + +diff --git a/HACKING b/CONTRIBUTING.md +similarity index 94% +rename from HACKING +rename to CONTRIBUTING.md +index 3435775..0a92431 100644 +--- a/HACKING ++++ b/CONTRIBUTING.md +@@ -1,5 +1,5 @@ +-hacking public-inbox +--------------------- ++contributing to public-inbox ++---------------------------- + + Send all patches and "git request-pull"-formatted emails to our + self-hosting inbox at meta@public-inbox.org +@@ -15,7 +15,7 @@ Please consider our goals in mind: + Decentralization, Accessibility, Compatibility, Performance + + These goals apply to everyone: users viewing over the web or NNTP, +-sysadmins running public-inbox, and other hackers working public-inbox. ++sysadmins running public-inbox, and other contributors working public-inbox. + + We will reject any feature which advocates or contributes to any + particular instance of a public-inbox becoming a single point of failure. diff --git a/t/solver_git.t b/t/solver_git.t new file mode 100644 index 00000000..197a003a --- /dev/null +++ b/t/solver_git.t @@ -0,0 +1,99 @@ +# Copyright (C) 2019 all contributors <meta@public-inbox.org> +# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt> +use strict; +use warnings; +use Test::More; +use File::Temp qw(tempdir); +use Cwd qw(abs_path); +require './t/common.perl'; +require_git(2.6); + +my @mods = qw(DBD::SQLite Search::Xapian HTTP::Request::Common Plack::Test + URI::Escape Plack::Builder); +foreach my $mod (@mods) { + eval "require $mod"; + plan skip_all => "$mod missing for $0" if $@; +} +chomp(my $git_dir = `git rev-parse --git-dir 2>/dev/null`); +plan skip_all => "$0 must be run from a git working tree" if $?; +$git_dir = abs_path($git_dir); + +use_ok "PublicInbox::$_" for (qw(Inbox V2Writable MIME Git SolverGit)); + +my $mainrepo = tempdir('pi-solver-XXXXXX', TMPDIR => 1, CLEANUP => 1); +my $opts = { + mainrepo => $mainrepo, + name => 'test-v2writable', + version => 2, + -primary_address => 'test@example.com', +}; +my $ibx = PublicInbox::Inbox->new($opts); +my $im = PublicInbox::V2Writable->new($ibx, 1); +$im->{parallel} = 0; + +sub deliver_patch ($) { + open my $fh, '<', $_[0] or die "open: $!"; + my $mime = PublicInbox::MIME->new(do { local $/; <$fh> }); + $im->add($mime); + $im->done; +} + +deliver_patch('t/solve/0001-simple-mod.patch'); + +$ibx->{-repo_objs} = [ PublicInbox::Git->new($git_dir) ]; +my $res; +my $solver = PublicInbox::SolverGit->new($ibx, sub { $res = $_[0] }); +open my $log, '+>>', "$mainrepo/solve.log" or die "open: $!"; +my $psgi_env = { 'psgi.url_scheme' => 'http', HTTP_HOST => 'example.com' }; +$solver->solve($psgi_env, $log, '69df7d5', {}); +ok($res, 'solved a blob!'); +my $wt_git = $res->[0]; +is(ref($wt_git), 'PublicInbox::Git', 'got a git object for the blob'); +my $expect = '69df7d565d49fbaaeb0a067910f03dc22cd52bd0'; +is($res->[1], $expect, 'resolved blob to unabbreviated identifier'); +is($res->[2], 'blob', 'type specified'); +is($res->[3], 4405, 'size returned'); + +is(ref($wt_git->cat_file($res->[1])), 'SCALAR', 'wt cat-file works'); +is_deeply([$expect, 'blob', 4405], + [$wt_git->check($res->[1])], 'wt check works'); + +if (0) { # TODO: check this? + seek($log, 0, 0); + my $z = do { local $/; <$log> }; + diag $z; +} + +$solver = undef; +$res = undef; +my $wt_git_dir = $wt_git->{git_dir}; +$wt_git = undef; +ok(!-d $wt_git_dir, 'no references to WT held'); + +$solver = PublicInbox::SolverGit->new($ibx, sub { $res = $_[0] }); +$solver->solve($psgi_env, $log, '0'x40, {}); +is($res, undef, 'no error on z40'); + +my $git_v2_20_1_tag = '7a95a1cd084cb665c5c2586a415e42df0213af74'; +$solver = PublicInbox::SolverGit->new($ibx, sub { $res = $_[0] }); +$solver->solve($psgi_env, $log, $git_v2_20_1_tag, {}); +is($res, undef, 'no error on a tag not in our repo'); + +deliver_patch('t/solve/0002-rename-with-modifications.patch'); +$solver = PublicInbox::SolverGit->new($ibx, sub { $res = $_[0] }); +$solver->solve($psgi_env, $log, '0a92431', {}); +ok($res, 'resolved without hints'); + +my $hints = { + oid_a => '3435775', + path_a => 'HACKING', + path_b => 'CONTRIBUTING' +}; +$solver = PublicInbox::SolverGit->new($ibx, sub { $res = $_[0] }); +$solver->solve($psgi_env, $log, '0a92431', $hints); +my $hinted = $res; +# don't compare ::Git objects: +shift @$res; shift @$hinted; +is_deeply($res, $hinted, 'hints work (or did not hurt :P'); + +done_testing(); @@ -6,6 +6,7 @@ use Test::More; use Email::MIME; use Plack::Util; use_ok 'PublicInbox::View'; +use_ok 'PublicInbox::Config'; # FIXME: make this test less fragile my $ctx = { @@ -18,6 +19,7 @@ my $ctx = { nntp_url => sub {[]}, max_git_part => sub { undef }, description => sub { '' }), + www => Plack::Util::inline_object(style => sub { '' }), }; $ctx->{-inbox}->{-primary_address} = 'test@example.com'; |