From 0b3e19584c90d958a723ac2d3dec3f84f5513688 Mon Sep 17 00:00:00 2001 From: "Eric Wong (Contractor, The Linux Foundation)" Date: Sun, 9 Jun 2019 04:31:03 +0000 Subject: wwwlisting: generate grokmirror-compatible manifest.js.gz Support on-demand generation of "/manifest.js.gz" for inboxes. By default, this matches inboxes with URLs matching the given request hostname by default. This makes it easier to create full mirrors of several inboxes without needing to configure static file serving. cf. https://git.kernel.org/pub/scm/utils/grokmirror/grokmirror.git --- t/www_listing.t | 138 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 138 insertions(+) create mode 100644 t/www_listing.t (limited to 't') diff --git a/t/www_listing.t b/t/www_listing.t new file mode 100644 index 00000000..f9d543e5 --- /dev/null +++ b/t/www_listing.t @@ -0,0 +1,138 @@ +# Copyright (C) 2019 all contributors +# License: AGPL-3.0+ +# manifest.js.gz generation and grok-pull integration test +use strict; +use warnings; +use Test::More; +use PublicInbox::Spawn qw(which); +use File::Temp qw/tempdir/; +require './t/common.perl'; +my @mods = qw(URI::Escape Plack::Builder IPC::Run Digest::SHA HTTP::Tiny + IO::Compress::Gzip IO::Uncompress::Gunzip Net::HTTP); +foreach my $mod (@mods) { + eval("require $mod") or plan skip_all => "$mod missing for $0"; +} +use_ok 'PublicInbox::WwwListing'; +use_ok 'PublicInbox::Git'; + +my $fi_data = './t/git.fast-import-data'; +my $tmpdir = tempdir('www_listing-tmp-XXXXXX', TMPDIR => 1, CLEANUP => 1); +my $bare = PublicInbox::Git->new("$tmpdir/bare.git"); +is(system(qw(git init -q --bare), $bare->{git_dir}), 0, 'git init --bare'); +is(PublicInbox::WwwListing::fingerprint($bare), undef, + 'empty repo has no fingerprint'); + +my $cmd = [ 'git', "--git-dir=$bare->{git_dir}", qw(fast-import --quiet) ]; +ok(IPC::Run::run($cmd, '<', $fi_data), 'fast-import'); + +like(PublicInbox::WwwListing::fingerprint($bare), qr/\A[a-f0-9]{40}\z/, + 'got fingerprint with non-empty repo'); + +my $pid; +END { kill 'TERM', $pid if defined $pid }; +SKIP: { + my $json = eval { PublicInbox::WwwListing::_json() }; + skip "JSON module missing: $@", 1 if $@; + my $err = "$tmpdir/stderr.log"; + my $out = "$tmpdir/stdout.log"; + my $alt = "$tmpdir/alt.git"; + my $cfgfile = "$tmpdir/config"; + my $v2 = "$tmpdir/v2"; + my $httpd = 'blib/script/public-inbox-httpd'; + use IO::Socket::INET; + my %opts = ( + LocalAddr => '127.0.0.1', + ReuseAddr => 1, + Proto => 'tcp', + Type => SOCK_STREAM, + Listen => 1024, + ); + my $sock = IO::Socket::INET->new(%opts); + ok($sock, 'sock created'); + my ($host, $port) = ($sock->sockhost, $sock->sockport); + my @clone = qw(git clone -q -s --bare); + is(system(@clone, $bare->{git_dir}, $alt), 0, 'clone shared repo'); + + for my $i (0..2) { + is(system(@clone, $alt, "$v2/git/$i.git"), 0, "clone epoch $i"); + } + ok(open(my $fh, '>', "$v2/inbox.lock"), 'mock a v2 inbox'); + open $fh, '>', "$alt/description" or die; + print $fh "we're all clones\n" or die; + close $fh or die; + is(system('git', "--git-dir=$alt", qw(config gitweb.owner lorelei)), 0, + 'set gitweb user'); + ok(unlink("$bare->{git_dir}/description"), 'removed bare/description'); + open $fh, '>', $cfgfile or die; + print $fh <<"" or die; +[publicinbox "bare"] + mainrepo = $bare->{git_dir} + url = http://$host/bare + address = bare\@example.com +[publicinbox "alt"] + mainrepo = $alt + url = http://$host/alt + address = alt\@example.com +[publicinbox "v2"] + mainrepo = $v2 + url = http://$host/v2 + address = v2\@example.com + + close $fh or die; + my $env = { PI_CONFIG => $cfgfile }; + my $cmd = [ $httpd, "--stdout=$out", "--stderr=$err" ]; + $pid = spawn_listener($env, $cmd, [$sock]); + $sock = undef; + my $http = Net::HTTP->new(Host => "$host:$port"); + $http->write_request(GET => '/manifest.js.gz'); + my ($code, undef, %h) = $http->read_response_headers; + is($code, 200, 'got manifest'); + my $tmp; + my $body = ''; + while (1) { + my $n = $http->read_entity_body(my $buf, 65536); + die unless defined $n; + last if $n == 0; + $body .= $buf; + } + IO::Uncompress::Gunzip::gunzip(\$body => \$tmp); + my $manifest = $json->decode($tmp); + ok(my $clone = $manifest->{'/alt'}, '/alt in manifest'); + is($clone->{owner}, 'lorelei', 'owner set'); + is($clone->{reference}, '/bare', 'reference detected'); + is($clone->{description}, "we're all clones", 'description read'); + ok(my $bare = $manifest->{'/bare'}, '/bare in manifest'); + is($bare->{description}, 'Unnamed repository', + 'missing $GIT_DIR/description fallback'); + + like($bare->{fingerprint}, qr/\A[a-f0-9]{40}\z/, 'fingerprint'); + is($clone->{fingerprint}, $bare->{fingerprint}, 'fingerprint matches'); + + is(HTTP::Date::time2str($bare->{modified}), $h{'Last-Modified'}, + 'modified field and Last-Modified header match'); + + ok($manifest->{'/v2/0'}, 'v2 epoch appeared'); + + skip 'skipping grok-pull integration test', 2 if !which('grok-pull'); + + ok(mkdir("$tmpdir/mirror"), 'prepare grok mirror dest'); + open $fh, '>', "$tmpdir/repos.conf" or die; + print $fh <<"" or die; +# You can pull from multiple grok mirrors, just create +# a separate section for each mirror. The name can be anything. +[test] +site = http://$host:$port +manifest = http://$host:$port/manifest.js.gz +toplevel = $tmpdir/mirror +mymanifest = $tmpdir/local-manifest.js.gz + + close $fh or die; + + system(qw(grok-pull -c), "$tmpdir/repos.conf"); + is($? >> 8, 127, 'grok-pull exit code as expected'); + for (qw(alt bare v2/0 v2/1 v2/2)) { + ok(-d "$tmpdir/mirror/$_", "grok-pull created $_"); + } +} + +done_testing(); -- cgit v1.2.3-24-ge0c7 From 279a47f3f64fc7a414247922b870e58a0b334b0f Mon Sep 17 00:00:00 2001 From: "Eric Wong (Contractor, The Linux Foundation)" Date: Sun, 9 Jun 2019 04:31:04 +0000 Subject: www: wire up /$INBOX/manifest.js.gz, too I can imagine myself just wanting to clone a single v2 inbox and all its epochs without thinking about include/exclude rules in a grokmirror config file. --- t/www_listing.t | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) (limited to 't') diff --git a/t/www_listing.t b/t/www_listing.t index f9d543e5..546c2f8f 100644 --- a/t/www_listing.t +++ b/t/www_listing.t @@ -133,6 +133,26 @@ mymanifest = $tmpdir/local-manifest.js.gz for (qw(alt bare v2/0 v2/1 v2/2)) { ok(-d "$tmpdir/mirror/$_", "grok-pull created $_"); } + + # support per-inbox manifests, handy for v2: + # /$INBOX/v2/manifest.js.gz + open $fh, '>', "$tmpdir/per-inbox.conf" or die; + print $fh <<"" or die; +# You can pull from multiple grok mirrors, just create +# a separate section for each mirror. The name can be anything. +[v2] +site = http://$host:$port +manifest = http://$host:$port/v2/manifest.js.gz +toplevel = $tmpdir/per-inbox +mymanifest = $tmpdir/per-inbox-manifest.js.gz + + close $fh or die; + ok(mkdir("$tmpdir/per-inbox"), 'prepare single-v2-inbox mirror'); + system(qw(grok-pull -c), "$tmpdir/per-inbox.conf"); + is($? >> 8, 127, 'grok-pull exit code as expected'); + for (qw(v2/0 v2/1 v2/2)) { + ok(-d "$tmpdir/per-inbox/$_", "grok-pull created $_"); + } } done_testing(); -- cgit v1.2.3-24-ge0c7 From 0886c264b24b8bc7626e5a7eb59598b0229f066d Mon Sep 17 00:00:00 2001 From: "Eric Wong (Contractor, The Linux Foundation)" Date: Sun, 9 Jun 2019 04:31:05 +0000 Subject: www: support $INBOX/git/$EPOCH.git for v2 cloning And use it in manifest.js. To ease maintaining mirrors with grokmirror(1), we can accept a "git/" directory prefix before the epoch, and ".git" suffix after the epoch number. We maintain compatibility with "$INBOX/$EPOCH" cloning, of course, and it's still easier-to-type on the command-line. --- t/psgi_v2.t | 2 ++ t/v2mirror.t | 8 +++++--- t/www_listing.t | 6 +++--- 3 files changed, 10 insertions(+), 6 deletions(-) (limited to 't') diff --git a/t/psgi_v2.t b/t/psgi_v2.t index 98112494..5c358cde 100644 --- a/t/psgi_v2.t +++ b/t/psgi_v2.t @@ -202,6 +202,8 @@ test_psgi(sub { $www->call(@_) }, sub { $res = $cb->(GET('/v2test/0/info/refs')); is($res->code, 200, 'got info refs for dumb clones'); + $res = $cb->(GET('/v2test/0.git/info/refs')); + is($res->code, 200, 'got info refs for dumb clones w/ .git suffix'); $res = $cb->(GET('/v2test/info/refs')); is($res->code, 404, 'unpartitioned git URL fails'); diff --git a/t/v2mirror.t b/t/v2mirror.t index fe05ec4d..c31dcd5b 100644 --- a/t/v2mirror.t +++ b/t/v2mirror.t @@ -80,11 +80,13 @@ $sock = undef; my @cmd; foreach my $i (0..$epoch_max) { - @cmd = (qw(git clone --mirror -q), "http://$host:$port/v2/$i", + my $sfx = $i == 0 ? '.git' : ''; + @cmd = (qw(git clone --mirror -q), + "http://$host:$port/v2/$i$sfx", "$tmpdir/m/git/$i.git"); - is(system(@cmd), 0, 'cloned OK'); - ok(-d "$tmpdir/m/git/$i.git", 'mirror OK'); + is(system(@cmd), 0, "cloned $i.git"); + ok(-d "$tmpdir/m/git/$i.git", "mirror $i OK"); } @cmd = ("$script-init", '-V2', 'm', "$tmpdir/m", 'http://example.com/m', diff --git a/t/www_listing.t b/t/www_listing.t index 546c2f8f..2741e1b8 100644 --- a/t/www_listing.t +++ b/t/www_listing.t @@ -111,7 +111,7 @@ SKIP: { is(HTTP::Date::time2str($bare->{modified}), $h{'Last-Modified'}, 'modified field and Last-Modified header match'); - ok($manifest->{'/v2/0'}, 'v2 epoch appeared'); + ok($manifest->{'/v2/git/0.git'}, 'v2 epoch appeared'); skip 'skipping grok-pull integration test', 2 if !which('grok-pull'); @@ -130,7 +130,7 @@ mymanifest = $tmpdir/local-manifest.js.gz system(qw(grok-pull -c), "$tmpdir/repos.conf"); is($? >> 8, 127, 'grok-pull exit code as expected'); - for (qw(alt bare v2/0 v2/1 v2/2)) { + for (qw(alt bare v2/git/0.git v2/git/1.git v2/git/2.git)) { ok(-d "$tmpdir/mirror/$_", "grok-pull created $_"); } @@ -150,7 +150,7 @@ mymanifest = $tmpdir/per-inbox-manifest.js.gz ok(mkdir("$tmpdir/per-inbox"), 'prepare single-v2-inbox mirror'); system(qw(grok-pull -c), "$tmpdir/per-inbox.conf"); is($? >> 8, 127, 'grok-pull exit code as expected'); - for (qw(v2/0 v2/1 v2/2)) { + for (qw(v2/git/0.git v2/git/1.git v2/git/2.git)) { ok(-d "$tmpdir/per-inbox/$_", "grok-pull created $_"); } } -- cgit v1.2.3-24-ge0c7 From d3c94cf92e8a4693aa691f3464c94c00be543cfc Mon Sep 17 00:00:00 2001 From: "Eric Wong (Contractor, The Linux Foundation)" Date: Sun, 9 Jun 2019 00:53:29 +0000 Subject: git: ensure ->modified returns an integer We don't want to serialize timestamps as strings to JSON. I only noticed this bug on a 32-bit system. --- t/www_listing.t | 1 + 1 file changed, 1 insertion(+) (limited to 't') diff --git a/t/www_listing.t b/t/www_listing.t index 2741e1b8..1f292980 100644 --- a/t/www_listing.t +++ b/t/www_listing.t @@ -96,6 +96,7 @@ SKIP: { $body .= $buf; } IO::Uncompress::Gunzip::gunzip(\$body => \$tmp); + unlike($tmp, qr/"modified":\s*"/, 'modified is an integer'); my $manifest = $json->decode($tmp); ok(my $clone = $manifest->{'/alt'}, '/alt in manifest'); is($clone->{owner}, 'lorelei', 'owner set'); -- cgit v1.2.3-24-ge0c7