diff options
| -rw-r--r-- | MANIFEST | 1 | ||||
| -rw-r--r-- | lib/PublicInbox/WWW.pm | 2 | ||||
| -rw-r--r-- | lib/PublicInbox/WwwListing.pm | 164 | ||||
| -rw-r--r-- | t/www_listing.t | 138 |
4 files changed, 282 insertions, 23 deletions
@@ -258,3 +258,4 @@ t/view.t t/watch_filter_rubylang.t t/watch_maildir.t t/watch_maildir_v2.t +t/www_listing.t diff --git a/lib/PublicInbox/WWW.pm b/lib/PublicInbox/WWW.pm index 7ea98204..614adad6 100644 --- a/lib/PublicInbox/WWW.pm +++ b/lib/PublicInbox/WWW.pm @@ -88,7 +88,7 @@ sub call { } # top-level indices and feeds - if ($path_info eq '/') { + if ($path_info eq '/' || $path_info eq '/manifest.js.gz') { www_listing($self)->call($env); } elsif ($path_info =~ m!$INBOX_RE\z!o) { invalid_inbox($ctx, $1) || r301($ctx, $1); diff --git a/lib/PublicInbox/WwwListing.pm b/lib/PublicInbox/WwwListing.pm index 6d6d3015..690976ac 100644 --- a/lib/PublicInbox/WwwListing.pm +++ b/lib/PublicInbox/WwwListing.pm @@ -9,6 +9,11 @@ use warnings; use PublicInbox::Hval qw(ascii_html); use PublicInbox::Linkify; use PublicInbox::View; +use bytes (); +use HTTP::Date qw(time2str); +require Digest::SHA; +require File::Spec; +{ no warnings 'once'; *try_cat = *PublicInbox::Inbox::try_cat }; sub list_all ($$$) { my ($self, $env, $hide_key) = @_; @@ -44,21 +49,27 @@ my %VALID = ( 404 => *list_404, ); +sub set_cb ($$$) { + my ($pi_config, $k, $default) = @_; + my $v = $pi_config->{lc $k} // $default; + $VALID{$v} || do { + warn <<""; +`$v' is not a valid value for `$k' +$k be one of `all', `match=domain', or `404' + + $VALID{$default}; + }; +} + sub new { my ($class, $www) = @_; - my $k = 'publicinbox.wwwListing'; my $pi_config = $www->{pi_config}; - my $v = $pi_config->{lc($k)} // 404; bless { pi_config => $pi_config, style => $www->style("\0"), - list_cb => $VALID{$v} || do { - warn <<""; -`$v' is not a valid value for `$k' -$k be one of `all', `match=domain', or `404' - - *list_404; - }, + www_cb => set_cb($pi_config, 'publicInbox.wwwListing', 404), + manifest_cb => set_cb($pi_config, 'publicInbox.grokManifest', + 'match=domain'), }, $class; } @@ -76,26 +87,20 @@ sub ibx_entry { $tmp; } -# not really a stand-alone PSGI app, but maybe it could be... -sub call { - my ($self, $env) = @_; - my $h = [ 'Content-Type', 'text/html; charset=UTF-8' ]; - my $hide_key = 'www'; - if ($env->{PATH_INFO} =~ m!/manifest\.js(?:\.gz)\z/!) { - $hide_key = 'manifest'; - } - my $list = $self->{list_cb}->($self, $env, $hide_key); - my $code = 404; +sub html ($$) { + my ($env, $list) = @_; my $title = 'public-inbox'; my $out = ''; + my $code = 404; if (@$list) { + $title .= ' - listing'; + $code = 200; + # Swartzian transform since ->modified is expensive @$list = sort { $b->[0] <=> $a->[0] } map { [ $_->modified, $_ ] } @$list; - $code = 200; - $title .= ' - listing'; my $tmp = join("\n", map { ibx_entry(@$_, $env) } @$list); my $l = PublicInbox::Linkify->new; $l->linkify_1($tmp); @@ -104,7 +109,122 @@ sub call { $out = "<html><head><title>$title</title></head><body>" . $out; $out .= '<pre>'. PublicInbox::WwwStream::code_footer($env) . '</pre></body></html>'; - [ $code, $h, [ $out ] ] + + my $h = [ 'Content-Type', 'text/html; charset=UTF-8' ]; + [ $code, $h, [ $out ] ]; +} + +my $json; +sub _json () { + for my $mod (qw(JSON::MaybeXS JSON JSON::PP)) { + eval "require $mod" or next; + # ->ascii encodes non-ASCII to "\uXXXX" + return $mod->new->ascii(1); + } + die; +} + +sub fingerprint ($) { + my ($git) = @_; + my $fh = $git->popen('show-ref') or + die "popen($git->{git_dir} show-ref) failed: $!"; + + my $dig = Digest::SHA->new(1); + while (read($fh, my $buf, 65536)) { + $dig->add($buf); + } + close $fh; + return if $?; # empty, uninitialized git repo + $dig->hexdigest; +} + +sub manifest_add ($$;$) { + my ($manifest, $ibx, $epoch) = @_; + my $url_path = "/$ibx->{name}"; + my $git_dir = $ibx->{mainrepo}; + if (defined $epoch) { + $git_dir .= "/git/$epoch.git"; + $url_path .= "/$epoch"; + } + return unless -d $git_dir; + my $git = PublicInbox::Git->new($git_dir); + my $fingerprint = fingerprint($git) or return; # no empty repos + + chomp(my $owner = $git->qx('config', 'gitweb.owner')); + chomp(my $desc = try_cat("$git_dir/description")); + $owner = undef if $owner eq ''; + $desc = 'Unnamed repository' if $desc eq ''; + + my $reference; + chomp(my $alt = try_cat("$git_dir/objects/info/alternates")); + if ($alt) { + # n.b.: GitPython doesn't seem to handle comments or C-quoted + # strings like native git does; and we don't for now, either. + my @alt = split(/\n+/, $alt); + + # grokmirror only supports 1 alternate for "reference", + if (scalar(@alt) == 1) { + my $objdir = "$git_dir/objects"; + $reference = File::Spec->rel2abs($alt[0], $objdir); + $reference =~ s!/[^/]+/?\z!!; # basename + } + } + $manifest->{-abs2urlpath}->{$git_dir} = $url_path; + my $modified = $git->modified; + if ($modified > $manifest->{-mtime}) { + $manifest->{-mtime} = $modified; + } + $manifest->{$url_path} = { + owner => $owner, + reference => $reference, + description => $desc, + modified => $modified, + fingerprint => $fingerprint, + }; +} + +# manifest.js.gz +sub js ($$) { + my ($env, $list) = @_; + eval { require IO::Compress::Gzip } or return [ 404, [], [] ]; + + my $manifest = { -abs2urlpath => {}, -mtime => 0 }; + for my $ibx (@$list) { + if (defined(my $max = $ibx->max_git_part)) { + for my $epoch (0..$max) { + manifest_add($manifest, $ibx, $epoch); + } + } else { + manifest_add($manifest, $ibx); + } + } + my $abs2urlpath = delete $manifest->{-abs2urlpath}; + my $mtime = delete $manifest->{-mtime}; + while (my ($url_path, $repo) = each %$manifest) { + defined(my $abs = $repo->{reference}) or next; + $repo->{reference} = $abs2urlpath->{$abs}; + } + my $out; + IO::Compress::Gzip::gzip(\(($json ||= _json())->encode($manifest)) => + \$out); + $manifest = undef; + [ 200, [ qw(Content-Type application/gzip), + 'Last-Modified', time2str($mtime), + 'Content-Length', bytes::length($out) ], [ $out ] ]; +} + +# not really a stand-alone PSGI app, but maybe it could be... +sub call { + my ($self, $env) = @_; + + if ($env->{PATH_INFO} eq '/manifest.js.gz') { + # grokmirror uses relative paths, so it's domain-dependent + my $list = $self->{manifest_cb}->($self, $env, 'manifest'); + js($env, $list); + } else { # / + my $list = $self->{www_cb}->($self, $env, 'www'); + html($env, $list); + } } 1; diff --git a/t/www_listing.t b/t/www_listing.t new file mode 100644 index 00000000..f9d543e5 --- /dev/null +++ b/t/www_listing.t @@ -0,0 +1,138 @@ +# Copyright (C) 2019 all contributors <meta@public-inbox.org> +# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt> +# manifest.js.gz generation and grok-pull integration test +use strict; +use warnings; +use Test::More; +use PublicInbox::Spawn qw(which); +use File::Temp qw/tempdir/; +require './t/common.perl'; +my @mods = qw(URI::Escape Plack::Builder IPC::Run Digest::SHA HTTP::Tiny + IO::Compress::Gzip IO::Uncompress::Gunzip Net::HTTP); +foreach my $mod (@mods) { + eval("require $mod") or plan skip_all => "$mod missing for $0"; +} +use_ok 'PublicInbox::WwwListing'; +use_ok 'PublicInbox::Git'; + +my $fi_data = './t/git.fast-import-data'; +my $tmpdir = tempdir('www_listing-tmp-XXXXXX', TMPDIR => 1, CLEANUP => 1); +my $bare = PublicInbox::Git->new("$tmpdir/bare.git"); +is(system(qw(git init -q --bare), $bare->{git_dir}), 0, 'git init --bare'); +is(PublicInbox::WwwListing::fingerprint($bare), undef, + 'empty repo has no fingerprint'); + +my $cmd = [ 'git', "--git-dir=$bare->{git_dir}", qw(fast-import --quiet) ]; +ok(IPC::Run::run($cmd, '<', $fi_data), 'fast-import'); + +like(PublicInbox::WwwListing::fingerprint($bare), qr/\A[a-f0-9]{40}\z/, + 'got fingerprint with non-empty repo'); + +my $pid; +END { kill 'TERM', $pid if defined $pid }; +SKIP: { + my $json = eval { PublicInbox::WwwListing::_json() }; + skip "JSON module missing: $@", 1 if $@; + my $err = "$tmpdir/stderr.log"; + my $out = "$tmpdir/stdout.log"; + my $alt = "$tmpdir/alt.git"; + my $cfgfile = "$tmpdir/config"; + my $v2 = "$tmpdir/v2"; + my $httpd = 'blib/script/public-inbox-httpd'; + use IO::Socket::INET; + my %opts = ( + LocalAddr => '127.0.0.1', + ReuseAddr => 1, + Proto => 'tcp', + Type => SOCK_STREAM, + Listen => 1024, + ); + my $sock = IO::Socket::INET->new(%opts); + ok($sock, 'sock created'); + my ($host, $port) = ($sock->sockhost, $sock->sockport); + my @clone = qw(git clone -q -s --bare); + is(system(@clone, $bare->{git_dir}, $alt), 0, 'clone shared repo'); + + for my $i (0..2) { + is(system(@clone, $alt, "$v2/git/$i.git"), 0, "clone epoch $i"); + } + ok(open(my $fh, '>', "$v2/inbox.lock"), 'mock a v2 inbox'); + open $fh, '>', "$alt/description" or die; + print $fh "we're all clones\n" or die; + close $fh or die; + is(system('git', "--git-dir=$alt", qw(config gitweb.owner lorelei)), 0, + 'set gitweb user'); + ok(unlink("$bare->{git_dir}/description"), 'removed bare/description'); + open $fh, '>', $cfgfile or die; + print $fh <<"" or die; +[publicinbox "bare"] + mainrepo = $bare->{git_dir} + url = http://$host/bare + address = bare\@example.com +[publicinbox "alt"] + mainrepo = $alt + url = http://$host/alt + address = alt\@example.com +[publicinbox "v2"] + mainrepo = $v2 + url = http://$host/v2 + address = v2\@example.com + + close $fh or die; + my $env = { PI_CONFIG => $cfgfile }; + my $cmd = [ $httpd, "--stdout=$out", "--stderr=$err" ]; + $pid = spawn_listener($env, $cmd, [$sock]); + $sock = undef; + my $http = Net::HTTP->new(Host => "$host:$port"); + $http->write_request(GET => '/manifest.js.gz'); + my ($code, undef, %h) = $http->read_response_headers; + is($code, 200, 'got manifest'); + my $tmp; + my $body = ''; + while (1) { + my $n = $http->read_entity_body(my $buf, 65536); + die unless defined $n; + last if $n == 0; + $body .= $buf; + } + IO::Uncompress::Gunzip::gunzip(\$body => \$tmp); + my $manifest = $json->decode($tmp); + ok(my $clone = $manifest->{'/alt'}, '/alt in manifest'); + is($clone->{owner}, 'lorelei', 'owner set'); + is($clone->{reference}, '/bare', 'reference detected'); + is($clone->{description}, "we're all clones", 'description read'); + ok(my $bare = $manifest->{'/bare'}, '/bare in manifest'); + is($bare->{description}, 'Unnamed repository', + 'missing $GIT_DIR/description fallback'); + + like($bare->{fingerprint}, qr/\A[a-f0-9]{40}\z/, 'fingerprint'); + is($clone->{fingerprint}, $bare->{fingerprint}, 'fingerprint matches'); + + is(HTTP::Date::time2str($bare->{modified}), $h{'Last-Modified'}, + 'modified field and Last-Modified header match'); + + ok($manifest->{'/v2/0'}, 'v2 epoch appeared'); + + skip 'skipping grok-pull integration test', 2 if !which('grok-pull'); + + ok(mkdir("$tmpdir/mirror"), 'prepare grok mirror dest'); + open $fh, '>', "$tmpdir/repos.conf" or die; + print $fh <<"" or die; +# You can pull from multiple grok mirrors, just create +# a separate section for each mirror. The name can be anything. +[test] +site = http://$host:$port +manifest = http://$host:$port/manifest.js.gz +toplevel = $tmpdir/mirror +mymanifest = $tmpdir/local-manifest.js.gz + + close $fh or die; + + system(qw(grok-pull -c), "$tmpdir/repos.conf"); + is($? >> 8, 127, 'grok-pull exit code as expected'); + for (qw(alt bare v2/0 v2/1 v2/2)) { + ok(-d "$tmpdir/mirror/$_", "grok-pull created $_"); + } +} + +done_testing(); |