user/dev discussion of public-inbox itself
 help / color / Atom feed
From: "Eric Wong (Contractor, The Linux Foundation)" <e@80x24.org>
To: meta@public-inbox.org
Subject: [PATCH 2/4] wwwlisting: generate grokmirror-compatible manifest.js.gz
Date: Sun,  9 Jun 2019 04:31:03 +0000
Message-ID: <20190609043105.22338-3-e@80x24.org> (raw)
In-Reply-To: <20190609043105.22338-1-e@80x24.org>

Support on-demand generation of "/manifest.js.gz" for inboxes.
By default, this matches inboxes with URLs matching the given
request hostname by default.

This makes it easier to create full mirrors of several inboxes
without needing to configure static file serving.

cf. https://git.kernel.org/pub/scm/utils/grokmirror/grokmirror.git
---
 MANIFEST                      |   1 +
 lib/PublicInbox/WWW.pm        |   2 +-
 lib/PublicInbox/WwwListing.pm | 164 +++++++++++++++++++++++++++++-----
 t/www_listing.t               | 138 ++++++++++++++++++++++++++++
 4 files changed, 282 insertions(+), 23 deletions(-)
 create mode 100644 t/www_listing.t

diff --git a/MANIFEST b/MANIFEST
index 5085bff..9a88f13 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -258,3 +258,4 @@ t/view.t
 t/watch_filter_rubylang.t
 t/watch_maildir.t
 t/watch_maildir_v2.t
+t/www_listing.t
diff --git a/lib/PublicInbox/WWW.pm b/lib/PublicInbox/WWW.pm
index 7ea9820..614adad 100644
--- a/lib/PublicInbox/WWW.pm
+++ b/lib/PublicInbox/WWW.pm
@@ -88,7 +88,7 @@ sub call {
 	}
 
 	# top-level indices and feeds
-	if ($path_info eq '/') {
+	if ($path_info eq '/' || $path_info eq '/manifest.js.gz') {
 		www_listing($self)->call($env);
 	} elsif ($path_info =~ m!$INBOX_RE\z!o) {
 		invalid_inbox($ctx, $1) || r301($ctx, $1);
diff --git a/lib/PublicInbox/WwwListing.pm b/lib/PublicInbox/WwwListing.pm
index 6d6d301..690976a 100644
--- a/lib/PublicInbox/WwwListing.pm
+++ b/lib/PublicInbox/WwwListing.pm
@@ -9,6 +9,11 @@ use warnings;
 use PublicInbox::Hval qw(ascii_html);
 use PublicInbox::Linkify;
 use PublicInbox::View;
+use bytes ();
+use HTTP::Date qw(time2str);
+require Digest::SHA;
+require File::Spec;
+{ no warnings 'once'; *try_cat = *PublicInbox::Inbox::try_cat };
 
 sub list_all ($$$) {
 	my ($self, $env, $hide_key) = @_;
@@ -44,21 +49,27 @@ my %VALID = (
 	404 => *list_404,
 );
 
+sub set_cb ($$$) {
+	my ($pi_config, $k, $default) = @_;
+	my $v = $pi_config->{lc $k} // $default;
+	$VALID{$v} || do {
+		warn <<"";
+`$v' is not a valid value for `$k'
+$k be one of `all', `match=domain', or `404'
+
+		$VALID{$default};
+	};
+}
+
 sub new {
 	my ($class, $www) = @_;
-	my $k = 'publicinbox.wwwListing';
 	my $pi_config = $www->{pi_config};
-	my $v = $pi_config->{lc($k)} // 404;
 	bless {
 		pi_config => $pi_config,
 		style => $www->style("\0"),
-		list_cb => $VALID{$v} || do {
-			warn <<"";
-`$v' is not a valid value for `$k'
-$k be one of `all', `match=domain', or `404'
-
-			*list_404;
-		},
+		www_cb => set_cb($pi_config, 'publicInbox.wwwListing', 404),
+		manifest_cb => set_cb($pi_config, 'publicInbox.grokManifest',
+					'match=domain'),
 	}, $class;
 }
 
@@ -76,26 +87,20 @@ sub ibx_entry {
 	$tmp;
 }
 
-# not really a stand-alone PSGI app, but maybe it could be...
-sub call {
-	my ($self, $env) = @_;
-	my $h = [ 'Content-Type', 'text/html; charset=UTF-8' ];
-	my $hide_key = 'www';
-	if ($env->{PATH_INFO} =~ m!/manifest\.js(?:\.gz)\z/!) {
-		$hide_key = 'manifest';
-	}
-	my $list = $self->{list_cb}->($self, $env, $hide_key);
-	my $code = 404;
+sub html ($$) {
+	my ($env, $list) = @_;
 	my $title = 'public-inbox';
 	my $out = '';
+	my $code = 404;
 	if (@$list) {
+		$title .= ' - listing';
+		$code = 200;
+
 		# Swartzian transform since ->modified is expensive
 		@$list = sort {
 			$b->[0] <=> $a->[0]
 		} map { [ $_->modified, $_ ] } @$list;
 
-		$code = 200;
-		$title .= ' - listing';
 		my $tmp = join("\n", map { ibx_entry(@$_, $env) } @$list);
 		my $l = PublicInbox::Linkify->new;
 		$l->linkify_1($tmp);
@@ -104,7 +109,122 @@ sub call {
 	$out = "<html><head><title>$title</title></head><body>" . $out;
 	$out .= '<pre>'. PublicInbox::WwwStream::code_footer($env) .
 		'</pre></body></html>';
-	[ $code, $h, [ $out ] ]
+
+	my $h = [ 'Content-Type', 'text/html; charset=UTF-8' ];
+	[ $code, $h, [ $out ] ];
+}
+
+my $json;
+sub _json () {
+	for my $mod (qw(JSON::MaybeXS JSON JSON::PP)) {
+		eval "require $mod" or next;
+		# ->ascii encodes non-ASCII to "\uXXXX"
+		return $mod->new->ascii(1);
+	}
+	die;
+}
+
+sub fingerprint ($) {
+	my ($git) = @_;
+	my $fh = $git->popen('show-ref') or
+		die "popen($git->{git_dir} show-ref) failed: $!";
+
+	my $dig = Digest::SHA->new(1);
+	while (read($fh, my $buf, 65536)) {
+		$dig->add($buf);
+	}
+	close $fh;
+	return if $?; # empty, uninitialized git repo
+	$dig->hexdigest;
+}
+
+sub manifest_add ($$;$) {
+	my ($manifest, $ibx, $epoch) = @_;
+	my $url_path = "/$ibx->{name}";
+	my $git_dir = $ibx->{mainrepo};
+	if (defined $epoch) {
+		$git_dir .= "/git/$epoch.git";
+		$url_path .= "/$epoch";
+	}
+	return unless -d $git_dir;
+	my $git = PublicInbox::Git->new($git_dir);
+	my $fingerprint = fingerprint($git) or return; # no empty repos
+
+	chomp(my $owner = $git->qx('config', 'gitweb.owner'));
+	chomp(my $desc = try_cat("$git_dir/description"));
+	$owner = undef if $owner eq '';
+	$desc = 'Unnamed repository' if $desc eq '';
+
+	my $reference;
+	chomp(my $alt = try_cat("$git_dir/objects/info/alternates"));
+	if ($alt) {
+		# n.b.: GitPython doesn't seem to handle comments or C-quoted
+		# strings like native git does; and we don't for now, either.
+		my @alt = split(/\n+/, $alt);
+
+		# grokmirror only supports 1 alternate for "reference",
+		if (scalar(@alt) == 1) {
+			my $objdir = "$git_dir/objects";
+			$reference = File::Spec->rel2abs($alt[0], $objdir);
+			$reference =~ s!/[^/]+/?\z!!; # basename
+		}
+	}
+	$manifest->{-abs2urlpath}->{$git_dir} = $url_path;
+	my $modified = $git->modified;
+	if ($modified > $manifest->{-mtime}) {
+		$manifest->{-mtime} = $modified;
+	}
+	$manifest->{$url_path} = {
+		owner => $owner,
+		reference => $reference,
+		description => $desc,
+		modified => $modified,
+		fingerprint => $fingerprint,
+	};
+}
+
+# manifest.js.gz
+sub js ($$) {
+	my ($env, $list) = @_;
+	eval { require IO::Compress::Gzip } or return [ 404, [], [] ];
+
+	my $manifest = { -abs2urlpath => {}, -mtime => 0 };
+	for my $ibx (@$list) {
+		if (defined(my $max = $ibx->max_git_part)) {
+			for my $epoch (0..$max) {
+				manifest_add($manifest, $ibx, $epoch);
+			}
+		} else {
+			manifest_add($manifest, $ibx);
+		}
+	}
+	my $abs2urlpath = delete $manifest->{-abs2urlpath};
+	my $mtime = delete $manifest->{-mtime};
+	while (my ($url_path, $repo) = each %$manifest) {
+		defined(my $abs = $repo->{reference}) or next;
+		$repo->{reference} = $abs2urlpath->{$abs};
+	}
+	my $out;
+	IO::Compress::Gzip::gzip(\(($json ||= _json())->encode($manifest)) =>
+				 \$out);
+	$manifest = undef;
+	[ 200, [ qw(Content-Type application/gzip),
+		 'Last-Modified', time2str($mtime),
+		 'Content-Length', bytes::length($out) ], [ $out ] ];
+}
+
+# not really a stand-alone PSGI app, but maybe it could be...
+sub call {
+	my ($self, $env) = @_;
+
+	if ($env->{PATH_INFO} eq '/manifest.js.gz') {
+		# grokmirror uses relative paths, so it's domain-dependent
+		my $list = $self->{manifest_cb}->($self, $env, 'manifest');
+		js($env, $list);
+	} else { # /
+		my $list = $self->{www_cb}->($self, $env, 'www');
+		html($env, $list);
+	}
 }
 
 1;
diff --git a/t/www_listing.t b/t/www_listing.t
new file mode 100644
index 0000000..f9d543e
--- /dev/null
+++ b/t/www_listing.t
@@ -0,0 +1,138 @@
+# Copyright (C) 2019 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+# manifest.js.gz generation and grok-pull integration test
+use strict;
+use warnings;
+use Test::More;
+use PublicInbox::Spawn qw(which);
+use File::Temp qw/tempdir/;
+require './t/common.perl';
+my @mods = qw(URI::Escape Plack::Builder IPC::Run Digest::SHA HTTP::Tiny
+		IO::Compress::Gzip IO::Uncompress::Gunzip Net::HTTP);
+foreach my $mod (@mods) {
+	eval("require $mod") or plan skip_all => "$mod missing for $0";
+}
+use_ok 'PublicInbox::WwwListing';
+use_ok 'PublicInbox::Git';
+
+my $fi_data = './t/git.fast-import-data';
+my $tmpdir = tempdir('www_listing-tmp-XXXXXX', TMPDIR => 1, CLEANUP => 1);
+my $bare = PublicInbox::Git->new("$tmpdir/bare.git");
+is(system(qw(git init -q --bare), $bare->{git_dir}), 0, 'git init --bare');
+is(PublicInbox::WwwListing::fingerprint($bare), undef,
+	'empty repo has no fingerprint');
+
+my $cmd = [ 'git', "--git-dir=$bare->{git_dir}", qw(fast-import --quiet) ];
+ok(IPC::Run::run($cmd, '<', $fi_data), 'fast-import');
+
+like(PublicInbox::WwwListing::fingerprint($bare), qr/\A[a-f0-9]{40}\z/,
+	'got fingerprint with non-empty repo');
+
+my $pid;
+END { kill 'TERM', $pid if defined $pid };
+SKIP: {
+	my $json = eval { PublicInbox::WwwListing::_json() };
+	skip "JSON module missing: $@", 1 if $@;
+	my $err = "$tmpdir/stderr.log";
+	my $out = "$tmpdir/stdout.log";
+	my $alt = "$tmpdir/alt.git";
+	my $cfgfile = "$tmpdir/config";
+	my $v2 = "$tmpdir/v2";
+	my $httpd = 'blib/script/public-inbox-httpd';
+	use IO::Socket::INET;
+	my %opts = (
+		LocalAddr => '127.0.0.1',
+		ReuseAddr => 1,
+		Proto => 'tcp',
+		Type => SOCK_STREAM,
+		Listen => 1024,
+	);
+	my $sock = IO::Socket::INET->new(%opts);
+	ok($sock, 'sock created');
+	my ($host, $port) = ($sock->sockhost, $sock->sockport);
+	my @clone = qw(git clone -q -s --bare);
+	is(system(@clone, $bare->{git_dir}, $alt), 0, 'clone shared repo');
+
+	for my $i (0..2) {
+		is(system(@clone, $alt, "$v2/git/$i.git"), 0, "clone epoch $i");
+	}
+	ok(open(my $fh, '>', "$v2/inbox.lock"), 'mock a v2 inbox');
+	open $fh, '>', "$alt/description" or die;
+	print $fh "we're all clones\n" or die;
+	close $fh or die;
+	is(system('git', "--git-dir=$alt", qw(config gitweb.owner lorelei)), 0,
+		'set gitweb user');
+	ok(unlink("$bare->{git_dir}/description"), 'removed bare/description');
+	open $fh, '>', $cfgfile or die;
+	print $fh <<"" or die;
+[publicinbox "bare"]
+	mainrepo = $bare->{git_dir}
+	url = http://$host/bare
+	address = bare\@example.com
+[publicinbox "alt"]
+	mainrepo = $alt
+	url = http://$host/alt
+	address = alt\@example.com
+[publicinbox "v2"]
+	mainrepo = $v2
+	url = http://$host/v2
+	address = v2\@example.com
+
+	close $fh or die;
+	my $env = { PI_CONFIG => $cfgfile };
+	my $cmd = [ $httpd, "--stdout=$out", "--stderr=$err" ];
+	$pid = spawn_listener($env, $cmd, [$sock]);
+	$sock = undef;
+	my $http = Net::HTTP->new(Host => "$host:$port");
+	$http->write_request(GET => '/manifest.js.gz');
+	my ($code, undef, %h) = $http->read_response_headers;
+	is($code, 200, 'got manifest');
+	my $tmp;
+	my $body = '';
+	while (1) {
+		my $n = $http->read_entity_body(my $buf, 65536);
+		die unless defined $n;
+		last if $n == 0;
+		$body .= $buf;
+	}
+	IO::Uncompress::Gunzip::gunzip(\$body => \$tmp);
+	my $manifest = $json->decode($tmp);
+	ok(my $clone = $manifest->{'/alt'}, '/alt in manifest');
+	is($clone->{owner}, 'lorelei', 'owner set');
+	is($clone->{reference}, '/bare', 'reference detected');
+	is($clone->{description}, "we're all clones", 'description read');
+	ok(my $bare = $manifest->{'/bare'}, '/bare in manifest');
+	is($bare->{description}, 'Unnamed repository',
+		'missing $GIT_DIR/description fallback');
+
+	like($bare->{fingerprint}, qr/\A[a-f0-9]{40}\z/, 'fingerprint');
+	is($clone->{fingerprint}, $bare->{fingerprint}, 'fingerprint matches');
+
+	is(HTTP::Date::time2str($bare->{modified}), $h{'Last-Modified'},
+		'modified field and Last-Modified header match');
+
+	ok($manifest->{'/v2/0'}, 'v2 epoch appeared');
+
+	skip 'skipping grok-pull integration test', 2 if !which('grok-pull');
+
+	ok(mkdir("$tmpdir/mirror"), 'prepare grok mirror dest');
+	open $fh, '>', "$tmpdir/repos.conf" or die;
+	print $fh <<"" or die;
+# You can pull from multiple grok mirrors, just create
+# a separate section for each mirror. The name can be anything.
+[test]
+site = http://$host:$port
+manifest = http://$host:$port/manifest.js.gz
+toplevel = $tmpdir/mirror
+mymanifest = $tmpdir/local-manifest.js.gz
+
+	close $fh or die;
+
+	system(qw(grok-pull -c), "$tmpdir/repos.conf");
+	is($? >> 8, 127, 'grok-pull exit code as expected');
+	for (qw(alt bare v2/0 v2/1 v2/2)) {
+		ok(-d "$tmpdir/mirror/$_", "grok-pull created $_");
+	}
+}
+
+done_testing();
-- 
EW


  parent reply index

Thread overview: 6+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2019-06-09  4:31 [PATCH 0/4] grokmirror-compatible manifests Eric Wong (Contractor, The Linux Foundation)
2019-06-09  4:31 ` [PATCH 1/4] wwwlisting: allow hiding entries from manifest Eric Wong (Contractor, The Linux Foundation)
2019-06-09  4:31 ` Eric Wong (Contractor, The Linux Foundation) [this message]
2019-06-09  4:31 ` [PATCH 3/4] www: wire up /$INBOX/manifest.js.gz, too Eric Wong (Contractor, The Linux Foundation)
2019-06-09  4:31 ` [PATCH 4/4] www: support $INBOX/git/$EPOCH.git for v2 cloning Eric Wong (Contractor, The Linux Foundation)
2019-06-10  6:21 ` [PATCH 5/4] git: ensure ->modified returns an integer Eric Wong (Contractor, The Linux Foundation)

Reply instructions:

You may reply publically to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

  List information: http://public-inbox.org/README

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20190609043105.22338-3-e@80x24.org \
    --to=e@80x24.org \
    --cc=meta@public-inbox.org \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link

user/dev discussion of public-inbox itself

Archives are clonable:
	git clone --mirror http://public-inbox.org/meta
	git clone --mirror http://czquwvybam4bgbro.onion/meta
	git clone --mirror http://hjrcffqmbrq6wope.onion/meta
	git clone --mirror http://ou63pmih66umazou.onion/meta

Newsgroups are available over NNTP:
	nntp://news.public-inbox.org/inbox.comp.mail.public-inbox.meta
	nntp://ou63pmih66umazou.onion/inbox.comp.mail.public-inbox.meta
	nntp://czquwvybam4bgbro.onion/inbox.comp.mail.public-inbox.meta
	nntp://hjrcffqmbrq6wope.onion/inbox.comp.mail.public-inbox.meta
	nntp://news.gmane.org/gmane.mail.public-inbox.general

 note: .onion URLs require Tor: https://www.torproject.org/

AGPL code for this site: git clone https://public-inbox.org/ public-inbox