[PATCH 05/10] www_coderepo: an alternative to cgit

[Eric Wong <e@80x24.org>]

This will allow it to easily map a single coderepo to multiple
inboxes (or multiple coderepos to any number of inboxes).
For now, this is just a summary, but $REPO/$OID/s/ support
will be added, along with archive downloads.

Indexing of coderepos will probably be supported via -extindex,
only.
---
 MANIFEST                       |   1 +
 lib/PublicInbox/Cgit.pm        |  14 +--
 lib/PublicInbox/Config.pm      |   2 +-
 lib/PublicInbox/Git.pm         |  14 ++-
 lib/PublicInbox/GitAsyncCat.pm |  17 +--
 lib/PublicInbox/WWW.pm         |  12 ++-
 lib/PublicInbox/WwwCoderepo.pm | 185 +++++++++++++++++++++++++++++++++
 lib/PublicInbox/WwwStream.pm   |  16 ++-
 t/solver_git.t                 |  33 +++++-
 9 files changed, 266 insertions(+), 28 deletions(-)
 create mode 100644 lib/PublicInbox/WwwCoderepo.pm

diff --git a/MANIFEST b/MANIFEST
index 35382d2d..cf6d97e1 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -342,6 +342,7 @@ lib/PublicInbox/Watch.pm
 lib/PublicInbox/WwwAltId.pm
 lib/PublicInbox/WwwAtomStream.pm
 lib/PublicInbox/WwwAttach.pm
+lib/PublicInbox/WwwCoderepo.pm
 lib/PublicInbox/WwwHighlight.pm
 lib/PublicInbox/WwwListing.pm
 lib/PublicInbox/WwwStatic.pm
diff --git a/lib/PublicInbox/Cgit.pm b/lib/PublicInbox/Cgit.pm
index a63f8902..1112d9f8 100644
--- a/lib/PublicInbox/Cgit.pm
+++ b/lib/PublicInbox/Cgit.pm
@@ -7,6 +7,7 @@
 
 package PublicInbox::Cgit;
 use v5.12;
+use parent qw(PublicInbox::WwwCoderepo);
 use PublicInbox::GitHTTPBackend;
 use PublicInbox::Git;
 # not bothering with Exporter for a one-off
@@ -52,10 +53,6 @@ sub locate_cgit ($) {
 sub new {
 	my ($class, $pi_cfg) = @_;
 	my ($cgit_bin, $cgit_data) = locate_cgit($pi_cfg);
-	# TODO: support gitweb and other repository viewers?
-	if (defined(my $cgitrc = $pi_cfg->{-cgitrc_unparsed})) {
-		$pi_cfg->parse_cgitrc($cgitrc, 0);
-	}
 	my $self = bless {
 		cmd => [ $cgit_bin ],
 		cgit_data => $cgit_data,
@@ -63,14 +60,7 @@ sub new {
 	}, $class;
 
 	# some cgit repos may not be mapped to inboxes, so ensure those exist:
-	my $code_repos = $pi_cfg->{-code_repos};
-	for my $k (grep(/\Acoderepo\.(?:.+)\.dir\z/, keys %$pi_cfg)) {
-		$k = substr($k, length('coderepo.'), -length('.dir'));
-		$code_repos->{$k} //= $pi_cfg->fill_code_repo($k);
-	}
-	while (my ($nick, $repo) = each %$code_repos) {
-		$self->{"\0$nick"} = $repo;
-	}
+	PublicInbox::WwwCoderepo::prepare_coderepos($self);
 	my $s = join('|', map { quotemeta } keys %{$pi_cfg->{-cgit_static}});
 	$self->{static} = qr/\A($s)\z/;
 	$self;
diff --git a/lib/PublicInbox/Config.pm b/lib/PublicInbox/Config.pm
index 1b5d87e2..42bd9438 100644
--- a/lib/PublicInbox/Config.pm
+++ b/lib/PublicInbox/Config.pm
@@ -343,7 +343,7 @@ sub fill_code_repo {
 		$git->{cgit_url} = $cgits = _array($cgits);
 		$self->{"$pfx.cgiturl"} = $cgits;
 	}
-
+	$git->{nick} = $nick;
 	$git;
 }
 
diff --git a/lib/PublicInbox/Git.pm b/lib/PublicInbox/Git.pm
index 2f0bb6a0..395add1f 100644
--- a/lib/PublicInbox/Git.pm
+++ b/lib/PublicInbox/Git.pm
@@ -463,6 +463,16 @@ sub host_prefix_url ($$) {
 	"$scheme://$host_port". ($env->{SCRIPT_NAME} || '/') . $url;
 }
 
+sub base_url { # for coderepos, PSGI-only
+	my ($self, $env) = @_; # env - PSGI env
+	my $url = host_prefix_url($env, '');
+	# for mount in Plack::Builder
+	$url .= '/' if substr($url, -1, 1) ne '/';
+	$url . $self->{nick} . '/';
+}
+
+sub isrch {} # TODO
+
 sub pub_urls {
 	my ($self, $env) = @_;
 	if (my $urls = $self->{cgit_url}) {
@@ -518,11 +528,11 @@ sub description {
 }
 
 sub cloneurl {
-	my ($self) = @_;
+	my ($self, $env) = @_;
 	$self->{cloneurl} // do {
 		my @urls = split(/\s+/s, try_cat("$self->{git_dir}/cloneurl"));
 		scalar(@urls) ? ($self->{cloneurl} = \@urls) : undef;
-	} // [];
+	} // [ substr(base_url($self, $env), 0, -1) ];
 }
 
 # for grokmirror, which doesn't read gitweb.description
diff --git a/lib/PublicInbox/GitAsyncCat.pm b/lib/PublicInbox/GitAsyncCat.pm
index b32c2fd3..613dbf7e 100644
--- a/lib/PublicInbox/GitAsyncCat.pm
+++ b/lib/PublicInbox/GitAsyncCat.pm
@@ -45,6 +45,16 @@ sub event_step {
 	}
 }
 
+sub watch_cat {
+	my ($git) = @_;
+	$git->{async_cat} //= do {
+		my $self = bless { git => $git }, __PACKAGE__;
+		$git->{in}->blocking(0);
+		$self->SUPER::new($git->{in}, EPOLLIN|EPOLLET);
+		\undef; # this is a true ref()
+	};
+}
+
 sub ibx_async_cat ($$$$) {
 	my ($ibx, $oid, $cb, $arg) = @_;
 	my $git = $ibx->{git} // $ibx->git;
@@ -60,12 +70,7 @@ sub ibx_async_cat ($$$$) {
 		\undef;
 	} else { # read-only end of git-cat-file pipe
 		$git->cat_async($oid, $cb, $arg);
-		$git->{async_cat} //= do {
-			my $self = bless { git => $git }, __PACKAGE__;
-			$git->{in}->blocking(0);
-			$self->SUPER::new($git->{in}, EPOLLIN|EPOLLET);
-			\undef; # this is a true ref()
-		};
+		watch_cat($git);
 	}
 }
 
diff --git a/lib/PublicInbox/WWW.pm b/lib/PublicInbox/WWW.pm
index 1df5572d..d0e20fb5 100644
--- a/lib/PublicInbox/WWW.pm
+++ b/lib/PublicInbox/WWW.pm
@@ -197,7 +197,9 @@ sub news_cgit_fallback ($) {
 	my $www = $ctx->{www};
 	my $env = $ctx->{env};
 	my $res = $www->news_www->call($env);
-	$res->[0] == 404 ? $www->cgit->call($env) : $res;
+	$res = $www->cgit->call($env) if $res->[0] == 404;
+	$res = $www->coderepo->srv($ctx) if $res->[0] == 404;
+	$res;
 }
 
 # returns undef if valid, array ref response if invalid
@@ -494,6 +496,14 @@ sub cgit {
 	}
 }
 
+sub coderepo {
+	my ($self) = @_;
+	$self->{coderepo} //= do {
+		require PublicInbox::WwwCoderepo;
+		PublicInbox::WwwCoderepo->new($self->{pi_cfg});
+	}
+}
+
 # GET $INBOX/manifest.js.gz
 sub get_inbox_manifest ($$$) {
 	my ($ctx, $inbox, $key) = @_;
diff --git a/lib/PublicInbox/WwwCoderepo.pm b/lib/PublicInbox/WwwCoderepo.pm
new file mode 100644
index 00000000..4b1a4f9b
--- /dev/null
+++ b/lib/PublicInbox/WwwCoderepo.pm
@@ -0,0 +1,185 @@
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+#
+# Standalone code repository viewer for users w/o cgit
+package PublicInbox::WwwCoderepo;
+use v5.12;
+use File::Temp 0.19 (); # newdir
+use PublicInbox::ViewVCS;
+use PublicInbox::WwwStatic qw(r);
+use PublicInbox::GitHTTPBackend;
+use PublicInbox::Git;
+use PublicInbox::GitAsyncCat;
+use PublicInbox::WwwStream;
+use PublicInbox::Hval qw(ascii_html);
+
+my $EACH_REF = "git for-each-ref --sort=-creatordate --format='%(HEAD)%00".
+	join('%00', map { "%($_)" }
+	qw(objectname refname:short subject creatordate:short))."'";
+
+# shared with PublicInbox::Cgit
+sub prepare_coderepos {
+	my ($self) = @_;
+	my $pi_cfg = $self->{pi_cfg};
+
+	# TODO: support gitweb and other repository viewers?
+	if (defined(my $cgitrc = $pi_cfg->{-cgitrc_unparsed})) {
+		$pi_cfg->parse_cgitrc($cgitrc, 0);
+	}
+	my $code_repos = $pi_cfg->{-code_repos};
+	for my $k (grep(/\Acoderepo\.(?:.+)\.dir\z/, keys %$pi_cfg)) {
+		$k = substr($k, length('coderepo.'), -length('.dir'));
+		$code_repos->{$k} //= $pi_cfg->fill_code_repo($k);
+	}
+	while (my ($nick, $repo) = each %$code_repos) {
+		$self->{"\0$nick"} = $repo;
+	}
+}
+
+sub new {
+	my ($cls, $pi_cfg) = @_;
+	my $self = bless { pi_cfg => $pi_cfg }, $cls;
+	prepare_coderepos($self);
+	$self->{$_} = 10 for qw(summary_branches summary_tags);
+	$self->{$_} = 10 for qw(summary_log);
+	$self;
+}
+
+sub summary_finish {
+	my ($ctx) = @_;
+	my $wcb = delete($ctx->{env}->{'qspawn.wcb'}) or return; # already done
+	my @x = split(/\n\n/sm, delete($ctx->{-each_refs}));
+	PublicInbox::WwwStream::html_init($ctx);
+	my $zfh = $ctx->zfh;
+
+	# git log
+	my @r = split(/\n/s, pop(@x) // '');
+	my $last = pop(@r) if scalar(@r) > $ctx->{wcr}->{summary_log};
+	print $zfh '<pre><a id=log>$</a> '.
+		"git log --pretty=format:'%h %s (%cs)%d'\n";
+	for (@r) {
+		my $d; # decorations
+		s/^ \(([^\)]+)\)// and $d = $1;
+		substr($_, 0, 1, '');
+		my ($H, $h, $cs, $s) = split(/ /, $_, 4);
+		print $zfh "<a\nhref=./$H/s/>$h</a> ", ascii_html($s),
+			" (", $cs, ")\n";
+		print $zfh "\t(", ascii_html($d), ")\n" if $d;
+	}
+	print $zfh "# no commits, yet\n" if !@r;
+	print $zfh "...\n" if $last;
+
+	# README
+	my ($bref, $oid, $ref_path) = @{delete $ctx->{-readme}};
+	if ($bref) {
+		my $l = PublicInbox::Linkify->new;
+		$$bref =~ s/\s*\z//sm;
+		print $zfh "\n<a id=readme>\$</a> " .
+			"git cat-file blob <a href=./$oid/s/>",
+			ascii_html($ref_path), "</a>\n",
+			$l->to_html($$bref), '</pre><hr><pre>';
+	}
+
+	# refs/heads
+	print $zfh "<a id=heads># heads (aka `branches'):</a>\n\$ " .
+		"git for-each-ref --sort=-creatordate refs/heads" .
+		" \\\n\t--format='%(HEAD) ". # no space for %(align:) hint
+		"%(refname:short) %(subject) (%(creatordate:short))'\n";
+	@r = split(/^/sm, shift(@x) // '');
+	$last = pop(@r) if scalar(@r) > $ctx->{wcr}->{summary_branches};
+	for (@r) {
+		my ($pfx, $oid, $ref, $s, $cd) = split(/\0/);
+		utf8::decode($_) for ($ref, $s);
+		chomp $cd;
+		my $align = length($ref) < 12 ? ' ' x (12 - length($ref)) : '';
+		print $zfh "$pfx <a\nhref=./$oid/s/>", ascii_html($ref),
+			"</a>$align ", ascii_html($s), " ($cd)\n";
+	}
+	print $zfh "# no heads (branches) yet...\n" if !@r;
+	print $zfh "...\n" if $last;
+	print $zfh "\n<a id=tags># tags:</a>\n\$ " .
+		"git for-each-ref --sort=-creatordate refs/tags" .
+		" \\\n\t--format='". # no space for %(align:) hint
+		"%(refname:short) %(subject) (%(creatordate:short))'\n";
+	@r = split(/^/sm, shift(@x) // '');
+	$last = pop(@r) if scalar(@r) > $ctx->{wcr}->{summary_tags};
+	for (@r) {
+		my (undef, $oid, $ref, $s, $cd) = split(/\0/);
+		utf8::decode($_) for ($ref, $s);
+		chomp $cd;
+		my $align = length($ref) < 12 ? ' ' x (12 - length($ref)) : '';
+		print $zfh "<a\nhref=./$oid/s/>", ascii_html($ref),
+			"</a>$align ", ascii_html($s), " ($cd)\n";
+	}
+	print $zfh "# no tags yet...\n" if !@r;
+	print $zfh "...\n" if $last;
+	$wcb->($ctx->html_done('</pre>'));
+}
+
+sub capture_refs ($$) { # psgi_qx callback to capture git-for-each-ref + git-log
+	my ($bref, $ctx) = @_;
+	my $qsp_err = delete $ctx->{-qsp_err};
+	$ctx->{-each_refs} = $$bref;
+	summary_finish($ctx) if $ctx->{-readme};
+}
+
+sub set_readme { # git->cat_async callback
+	my ($bref, $oid, $type, $size, $ctx) = @_;
+	my $ref_path = shift @{$ctx->{-nr_readme_tries}}; # e.g. HEAD:README
+	if ($type eq 'blob' && !$ctx->{-readme}) {
+		$ctx->{-readme} = [ $bref, $oid, $ref_path ];
+	} elsif (scalar @{$ctx->{-nr_readme_tries}} == 0) {
+		$ctx->{-readme} //= []; # nothing left to try
+	} # or try another README...
+	summary_finish($ctx) if $ctx->{-each_refs} && $ctx->{-readme};
+}
+
+sub summary {
+	my ($self, $ctx) = @_;
+	$ctx->{wcr} = $self;
+	my $nb = $self->{summary_branches} + 1;
+	my $nt = $self->{summary_tags} + 1;
+	my $nl = $self->{summary_log} + 1;
+	my $qsp = PublicInbox::Qspawn->new([qw(/bin/sh -c),
+		"$EACH_REF --count=$nb refs/heads; echo && " .
+		"$EACH_REF --count=$nt refs/tags; echo && " .
+		"git log -$nl --pretty=format:'%d %H %h %cs %s' --" ],
+		{ GIT_DIR => $ctx->{git}->{git_dir} });
+	$qsp->{qsp_err} = \($ctx->{-qsp_err} = '');
+	my @try = qw(HEAD:README HEAD:README.md); # TODO: configurable
+	$ctx->{-nr_readme_tries} = [ @try ];
+	$ctx->{git}->cat_async($_, \&set_readme, $ctx) for @try;
+	if ($ctx->{env}->{'pi-httpd.async'}) {
+		PublicInbox::GitAsyncCat::watch_cat($ctx->{git});
+	} else { # synchronous
+		$ctx->{git}->cat_async_wait;
+	}
+	sub { # $_[0] => PublicInbox::HTTP::{Identity,Chunked}
+		$ctx->{env}->{'qspawn.wcb'} = $_[0];
+		$qsp->psgi_qx($ctx->{env}, undef, \&capture_refs, $ctx);
+	}
+}
+
+sub srv { # endpoint called by PublicInbox::WWW
+	my ($self, $ctx) = @_;
+	my $path_info = $ctx->{env}->{PATH_INFO};
+	my $git;
+	# handle clone requests
+	if ($path_info =~ m!\A/(.+?)/($PublicInbox::GitHTTPBackend::ANY)\z!x) {
+		$git = $self->{"\0$1"} and return
+			PublicInbox::GitHTTPBackend::serve($ctx->{env},$git,$2);
+	}
+	$path_info =~ m!\A/(.+?)/\z! and
+		($ctx->{git} = $self->{"\0$1"}) and return summary($self, $ctx);
+	if ($path_info =~ m!\A/(.+?)\z! and ($git = $self->{"\0$1"})) {
+		my $qs = $ctx->{env}->{QUERY_STRING};
+		my $url = $git->base_url($ctx->{env});
+		$url .= "?$qs" if $qs ne '';
+		[ 301, [ Location => $url, 'Content-Type' => 'text/plain' ],
+			[ "Redirecting to $url\n" ] ];
+	} else {
+		r(404);
+	}
+}
+
+1;
diff --git a/lib/PublicInbox/WwwStream.pm b/lib/PublicInbox/WwwStream.pm
index 16442d51..92d243eb 100644
--- a/lib/PublicInbox/WwwStream.pm
+++ b/lib/PublicInbox/WwwStream.pm
@@ -18,7 +18,7 @@ https://public-inbox.org/public-inbox.git) ];
 
 sub base_url ($) {
 	my $ctx = shift;
-	my $base_url = $ctx->{ibx}->base_url($ctx->{env});
+	my $base_url = ($ctx->{ibx} // $ctx->{git})->base_url($ctx->{env});
 	chop $base_url; # no trailing slash for clone
 	$base_url;
 }
@@ -40,7 +40,7 @@ sub async_eml { # for async_blob_cb
 
 sub html_top ($) {
 	my ($ctx) = @_;
-	my $ibx = $ctx->{ibx};
+	my $ibx = $ctx->{ibx} // $ctx->{git};
 	my $desc = ascii_html($ibx->description);
 	my $title = delete($ctx->{-title_html}) // $desc;
 	my $upfx = $ctx->{-upfx} || '';
@@ -84,8 +84,11 @@ sub html_top ($) {
 		'</head><body>'. $top . (delete($ctx->{-html_tip}) // '');
 }
 
+sub inboxes { () } # TODO
+
 sub coderepos ($) {
 	my ($ctx) = @_;
+	$ctx->{ibx} // return inboxes($ctx);
 	my $cr = $ctx->{ibx}->{coderepo} // return ();
 	my $cfg = $ctx->{www}->{pi_cfg};
 	my $upfx = ($ctx->{-upfx} // ''). '../';
@@ -114,8 +117,8 @@ sub _html_end {
 	my ($ctx) = @_;
 	my $upfx = $ctx->{-upfx} || '';
 	my $m = "${upfx}_/text/mirror/";
-	my $x;
-	if ($ctx->{ibx}->can('cloneurl')) {
+	my $x = '';
+	if ($ctx->{ibx} && $ctx->{ibx}->can('cloneurl')) {
 		$x = <<EOF;
 This is a public inbox, see <a
 href="$m">mirroring instructions</a>
@@ -139,12 +142,15 @@ as well as URLs for IMAP folder(s).
 EOM
 			}
 		}
-	} else {
+	} elsif ($ctx->{ibx}) { # extindex
 		$x = <<EOF;
 This is an external index of several public inboxes,
 see <a href="$m">mirroring instructions</a> on how to clone and mirror
 all data and code used by this external index.
 EOF
+	} elsif ($ctx->{git}) { # coderepo
+		$x = join('', map { "git clone $_\n" }
+			@{$ctx->{git}->cloneurl($ctx->{env})});
 	}
 	chomp $x;
 	'<hr><pre>'.join("\n\n", coderepos($ctx), $x).'</pre></body></html>'
diff --git a/t/solver_git.t b/t/solver_git.t
index e347c711..d6936c47 100644
--- a/t/solver_git.t
+++ b/t/solver_git.t
@@ -9,7 +9,9 @@ require_git(2.6);
 use PublicInbox::ContentHash qw(git_sha);
 use PublicInbox::Spawn qw(popen_rd);
 require_mods(qw(DBD::SQLite Search::Xapian Plack::Util));
-my $git_dir = xqx([qw(git rev-parse --git-dir)], undef, {2 => \(my $null)});
+my $rdr = { 2 => \(my $null) };
+my $git_dir = xqx([qw(git rev-parse --git-common-dir)], undef, $rdr);
+$git_dir = xqx([qw(git rev-parse --git-dir)], undef, $rdr) if $? != 0;
 $? == 0 or plan skip_all => "$0 must be run from a git working tree";
 chomp $git_dir;
 
@@ -300,6 +302,35 @@ EOF
 		is($res->code, 200, 'shows commit w/ utf8.eml');
 		like($res->content, qr/El&#233;anor/,
 				'UTF-8 commit shown properly');
+
+		# WwwCoderepo
+		my $olderr;
+		if (defined $ENV{PLACK_TEST_EXTERNALSERVER_URI}) {
+			ok(!-s "$tmpdir/stderr.log",
+				'nothing in stderr.log, yet');
+		} else {
+			open $olderr, '>&', \*STDERR or xbail "open: $!";
+			open STDERR, '+>>', "$tmpdir/stderr.log" or
+				xbail "open: $!";
+		}
+		$res = $cb->(GET('/binfoo/'));
+		defined($ENV{PLACK_TEST_EXTERNALSERVER_URI}) or
+			open STDERR, '>&', $olderr or xbail "open: $!";
+		is($res->code, 200, 'coderepo summary (binfoo)');
+		if (ok(-s "$tmpdir/stderr.log")) {
+			open my $fh, '<', "$tmpdir/stderr.log" or xbail $!;
+			my $s = do { local $/; <$fh> };
+			open $fh, '>', "$tmpdir/stderr.log" or xbail $!;
+			ok($s =~ s/^fatal: your current branch.*?\n//sm,
+				'got current branch warning');
+			ok($s =~ s/^.*? exit status=[1-9]+ .*?\n//sm,
+				'got exit status warning');
+			is($s, '', 'no unexpected warnings on empty coderepo');
+		}
+		$res = $cb->(GET('/public-inbox/'));
+		is($res->code, 200, 'coderepo summary (public-inbox)');
+		$res = $cb->(GET('/public-inbox'));
+		is($res->code, 301, 'redirected');
 	};
 	test_psgi(sub { $www->call(@_) }, $client);
 	my $env = { PI_CONFIG => $cfgpath, TMPDIR => $tmpdir };