[PATCH 08/14] www: load and use cindex join data

[Eric Wong <e@80x24.org>]

This is a major step in solving the problem of having to
manually associate hundreds/thousands of coderepos with
hundreds/thousands of public-inboxes to power solver
(and more).
---
 lib/PublicInbox/CodeSearch.pm    | 153 +++++++++++++++++++++++++++++--
 lib/PublicInbox/CodeSearchIdx.pm |  42 ++++-----
 lib/PublicInbox/Config.pm        |  39 +++++++-
 lib/PublicInbox/Search.pm        |  17 ++++
 lib/PublicInbox/SearchIdx.pm     |  10 +-
 lib/PublicInbox/SolverGit.pm     |   6 +-
 lib/PublicInbox/View.pm          |   7 +-
 lib/PublicInbox/WWW.pm           |   1 +
 lib/PublicInbox/WwwCoderepo.pm   |  41 ++++++++-
 lib/PublicInbox/WwwText.pm       |  19 +++-
 t/cindex.t                       |  28 +++++-
 xt/solver.t                      |   3 +-
 12 files changed, 312 insertions(+), 54 deletions(-)

diff --git a/lib/PublicInbox/CodeSearch.pm b/lib/PublicInbox/CodeSearch.pm
index eb057525..7d7f6df6 100644
--- a/lib/PublicInbox/CodeSearch.pm
+++ b/lib/PublicInbox/CodeSearch.pm
@@ -21,7 +21,7 @@ use constant {
 our @CODE_NRP;
 our @CODE_VMAP = (
 	[ AT, 'd:' ], # mairix compat
-	[ AT, 'dt:' ], # mail compat
+	[ AT, 'dt:' ], # public-inbox mail compat
 	[ CT, 'ct:' ],
 );
 
@@ -51,7 +51,7 @@ my %prob_prefix = ( # copied from PublicInbox::Search
 sub new {
 	my ($cls, $dir, $cfg) = @_;
 	# can't have a PublicInbox::Config here due to circular refs
-	bless { xpfx => "$dir/cidx".CIDX_SCHEMA_VER,
+	bless { topdir => $dir, xpfx => "$dir/cidx".CIDX_SCHEMA_VER,
 		-cfg_f => $cfg->{-f} }, $cls;
 }
 
@@ -63,7 +63,20 @@ sub join_data {
 	my $cur = $self->xdb->get_metadata($key) or return;
 	$cur = eval { PublicInbox::Config::json()->decode(uncompress($cur)) };
 	warn "E: $@ (corrupt metadata in `$key' key?)" if $@;
-	$cur;
+	my @m = grep { ref($cur->{$_}) ne 'ARRAY' } qw(ekeys roots ibx2root);
+	if (@m) {
+		warn <<EOM;
+W: $self->{topdir} join data for $self->{-cfg_f} missing: @m
+EOM
+		undef;
+	} elsif (@{$cur->{ekeys}} != @{$cur->{ibx2root}}) {
+		warn <<EOM;
+W: $self->{topdir} join data for $self->{-cfg_f} mismatched ekeys and ibx2root
+EOM
+		undef;
+	} else {
+		$cur;
+	}
 }
 
 sub qparse_new ($) {
@@ -196,16 +209,136 @@ sub roots2paths { # for diagnostics
 	\%ret;
 }
 
-sub paths2roots { # for diagnostics
-	my ($self) = @_;
+sub root_oids ($$) {
+	my ($self, $git_dir) = @_;
+	my @ids = $self->docids_by_postlist('P'.$git_dir);
+	@ids or warn <<"";
+BUG? (non-fatal) `$git_dir' not indexed in $self->{topdir}
+
+	warn <<"" if @ids > 1;
+BUG: (non-fatal) $git_dir indexed multiple times in $self->{topdir}
+
 	my %ret;
-	my $tmp = roots2paths($self);
-	for my $root_oidhex (keys %$tmp) {
-		my $paths = delete $tmp->{$root_oidhex};
-		push @{$ret{$_}}, $root_oidhex for @$paths;
+	for my $docid (@ids) {
+		my @oids = xap_terms('G', $self->xdb, $docid);
+		@ret{@oids} = @oids;
+	}
+	sort keys %ret;
+}
+
+sub paths2roots {
+	my ($self, $paths) = @_;
+	my %ret;
+	if ($paths) {
+		for my $p (keys %$paths) { @{$ret{$p}} = root_oids($self, $p) }
+	} else {
+		my $tmp = roots2paths($self);
+		for my $root_oidhex (keys %$tmp) {
+			my $paths = delete $tmp->{$root_oidhex};
+			push @{$ret{$_}}, $root_oidhex for @$paths;
+		}
+		@$_ = sort(@$_) for values %ret;
 	}
-	@$_ = sort(@$_) for values %ret;
 	\%ret;
 }
 
+sub load_commit_times { # each_cindex callback
+	my ($self, $todo) = @_; # todo = [ [ time, git ], [ time, git ] ...]
+	my (@pending, $rec, $dir, @ids, $doc);
+	while ($rec = shift @$todo) {
+		@ids = $self->docids_by_postlist('P'.$rec->[1]->{git_dir});
+		if (@ids) {
+			warn <<EOM if @ids > 1;
+W: $rec->[1]->{git_dir} indexed multiple times in $self->{topdir}
+EOM
+			for (@ids) {
+				$doc = $self->get_doc($_) // next;
+				$rec->[0] = int_val($doc, CT);
+				last;
+			}
+		} else { # may be in another cindex:
+			push @pending, $rec;
+		}
+	}
+	@$todo = @pending;
+}
+
+sub load_coderepos { # each_cindex callback
+	my ($self, $pi_cfg) = @_;
+	my $name = $self->{name};
+	my $cfg_f = $pi_cfg->{-f};
+	my $lpfx = $self->{localprefix} or return warn <<EOM;
+W: cindex.$name.localprefix unset in $cfg_f, ignoring cindex.$name
+EOM
+	my $lre = join('|', map { $_ .= '/'; tr!/!/!s; quotemeta } @$lpfx);
+	$lre = qr!\A(?:$lre)!;
+	my $coderepos = $pi_cfg->{-coderepos};
+	my $nick_pfx = $name eq '' ? '' : "$name/";
+	my %dir2cr;
+	for my $p ($self->all_terms('P')) {
+		my $nick = $p;
+		$nick =~ s!$lre!$nick_pfx!s or next;
+		$dir2cr{$p} = $coderepos->{$nick} //= do {
+			my $git = PublicInbox::Git->new($p);
+			$git->{nick} = $nick; # for git->pub_urls
+			$git;
+		};
+	}
+	my $jd = join_data($self) or return warn <<EOM;
+W: cindex.$name.topdir=$self->{topdir} has no usable join data for $cfg_f
+EOM
+	my ($ekeys, $roots, $ibx2root) = @$jd{qw(ekeys roots ibx2root)};
+	my $roots2paths = roots2paths($self);
+	for my $root_offs (@$ibx2root) {
+		my $ekey = shift(@$ekeys) // die 'BUG: {ekeys} empty';
+		scalar(@$root_offs) or next;
+		my $ibx = $pi_cfg->lookup_eidx_key($ekey) // do {
+			warn "W: `$ekey' gone from $cfg_f\n";
+			next;
+		};
+		my $gits = $ibx->{-repo_objs} //= [];
+		my $cr_score = $ibx->{-cr_score} //= {};
+		my %ibx_p2g = map { $_->{git_dir} => $_ } @$gits;
+		my $ibx2self; # cindex has an association w/ inbox?
+		for (@$root_offs) { # sorted by $nr descending
+			my ($nr, $root_off) = @$_;
+			my $root_oid = $roots->[$root_off] // do {
+				warn <<EOM;
+BUG: root #$root_off invalid in join data for `$ekey' with $cfg_f
+EOM
+				next;
+			};
+			my $git_dirs = $roots2paths->{$root_oid};
+			my @gits = map { $dir2cr{$_} // () } @$git_dirs;
+			$cr_score->{$_->{nick}} //= $nr for @gits;
+			@$git_dirs = grep { !$ibx_p2g{$_} } @$git_dirs;
+			# @$git_dirs or warn "W: no matches for $root_oid\n";
+			for (@$git_dirs) {
+				if (my $git = $dir2cr{$_}) {
+					$ibx_p2g{$_} = $git;
+					$ibx2self = 1;
+					$ibx->{-hide}->{www} or
+						push @{$git->{ibx_score}},
+							[ $nr, $ibx->{name} ];
+					push @$gits, $git;
+				} else {
+					warn <<EOM;
+W: no coderepo available for $_ (localprefix=@$lpfx)
+EOM
+				}
+			}
+		}
+		if (@$gits) {
+			push @{$ibx->{-csrch}}, $self if $ibx2self;
+		} else {
+			delete $ibx->{-repo_objs};
+			delete $ibx->{-cr_score};
+		}
+	}
+	for my $git (values %dir2cr) {
+		my $s = $git->{ibx_score};
+		@$s = sort { $b->[0] <=> $a->[0] } @$s if $s;
+	}
+}
+
 1;
diff --git a/lib/PublicInbox/CodeSearchIdx.pm b/lib/PublicInbox/CodeSearchIdx.pm
index bb1d698b..a6cbe0b0 100644
--- a/lib/PublicInbox/CodeSearchIdx.pm
+++ b/lib/PublicInbox/CodeSearchIdx.pm
@@ -172,7 +172,7 @@ sub count_shards { scalar($_[0]->xdb_shards_flat) }
 sub update_commit ($$$) {
 	my ($self, $cmt, $roots) = @_; # fields from @FMT
 	my $x = 'Q'.$cmt->{H};
-	my ($docid, @extra) = sort { $a <=> $b } docids_by_postlist($self, $x);
+	my ($docid, @extra) = sort { $a <=> $b } $self->docids_by_postlist($x);
 	@extra and warn "W: $cmt->{H} indexed multiple times, pruning ",
 			join(', ', map { "#$_" } @extra), "\n";
 	$self->{xdb}->delete_document($_) for @extra;
@@ -377,15 +377,6 @@ sub seen ($$) {
 # used to select the shard for a GIT_DIR
 sub git_dir_hash ($) { hex(substr(sha256_hex($_[0]), 0, 8)) }
 
-sub docids_by_postlist ($$) { # consider moving to PublicInbox::Search
-	my ($self, $q) = @_;
-	my $cur = $self->{xdb}->postlist_begin($q);
-	my $end = $self->{xdb}->postlist_end($q);
-	my @ids;
-	for (; $cur != $end; $cur++) { push(@ids, $cur->get_docid) };
-	@ids;
-}
-
 sub _cb { # run_await cb
 	my ($pid, $cmd, undef, $opt, $cb, $self, $git, @arg) = @_;
 	return if $DO_QUIT;
@@ -452,7 +443,7 @@ sub prep_repo ($$) {
 
 sub check_existing { # retry_reopen callback
 	my ($shard, $self, $git) = @_;
-	my @docids = docids_by_postlist($shard, 'P'.$git->{git_dir});
+	my @docids = $shard->docids_by_postlist('P'.$git->{git_dir});
 	my $docid = shift(@docids) // return get_roots($self, $git);
 	my $doc = $shard->get_doc($docid) //
 			die "BUG: no #$docid ($git->{git_dir})";
@@ -778,7 +769,7 @@ sub prune_init { # via wq_io_do in IDX_SHARDS
 
 sub prune_one { # via wq_io_do in IDX_SHARDS
 	my ($self, $term) = @_;
-	my @docids = docids_by_postlist($self, $term);
+	my @docids = $self->docids_by_postlist($term);
 	for (@docids) {
 		$TXN_BYTES -= $self->{xdb}->get_doclength($_) * 42;
 		$self->{xdb}->delete_document($_);
@@ -894,10 +885,9 @@ sub current_join_data ($) {
 sub score_old_join_data ($$$) {
 	my ($self, $score, $ekeys_new) = @_;
 	my $old = ($JOIN{reset} ? undef : current_join_data($self)) or return;
-	my @old = @$old{qw(ekeys roots ibx2root)};
-	@old == 3 or return warn "W: ekeys/roots missing from old JOIN data\n";
 	progress($self, 'merging old join data...');
-	my ($ekeys_old, $roots_old, $ibx2root_old) = @old;
+	my ($ekeys_old, $roots_old, $ibx2root_old) =
+					@$old{qw(ekeys roots ibx2root)};
 	# score: "ibx_off root_off" => nr
 	my $i = -1;
 	my %root2id_new = map { $_ => ++$i } @OFF2ROOT;
@@ -905,16 +895,24 @@ sub score_old_join_data ($$$) {
 	my %ekey2id_new = map { $_ => ++$i } @$ekeys_new;
 	for my $ibx_off_old (0..$#$ibx2root_old) {
 		my $root_offs_old = $ibx2root_old->[$ibx_off_old];
-		my $ekey = $ekeys_old->[$ibx_off_old] //
-			warn "W: no ibx #$ibx_off_old in old JOIN data\n";
-		my $ibx_off_new = $ekey2id_new{$ekey // next} //
+		my $ekey = $ekeys_old->[$ibx_off_old] // do {
+			warn "W: no ibx #$ibx_off_old in old join data\n";
+			next;
+		};
+		my $ibx_off_new = $ekey2id_new{$ekey} // do {
 			warn "W: `$ekey' no longer exists\n";
+			next;
+		};
 		for (@$root_offs_old) {
 			my ($nr, $rid_old) = @$_;
-			my $root_old = $roots_old->[$rid_old] //
-				warn "W: no root #$rid_old in old JOIN data\n";
-			my $rid_new = $root2id_new{$root_old // next} //
+			my $root_old = $roots_old->[$rid_old] // do {
+				warn "W: no root #$rid_old in old data\n";
+				next;
+			};
+			my $rid_new = $root2id_new{$root_old} // do {
 				warn "W: root `$root_old' no longer exists\n";
+				next;
+			};
 			$score->{"$ibx_off_new $rid_new"} += $nr;
 		}
 	}
@@ -963,7 +961,7 @@ sub do_join {
 		progress($self, "$ekey => $root has $nr matches");
 		push @{$new->{ibx2root}->[$ibx_off]}, [ $nr, $root_off ];
 	}
-	for my $ary (values %$new) { # sort by nr
+	for my $ary (values %$new) { # sort by nr (largest first)
 		for (@$ary) { @$_ = sort { $b->[0] <=> $a->[0] } @$_ }
 	}
 	$new->{ekeys} = \@ekeys;
diff --git a/lib/PublicInbox/Config.pm b/lib/PublicInbox/Config.pm
index 9bee94b8..779e3140 100644
--- a/lib/PublicInbox/Config.pm
+++ b/lib/PublicInbox/Config.pm
@@ -412,8 +412,8 @@ sub get_1 {
 
 sub repo_objs {
 	my ($self, $ibxish) = @_;
-	my $ibx_coderepos = $ibxish->{coderepo} // return;
 	$ibxish->{-repo_objs} // do {
+		my $ibx_coderepos = $ibxish->{coderepo} // return;
 		parse_cgitrc($self, undef, 0);
 		my $coderepos = $self->{-coderepos};
 		my @repo_objs;
@@ -568,6 +568,43 @@ sub _fill_ei ($$) {
 	$es;
 }
 
+sub _fill_csrch ($$) {
+	my ($self, $name) = @_; # "" is a valid name for cindex
+	return if $name ne '' && !valid_foo_name($name, 'cindex');
+	eval { require PublicInbox::CodeSearch } or return;
+	my $pfx = "cindex.$name";
+	my $d = $self->{"$pfx.topdir"} // return;
+	-d $d or return;
+	if (index($d, "\n") >= 0) {
+		warn "E: `$d' must not contain `\\n'\n";
+		return;
+	}
+	my $csrch = PublicInbox::CodeSearch->new($d, $self);
+	for my $k (qw(localprefix)) {
+		my $v = $self->{"$pfx.$k"} // next;
+		$csrch->{$k} = _array($v);
+	}
+	$csrch->{name} = $name;
+	$csrch;
+}
+
+sub lookup_cindex ($$) {
+	my ($self, $name) = @_;
+	$self->{-csrch_by_name}->{$name} //= _fill_csrch($self, $name);
+}
+
+sub each_cindex {
+	my ($self, $cb, @arg) = @_;
+	my @csrch = map {
+		lookup_cindex($self, substr($_, length('cindex.'))) // ()
+	} grep(m!\Acindex\.[^\./]*\z!, @{$self->{-section_order}});
+	if (ref($cb) eq 'CODE') {
+		$cb->($_, @arg) for @csrch;
+	} else { # string function
+		$_->$cb(@arg) for @csrch;
+	}
+}
+
 sub config_cmd {
 	my ($self, $env, $opt) = @_;
 	my $f = $self->{-f} // default_file();
diff --git a/lib/PublicInbox/Search.pm b/lib/PublicInbox/Search.pm
index 6145b027..8ef17d58 100644
--- a/lib/PublicInbox/Search.pm
+++ b/lib/PublicInbox/Search.pm
@@ -649,4 +649,21 @@ sub xh_args { # prep getopt args to feed to xap_helper.h socket
 	map { ('-d', $_) } shard_dirs($_[0]);
 }
 
+sub docids_by_postlist ($$) {
+	my ($self, $q) = @_;
+	my $cur = $self->xdb->postlist_begin($q);
+	my $end = $self->{xdb}->postlist_end($q);
+	my @ids;
+	for (; $cur != $end; $cur++) { push(@ids, $cur->get_docid) };
+	@ids;
+}
+
+sub get_doc ($$) {
+	my ($self, $docid) = @_;
+	eval { $self->{xdb}->get_document($docid) } // do {
+		die $@ if $@ && ref($@) !~ /\bDocNotFoundError\b/;
+		undef;
+	}
+}
+
 1;
diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm
index f569428c..17538027 100644
--- a/lib/PublicInbox/SearchIdx.pm
+++ b/lib/PublicInbox/SearchIdx.pm
@@ -545,17 +545,9 @@ sub add_message {
 	$smsg->{num};
 }
 
-sub get_doc ($$) {
-	my ($self, $docid) = @_;
-	eval { $self->{xdb}->get_document($docid) } // do {
-		die $@ if $@ && ref($@) !~ /\bDocNotFoundError\b/;
-		undef;
-	}
-}
-
 sub _get_doc ($$) {
 	my ($self, $docid) = @_;
-	get_doc($self, $docid) // do {
+	$self->get_doc($docid) // do {
 		warn "E: #$docid missing in Xapian\n";
 		undef;
 	}
diff --git a/lib/PublicInbox/SolverGit.pm b/lib/PublicInbox/SolverGit.pm
index 7cc10198..4e79f750 100644
--- a/lib/PublicInbox/SolverGit.pm
+++ b/lib/PublicInbox/SolverGit.pm
@@ -643,9 +643,13 @@ sub resolve_patch ($$) {
 # so user_cb never references the SolverGit object
 sub new {
 	my ($class, $ibx, $user_cb, $uarg) = @_;
+	my $gits = $ibx ? $ibx->{-repo_objs} : undef;
+
+	# FIXME: cindex --join= is super-aggressive and may hit too many
+	$gits = [ @$gits[0..2] ] if $gits && @$gits > 3;
 
 	bless { # $ibx is undef if coderepo only (see WwwCoderepo)
-		gits => $ibx ? $ibx->{-repo_objs} : undef,
+		gits => $gits,
 		user_cb => $user_cb,
 		uarg => $uarg,
 		# -cur_di, -qsp_err, -msg => temp fields for Qspawn callbacks
diff --git a/lib/PublicInbox/View.pm b/lib/PublicInbox/View.pm
index e5f748f7..d81c66b7 100644
--- a/lib/PublicInbox/View.pm
+++ b/lib/PublicInbox/View.pm
@@ -80,7 +80,7 @@ sub msg_page {
 	# allow user to easily browse the range around this message if
 	# they have ->over
 	$ctx->{-t_max} = $smsg->{ts};
-	$ctx->{-spfx} = '../' if $ibx->{coderepo};
+	$ctx->{-spfx} = '../' if $ibx->{-repo_objs};
 	PublicInbox::WwwStream::aresponse($ctx, \&msg_page_i);
 }
 
@@ -443,7 +443,7 @@ sub thread_html {
 	my $ibx = $ctx->{ibx};
 	my ($nr, $msgs) = $ibx->over->get_thread($mid);
 	return missing_thread($ctx) if $nr == 0;
-	$ctx->{-spfx} = '../../' if $ibx->{coderepo};
+	$ctx->{-spfx} = '../../' if $ibx->{-repo_objs};
 
 	# link $INBOX_DIR/description text to "index_topics" view around
 	# the newest message in this thread
@@ -779,6 +779,9 @@ href=#t>this message</a>:
 <input type=submit value=search
 />\t(<a href=${upfx}_/text/help/#search>help</a>)</pre></form>
 EOM
+		# TODO: related codesearch
+		# my $csrchv = $ctx->{ibx}->{-csrch} // [];
+		# push @related, '<pre>'.ascii_html(Dumper($csrchv)).'</pre>';
 	}
 	if ($ctx->{ibx}->over) {
 		my $t = ts2str($ctx->{-t_max});
diff --git a/lib/PublicInbox/WWW.pm b/lib/PublicInbox/WWW.pm
index 6b616bd4..289599b8 100644
--- a/lib/PublicInbox/WWW.pm
+++ b/lib/PublicInbox/WWW.pm
@@ -189,6 +189,7 @@ sub preload {
 		}
 		$pi_cfg->ALL and require PublicInbox::Isearch;
 		$self->cgit;
+		$self->coderepo;
 		$self->stylesheets_prepare($_) for ('', '../', '../../');
 		$self->news_www;
 	}
diff --git a/lib/PublicInbox/WwwCoderepo.pm b/lib/PublicInbox/WwwCoderepo.pm
index 0eb4a2d6..8ab4911f 100644
--- a/lib/PublicInbox/WwwCoderepo.pm
+++ b/lib/PublicInbox/WwwCoderepo.pm
@@ -14,12 +14,14 @@ use PublicInbox::ViewVCS;
 use PublicInbox::WwwStatic qw(r);
 use PublicInbox::GitHTTPBackend;
 use PublicInbox::WwwStream;
-use PublicInbox::Hval qw(ascii_html utf8_maybe);
+use PublicInbox::Hval qw(prurl ascii_html utf8_maybe);
 use PublicInbox::ViewDiff qw(uri_escape_path);
 use PublicInbox::RepoSnapshot;
 use PublicInbox::RepoAtom;
 use PublicInbox::RepoTree;
 use PublicInbox::OnDestroy;
+use URI::Escape qw(uri_escape_utf8);
+use File::Spec;
 
 my @EACH_REF = (qw(git for-each-ref --sort=-creatordate),
 		"--format=%(HEAD)%00".join('%00', map { "%($_)" }
@@ -62,6 +64,7 @@ sub prepare_coderepos {
 		my $eidx = $pi_cfg->lookup_ei($k) // next;
 		$pi_cfg->repo_objs($eidx);
 	}
+	$pi_cfg->each_cindex('load_coderepos', $pi_cfg);
 }
 
 sub new {
@@ -119,6 +122,41 @@ sub _refs_tags_link {
 		"</a>$align ", ascii_html($s), " ($cd)", @snap_fmt, "\n");
 }
 
+sub emit_joined_inboxes ($) {
+	my ($ctx) = @_;
+	my $names = $ctx->{git}->{ibx_names}; # coderepo directives in config
+	my $score = $ctx->{git}->{ibx_score}; # generated w/ cindex --join
+	($names || $score) or return;
+	my $pi_cfg = $ctx->{wcr}->{pi_cfg};
+	my ($u, $h);
+	my $zfh = $ctx->zfh;
+	print $zfh "\n# associated public inboxes:",
+		"\n# (number on the left is used for dev purposes)";
+	my @ns = map { [ 0, $_ ] } @$names;
+	my $env = $ctx->{env};
+	for (@ns, @$score) {
+		my ($nr, $name) = @$_;
+		my $ibx = $pi_cfg->lookup_name($name) // do {
+			warn "W: inbox `$name' gone for $ctx->{git}->{git_dir}";
+			say $zfh '# ', ascii_html($name), ' (missing inbox?)';
+			next;
+		};
+		if (scalar(@{$ibx->{url} // []})) {
+			$u = $h = ascii_html(prurl($env, $ibx->{url}));
+		} else {
+			$h = ascii_html(prurl($env, uri_escape_utf8($name)));
+			$h .= '/';
+			$u = ascii_html($name);
+		}
+		if ($nr) {
+			printf $zfh "\n% 11u", $nr;
+		} else {
+			print $zfh "\n", ' 'x11;
+		}
+		print $zfh qq{ <a\nhref="$h">$u</a>};
+	}
+}
+
 sub summary_END { # called via OnDestroy
 	my ($ctx) = @_;
 	my $wcb = delete($ctx->{-wcb}) or return; # already done
@@ -174,6 +212,7 @@ EOM
 	for (@r) { print $zfh _refs_tags_link($_, './', $snap_pfx, @snap_fmt) }
 	print $zfh $NO_TAGS if !@r;
 	print $zfh qq(<a href="refs/tags/">...</a>\n) if $last;
+	emit_joined_inboxes $ctx;
 	$wcb->($ctx->html_done('</pre>'));
 }
 
diff --git a/lib/PublicInbox/WwwText.pm b/lib/PublicInbox/WwwText.pm
index f4508b3f..4b4b2e4c 100644
--- a/lib/PublicInbox/WwwText.pm
+++ b/lib/PublicInbox/WwwText.pm
@@ -7,7 +7,7 @@ use strict;
 use v5.10.1;
 use PublicInbox::Linkify;
 use PublicInbox::WwwStream;
-use PublicInbox::Hval qw(ascii_html prurl);
+use PublicInbox::Hval qw(ascii_html prurl fmt_ts);
 use HTTP::Date qw(time2str);
 use URI::Escape qw(uri_escape_utf8);
 use PublicInbox::GzipFilter qw(gzf_maybe);
@@ -248,14 +248,23 @@ EOS
 
 sub coderepos_raw ($$) {
 	my ($ctx, $top_url) = @_;
-	my $cr = $ctx->{ibx}->{coderepo} // return ();
 	my $cfg = $ctx->{www}->{pi_cfg};
+	my $cr = $cfg->repo_objs($ctx->{ibx}) or return ();
 	my $buf = 'Code repositories for project(s) associated with this '.
-		$ctx->{ibx}->thing_type . "\n";
-	for my $git (@{$ctx->{www}->{pi_cfg}->repo_objs($ctx->{ibx})}) {
+		$ctx->{ibx}->thing_type . ":\n";
+	my @recs = map { [ 0, $_ ] } @$cr;
+	my @todo = @recs;
+	$cfg->each_cindex('load_commit_times', \@todo);
+	@recs = sort { $b->[0] <=> $a->[0] } @recs;
+	my $cr_score = $ctx->{ibx}->{-cr_score};
+	for (@recs) {
+		my ($t, $git) = @$_;
 		for ($git->pub_urls($ctx->{env})) {
 			my $u = m!\A(?:[a-z\+]+:)?//!i ? $_ : $top_url.$_;
-			$buf .= "\n\t" . prurl($ctx->{env}, $u);
+			my $nr = $cr_score->{$git->{nick}};
+			$buf .= "\n";
+			$buf .= $nr ? sprintf('% 9u', $nr) : (' 'x9);
+			$buf .= ' '.fmt_ts($t).' '.prurl($ctx->{env}, $u);
 		}
 	}
 	($buf);
diff --git a/t/cindex.t b/t/cindex.t
index a9075092..29d88ca8 100644
--- a/t/cindex.t
+++ b/t/cindex.t
@@ -5,7 +5,7 @@ use v5.12;
 use PublicInbox::TestCommon;
 use Cwd qw(getcwd abs_path);
 use List::Util qw(sum);
-use autodie qw(close open rename);
+use autodie qw(close mkdir open rename);
 require_mods(qw(json Xapian +SCM_RIGHTS));
 use_ok 'PublicInbox::CodeSearchIdx';
 use PublicInbox::Import;
@@ -227,7 +227,7 @@ SKIP: { # --prune
 }
 
 File::Path::remove_tree("$tmp/ext");
-ok(mkdir("$tmp/ext", 0707), 'create $tmp/ext with odd permissions');
+mkdir("$tmp/ext", 0707);
 ok(run_script([qw(-cindex --dangerous -q -d), "$tmp/ext", $zp]),
 	'external on existing dir');
 {
@@ -265,4 +265,28 @@ EOM
 		'non-Xapian-enabled inbox noted');
 }
 
+# we need to support blank sections for a top-level repos
+# (e.g. <https://example.com/my-project>
+# git.kernel.org could use "pub" as section name, though, since all git repos
+# are currently under //git.kernel.org/pub/**/*
+{
+	mkdir(my $d = "$tmp/blanksection");
+	my $cfg = cfg_new($d, <<EOM);
+[cindex ""]
+	topdir = $tmp/ext
+	localprefix = $tmp
+EOM
+	my $csrch = $cfg->lookup_cindex('');
+	is ref($csrch), 'PublicInbox::CodeSearch', 'codesearch w/ blank name';
+	is_deeply $csrch->{localprefix}, [ "$tmp" ], 'localprefix respected';
+	my $nr = 0;
+	$cfg->each_cindex(sub {
+		my ($cs, @rest) = @_;
+		is $cs->{topdir}, $csrch->{topdir}, 'each_cindex works';
+		is_deeply \@rest, [ '.' ], 'got expected arg';
+		++$nr;
+	}, '.');
+	is $nr, 1, 'iterated through cindices';
+}
+
 done_testing;
diff --git a/xt/solver.t b/xt/solver.t
index 51b4144c..372d003b 100644
--- a/xt/solver.t
+++ b/xt/solver.t
@@ -10,6 +10,7 @@ use_ok($_) for @psgi;
 use_ok 'PublicInbox::WWW';
 my $cfg = PublicInbox::Config->new;
 my $www = PublicInbox::WWW->new($cfg);
+$www->preload;
 my $app = sub {
 	my $env = shift;
 	$env->{'psgi.errors'} = \*STDERR;
@@ -63,7 +64,7 @@ while (my ($ibx_name, $urls) = each %$todo) {
 			skip(qq{[publicinbox "$ibx_name"] not configured},
 				scalar(@$urls));
 		}
-		if (!defined($ibx->{coderepo})) {
+		if (!defined($ibx->{-repo_objs})) {
 			push @gone, $ibx_name;
 			skip(qq{publicinbox.$ibx_name.coderepo not configured},
 				scalar(@$urls));