about summary refs log tree commit homepage
diff options
context:
space:
mode:
-rw-r--r--Documentation/RelNotes/v1.9.0.eml68
-rw-r--r--Documentation/lei-add-external.pod4
-rw-r--r--Documentation/lei-blob.pod2
-rw-r--r--Documentation/lei-convert.pod2
-rw-r--r--Documentation/lei-edit-search.pod6
-rw-r--r--Documentation/lei-forget-search.pod4
-rw-r--r--Documentation/lei-import.pod7
-rw-r--r--Documentation/lei-lcat.pod2
-rw-r--r--Documentation/lei-ls-search.pod5
-rw-r--r--Documentation/lei-q.pod19
-rw-r--r--Documentation/lei-rediff.pod2
-rw-r--r--Documentation/lei-reindex.pod47
-rw-r--r--Documentation/lei-up.pod4
-rwxr-xr-xDocumentation/mknews.perl3
-rw-r--r--Documentation/public-inbox-clone.pod76
-rw-r--r--Documentation/public-inbox-config.pod30
-rw-r--r--Documentation/public-inbox-daemon.pod4
-rw-r--r--Documentation/public-inbox-fetch.pod6
-rw-r--r--Documentation/public-inbox-imapd.pod6
-rw-r--r--Documentation/public-inbox-netd.pod4
-rw-r--r--Documentation/public-inbox-nntpd.pod4
-rw-r--r--Documentation/public-inbox-pop3d.pod6
-rwxr-xr-xDocumentation/txt2pre19
-rw-r--r--INSTALL6
-rw-r--r--MANIFEST18
-rw-r--r--Makefile.PL8
-rw-r--r--README6
-rw-r--r--TODO9
-rwxr-xr-xdevel/syscall-list8
-rw-r--r--examples/README4
-rw-r--r--examples/apache2_cgi.conf34
-rw-r--r--examples/apache2_perl.conf25
-rw-r--r--examples/apache2_perl_old.conf38
-rw-r--r--examples/cgi-webrick.rb25
-rwxr-xr-xexamples/grok-pull.post_update_hook.sh2
-rw-r--r--examples/nginx_proxy9
-rw-r--r--examples/public-inbox-httpd.socket3
-rw-r--r--examples/public-inbox-httpd@.service3
-rw-r--r--examples/public-inbox-imap-onion.socket12
-rw-r--r--examples/public-inbox-imapd.socket17
-rw-r--r--examples/public-inbox-imapd@.service12
-rw-r--r--examples/public-inbox-imaps.socket12
-rw-r--r--examples/public-inbox-netd.socket45
-rw-r--r--examples/public-inbox-netd@.service60
-rw-r--r--examples/public-inbox-nntpd.socket21
-rw-r--r--examples/public-inbox-nntpd@.service9
-rw-r--r--examples/public-inbox-nntps.socket12
-rw-r--r--lib/PublicInbox/Cgit.pm38
-rw-r--r--lib/PublicInbox/CompressNoop.pm4
-rw-r--r--lib/PublicInbox/Config.pm43
-rw-r--r--lib/PublicInbox/ContentHash.pm7
-rw-r--r--lib/PublicInbox/DS.pm6
-rw-r--r--lib/PublicInbox/DSKQXS.pm11
-rw-r--r--lib/PublicInbox/DSdeflate.pm2
-rw-r--r--lib/PublicInbox/Daemon.pm106
-rw-r--r--lib/PublicInbox/DirIdle.pm4
-rw-r--r--lib/PublicInbox/Eml.pm8
-rw-r--r--lib/PublicInbox/ExtMsg.pm4
-rw-r--r--lib/PublicInbox/ExtSearch.pm7
-rw-r--r--lib/PublicInbox/ExtSearchIdx.pm26
-rw-r--r--lib/PublicInbox/FakeInotify.pm13
-rw-r--r--lib/PublicInbox/Feed.pm18
-rw-r--r--lib/PublicInbox/Fetch.pm48
-rw-r--r--lib/PublicInbox/Filter/RubyLang.pm29
-rw-r--r--lib/PublicInbox/Gcf2.pm17
-rw-r--r--lib/PublicInbox/Git.pm57
-rw-r--r--lib/PublicInbox/GitAsyncCat.pm80
-rw-r--r--lib/PublicInbox/GitHTTPBackend.pm36
-rw-r--r--lib/PublicInbox/GzipFilter.pm63
-rw-r--r--lib/PublicInbox/HTTP.pm2
-rw-r--r--lib/PublicInbox/HTTPD.pm1
-rw-r--r--lib/PublicInbox/HTTPD/Async.pm9
-rw-r--r--lib/PublicInbox/IMAP.pm36
-rw-r--r--lib/PublicInbox/IMAPD.pm57
-rw-r--r--lib/PublicInbox/Import.pm30
-rw-r--r--lib/PublicInbox/In2Tie.pm4
-rw-r--r--lib/PublicInbox/Inbox.pm37
-rw-r--r--lib/PublicInbox/InboxIdle.pm14
-rw-r--r--lib/PublicInbox/KQNotify.pm12
-rw-r--r--lib/PublicInbox/LEI.pm24
-rw-r--r--lib/PublicInbox/LeiBlob.pm4
-rw-r--r--lib/PublicInbox/LeiCurl.pm11
-rw-r--r--lib/PublicInbox/LeiExternal.pm4
-rw-r--r--lib/PublicInbox/LeiInspect.pm5
-rw-r--r--lib/PublicInbox/LeiLsExternal.pm1
-rw-r--r--lib/PublicInbox/LeiLsMailSync.pm7
-rw-r--r--lib/PublicInbox/LeiMirror.pm1101
-rw-r--r--lib/PublicInbox/LeiQuery.pm34
-rw-r--r--lib/PublicInbox/LeiReindex.pm49
-rw-r--r--lib/PublicInbox/LeiSavedSearch.pm1
-rw-r--r--lib/PublicInbox/LeiStore.pm48
-rw-r--r--lib/PublicInbox/LeiStoreErr.pm21
-rw-r--r--lib/PublicInbox/LeiToMail.pm44
-rw-r--r--lib/PublicInbox/LeiUp.pm11
-rw-r--r--lib/PublicInbox/LeiXSearch.pm21
-rw-r--r--lib/PublicInbox/Linkify.pm27
-rw-r--r--lib/PublicInbox/ManifestJsGz.pm8
-rw-r--r--lib/PublicInbox/Mbox.pm14
-rw-r--r--lib/PublicInbox/MboxGz.pm6
-rw-r--r--lib/PublicInbox/MiscIdx.pm2
-rw-r--r--lib/PublicInbox/NNTP.pm14
-rw-r--r--lib/PublicInbox/NNTPD.pm1
-rw-r--r--lib/PublicInbox/NetReader.pm8
-rw-r--r--lib/PublicInbox/OnDestroy.pm5
-rw-r--r--lib/PublicInbox/OverIdx.pm18
-rw-r--r--lib/PublicInbox/POP3.pm74
-rw-r--r--lib/PublicInbox/POP3D.pm10
-rw-r--r--lib/PublicInbox/Qspawn.pm41
-rw-r--r--lib/PublicInbox/RepoSnapshot.pm97
-rw-r--r--lib/PublicInbox/SaPlugin/ListMirror.pm10
-rw-r--r--lib/PublicInbox/SaPlugin/ListMirror.pod27
-rw-r--r--lib/PublicInbox/SearchIdx.pm2
-rw-r--r--lib/PublicInbox/SearchView.pm8
-rw-r--r--lib/PublicInbox/Sigfd.pm13
-rw-r--r--lib/PublicInbox/Smsg.pm9
-rw-r--r--lib/PublicInbox/SolverGit.pm82
-rw-r--r--lib/PublicInbox/Syscall.pm26
-rw-r--r--lib/PublicInbox/TestCommon.pm37
-rw-r--r--lib/PublicInbox/View.pm426
-rw-r--r--lib/PublicInbox/ViewDiff.pm154
-rw-r--r--lib/PublicInbox/ViewVCS.pm512
-rw-r--r--lib/PublicInbox/WWW.pm44
-rw-r--r--lib/PublicInbox/Watch.pm4
-rw-r--r--lib/PublicInbox/WwwAltId.pm6
-rw-r--r--lib/PublicInbox/WwwAtomStream.pm22
-rw-r--r--lib/PublicInbox/WwwCoderepo.pm243
-rw-r--r--lib/PublicInbox/WwwListing.pm40
-rw-r--r--lib/PublicInbox/WwwStatic.pm32
-rw-r--r--lib/PublicInbox/WwwStream.pm87
-rw-r--r--lib/PublicInbox/WwwText.pm63
-rwxr-xr-xscript/public-inbox-clone23
-rwxr-xr-xscript/public-inbox-fetch4
-rwxr-xr-xscript/public-inbox-watch6
-rw-r--r--t/altid_v2.t10
-rw-r--r--t/convert-compact.t2
-rw-r--r--t/data/attached-mbox-with-utf8.eml45
-rw-r--r--t/extsearch.t10
-rw-r--r--t/filter_rubylang.t16
-rw-r--r--t/git.t4
-rw-r--r--t/hl_mod.t4
-rw-r--r--t/httpd-corner.t8
-rw-r--r--t/imapd.t34
-rw-r--r--t/import.t4
-rw-r--r--t/indexlevels-mirror.t10
-rw-r--r--t/init.t2
-rw-r--r--t/lei-index.t12
-rw-r--r--t/lei-mirror.t10
-rw-r--r--t/lei-q-save.t13
-rw-r--r--t/lei-reindex.t12
-rw-r--r--t/lei-up.t43
-rw-r--r--t/lei.t3
-rw-r--r--t/lei_store.t5
-rw-r--r--t/lei_to_mail.t10
-rw-r--r--t/lei_xsearch.t2
-rw-r--r--t/linkify.t5
-rw-r--r--t/mda.t4
-rw-r--r--t/nntpd.t14
-rw-r--r--t/on_destroy.t8
-rw-r--r--t/plack.t190
-rw-r--r--t/pop3d.t61
-rw-r--r--t/psgi_attach.t13
-rw-r--r--t/psgi_search.t7
-rw-r--r--t/psgi_v2.t4
-rw-r--r--t/qspawn.t14
-rw-r--r--t/replace.t4
-rw-r--r--t/sigfd.t5
-rw-r--r--t/solver_git.t86
-rw-r--r--t/v1-add-remove-add.t2
-rw-r--r--t/v2-add-remove-add.t2
-rw-r--r--t/v2mirror.t3
-rw-r--r--t/v2writable.t18
-rw-r--r--t/www_altid.t13
-rw-r--r--t/www_listing.t71
-rw-r--r--xt/cmp-msgview.t94
-rw-r--r--xt/git-http-backend.t4
-rw-r--r--xt/perf-msgview.t30
-rw-r--r--xt/perf-obfuscate.t64
-rw-r--r--xt/solver.t37
178 files changed, 4156 insertions, 2004 deletions
diff --git a/Documentation/RelNotes/v1.9.0.eml b/Documentation/RelNotes/v1.9.0.eml
new file mode 100644
index 00000000..2d83cbfe
--- /dev/null
+++ b/Documentation/RelNotes/v1.9.0.eml
@@ -0,0 +1,68 @@
+From: Eric Wong <e@80x24.org>
+To: meta@public-inbox.org
+Subject: [ANNOUNCE] public-inbox 1.9.0
+Date: Sun, 21 Aug 2022 02:36:59 +0000
+MIME-Version: 1.0
+Content-Type: text/plain; charset=utf-8
+Content-Disposition: inline
+Message-ID: <2022-08-21T023659Z-public-inbox-1.9.0-rele@sed>
+
+Upgrading:
+
+  lei users need to "lei daemon-kill" after installation to load
+  new code.  Normal daemons (read-only, and public-inbox-watch)
+  will also need restarts, of course, but there's no
+  backwards-incompatible data format changes so rolling back to
+  older versions is harmless.
+
+Major bugfixes:
+
+  * lei no longer freezes from inotify/EVFILT_VNODE handling,
+    user interrupts (Ctrl-C), nor excessive errors/warnings
+
+  * IMAP server fairness improved to avoid excessive blob prefetch
+
+New features:
+
+  * POP3 server support added, use either public-inbox-pop3d or
+    the new public-inbox-netd superserver
+
+  * public-inbox-netd superserver supporting any combination of HTTP,
+    IMAP, POP3, and NNTP services; simplifying management and allowing
+    more sharing of memory used for various data structures.
+
+  * public-inbox-httpd and -netd support per-listener .psgi files
+
+  * SIGHUP reloads TLS certs and keys in addition to config and .psgi files
+
+  * "lei reindex" command for lei users to update personal index
+    in ~/.local/share/lei/store for search improvements below:
+
+Search improvements:
+
+  These will require --reindex with public-inbox-index and/or
+  public-inbox-extindex for public inboxes.
+
+  * patchid: prefix search support added to WWW and lei for
+    "git patch-id --stable" support
+
+  * text inside base-85 binary patches are no longer indexed
+    to avoid false positives
+
+  * for lei users, "lei reindex" now exists and is required
+    to take advantage of aforementioned indexing changes
+
+Performance improvements:
+
+  * IMAP server startup is faster with many mailboxes when using
+    "public-inbox-extindex --all"
+
+  * NNTP group listings are also faster with many inboxes when
+    using "public-inbox-extindex --all"
+
+  * various small opcode and memory usage reductions
+
+Please report bugs via plain-text mail to: meta@public-inbox.org
+
+See archives at https://public-inbox.org/meta/ for all history.
+See https://public-inbox.org/TODO for what the future holds.
diff --git a/Documentation/lei-add-external.pod b/Documentation/lei-add-external.pod
index 7afcad63..2a131b55 100644
--- a/Documentation/lei-add-external.pod
+++ b/Documentation/lei-add-external.pod
@@ -75,7 +75,9 @@ Default: C<auto>
 
 =item --inbox-version=NUM
 
-Force a public-inbox version (must be C<1> or C<2>).
+Force a remote public-inbox version (must be C<1> or C<2>).
+This is auto-detected by default, and this option exists mainly
+for testing.
 
 =back
 
diff --git a/Documentation/lei-blob.pod b/Documentation/lei-blob.pod
index e401bb47..558fc54c 100644
--- a/Documentation/lei-blob.pod
+++ b/Documentation/lei-blob.pod
@@ -86,7 +86,7 @@ reconstructed from patch emails.
 
 =item --no-torsocks
 
-=item --proxy=PROTO://HOST[:PORT]
+=item --proxy=PROTOCOL://HOST[:PORT]
 
 =back
 
diff --git a/Documentation/lei-convert.pod b/Documentation/lei-convert.pod
index c113db18..b3e29824 100644
--- a/Documentation/lei-convert.pod
+++ b/Documentation/lei-convert.pod
@@ -48,7 +48,7 @@ L<lei-q(1)>.
 
 =item --no-torsocks
 
-=item --proxy=PROTO://HOST[:PORT]
+=item --proxy=PROTOCOL://HOST[:PORT]
 
 =back
 
diff --git a/Documentation/lei-edit-search.pod b/Documentation/lei-edit-search.pod
index 21cb11aa..7f447ca2 100644
--- a/Documentation/lei-edit-search.pod
+++ b/Documentation/lei-edit-search.pod
@@ -8,7 +8,9 @@ lei edit-search [OPTIONS] OUTPUT
 
 =head1 DESCRIPTION
 
-Invoke C<git config --edit> to edit the saved search at C<OUTPUT>.
+Invoke C<git config --edit> to edit the saved search at C<OUTPUT>,
+where C<OUTPUT> was supplied for argument of C<lei q -o OUTPUT ...>
+A listing of outputs is available via C<lei ls-search>.
 
 =head1 CONTACT
 
@@ -19,7 +21,7 @@ and L<http://4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/meta
 
 =head1 COPYRIGHT
 
-Copyright 2021 all contributors L<mailto:meta@public-inbox.org>
+Copyright all contributors L<mailto:meta@public-inbox.org>
 
 License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>
 
diff --git a/Documentation/lei-forget-search.pod b/Documentation/lei-forget-search.pod
index adbe7638..5ff526f1 100644
--- a/Documentation/lei-forget-search.pod
+++ b/Documentation/lei-forget-search.pod
@@ -8,7 +8,9 @@ lei forget-search [OPTIONS] OUTPUT
 
 =head1 DESCRIPTION
 
-Forget a saved search at C<OUTPUT>.
+Forget a saved search at C<OUTPUT>,
+where C<OUTPUT> was supplied for argument of C<lei q -o OUTPUT ...>
+A listing of outputs is available via C<lei ls-search>.
 
 =head1 OPTIONS
 
diff --git a/Documentation/lei-import.pod b/Documentation/lei-import.pod
index ad769084..69ec6497 100644
--- a/Documentation/lei-import.pod
+++ b/Documentation/lei-import.pod
@@ -10,7 +10,8 @@ lei import [OPTIONS] (--stdin|-)
 
 =head1 DESCRIPTION
 
-Import messages into the local storage of L<lei(1)>.  C<LOCATION> is a
+Import messages into the local storage of L<lei(1)>
+(aka L<leiE<sol>store|lei-store-format(5)>).  C<LOCATION> is a
 source of messages: a directory (Maildir), a file, or a URL
 (C<imap://>, C<imaps://>, C<nntp://>, or C<nntps://>).  URLs requiring
 authentication use L<git-credential(1)> to
@@ -81,7 +82,7 @@ Whether to wrap L<git(1)> and L<curl(1)> commands with L<torsocks(1)>.
 
 Default: C<auto>
 
-=item --proxy=PROTO://HOST[:PORT]
+=item --proxy=PROTOCOL://HOST[:PORT]
 
 Use the specified proxy (e.g., C<socks5h://0:9050>).
 
@@ -102,4 +103,4 @@ License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>
 
 =head1 SEE ALSO
 
-L<lei-index(1)>
+L<lei-index(1)>, L<lei-store-format(5)>
diff --git a/Documentation/lei-lcat.pod b/Documentation/lei-lcat.pod
index e85e5e67..e8073862 100644
--- a/Documentation/lei-lcat.pod
+++ b/Documentation/lei-lcat.pod
@@ -52,7 +52,7 @@ which lets you pipe arbitrary lines to arbitrary commands).
 
 =item --torsocks=auto|no|yes, --no-torsocks
 
-=item --proxy=PROTO://HOST[:PORT]
+=item --proxy=PROTOCOL://HOST[:PORT]
 
 =item -o MFOLDER, --output=MFOLDER
 
diff --git a/Documentation/lei-ls-search.pod b/Documentation/lei-ls-search.pod
index a56611bf..0fe4b759 100644
--- a/Documentation/lei-ls-search.pod
+++ b/Documentation/lei-ls-search.pod
@@ -8,7 +8,8 @@ lei ls-search [OPTIONS] [PREFIX]
 
 =head1 DESCRIPTION
 
-List saved search queries.  If C<PREFIX> is given, restrict the output
+List saved search queries (generated from C<lei q -o OUTPUT>).
+If C<PREFIX> is given, restrict the output
 to entries that start with the specified value.
 
 =head1 OPTIONS
@@ -55,7 +56,7 @@ and L<http://4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/meta
 
 =head1 COPYRIGHT
 
-Copyright 2021 all contributors L<mailto:meta@public-inbox.org>
+Copyright all contributors L<mailto:meta@public-inbox.org>
 
 License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>
 
diff --git a/Documentation/lei-q.pod b/Documentation/lei-q.pod
index 1cbffba4..d52c5b04 100644
--- a/Documentation/lei-q.pod
+++ b/Documentation/lei-q.pod
@@ -124,6 +124,23 @@ of the same thread.
 TODO: Warning: this flag may become persistent and saved in
 lei/store unless an MUA unflags it!  (Behavior undecided)
 
+=item --jobs=QUERY_WORKERS[,WRITE_WORKERS]
+=item --jobs=,WRITE_WORKERS
+
+=item -j QUERY_WORKERS[,WRITE_WORKERS]
+=item -j ,WRITE_WORKERS
+
+Set the number of query and write worker processes for parallelism.
+
+C<QUERY_WORKERS> defaults to the number of CPUs available, but 4 per
+remote (HTTP/HTTPS) host.
+
+C<WRITE_WORKERS> defaults to 75% of the number of CPUs available for
+Maildir and mbox* destinations, but 4 per IMAP/IMAPS host.
+
+Omitting C<QUERY_WORKERS> but leaving the comma (C<,>) allows
+one to only set C<WRITE_WORKERS>
+
 =item --dedupe=STRATEGY
 
 =item -d STRATEGY
@@ -241,7 +258,7 @@ Whether to wrap L<git(1)> and L<curl(1)> commands with L<torsocks(1)>.
 
 Default: C<auto>
 
-=item --proxy=PROTO://HOST[:PORT]
+=item --proxy=PROTOCOL://HOST[:PORT]
 
 =back
 
diff --git a/Documentation/lei-rediff.pod b/Documentation/lei-rediff.pod
index 4d5e8168..f18548d3 100644
--- a/Documentation/lei-rediff.pod
+++ b/Documentation/lei-rediff.pod
@@ -104,7 +104,7 @@ The options below, described in L<lei-q(1)>, are also supported.
 
 =item --torsocks=auto|no|yes, --no-torsocks
 
-=item --proxy=PROTO://HOST[:PORT]
+=item --proxy=PROTOCOL://HOST[:PORT]
 
 =back
 
diff --git a/Documentation/lei-reindex.pod b/Documentation/lei-reindex.pod
new file mode 100644
index 00000000..3a5861c4
--- /dev/null
+++ b/Documentation/lei-reindex.pod
@@ -0,0 +1,47 @@
+=head1 NAME
+
+lei-reindex - reindex messages already in lei/store
+
+=head1 SYNOPSIS
+
+lei reindex [OPTIONS]
+
+=head1 DESCRIPTION
+
+Forces a re-index of all messages previously-indexed by L<lei-import(1)>
+or L<lei-index(1)>.  This can be used for in-place upgrades and bugfixes
+while other processes are querying the store.  Keep in mind this roughly
+doubles the size of the already-large Xapian database.
+
+It does not re-index messages in externals, using the C<--reindex>
+switch of L<public-inbox-index(1)> or L<public-inbox-extindex(1)> is
+needed for that.
+
+=head1 OPTIONS
+
+=over
+
+=item -q
+
+=item --quiet
+
+Suppress feedback messages.
+
+=back
+
+=head1 CONTACT
+
+Feedback welcome via plain-text mail to L<mailto:meta@public-inbox.org>
+
+The mail archives are hosted at L<https://public-inbox.org/meta/> and
+L<http://4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/meta/>
+
+=head1 COPYRIGHT
+
+Copyright all contributors L<mailto:meta@public-inbox.org>
+
+License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>
+
+=head1 SEE ALSO
+
+L<lei-index(1)>, L<lei-import(1)>
diff --git a/Documentation/lei-up.pod b/Documentation/lei-up.pod
index ac644a96..3b7c6f46 100644
--- a/Documentation/lei-up.pod
+++ b/Documentation/lei-up.pod
@@ -64,7 +64,9 @@ specified via C<lei q --only>.
 
 =item --mua=CMD
 
-C<--lock>, C<--alert>, and C<--mua> are all supported and
+=item --jobs QUERY_WORKERS[,WRITE_WORKERS]
+
+C<--lock>, C<--alert>, C<--mua>, and C<--jobs> are all supported and
 documented in L<lei-q(1)>.
 
 C<--mua> is incompatible with C<--all>.
diff --git a/Documentation/mknews.perl b/Documentation/mknews.perl
index 1936cea7..68866f44 100755
--- a/Documentation/mknews.perl
+++ b/Documentation/mknews.perl
@@ -1,5 +1,5 @@
 #!/usr/bin/perl -w
-# Copyright (C) 2019-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 # Generates NEWS, NEWS.atom, and NEWS.html files using release emails
 # this uses unstable internal APIs of public-inbox, and this script
@@ -46,6 +46,7 @@ if ($dst eq 'NEWS') {
                 ibx => $ibx,
                 -upfx => "$base_url/",
                 -hr => 1,
+                zfh => $out,
         };
         if ($dst eq 'NEWS.html') {
                 html_start($out, $ctx);
diff --git a/Documentation/public-inbox-clone.pod b/Documentation/public-inbox-clone.pod
index c80c3c5f..2a7081ac 100644
--- a/Documentation/public-inbox-clone.pod
+++ b/Documentation/public-inbox-clone.pod
@@ -6,6 +6,8 @@ public-inbox-clone - "git clone --mirror" wrapper
 
 public-inbox-clone INBOX_URL [INBOX_DIR]
 
+public-inbox-clone ROOT_URL [DESTINATION]
+
 =head1 DESCRIPTION
 
 public-inbox-clone is a wrapper around C<git clone --mirror> for
@@ -51,6 +53,80 @@ C<--epoch=~2..> clones the three latest epochs.
 Default: C<0..~0> or C<0..> or C<..~0>
 (all epochs, all three examples are equivalent)
 
+=item -I PATTERN
+
+=item --include=PATTERN
+
+When cloning a top-level with multiple inboxes, only clone inboxes and
+repositories matching a given wildcard pattern (using C<*?> and C<[]> is
+supported).
+
+=item --exclude=PATTERN
+
+When cloning a top-level with multiple inboxes, ignore inboxes and
+repositories matching the given wildcard pattern.  Supports the same
+wildcards as L</--include>
+
+=item --inbox-config=always|v2|v1|never
+
+Whether or not to retrieve the C<$INBOX/_/text/config/raw> HTTP(S)
+endpoint when cloning.
+
+Since we can't deduce v1 inboxes from code repositories, setting this
+to C<v2> or C<never> can allow faster clones of code repositories if
+no v1 inboxes are present.
+
+Default: C<always>
+
+=item --inbox-version=NUM
+
+Force a remote public-inbox version (must be C<1> or C<2>).
+This is auto-detected by default, and this option exists mainly
+for testing.
+
+=item --objstore=DIR
+
+Enables space savings when the remote C<manifest.js.gz>
+includes C<forkgroup> entries as generated by grokmirror 2.x.
+
+If C<DIR> does not start with C</>, C<./>, or C<../>, it is treated
+as relative to the C<DESTINATION> directory.  If only C<--objstore=>
+is specified where C<DIR> is an empty string (C<"">), then C<objstore>
+(C<$DESTINATION/objstore>) is the implied value of C<DIR>.
+
+=item --manifest=FILE
+
+When incrementally updating an existing mirror, load the given
+manifest (typically C<manifest.js.gz>) to speed up updates.
+
+By default, public-inbox writes the retrieved manifest to
+C<$DESTINATION/manifest.js.gz>, this directive also
+changes the destination to the specified C<FILE>
+
+If C<FILE> does not start with C</>, C<./>, or C<../>, it is treated
+as relative to the C<DESTINATION> directory.  If only C<--manifest=>
+is specified where C<FILE> is an empty string (C<"">), then C<manifest.js.gz>
+(C<$DESTINATION/manifest.js.gz>) is the implied value of C<FILE>.
+
+=item -p
+
+=item --prune
+
+Pass the C<--prune> and C<--prune-tags> flags to L<git-fetch(1)>
+calls on incremental clones.
+
+=item -k
+
+=item --keep-going
+
+Continue as much as possible after an error.
+
+=item -n
+
+=item --dry-run
+
+Show what would be done, without making any changes.
+
 =item -q
 
 =item --quiet
diff --git a/Documentation/public-inbox-config.pod b/Documentation/public-inbox-config.pod
index d8504e61..d175d2d7 100644
--- a/Documentation/public-inbox-config.pod
+++ b/Documentation/public-inbox-config.pod
@@ -265,6 +265,10 @@ The URL of the cgit instance associated with the coderepo.
 
 Default: none
 
+=item coderepo.snapshots
+
+See C<snapshots> in L<cgitrc(5)>
+
 =item publicinbox.cgitrc
 
 A path to a L<cgitrc(5)> file.  "repo.url" directives in the cgitrc
@@ -293,6 +297,32 @@ C<publicinbox.cgitbin>, but may be overridden.
 Default: basename of C<publicinbox.cgitbin>, /var/www/htdocs/cgit/
 or /usr/share/cgit/
 
+=item publicinbox.cgit
+
+Controls whether or not and how C<cgit> is used for serving coderepos.
+New in public-inbox 2.0.0 (PENDING).
+
+=over 8
+
+=item * first
+
+Try using C<cgit> as the first choice, this is the default.
+
+=item * fallback
+
+Fall back to using C<cgit> only if our native, inbox-aware
+git code repository viewer doesn't recognized the URL.
+
+=item * rewrite
+
+Rewrite C<cgit> URLs for our native, inbox-aware code repository viewer.
+This implies C<fallback> for URLs the native viewer does not recognize.
+
+=back
+
+Default: C<first>  (C<cgit> will be used iff C<publicinbox.cgitrc>
+is set and the C<cgit> binary exists).
+
 =item publicinbox.mailEditor
 
 See L<public-inbox-edit(1)>
diff --git a/Documentation/public-inbox-daemon.pod b/Documentation/public-inbox-daemon.pod
index 5d26ce56..81a79a10 100644
--- a/Documentation/public-inbox-daemon.pod
+++ b/Documentation/public-inbox-daemon.pod
@@ -31,9 +31,9 @@ processes to take advantage of multiple CPUs.
 
 =over
 
-=item -l [PROTO://]ADDRESS[?opt1=val1,opt2=val2]
+=item -l [PROTOCOL://]ADDRESS[?opt1=val1,opt2=val2]
 
-=item --listen [PROTO://]ADDRESS[?opt1=val1,opt2=val2]
+=item --listen [PROTOCOL://]ADDRESS[?opt1=val1,opt2=val2]
 
 This takes an absolute path to a Unix socket or HOST:PORT
 to listen on.  For example, to listen to TCP connections on
diff --git a/Documentation/public-inbox-fetch.pod b/Documentation/public-inbox-fetch.pod
index c78ffc0b..c5e73d38 100644
--- a/Documentation/public-inbox-fetch.pod
+++ b/Documentation/public-inbox-fetch.pod
@@ -61,6 +61,12 @@ there are no updates:
         public-inbox-fetch -q --exit-code && public-inbox-index
         test $? -eq 0 || exit $?
 
+=item -p
+
+=item --prune
+
+Pass the C<--prune> and C<--prune-tags> flags to L<git-fetch(1)> calls.
+
 =item -v
 
 =item --verbose
diff --git a/Documentation/public-inbox-imapd.pod b/Documentation/public-inbox-imapd.pod
index 23577a69..85bf3651 100644
--- a/Documentation/public-inbox-imapd.pod
+++ b/Documentation/public-inbox-imapd.pod
@@ -27,12 +27,12 @@ are supported and documented below.
 
 =over
 
-=item -l PROTO://ADDRESS/?cert=/path/to/cert,key=/path/to/key
+=item -l PROTOCOL://ADDRESS/?cert=/path/to/cert,key=/path/to/key
 
-=item --listen PROTO://ADDRESS/?cert=/path/to/cert,key=/path/to/key
+=item --listen PROTOCOL://ADDRESS/?cert=/path/to/cert,key=/path/to/key
 
 In addition to the normal C<-l>/C<--listen> switch described in
-L<public-inbox-daemon(8)>, the C<PROTO> prefix (e.g. C<imap://> or
+L<public-inbox-daemon(8)>, the C<PROTOCOL> prefix (e.g. C<imap://> or
 C<imaps://>) may be specified to force a given protocol.
 
 For STARTTLS and IMAPS support, the C<cert> and C<key> may be specified
diff --git a/Documentation/public-inbox-netd.pod b/Documentation/public-inbox-netd.pod
index 4dc27749..71425e3c 100644
--- a/Documentation/public-inbox-netd.pod
+++ b/Documentation/public-inbox-netd.pod
@@ -25,9 +25,9 @@ See common options in L<public-inbox-daemon(8)/OPTIONS>.
 
 =over
 
-=item -l PROTO://ADDRESS/?cert=/path/to/cert,key=/path/to/key
+=item -l PROTOCOL://ADDRESS/?cert=/path/to/cert,key=/path/to/key
 
-=item --listen PROTO://ADDRESS/?cert=/path/to/cert,key=/path/to/key
+=item --listen PROTOCOL://ADDRESS/?cert=/path/to/cert,key=/path/to/key
 
 =item -l http://ADDRESS/?env.PI_CONFIG=/path/to/cfg,psgi=/path/to/app.psgi
 
diff --git a/Documentation/public-inbox-nntpd.pod b/Documentation/public-inbox-nntpd.pod
index cf53da59..59111f92 100644
--- a/Documentation/public-inbox-nntpd.pod
+++ b/Documentation/public-inbox-nntpd.pod
@@ -26,9 +26,9 @@ are supported and documented below.
 
 =over
 
-=item -l PROTO://ADDRESS/?cert=/path/to/cert,key=/path/to/key
+=item -l PROTOCOL://ADDRESS/?cert=/path/to/cert,key=/path/to/key
 
-=item --listen PROTO://ADDRESS/?cert=/path/to/cert,key=/path/to/key
+=item --listen PROTOCOL://ADDRESS/?cert=/path/to/cert,key=/path/to/key
 
 In addition to the normal C<-l>/C<--listen> switch described in
 L<public-inbox-daemon(8)>, the protocol prefix (e.g. C<nntp://> or
diff --git a/Documentation/public-inbox-pop3d.pod b/Documentation/public-inbox-pop3d.pod
index 0404c2a7..fb16fb96 100644
--- a/Documentation/public-inbox-pop3d.pod
+++ b/Documentation/public-inbox-pop3d.pod
@@ -50,12 +50,12 @@ See common options in L<public-inbox-daemon(8)/OPTIONS>.
 
 =over
 
-=item -l PROTO://ADDRESS/?cert=/path/to/cert,key=/path/to/key
+=item -l PROTOCOL://ADDRESS/?cert=/path/to/cert,key=/path/to/key
 
-=item --listen PROTO://ADDRESS/?cert=/path/to/cert,key=/path/to/key
+=item --listen PROTOCOL://ADDRESS/?cert=/path/to/cert,key=/path/to/key
 
 In addition to the normal C<-l>/C<--listen> switch described in
-L<public-inbox-daemon(8)>, the C<PROTO> prefix (e.g. C<pop3://> or
+L<public-inbox-daemon(8)>, the C<PROTOCOL> prefix (e.g. C<pop3://> or
 C<pop3s://>) may be specified to force a given protocol.
 
 For STARTTLS and POP3S support, the C<cert> and C<key> may be specified
diff --git a/Documentation/txt2pre b/Documentation/txt2pre
index 3ecd9100..62175f34 100755
--- a/Documentation/txt2pre
+++ b/Documentation/txt2pre
@@ -1,15 +1,15 @@
-#!/usr/bin/env perl
-# Copyright (C) 2014-2021 all contributors <meta@public-inbox.org>
+#!perl -w
+# n.b. this is invoked via $(PERL) in makefiles
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 #
 # Stupid script to make HTML from preformatted, utf-8 text versions,
 # only generating links for http(s).  Markdown does too much
 # and requires indentation to output preformatted text.
-use strict;
-use warnings;
+use v5.12;
 use PublicInbox::Linkify;
 use PublicInbox::Hval qw(ascii_html);
-my %xurls;
+my (%xurls, %lei);
 for (qw[lei(1)
         lei-add-external(1)
         lei-add-watch(1)
@@ -42,6 +42,7 @@ for (qw[lei(1)
         lei-q(1)
         lei-rediff(1)
         lei-refresh-mail-sync(1)
+        lei-reindex(1)
         lei-rm(1)
         lei-rm-watch(1)
         lei-security(7)
@@ -63,8 +64,10 @@ for (qw[lei(1)
         public-inbox-init(1)
         public-inbox-learn(1)
         public-inbox-mda(1)
+        public-inbox-netd(1)
         public-inbox-nntpd(1)
         public-inbox-overview(7)
+        public-inbox-pop3d(1)
         public-inbox-purge(1)
         public-inbox-v1-format(5)
         public-inbox-v2-format(5)
@@ -74,8 +77,11 @@ for (qw[lei(1)
         my ($n) = (/([\w\-\.]+)/);
         $xurls{$_} = "$n.html";
         $xurls{$n} = "$n.html";
+        /\Alei-(.+?)\(1\)\z/ and $xurls{"lei $1"} = "$n.html";
 }
 
+$xurls{'lei/store'} = 'lei-store-format.html';
+
 for (qw[make(1) flock(2) setrlimit(2) vfork(2) tmpfs(5) inotify(7) unix(7)
                 syslog(3)]) {
         my ($n, $s) = (/([\w\-]+)\((\d)\)/);
@@ -158,6 +164,9 @@ if ($str =~ /^NAME\n\s+([^\n]+)/sm) {
         if ($title =~ /([\w\.\-]+)/) {
                 delete $xurls{$1};
         }
+        if ($title =~ /\blei-([\w\-]+)\b/) {
+                delete $xurls{"lei $1"};
+        }
 }
 $title = ascii_html($title);
 my $l = PublicInbox::Linkify->new;
diff --git a/INSTALL b/INSTALL
index 0974028d..aa9a502d 100644
--- a/INSTALL
+++ b/INSTALL
@@ -5,7 +5,7 @@ This is for folks who want to setup their own public-inbox instance.
 Clients should use normal git-clone/git-fetch, IMAP or NNTP clients
 if they want to import mail into their personal inboxes.
 
-As of 2021, public-inbox is packaged by several OS distributions,
+As of 2022, public-inbox is packaged by several OS distributions,
 listed in alphabetical order: Debian, GNU Guix, NixOS, and Void Linux.
 
 public-inbox is developed on Debian GNU/Linux systems and will
@@ -28,7 +28,7 @@ public-inbox requires a number of other packages to access its full
 functionality.  The core tools are, of course:
 
 * Git (1.8.0+, 2.6+ for writing v2 inboxes)
-* Perl 5.10.1+
+* Perl 5.12.0+
 * DBD::SQLite (needed for IMAP, NNTP, message threading, and v2 inboxes)
 
 To accept incoming mail into a public inbox, you'll likely want:
@@ -210,5 +210,5 @@ RPM-based distros split them out into separate packages:
 Copyright
 ---------
 
-Copyright 2013-2021 all contributors <meta@public-inbox.org>
+Copyright all contributors <meta@public-inbox.org>
 License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
diff --git a/MANIFEST b/MANIFEST
index e5880cc3..29f368de 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -14,6 +14,7 @@ Documentation/RelNotes/v1.6.0.eml
 Documentation/RelNotes/v1.6.1.eml
 Documentation/RelNotes/v1.7.0.eml
 Documentation/RelNotes/v1.8.0.eml
+Documentation/RelNotes/v1.9.0.eml
 Documentation/clients.txt
 Documentation/common.perl
 Documentation/dc-dlvr-spam-flow.txt
@@ -55,6 +56,7 @@ Documentation/lei-p2q.pod
 Documentation/lei-q.pod
 Documentation/lei-rediff.pod
 Documentation/lei-refresh-mail-sync.pod
+Documentation/lei-reindex.pod
 Documentation/lei-rm-watch.pod
 Documentation/lei-rm.pod
 Documentation/lei-security.pod
@@ -121,10 +123,6 @@ devel/README
 devel/syscall-list
 examples/README
 examples/README.unsubscribe
-examples/apache2_cgi.conf
-examples/apache2_perl.conf
-examples/apache2_perl_old.conf
-examples/cgi-webrick.rb
 examples/cgit-commit-filter.lua
 examples/cgit-wwwhighlight-filter.lua
 examples/cgit.psgi
@@ -137,13 +135,12 @@ examples/nginx_proxy
 examples/public-inbox-config
 examples/public-inbox-httpd.socket
 examples/public-inbox-httpd@.service
-examples/public-inbox-imap-onion.socket
 examples/public-inbox-imapd.socket
 examples/public-inbox-imapd@.service
-examples/public-inbox-imaps.socket
+examples/public-inbox-netd.socket
+examples/public-inbox-netd@.service
 examples/public-inbox-nntpd.socket
 examples/public-inbox-nntpd@.service
-examples/public-inbox-nntps.socket
 examples/public-inbox-watch.service
 examples/public-inbox.psgi
 examples/unsubscribe-milter.socket
@@ -260,6 +257,7 @@ lib/PublicInbox/LeiPmdir.pm
 lib/PublicInbox/LeiQuery.pm
 lib/PublicInbox/LeiRediff.pm
 lib/PublicInbox/LeiRefreshMailSync.pm
+lib/PublicInbox/LeiReindex.pm
 lib/PublicInbox/LeiRemote.pm
 lib/PublicInbox/LeiRm.pm
 lib/PublicInbox/LeiRmWatch.pm
@@ -308,6 +306,7 @@ lib/PublicInbox/PktOp.pm
 lib/PublicInbox/ProcessPipe.pm
 lib/PublicInbox/Qspawn.pm
 lib/PublicInbox/Reply.pm
+lib/PublicInbox/RepoSnapshot.pm
 lib/PublicInbox/SaPlugin/ListMirror.pm
 lib/PublicInbox/SaPlugin/ListMirror.pod
 lib/PublicInbox/Search.pm
@@ -344,6 +343,7 @@ lib/PublicInbox/Watch.pm
 lib/PublicInbox/WwwAltId.pm
 lib/PublicInbox/WwwAtomStream.pm
 lib/PublicInbox/WwwAttach.pm
+lib/PublicInbox/WwwCoderepo.pm
 lib/PublicInbox/WwwHighlight.pm
 lib/PublicInbox/WwwListing.pm
 lib/PublicInbox/WwwStatic.pm
@@ -402,6 +402,7 @@ t/content_hash.t
 t/convert-compact.t
 t/data-gen/.gitignore
 t/data/0001.patch
+t/data/attached-mbox-with-utf8.eml
 t/data/binary.patch
 t/data/message_embed.eml
 t/dir_idle.t
@@ -479,6 +480,7 @@ t/lei-q-remote-import.t
 t/lei-q-save.t
 t/lei-q-thread.t
 t/lei-refresh-mail-sync.t
+t/lei-reindex.t
 t/lei-sigpipe.t
 t/lei-tag.t
 t/lei-up.t
@@ -583,7 +585,6 @@ t/x-unknown-alpine.eml
 t/xcpdb-reshard.t
 version-gen.perl
 xt/cmp-msgstr.t
-xt/cmp-msgview.t
 xt/create-many-inboxes.t
 xt/eml_check_limits.t
 xt/eml_octet-stream.t
@@ -604,7 +605,6 @@ xt/nntpd-validate.t
 xt/over-fsck.perl
 xt/perf-msgview.t
 xt/perf-nntpd.t
-xt/perf-obfuscate.t
 xt/perf-threading.t
 xt/pop3d-mpop.t
 xt/solver.t
diff --git a/Makefile.PL b/Makefile.PL
index 848eb702..9233ac9d 100644
--- a/Makefile.PL
+++ b/Makefile.PL
@@ -11,7 +11,8 @@ my $v = {};
 my $t = {};
 
 # do not sort
-my @RELEASES = qw(v1.8.0 v1.7.0 v1.6.1 v1.6.0 v1.5.0 v1.4.0 v1.3.0 v1.2.0
+my @RELEASES = qw(v1.9.0
+        v1.8.0 v1.7.0 v1.6.1 v1.6.0 v1.5.0 v1.4.0 v1.3.0 v1.2.0
         v1.1.0-pre1 v1.0.0);
 
 $v->{news_deps} = [ map { "Documentation/RelNotes/$_.eml" } @RELEASES ];
@@ -52,7 +53,8 @@ $v->{-m1} = [ map {
         lei-import lei-index lei-init lei-inspect lei-lcat
         lei-ls-external lei-ls-label lei-ls-mail-source lei-ls-mail-sync
         lei-ls-search lei-ls-watch lei-mail-diff lei-p2q lei-q
-        lei-rediff lei-refresh-mail-sync lei-rm lei-rm-watch lei-tag
+        lei-rediff lei-refresh-mail-sync lei-reindex
+        lei-rm lei-rm-watch lei-tag
         lei-up)];
 $v->{-m5} = [ qw(public-inbox-config public-inbox-v1-format
                 public-inbox-v2-format public-inbox-extindex-format
@@ -131,7 +133,7 @@ WriteMakefile(
         NAME => 'PublicInbox', # n.b. camel-case is not our choice
 
         # XXX drop "PENDING" in .pod before updating this!
-        VERSION => '1.9.0.PENDING',
+        VERSION => '2.0.0.PENDING',
 
         AUTHOR => 'public-inbox hackers <meta@public-inbox.org>',
         ABSTRACT => 'an "archives first" approach to mailing lists',
diff --git a/README b/README
index 364ef7e0..01089314 100644
--- a/README
+++ b/README
@@ -117,16 +117,16 @@ on git@vger.kernel.org).
 The archives are readable via IMAP, NNTP or HTTP:
 
         nntps://news.public-inbox.org/inbox.comp.mail.public-inbox.meta
-        imaps://news.public-inbox.org/inbox.comp.mail.public-inbox.meta.0
+        imaps://;AUTH=ANONYMOUS@public-inbox.org/inbox.comp.mail.public-inbox.meta.0
         https://public-inbox.org/meta/
 
-AUTH=ANONYMOUS is supported for IMAP, but any username + password works
+AUTH=ANONYMOUS is recommended for IMAP, but any username + password works
 
 And as Tor hidden services:
 
         http://4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/meta/
         nntp://4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/inbox.comp.mail.public-inbox.meta
-        imap://4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/inbox.comp.mail.public-inbox.meta.0
+        imap://;AUTH=ANONYMOUS@4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/inbox.comp.mail.public-inbox.meta.0
 
 You may also clone all messages via git:
 
diff --git a/TODO b/TODO
index 36055911..1537179e 100644
--- a/TODO
+++ b/TODO
@@ -111,6 +111,13 @@ all need to be considered for everything we introduce)
 * improve performance and avoid head-of-line blocking on slow storage
   (done for most git blob retrievals, Xapian needs work)
 
+* allow optional use of separate Xapian worker process to implement
+  timeouts and avoid head-of-line blocking problems.  Consider
+  just-ahead-of-time builds to take advantage of custom date parsers
+  (approxidate) and other features not available to Perl bindings.
+
+* integrate git approxidate parsing into Xapian w/o spawning git
+
 * HTTP(S) search API (likely JMAP, but GraphQL could be an option)
   It should support git-specific prefixes (dfpre:, dfpost:, dfn:, etc)
   as extensions.  If JMAP, it should have HTTP(S) analogues to
@@ -149,3 +156,5 @@ all need to be considered for everything we introduce)
 * expose lei contents via read/write IMAP/JMAP server for personal use
 
 * git SHA-256 migration/coexistence path
+
+* decode RFC 3676 format=flowed + DelSp properly (see mflow (mblaze), mutt, ...)
diff --git a/devel/syscall-list b/devel/syscall-list
index d33a8a78..0b36c0e2 100755
--- a/devel/syscall-list
+++ b/devel/syscall-list
@@ -26,9 +26,13 @@ system($cc, '-o', $x, $f, @cflags) == 0 or die "cc failed \$?=$?";
 exec($x);
 __DATA__
 #define _GNU_SOURCE
+#include <signal.h>
 #include <sys/syscall.h>
 #include <sys/ioctl.h>
+#ifdef __linux__
 #include <linux/fs.h>
+#endif
+#include <sys/types.h>
 #include <unistd.h>
 #include <stdio.h>
 
@@ -60,6 +64,8 @@ int main(void)
         D(SYS_renameat2);
 #endif
 #endif /* Linux, any other OSes with stable syscalls? */
-        printf("size_t=%zu off_t=%zu\n", sizeof(size_t), sizeof(off_t));
+        printf("size_t=%zu off_t=%zu pid_t=%zu\n",
+                 sizeof(size_t), sizeof(off_t), sizeof(pid_t));
+        D(SIGWINCH);
         return 0;
 }
diff --git a/examples/README b/examples/README
index 1d5dcd34..5674d7ed 100644
--- a/examples/README
+++ b/examples/README
@@ -9,10 +9,6 @@ For PSGI/Plack (HTTP) servers
 -----------------------------
 public-inbox.psgi - starting point for PSGI/Plack users in production and dev
 
-For Apache2 users
------------------
-apache2_perl.conf - intended to be the basis of a production config
-
 Contact
 -------
 Please send any related feedback to public-inbox: meta@public-inbox.org
diff --git a/examples/apache2_cgi.conf b/examples/apache2_cgi.conf
deleted file mode 100644
index 5ec64d72..00000000
--- a/examples/apache2_cgi.conf
+++ /dev/null
@@ -1,34 +0,0 @@
-# Example Apache2 configuration using CGI mod_cgi
-# If possible, use mod_perl (see apache2_perl.conf) or
-# a standalone PSGI/Plack # server instead of this.
-# Adjust paths to your installation.
-
-ServerName "public-inbox"
-ServerRoot "/var/www/cgi-bin"
-DocumentRoot "/var/www/cgi-bin"
-ErrorLog "/tmp/public-inbox-error.log"
-PidFile "/tmp/public-inbox.pid"
-Listen 127.0.0.1:8080
-LoadModule cgi_module /usr/lib/apache2/modules/mod_cgi.so
-LoadModule env_module /usr/lib/apache2/modules/mod_env.so
-LoadModule rewrite_module /usr/lib/apache2/modules/mod_rewrite.so
-LoadModule dir_module /usr/lib/apache2/modules/mod_dir.so
-LoadModule mime_module /usr/lib/apache2/modules/mod_mime.so
-TypesConfig "/dev/null"
-
-<Directory /var/www/cgi-bin>
-        Options +ExecCGI
-        AddHandler cgi-script .cgi
-
-        # we use this hack to ensure "public-inbox.cgi" doesn't show up
-        # in any of our redirects:
-        SetEnv NO_SCRIPT_NAME 1
-
-        # our public-inbox.cgi requires PATH_INFO-based URLs with minimal
-        # use of query parameters
-        DirectoryIndex public-inbox.cgi
-        RewriteEngine On
-        RewriteCond %{REQUEST_FILENAME} !-f
-        RewriteCond %{REQUEST_FILENAME} !-d
-        RewriteRule ^.* /public-inbox.cgi/$0 [L,PT]
-</Directory>
diff --git a/examples/apache2_perl.conf b/examples/apache2_perl.conf
deleted file mode 100644
index a4721b5b..00000000
--- a/examples/apache2_perl.conf
+++ /dev/null
@@ -1,25 +0,0 @@
-# Example Apache2 configuration using Plack::Handler::Apache2
-# Adjust paths to your installation
-
-ServerName "public-inbox"
-ServerRoot "/var/www"
-DocumentRoot "/var/www"
-ErrorLog "/tmp/public-inbox-error.log"
-PidFile "/tmp/public-inbox.pid"
-Listen 127.0.0.1:8080
-LoadModule perl_module /usr/lib/apache2/modules/mod_perl.so
-
-# no need to set no rely on HOME if using this:
-PerlSetEnv PI_CONFIG /home/pi/.public-inbox/config
-
-<Location />
-        SetHandler perl-script
-        PerlResponseHandler Plack::Handler::Apache2
-        PerlSetVar psgi_app /path/to/public-inbox.psgi
-</Location>
-
-# Optional, preload the application in the parent like startup.pl
-<Perl>
-        use Plack::Handler::Apache2;
-        Plack::Handler::Apache2->preload("/path/to/public-inbox.psgi");
-</Perl>
diff --git a/examples/apache2_perl_old.conf b/examples/apache2_perl_old.conf
deleted file mode 100644
index a6de2304..00000000
--- a/examples/apache2_perl_old.conf
+++ /dev/null
@@ -1,38 +0,0 @@
-# Example legacy Apache2 configuration using CGI + mod_perl2
-# Consider using Plack::Handler::Apache2 instead (see apache2_perl.conf)
-# Adjust paths to your installation
-
-ServerName "public-inbox"
-ServerRoot "/var/www/cgi-bin"
-DocumentRoot "/var/www/cgi-bin"
-ErrorLog "/tmp/public-inbox-error.log"
-PidFile "/tmp/public-inbox.pid"
-Listen 127.0.0.1:8080
-LoadModule perl_module /usr/lib/apache2/modules/mod_perl.so
-LoadModule rewrite_module /usr/lib/apache2/modules/mod_rewrite.so
-LoadModule dir_module /usr/lib/apache2/modules/mod_dir.so
-LoadModule mime_module /usr/lib/apache2/modules/mod_mime.so
-TypesConfig "/dev/null"
-
-# PerlPassEnv PATH # this is implicit
-<Directory /var/www/cgi-bin>
-        Options +ExecCGI
-        AddHandler perl-script .cgi
-        PerlResponseHandler ModPerl::Registry
-        PerlOptions +ParseHeaders
-
-        # we use this hack to ensure "public-inbox.cgi" doesn't show up
-        # in any of our redirects:
-        PerlSetEnv NO_SCRIPT_NAME 1
-
-        # no need to set no rely on HOME if using this:
-        PerlSetEnv PI_CONFIG /home/pi/.public-inbox/config
-
-        # our public-inbox.cgi requires PATH_INFO-based URLs with minimal
-        # use of query parameters
-        DirectoryIndex public-inbox.cgi
-        RewriteEngine On
-        RewriteCond %{REQUEST_FILENAME} !-f
-        RewriteCond %{REQUEST_FILENAME} !-d
-        RewriteRule ^.* /public-inbox.cgi/$0 [L,PT]
-</Directory>
diff --git a/examples/cgi-webrick.rb b/examples/cgi-webrick.rb
deleted file mode 100644
index 5554a012..00000000
--- a/examples/cgi-webrick.rb
+++ /dev/null
@@ -1,25 +0,0 @@
-#!/usr/bin/env ruby
-# Sample configuration using WEBrick, mainly intended dev/testing
-# for folks familiar with Ruby and not various Perl webserver
-# deployment options.  For those familiar with Perl web servers,
-# plackup(1) is recommended for development and public-inbox-httpd(1)
-# is our production deployment server.
-require 'webrick'
-require 'logger'
-options = {
-  :BindAddress => '127.0.0.1',
-  :Port => 8080,
-  :Logger => Logger.new($stderr),
-  :CGIPathEnv => ENV['PATH'], # need to run 'git' commands
-  :AccessLog => [
-    [ Logger.new($stdout), WEBrick::AccessLog::COMBINED_LOG_FORMAT ]
-  ],
-}
-server = WEBrick::HTTPServer.new(options)
-server.mount("/",
-             WEBrick::HTTPServlet::CGIHandler,
-            "/var/www/cgi-bin/public-inbox.cgi")
-['INT', 'TERM'].each do |signal|
-  trap(signal) {exit!(0)}
-end
-server.start
diff --git a/examples/grok-pull.post_update_hook.sh b/examples/grok-pull.post_update_hook.sh
index 77489472..4d303c03 100755
--- a/examples/grok-pull.post_update_hook.sh
+++ b/examples/grok-pull.post_update_hook.sh
@@ -111,7 +111,7 @@ case $cfg_dir in
                         "publicinbox.$inbox_name.infourl" "$url"
         done
         curl -sSfv "$remote_inbox_url"/description >"$inbox_dir"/description
-        echo "I: $inbox_name at $inbox_dir ($addresses) $local_url"
+        echo "# $inbox_name at $inbox_dir ($addresses) $local_url"
         ;;
 esac
 
diff --git a/examples/nginx_proxy b/examples/nginx_proxy
index d8d1e6df..754a4931 100644
--- a/examples/nginx_proxy
+++ b/examples/nginx_proxy
@@ -1,8 +1,14 @@
 # Example NGINX configuration to proxy-pass requests
-# to public-inbox-httpd or to a standalone PSGI/Plack server.
+# to varnish, public-inbox-(httpd|netd) or any PSGI/Plack server.
 # The daemon is assumed to be running locally on port 8001.
 # Adjust ssl certificate paths if you use any, or remove
 # the ssl configuration directives if you don't.
+#
+# Note: public-inbox-httpd and -netd both support HTTPS, but they
+# don't support caching which Varnish provides.  The recommended
+# setup is currently:
+#
+#   (nginx|any-HTTPS-proxy) <-> varnish <-> public-inbox-(httpd|netd)
 server {
         server_name _;
         listen 80;
@@ -14,6 +20,7 @@ server {
                 proxy_set_header    HOST $host;
                 proxy_set_header    X-Real-IP $remote_addr;
                 proxy_set_header    X-Forwarded-Proto $scheme;
+                proxy_buffering off; # lowers response latency
                 proxy_pass          http://127.0.0.1:8001$request_uri;
         }
 
diff --git a/examples/public-inbox-httpd.socket b/examples/public-inbox-httpd.socket
index 1a1ed735..3a6e4432 100644
--- a/examples/public-inbox-httpd.socket
+++ b/examples/public-inbox-httpd.socket
@@ -1,4 +1,7 @@
 # ==> /etc/systemd/system/public-inbox-httpd.socket <==
+# Consider looking at public-inbox-netd.socket instead of this file
+# to simplify management when serving multiple protocols.
+
 [Unit]
 Description = public-inbox-httpd socket
 
diff --git a/examples/public-inbox-httpd@.service b/examples/public-inbox-httpd@.service
index 147f7c6d..73731533 100644
--- a/examples/public-inbox-httpd@.service
+++ b/examples/public-inbox-httpd@.service
@@ -1,4 +1,7 @@
 # ==> /etc/systemd/system/public-inbox-httpd@.service <==
+# Consider looking at public-inbox-netd@.service instead of this file
+# to simplify management when serving multiple protocols.
+#
 # Since SIGUSR2 upgrades do not work under systemd, this service file
 # allows starting two simultaneous services during upgrade time
 # (e.g. public-inbox-httpd@1 public-inbox-httpd@2) with the intention
diff --git a/examples/public-inbox-imap-onion.socket b/examples/public-inbox-imap-onion.socket
deleted file mode 100644
index 76b4e7ca..00000000
--- a/examples/public-inbox-imap-onion.socket
+++ /dev/null
@@ -1,12 +0,0 @@
-# ==> /etc/systemd/system/public-inbox-imap-onion.socket <==
-# This unit is for the corresponding line in torrc(5):
-# HiddenServicePort 143 unix:/run/imapd.onion.sock
-[Unit]
-Description = public-inbox-imap .onion socket
-
-[Socket]
-ListenStream = /run/imapd.onion.sock
-Service = public-inbox-imapd@1.service
-
-[Install]
-WantedBy = sockets.target
diff --git a/examples/public-inbox-imapd.socket b/examples/public-inbox-imapd.socket
index fcd924fd..22ce16fb 100644
--- a/examples/public-inbox-imapd.socket
+++ b/examples/public-inbox-imapd.socket
@@ -1,11 +1,26 @@
 # ==> /etc/systemd/system/public-inbox-imapd.socket <==
+# Consider looking at public-inbox-netd.socket instead of this file
+# to simplify management when serving multiple protocols.
+#
+# This contains 5 sockets for an public-inbox-imapd instance.
+# The TCP ports are well-known ports registered in /etc/services.
+# The /run/imapd.onion.sock entry is meant for the Tor hidden service
+# enabled by the following line in the torrc(5) file:
+#   HiddenServicePort 143 unix:/run/imapd.onion.sock
 [Unit]
-Description = public-inbox-imapd socket
+Description = public-inbox-imapd sockets
 
 [Socket]
 ListenStream = 0.0.0.0:143
+ListenStream = 0.0.0.0:993
+ListenStream = /run/imapd.onion.sock
+
+# Separating IPv4 from IPv6 listeners makes for nicer output
+# of IPv4 addresses in various reporting/monitoring tools
 BindIPv6Only = ipv6-only
 ListenStream = [::]:143
+ListenStream = [::]:993
+
 Service = public-inbox-imapd@1.service
 
 [Install]
diff --git a/examples/public-inbox-imapd@.service b/examples/public-inbox-imapd@.service
index e0446ed3..300019a8 100644
--- a/examples/public-inbox-imapd@.service
+++ b/examples/public-inbox-imapd@.service
@@ -1,4 +1,7 @@
 # ==> /etc/systemd/system/public-inbox-imapd@.service <==
+# Consider looking at public-inbox-netd@.service instead of this file
+# to simplify management when serving multiple protocols.
+#
 # Since SIGUSR2 upgrades do not work under systemd, this service file
 # allows starting two simultaneous services during upgrade time
 # (e.g. public-inbox-imapd@1 public-inbox-imapd@2) with the intention
@@ -7,10 +10,8 @@
 
 [Unit]
 Description = public-inbox-imapd IMAP server %i
-Wants = public-inbox-imapd.socket public-inbox-imaps.socket \
-public-inbox-imap-onion.socket
-After = public-inbox-imapd.socket public-inbox-imaps.socket \
-public-inbox-imap-onion.socket
+Wants = public-inbox-imapd.socket
+After = public-inbox-imapd.socket
 
 [Service]
 Environment = PI_CONFIG=/home/pi/.public-inbox/config \
@@ -29,8 +30,7 @@ StandardError = syslog
 # simultaneous services
 NonBlocking = true
 
-Sockets = public-inbox-imapd.socket public-inbox-imaps.socket \
-public-inbox-imap-onion.socket
+Sockets = public-inbox-imapd.socket
 
 KillSignal = SIGQUIT
 User = nobody
diff --git a/examples/public-inbox-imaps.socket b/examples/public-inbox-imaps.socket
deleted file mode 100644
index b61cc742..00000000
--- a/examples/public-inbox-imaps.socket
+++ /dev/null
@@ -1,12 +0,0 @@
-# ==> /etc/systemd/system/public-inbox-imaps.socket <==
-[Unit]
-Description = public-inbox-imaps socket
-
-[Socket]
-ListenStream = 0.0.0.0:993
-BindIPv6Only = ipv6-only
-ListenStream = [::]:993
-Service = public-inbox-imapd@1.service
-
-[Install]
-WantedBy = sockets.target
diff --git a/examples/public-inbox-netd.socket b/examples/public-inbox-netd.socket
new file mode 100644
index 00000000..9a19602e
--- /dev/null
+++ b/examples/public-inbox-netd.socket
@@ -0,0 +1,45 @@
+# ==> /etc/systemd/system/public-inbox-netd.socket <==
+# This contains all the services that public-inbox-netd can run;
+# allowing it to replace (or run in parallel to) any existing -httpd,
+# -imapd, -nntpd, or -pop3d instances.
+#
+# The TCP ports are well-known ports registered in /etc/services.
+# The /run/*.sock entries are meant for the Tor hidden service
+# enabled by the following lines in the torrc(5) file:
+#   HiddenServicePort 110 unix:/run/pop3.sock
+#   HiddenServicePort 119 unix:/run/nntp.sock
+#   HiddenServicePort 143 unix:/run/imap.sock
+[Unit]
+Description = public-inbox-netd sockets
+
+[Socket]
+# for tor (see torrc(5))
+ListenStream = /run/imap.sock
+ListenStream = /run/pop3.sock
+ListenStream = /run/nntp.sock
+
+# this is for varnish:
+ListenStream = 127.0.0.1:280
+
+# public facing
+ListenStream = 0.0.0.0:110
+ListenStream = 0.0.0.0:119
+ListenStream = 0.0.0.0:143
+ListenStream = 0.0.0.0:563
+ListenStream = 0.0.0.0:993
+ListenStream = 0.0.0.0:995
+
+# Separating IPv4 from IPv6 listeners makes for nicer output
+# of IPv4 addresses in various reporting/monitoring tools
+BindIPv6Only = ipv6-only
+ListenStream = [::]:110
+ListenStream = [::]:119
+ListenStream = [::]:143
+ListenStream = [::]:563
+ListenStream = [::]:993
+ListenStream = [::]:995
+
+Service = public-inbox-netd@1.service
+
+[Install]
+WantedBy = sockets.target
diff --git a/examples/public-inbox-netd@.service b/examples/public-inbox-netd@.service
new file mode 100644
index 00000000..de5feea6
--- /dev/null
+++ b/examples/public-inbox-netd@.service
@@ -0,0 +1,60 @@
+# ==> /etc/systemd/system/public-inbox-netd@.service <==
+# Since SIGUSR2 upgrades do not work under systemd, this service file
+# allows starting two simultaneous services during upgrade time
+# (e.g. public-inbox-netd@1 public-inbox-netd@2) with the intention
+# that they take turns running in-between upgrades.  This should
+# allow upgrading without downtime.
+# For servers expecting visitors from multiple timezones, TZ=UTC
+# is needed to ensure a consistent approxidate experience with search.
+[Unit]
+Description = public-inbox-netd server %i
+Wants = public-inbox-netd.socket
+After = public-inbox-netd.socket
+
+[Service]
+Environment = PI_CONFIG=/home/pi/.public-inbox/config \
+PATH=/usr/local/bin:/usr/bin:/bin \
+TZ=UTC \
+PERL_INLINE_DIRECTORY=/tmp/.netd-inline
+
+LimitNOFILE = 30000
+LimitCORE = infinity
+ExecStartPre = /bin/mkdir -p -m 1777 /tmp/.netd-inline
+
+# The '-l' args below map each socket in public-inbox-netd.socket to
+# the appropriate IANA service name:
+ExecStart = /usr/local/bin/public-inbox-netd -W0 \
+-1 /var/log/netd/stdout.out.log \
+--cert /etc/ssl/certs/news.example.com.pem \
+--key /etc/ssl/private/news.example.com.key
+-l imap:///run/imap.sock?out=/var/log/netd/imap.out,err=/var/log/netd/imap.err \
+-l nntp:///run/nntp.sock?out=/var/log/netd/nntp.out,err=/var/log/netd/nntp.err \
+-l pop3:///run/pop3.sock?out=/var/log/netd/pop3.out,err=/var/log/netd/pop3.err \
+-l imap://0.0.0.0/?out=/var/log/netd/imap.out,err=/var/log/netd/imap.err \
+-l nntp://0.0.0.0/?out=/var/log/netd/nntp.out,err=/var/log/netd/nntp.err \
+-l pop3://0.0.0.0/?out=/var/log/netd/pop3.out,err=/var/log/netd/pop3.err \
+-l imap://[::]/?out=/var/log/netd/imap.out,err=/var/log/netd/imap.err \
+-l nntp://[::]/?out=/var/log/netd/nntp.out,err=/var/log/netd/nntp.err \
+-l pop3://[::]/?out=/var/log/netd/pop3.out,err=/var/log/netd/pop3.err \
+-l imaps://0.0.0.0/?out=/var/log/netd/imap.out,err=/var/log/netd/imap.err \
+-l nntps://0.0.0.0/?out=/var/log/netd/nntp.out,err=/var/log/netd/nntp.err \
+-l pop3s://0.0.0.0/?out=/var/log/netd/pop3.out,err=/var/log/netd/pop3.err \
+-l imaps://[::]/?out=/var/log/netd/imap.out,err=/var/log/netd/imap.err \
+-l nntps://[::]/?out=/var/log/netd/nntp.out,err=/var/log/netd/nntp.err \
+-l pop3s://[::]/?out=/var/log/netd/pop3.out,err=/var/log/netd/pop3.err \
+-l http://127.0.0.1:280/?psgi=/etc/public.psgi,err=/var/log/netd/http.err
+
+# NonBlocking is REQUIRED to avoid a race condition if running
+# simultaneous services
+NonBlocking = true
+
+Sockets = public-inbox-netd.socket
+KillSignal = SIGQUIT
+User = news
+Group = ssl-cert
+ExecReload = /bin/kill -HUP $MAINPID
+TimeoutStopSec = 30
+KillMode = process
+
+[Install]
+WantedBy = multi-user.target
diff --git a/examples/public-inbox-nntpd.socket b/examples/public-inbox-nntpd.socket
index eeddf343..10335d8d 100644
--- a/examples/public-inbox-nntpd.socket
+++ b/examples/public-inbox-nntpd.socket
@@ -1,9 +1,26 @@
 # ==> /etc/systemd/system/public-inbox-nntpd.socket <==
+# Consider looking at public-inbox-netd.socket instead of this file
+# to simplify management when serving multiple protocols.
+#
+# This contains 5 sockets for an public-inbox-nntpd instance.
+# The TCP ports are well-known ports registered in /etc/services.
+# The /run/nntpd.onion.sock entry is meant for the Tor hidden service
+# enabled by the following line in the torrc(5) file:
+#   HiddenServicePort 119 unix:/run/nntpd.onion.sock
 [Unit]
-Description = public-inbox-nntpd socket
+Description = public-inbox-nntpd sockets
 
 [Socket]
-ListenStream = 119
+ListenStream = 0.0.0.0:119
+ListenStream = 0.0.0.0:563
+ListenStream = /run/nntpd.onion.sock
+
+# Separating IPv4 from IPv6 listeners makes for nicer output
+# of IPv4 addresses in various reporting/monitoring tools
+BindIPv6Only = ipv6-only
+ListenStream = [::]:119
+ListenStream = [::]:563
+
 Service = public-inbox-nntpd@1.service
 
 [Install]
diff --git a/examples/public-inbox-nntpd@.service b/examples/public-inbox-nntpd@.service
index 4dd2f5d7..56e1cc8f 100644
--- a/examples/public-inbox-nntpd@.service
+++ b/examples/public-inbox-nntpd@.service
@@ -1,4 +1,7 @@
 # ==> /etc/systemd/system/public-inbox-nntpd@.service <==
+# Consider looking at public-inbox-netd@.service instead of this file
+# to simplify management when serving multiple protocols.
+#
 # Since SIGUSR2 upgrades do not work under systemd, this service file
 # allows starting two simultaneous services during upgrade time
 # (e.g. public-inbox-nntpd@1 public-inbox-nntpd@2) with the intention
@@ -7,8 +10,8 @@
 
 [Unit]
 Description = public-inbox NNTP server %i
-Wants = public-inbox-nntpd.socket public-inbox-nntps.socket
-After = public-inbox-nntpd.socket public-inbox-nntps.socket
+Wants = public-inbox-nntpd.socket
+After = public-inbox-nntpd.socket
 
 [Service]
 Environment = PI_CONFIG=/home/pi/.public-inbox/config \
@@ -27,7 +30,7 @@ StandardError = syslog
 # simultaneous services
 NonBlocking = true
 
-Sockets = public-inbox-nntpd.socket public-inbox-nntps.socket
+Sockets = public-inbox-nntpd.socket
 
 KillSignal = SIGQUIT
 User = nobody
diff --git a/examples/public-inbox-nntps.socket b/examples/public-inbox-nntps.socket
deleted file mode 100644
index fa678196..00000000
--- a/examples/public-inbox-nntps.socket
+++ /dev/null
@@ -1,12 +0,0 @@
-# ==> /etc/systemd/system/public-inbox-nntps.socket <==
-[Unit]
-Description = public-inbox-nntps socket
-
-[Socket]
-ListenStream = 0.0.0.0:563
-BindIPv6Only = ipv6-only
-ListenStream = [::]:563
-Service = public-inbox-nntpd@1.service
-
-[Install]
-WantedBy = sockets.target
diff --git a/lib/PublicInbox/Cgit.pm b/lib/PublicInbox/Cgit.pm
index cc729aa2..336098ca 100644
--- a/lib/PublicInbox/Cgit.pm
+++ b/lib/PublicInbox/Cgit.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2019-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # wrapper for cgit(1) and git-http-backend(1) for browsing and
@@ -6,7 +6,8 @@
 # directive to be set in the public-inbox config file.
 
 package PublicInbox::Cgit;
-use strict;
+use v5.12;
+use parent qw(PublicInbox::WwwCoderepo);
 use PublicInbox::GitHTTPBackend;
 use PublicInbox::Git;
 # not bothering with Exporter for a one-off
@@ -40,10 +41,9 @@ sub locate_cgit ($) {
                 if (defined($cgit_bin) && $cgit_bin =~ m!\A(.+?)/[^/]+\z!) {
                         unshift @dirs, $1 if -d $1;
                 }
-                foreach my $d (@dirs) {
-                        my $f = "$d/cgit.css";
-                        next unless -f $f;
-                        $cgit_data = $d;
+                for (@dirs) {
+                        next unless -f "$_/cgit.css";
+                        $cgit_data = $_;
                         last;
                 }
         }
@@ -53,10 +53,7 @@ sub locate_cgit ($) {
 sub new {
         my ($class, $pi_cfg) = @_;
         my ($cgit_bin, $cgit_data) = locate_cgit($pi_cfg);
-        # TODO: support gitweb and other repository viewers?
-        if (defined(my $cgitrc = $pi_cfg->{-cgitrc_unparsed})) {
-                $pi_cfg->parse_cgitrc($cgitrc, 0);
-        }
+        $cgit_bin // return; # fall back in WWW->cgit
         my $self = bless {
                 cmd => [ $cgit_bin ],
                 cgit_data => $cgit_data,
@@ -64,18 +61,9 @@ sub new {
         }, $class;
 
         # some cgit repos may not be mapped to inboxes, so ensure those exist:
-        my $code_repos = $pi_cfg->{-code_repos};
-        foreach my $k (keys %$pi_cfg) {
-                $k =~ /\Acoderepo\.(.+)\.dir\z/ or next;
-                my $dir = $pi_cfg->{$k};
-                $code_repos->{$1} ||= $pi_cfg->fill_code_repo($1);
-        }
-        while (my ($nick, $repo) = each %$code_repos) {
-                $self->{"\0$nick"} = $repo;
-        }
-        my $cgit_static = $pi_cfg->{-cgit_static};
-        my $static = join('|', map { quotemeta $_ } keys %$cgit_static);
-        $self->{static} = qr/\A($static)\z/;
+        PublicInbox::WwwCoderepo::prepare_coderepos($self);
+        my $s = join('|', map { quotemeta } keys %{$pi_cfg->{-cgit_static}});
+        $self->{static} = qr/\A($s)\z/;
         $self;
 }
 
@@ -96,7 +84,7 @@ my @PASS_ENV = qw(
 my $parse_cgi_headers = \&PublicInbox::GitHTTPBackend::parse_cgi_headers;
 
 sub call {
-        my ($self, $env) = @_;
+        my ($self, $env, $ctx) = @_; # $ctx is optional, used by WWW
         my $path_info = $env->{PATH_INFO};
         my $cgit_data;
 
@@ -114,7 +102,7 @@ sub call {
 
         my $cgi_env = { PATH_INFO => $path_info };
         foreach (@PASS_ENV) {
-                defined(my $v = $env->{$_}) or next;
+                my $v = $env->{$_} // next;
                 $cgi_env->{$_} = $v;
         }
         $cgi_env->{'HTTPS'} = 'on' if $env->{'psgi.url_scheme'} eq 'https';
@@ -122,7 +110,7 @@ sub call {
         my $rdr = input_prepare($env) or return r(500);
         my $qsp = PublicInbox::Qspawn->new($self->{cmd}, $cgi_env, $rdr);
         my $limiter = $self->{pi_cfg}->limiter('-cgit');
-        $qsp->psgi_return($env, $limiter, $parse_cgi_headers);
+        $qsp->psgi_return($env, $limiter, $parse_cgi_headers, $ctx);
 }
 
 1;
diff --git a/lib/PublicInbox/CompressNoop.pm b/lib/PublicInbox/CompressNoop.pm
index e3301473..5135299f 100644
--- a/lib/PublicInbox/CompressNoop.pm
+++ b/lib/PublicInbox/CompressNoop.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # Provide the same methods as Compress::Raw::Zlib::Deflate but
@@ -10,7 +10,7 @@ use Compress::Raw::Zlib qw(Z_OK);
 sub new { bless \(my $self), __PACKAGE__ }
 
 sub deflate { # ($self, $input, $output)
-        $_[2] .= $_[1];
+        $_[2] .= ref($_[1]) ? ${$_[1]} : $_[1];
         Z_OK;
 }
 
diff --git a/lib/PublicInbox/Config.pm b/lib/PublicInbox/Config.pm
index a31b5b74..5620bd0e 100644
--- a/lib/PublicInbox/Config.pm
+++ b/lib/PublicInbox/Config.pm
@@ -13,6 +13,7 @@ use v5.10.1;
 use PublicInbox::Inbox;
 use PublicInbox::Spawn qw(popen_rd);
 our $LD_PRELOAD = $ENV{LD_PRELOAD}; # only valid at startup
+our $DEDUPE; # set to {} to dedupe or clear cache
 
 sub _array ($) { ref($_[0]) eq 'ARRAY' ? $_[0] : [ $_[0] ] }
 
@@ -22,11 +23,17 @@ sub new {
         my ($class, $file, $errfh) = @_;
         $file //= default_file();
         my $self;
+        my $set_dedupe;
         if (ref($file) eq 'SCALAR') { # used by some tests
                 open my $fh, '<', $file or die;  # PerlIO::scalar
                 $self = config_fh_parse($fh, "\n", '=');
                 bless $self, $class;
         } else {
+                if (-f $file && $DEDUPE) {
+                        $file = rel2abs_collapsed($file);
+                        $self = $DEDUPE->{$file} and return $self;
+                        $set_dedupe = 1;
+                }
                 $self = git_config_dump($class, $file, $errfh);
                 $self->{'-f'} = $file;
         }
@@ -39,7 +46,6 @@ sub new {
         $self->{-no_obfuscate} = {};
         $self->{-limiters} = {};
         $self->{-code_repos} = {}; # nick => PublicInbox::Git object
-        $self->{-cgitrc_unparsed} = $self->{'publicinbox.cgitrc'};
 
         if (my $no = delete $self->{'publicinbox.noobfuscate'}) {
                 $no = _array($no);
@@ -62,7 +68,7 @@ sub new {
         if (my $css = delete $self->{'publicinbox.css'}) {
                 $self->{css} = _array($css);
         }
-
+        $DEDUPE->{$file} = $self if $set_dedupe;
         $self;
 }
 
@@ -270,6 +276,7 @@ sub scan_projects_coderepo ($$$) {
 
 sub parse_cgitrc {
         my ($self, $cgitrc, $nesting) = @_;
+        $cgitrc //= $self->{'publicinbox.cgitrc'};
         if ($nesting == 0) {
                 # defaults:
                 my %s = map { $_ => 1 } qw(/cgit.css /cgit.png
@@ -318,6 +325,8 @@ sub parse_cgitrc {
                 } elsif (m!\A(?:css|favicon|logo|repo\.logo)=(/.+)\z!) {
                         # absolute paths for static files via PublicInbox::Cgit
                         $self->{-cgit_static}->{$1} = 1;
+                } elsif (s!\Asnapshots=\s*!!) {
+                        $self->{'coderepo.snapshots'} = $_;
                 }
         }
         cgit_repo_merge($self, $repo->{dir}, $repo) if $repo;
@@ -336,7 +345,7 @@ sub fill_code_repo {
                 $git->{cgit_url} = $cgits = _array($cgits);
                 $self->{"$pfx.cgiturl"} = $cgits;
         }
-
+        $git->{nick} = $nick;
         $git;
 }
 
@@ -395,7 +404,12 @@ sub repo_objs {
                         push @repo_objs, $repo if $repo;
                 }
                 if (scalar @repo_objs) {
-                        $ibxish ->{-repo_objs} = \@repo_objs;
+                        require Scalar::Util;
+                        for (@repo_objs) {
+                                push @{$_->{-ibxs}}, $ibxish;
+                                Scalar::Util::weaken($_->{-ibxs}->[-1]);
+                        }
+                        $ibxish->{-repo_objs} = \@repo_objs;
                 } else {
                         delete $ibxish->{coderepo};
                 }
@@ -519,7 +533,7 @@ sub _fill_ei ($$) {
 }
 
 sub urlmatch {
-        my ($self, $key, $url) = @_;
+        my ($self, $key, $url, $try_git) = @_;
         state $urlmatch_broken; # requires git 1.8.5
         return if $urlmatch_broken;
         my $file = $self->{'-f'} // default_file();
@@ -528,13 +542,20 @@ sub urlmatch {
         my $fh = popen_rd($cmd);
         local $/ = "\0";
         my $val = <$fh>;
-        if (close($fh)) {
-                chomp($val);
-                $val;
-        } else {
-                $urlmatch_broken = 1 if (($? >> 8) != 1);
-                undef;
+        if (!close($fh)) {
+                undef $val;
+                if (($? >> 8) != 1) {
+                        $urlmatch_broken = 1;
+                } elsif ($try_git) { # n.b. this takes cwd into account
+                        $cmd = [qw(git config -z --get-urlmatch), $key, $url];
+                        $fh = popen_rd($cmd);
+                        $val = <$fh>;
+                        close($fh) or undef($val);
+                }
         }
+        $? = 0; # don't influence lei exit status
+        chomp $val if defined $val;
+        $val;
 }
 
 sub json {
diff --git a/lib/PublicInbox/ContentHash.pm b/lib/PublicInbox/ContentHash.pm
index bacc9cdd..1afbb413 100644
--- a/lib/PublicInbox/ContentHash.pm
+++ b/lib/PublicInbox/ContentHash.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2018-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # Unstable internal API.
@@ -63,8 +63,9 @@ sub content_digest ($;$) {
         # do NOT consider the Message-ID as part of the content_hash
         # if we got here, we've already got Message-ID reuse
         my %seen = map { $_ => 1 } @{mids($eml)};
-        foreach my $mid (@{references($eml)}) {
-                $dig->add("ref\0$mid\0") unless $seen{$mid}++;
+        for (grep { !$seen{$_}++ } @{references($eml)}) {
+                utf8::encode($_);
+                $dig->add("ref\0$_\0");
         }
 
         # Only use Sender: if From is not present
diff --git a/lib/PublicInbox/DS.pm b/lib/PublicInbox/DS.pm
index 77e2e5e9..26840662 100644
--- a/lib/PublicInbox/DS.pm
+++ b/lib/PublicInbox/DS.pm
@@ -660,8 +660,8 @@ sub long_step {
         if ($@ || !$self->{sock}) { # something bad happened...
                 delete $self->{long_cb};
                 my $elapsed = now() - $t0;
-                $@ and $self->err("%s during long response[$fd] - %0.6f",
-                                    $@, $elapsed);
+                $@ and warn("$@ during long response[$fd] - ",
+                                sprintf('%0.6f', $elapsed),"\n");
                 $self->out(" deferred[$fd] aborted - %0.6f", $elapsed);
                 $self->close;
         } elsif ($more) { # $self->{wbuf}:
@@ -688,7 +688,7 @@ sub requeue_once {
         # but only after all pending writes are done.
         # autovivify wbuf.  wbuf may be populated by $cb,
         # no need to rearm if so: (push returns new size of array)
-        requeue($self) if push(@{$self->{wbuf}}, \&long_step) == 1;
+        $self->requeue if push(@{$self->{wbuf}}, \&long_step) == 1;
 }
 
 sub long_response ($$;@) {
diff --git a/lib/PublicInbox/DSKQXS.pm b/lib/PublicInbox/DSKQXS.pm
index eccfa56d..7bd7773e 100644
--- a/lib/PublicInbox/DSKQXS.pm
+++ b/lib/PublicInbox/DSKQXS.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2019-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # Licensed the same as Danga::Socket (and Perl5)
 # License: GPL-1.0+ or Artistic-1.0-Perl
 #  <https://www.gnu.org/licenses/gpl-1.0.txt>
@@ -11,8 +11,7 @@
 #
 # It also implements signalfd(2) emulation via "tie".
 package PublicInbox::DSKQXS;
-use strict;
-use warnings;
+use v5.12;
 use parent qw(Exporter);
 use Symbol qw(gensym);
 use IO::KQueue;
@@ -71,7 +70,7 @@ sub READ { # called by sysread() for signalfd compatibility
         my $nr = $len / 128;
         my $r = 0;
         $_[1] = '';
-        do {
+        while (1) {
                 while ($nr--) {
                         my $signo = shift(@$sigbuf) or last;
                         # caller only cares about signalfd_siginfo.ssi_signo:
@@ -94,7 +93,7 @@ sub READ { # called by sysread() for signalfd compatibility
                 # field shows coalesced signals, and maybe we'll use it
                 # in the future...
                 @$sigbuf = map { $_->[0] } @events;
-        } while (1);
+        }
 }
 
 # for fileno() calls in PublicInbox::DS
@@ -107,6 +106,8 @@ sub epoll_ctl {
                 $kq->EV_SET($fd, EVFILT_READ, kq_flag(EPOLLIN, $ev));
                 eval { $kq->EV_SET($fd, EVFILT_WRITE, kq_flag(EPOLLOUT, $ev)) };
         } elsif ($op == EPOLL_CTL_DEL) {
+                use Carp ();
+                $kq // Carp::confess("nokq $fd");
                 $kq->EV_SET($fd, EVFILT_READ, EV_DISABLE);
                 eval { $kq->EV_SET($fd, EVFILT_WRITE, EV_DISABLE) };
         } else { # EPOLL_CTL_ADD
diff --git a/lib/PublicInbox/DSdeflate.pm b/lib/PublicInbox/DSdeflate.pm
index 639690e2..539adf0f 100644
--- a/lib/PublicInbox/DSdeflate.pm
+++ b/lib/PublicInbox/DSdeflate.pm
@@ -46,7 +46,7 @@ sub enable {
         my ($class, $self) = @_;
         my ($in, $err) = Compress::Raw::Zlib::Inflate->new(%IN_OPT);
         if ($err != Z_OK) {
-                $self->err("Inflate->new failed: $err");
+                warn("Inflate->new failed: $err\n");
                 return;
         }
         bless $self, $class;
diff --git a/lib/PublicInbox/Daemon.pm b/lib/PublicInbox/Daemon.pm
index 86234771..16bae231 100644
--- a/lib/PublicInbox/Daemon.pm
+++ b/lib/PublicInbox/Daemon.pm
@@ -22,6 +22,7 @@ use PublicInbox::Sigfd;
 use PublicInbox::Git;
 use PublicInbox::GitAsyncCat;
 use PublicInbox::Eml;
+use PublicInbox::Config;
 our $SO_ACCEPTFILTER = 0x1000;
 my @CMD;
 my ($set_user, $oldset);
@@ -133,6 +134,8 @@ sub load_mod ($;$$) {
                 $tlsd->{$f} = $logs{$p} //= open_log_path(my $fh, $p);
                 warn "# $scheme://$addr $f=$p\n";
         }
+        my $err = $tlsd->{err};
+        $tlsd->{warn_cb} = sub { print $err @_ }; # for local $SIG{__WARN__}
         \%xn;
 }
 
@@ -176,10 +179,7 @@ EOF
                 die "--pid-file cannot end with '.oldbin'\n";
         }
         @listeners = inherit($listener_names);
-
-        # allow socket-activation users to set certs once and not
-        # have to configure each socket:
-        my @inherited_names = keys(%$listener_names) if defined($default_cert);
+        my @inherited_names = keys(%$listener_names);
 
         # ignore daemonize when inheriting
         $daemonize = undef if scalar @listeners;
@@ -188,23 +188,24 @@ EOF
                 $default_listen // die "no listeners specified\n";
                 push @cfg_listen, $default_listen
         }
-
+        my ($default_scheme) = (($default_listen // '') =~ m!\A([^:]+)://!);
         foreach my $l (@cfg_listen) {
                 my $orig = $l;
-                my $scheme = '';
-                my $port;
-                if ($l =~ s!\A([^:]+)://!!) { $scheme = $1 }
+                my ($scheme, $port, $opt);
+                $l =~ s!\A([a-z0-9]+)://!! and $scheme = $1;
+                $scheme //= $default_scheme;
                 if ($l =~ /\A(?:\[[^\]]+\]|[^:]+):([0-9]+)/) {
                         $port = $1 + 0;
-                        my $s = $KNOWN_TLS{$port} // $KNOWN_STARTTLS{$port};
-                        $scheme //= $s if defined $s;
-                } elsif (index($l, '/') != 0) { # unix socket
-                        $port //= $SCHEME2PORT{$scheme} if $scheme;
-                        $port // die "no port in listen=$l\n";
+                        $scheme //= $KNOWN_TLS{$port} // $KNOWN_STARTTLS{$port};
+                }
+                $scheme // die "unable to determine URL scheme of $orig\n";
+                if (!defined($port) && index($l, '/') != 0) { # AF_UNIX socket
+                        $port = $SCHEME2PORT{$scheme} //
+                                die "no port in listen=$orig\n";
                         $l =~ s!\A([^/]+)!$1:$port! or
                                 die "unable to add port=$port to $l\n";
                 }
-                my $opt; # non-TLS options
+                $l =~ s!/\z!!; # chop one trailing slash
                 if ($l =~ s!/?\?(.+)\z!!) {
                         $opt = listener_opt($1);
                         $tls_opt{"$scheme://$l"} = accept_tls_opt($opt);
@@ -213,10 +214,10 @@ EOF
                 } elsif ($scheme =~ /\A(?:https|imaps|nntps|pop3s)\z/) {
                         die "$orig specified w/o cert=\n";
                 }
-                $scheme =~ /\A(?:http|imap|nntp|pop3)/ and
+                if ($listener_names->{$l}) { # already inherited
                         $xnetd->{$l} = load_mod($scheme, $opt, $l);
-
-                next if $listener_names->{$l}; # already inherited
+                        next;
+                }
                 my (%o, $sock_pkg);
                 if (index($l, '/') == 0) {
                         $sock_pkg = 'IO::Socket::UNIX';
@@ -243,35 +244,42 @@ EOF
                 }
                 $o{Listen} = 1024;
                 my $prev = umask 0000;
-                my $s = eval { $sock_pkg->new(%o) };
-                warn "error binding $l: $! ($@)\n" unless $s;
+                my $s = eval { $sock_pkg->new(%o) } or
+                        warn "error binding $l: $! ($@)\n";
                 umask $prev;
-                if ($s) {
-                        $s->blocking(0);
-                        my $k = sockname($s);
-                        warn "# bound $scheme://$k\n";
-                        $listener_names->{$k} = $s;
-                        push @listeners, $s;
-                }
+                $s // next;
+                $s->blocking(0);
+                my $sockname = sockname($s);
+                warn "# bound $scheme://$sockname\n";
+                $xnetd->{$sockname} //= load_mod($scheme);
+                $listener_names->{$sockname} = $s;
+                push @listeners, $s;
         }
 
         # cert/key options in @cfg_listen takes precedence when inheriting,
         # but map well-known inherited ports if --listen isn't specified
-        # at all
-        for my $sockname (@inherited_names) {
-                $sockname =~ /:([0-9]+)\z/ or next;
-                if (my $scheme = $KNOWN_TLS{$1}) {
-                        $xnetd->{$sockname} //= load_mod($scheme);
-                        $tls_opt{"$scheme://$sockname"} ||= accept_tls_opt('');
-                } elsif (($scheme = $KNOWN_STARTTLS{$1})) {
-                        $xnetd->{$sockname} //= load_mod($scheme);
-                        $tls_opt{"$scheme://$sockname"} ||= accept_tls_opt('');
-                        $tls_opt{''} ||= accept_tls_opt('');
+        # at all.  This allows socket-activation users to set certs once
+        # and not have to configure each socket:
+        if (defined $default_cert) {
+                my ($stls) = (($default_scheme // '') =~ /\A(pop3|nntp|imap)/);
+                for my $x (@inherited_names) {
+                        $x =~ /:([0-9]+)\z/ or next; # no TLS for AF_UNIX
+                        if (my $scheme = $KNOWN_TLS{$1}) {
+                                $xnetd->{$x} //= load_mod($scheme);
+                                $tls_opt{"$scheme://$x"} ||= accept_tls_opt('');
+                        } elsif (($scheme = $KNOWN_STARTTLS{$1})) {
+                                $xnetd->{$x} //= load_mod($scheme);
+                                $tls_opt{"$scheme://$x"} ||= accept_tls_opt('');
+                        } elsif (defined $stls) {
+                                $tls_opt{"$stls://$x"} ||= accept_tls_opt('');
+                        }
+                }
+        }
+        if (defined $default_scheme) {
+                for my $x (@inherited_names) {
+                        $xnetd->{$x} //= load_mod($default_scheme);
                 }
         }
-        my @d;
-        while (my ($k, $v) = each %tls_opt) { push(@d, $k) if !defined($v) }
-        delete @tls_opt{@d};
         die "No listeners bound\n" unless @listeners;
 }
 
@@ -653,8 +661,10 @@ sub defer_accept ($$) {
 
 sub daemon_loop ($) {
         my ($xnetd) = @_;
+        local $PublicInbox::Config::DEDUPE = {}; # enable dedupe cache
         my $refresh = sub {
                 my ($sig) = @_;
+                %$PublicInbox::Config::DEDUPE = (); # clear cache
                 for my $xn (values %$xnetd) {
                         delete $xn->{tlsd}->{ssl_ctx}; # PublicInbox::TLS::start
                         eval { $xn->{refresh}->($sig) };
@@ -663,14 +673,14 @@ sub daemon_loop ($) {
         };
         my %post_accept;
         while (my ($k, $ctx_opt) = each %tls_opt) {
-                my $l = $k;
-                $l =~ s!\A([^:]+)://!!;
-                my $scheme = $1 // '';
-                my $xn = $xnetd->{$l} // $xnetd->{''};
+                $ctx_opt // next;
+                my ($scheme, $l) = split(m!://!, $k, 2);
+                my $xn = $xnetd->{$l} // die "BUG: no xnetd for $k";
                 $xn->{tlsd}->{ssl_ctx_opt} //= $ctx_opt;
                 $scheme =~ m!\A(?:https|imaps|nntps|pop3s)! and
                         $post_accept{$l} = tls_cb(@$xn{qw(post_accept tlsd)});
         }
+        undef %tls_opt;
         my $sig = {
                 HUP => $refresh,
                 INT => \&worker_quit,
@@ -698,7 +708,7 @@ sub daemon_loop ($) {
         @listeners = map {;
                 my $l = sockname($_);
                 my $tls_cb = $post_accept{$l};
-                my $xn = $xnetd->{$l} // $xnetd->{''};
+                my $xn = $xnetd->{$l} // die "BUG: no xnetd for $l";
 
                 # NNTPS, HTTPS, HTTP, IMAPS and POP3S are client-first traffic
                 # IMAP, NNTP and POP3 are server-first
@@ -712,13 +722,7 @@ sub daemon_loop ($) {
 
 sub run {
         my ($default_listen) = @_;
-        my $xnetd = {};
-        if ($default_listen) {
-                $default_listen =~ /\A(http|imap|nntp|pop3)/ or
-                        die "BUG: $default_listen";
-                $xnetd->{''} = load_mod($1);
-        }
-        daemon_prepare($default_listen, $xnetd);
+        daemon_prepare($default_listen, my $xnetd = {});
         my $for_destroy = daemonize();
 
         # localize GCF2C for tests:
diff --git a/lib/PublicInbox/DirIdle.pm b/lib/PublicInbox/DirIdle.pm
index 9206da9c..55c3982f 100644
--- a/lib/PublicInbox/DirIdle.pm
+++ b/lib/PublicInbox/DirIdle.pm
@@ -1,9 +1,9 @@
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # Used by public-inbox-watch for Maildir (and possibly MH in the future)
 package PublicInbox::DirIdle;
-use strict;
+use v5.12;
 use parent 'PublicInbox::DS';
 use PublicInbox::Syscall qw(EPOLLIN);
 use PublicInbox::In2Tie;
diff --git a/lib/PublicInbox/Eml.pm b/lib/PublicInbox/Eml.pm
index 485f637a..8b999e1a 100644
--- a/lib/PublicInbox/Eml.pm
+++ b/lib/PublicInbox/Eml.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 #
 # Lazy MIME parser, it still slurps the full message but keeps short
@@ -144,6 +144,7 @@ sub header_raw {
         my $re = re_memo($_[1]);
         my @v = (${ $_[0]->{hdr} } =~ /$re/g);
         for (@v) {
+                utf8::decode($_); # SMTPUTF8
                 # for compatibility w/ Email::Simple::Header,
                 s/\s+\z//s;
                 s/\A\s+//s;
@@ -359,14 +360,15 @@ sub header_set {
         $pfx .= ': ';
         my $len = 78 - length($pfx);
         @vals = map {;
+                utf8::encode(my $v = $_); # to bytes, support SMTPUTF8
                 # folding differs from Email::Simple::Header,
                 # we favor tabs for visibility (and space savings :P)
                 if (length($_) >= $len && (/\n[^ \t]/s || !/\n/s)) {
                         local $Text::Wrap::columns = $len;
                         local $Text::Wrap::huge = 'overflow';
-                        $pfx . wrap('', "\t", $_) . $self->{crlf};
+                        $pfx . wrap('', "\t", $v) . $self->{crlf};
                 } else {
-                        $pfx . $_ . $self->{crlf};
+                        $pfx . $v . $self->{crlf};
                 }
         } @vals;
         $$hdr =~ s!$re!shift(@vals) // ''!ge; # replace current headers, first
diff --git a/lib/PublicInbox/ExtMsg.pm b/lib/PublicInbox/ExtMsg.pm
index 72cae005..95feb885 100644
--- a/lib/PublicInbox/ExtMsg.pm
+++ b/lib/PublicInbox/ExtMsg.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2015-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 #
 # Used by the web interface to link to messages outside of the our
@@ -11,7 +11,7 @@ use warnings;
 use PublicInbox::Hval qw(ascii_html prurl mid_href);
 use PublicInbox::WwwStream qw(html_oneshot);
 use PublicInbox::Smsg;
-our $MIN_PARTIAL_LEN = 16;
+our $MIN_PARTIAL_LEN = 14; # for 'XXXXXXXXXX.fsf' msgids gnus generates
 
 # TODO: user-configurable
 our @EXT_URL = map { ascii_html($_) } (
diff --git a/lib/PublicInbox/ExtSearch.pm b/lib/PublicInbox/ExtSearch.pm
index 2460d74f..fa49a1d0 100644
--- a/lib/PublicInbox/ExtSearch.pm
+++ b/lib/PublicInbox/ExtSearch.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # Read-only external (detached) index for cross inbox search.
@@ -108,7 +108,7 @@ sub altid_map { {} }
 sub description {
         my ($self) = @_;
         ($self->{description} //=
-                PublicInbox::Inbox::cat_desc("$self->{topdir}/description")) //
+                PublicInbox::Git::cat_desc("$self->{topdir}/description")) //
                 '$EXTINDEX_DIR/description missing';
 }
 
@@ -117,13 +117,14 @@ sub search {
         $_[0];
 }
 
+sub thing_type { 'external index' }
+
 no warnings 'once';
 *base_url = \&PublicInbox::Inbox::base_url;
 *smsg_eml = \&PublicInbox::Inbox::smsg_eml;
 *smsg_by_mid = \&PublicInbox::Inbox::smsg_by_mid;
 *msg_by_mid = \&PublicInbox::Inbox::msg_by_mid;
 *modified = \&PublicInbox::Inbox::modified;
-*recent = \&PublicInbox::Inbox::recent;
 
 *max_git_epoch = *nntp_usable = *msg_by_path = \&mm; # undef
 *isrch = \&search;
diff --git a/lib/PublicInbox/ExtSearchIdx.pm b/lib/PublicInbox/ExtSearchIdx.pm
index 7c44a1a4..401b18d0 100644
--- a/lib/PublicInbox/ExtSearchIdx.pm
+++ b/lib/PublicInbox/ExtSearchIdx.pm
@@ -406,14 +406,14 @@ EOM
         while (my ($ibx_id, $eidx_key) = $ibx_ck->fetchrow_array) {
                 next if $self->{ibx_map}->{$eidx_key};
                 $self->{midx}->remove_eidx_key($eidx_key);
-                warn "I: deleting messages for $eidx_key...\n";
+                warn "# deleting messages for $eidx_key...\n";
                 $x3_doc->execute($ibx_id);
                 my $ibx = { -ibx_id => $ibx_id, -gc_eidx_key => $eidx_key };
                 while (my ($docid, $xnum, $oid) = $x3_doc->fetchrow_array) {
                         my $r = _unref_doc($sync, $docid, $ibx, $xnum, $oid);
                         $oid = unpack('H*', $oid);
                         $r = $r ? 'unref' : 'remove';
-                        warn "I: $r #$docid $eidx_key $oid\n";
+                        warn "# $r #$docid $eidx_key $oid\n";
                         if (checkpoint_due($sync)) {
                                 $x3_doc = $ibx_ck = undef;
                                 reindex_checkpoint($self, $sync);
@@ -433,12 +433,12 @@ SELECT key FROM eidx_meta WHERE key LIKE ? ESCAPE ?
                 $lc_i->execute("lc-%:$pat//%", '\\');
                 while (my ($key) = $lc_i->fetchrow_array) {
                         next if $key !~ m!\Alc-v[1-9]+:\Q$eidx_key\E//!;
-                        warn "I: removing $key\n";
+                        warn "# removing $key\n";
                         $self->{oidx}->dbh->do(<<'', undef, $key);
 DELETE FROM eidx_meta WHERE key = ?
 
                 }
-                warn "I: $eidx_key removed\n";
+                warn "# $eidx_key removed\n";
         }
 }
 
@@ -447,20 +447,20 @@ sub eidx_gc_scan_shards ($$) { # TODO: use for lei/store
         my $nr = $self->{oidx}->dbh->do(<<'');
 DELETE FROM xref3 WHERE docid NOT IN (SELECT num FROM over)
 
-        warn "I: eliminated $nr stale xref3 entries\n" if $nr != 0;
+        warn "# eliminated $nr stale xref3 entries\n" if $nr != 0;
         reindex_checkpoint($self, $sync) if checkpoint_due($sync);
 
         # fixup from old bugs:
         $nr = $self->{oidx}->dbh->do(<<'');
 DELETE FROM over WHERE num > 0 AND num NOT IN (SELECT docid FROM xref3)
 
-        warn "I: eliminated $nr stale over entries\n" if $nr != 0;
+        warn "# eliminated $nr stale over entries\n" if $nr != 0;
         reindex_checkpoint($self, $sync) if checkpoint_due($sync);
 
         $nr = $self->{oidx}->dbh->do(<<'');
 DELETE FROM eidxq WHERE docid NOT IN (SELECT num FROM over)
 
-        warn "I: eliminated $nr stale reindex queue entries\n" if $nr != 0;
+        warn "# eliminated $nr stale reindex queue entries\n" if $nr != 0;
         reindex_checkpoint($self, $sync) if checkpoint_due($sync);
 
         my ($cur) = $self->{oidx}->dbh->selectrow_array(<<EOM);
@@ -490,7 +490,7 @@ SELECT num FROM over WHERE num >= ? ORDER BY num ASC LIMIT 10000
                         reindex_checkpoint($self, $sync);
                 }
         }
-        warn "I: eliminated $nr stale Xapian documents\n" if $nr != 0;
+        warn "# eliminated $nr stale Xapian documents\n" if $nr != 0;
 }
 
 sub eidx_gc {
@@ -731,13 +731,11 @@ sub eidxq_lock_acquire ($) {
         my $t = strftime('%Y-%m-%d %k:%M %z', localtime($time));
         local $self->{current_info} = 'eidxq';
         if ($euid == $> && $ident eq host_ident) {
-                if (kill(0, $pid)) {
-                        warn <<EOM; return;
-I: PID:$pid (re)indexing since $t, it will continue our work
+                kill(0, $pid) and warn <<EOM and return;
+# PID:$pid (re)indexing since $t, it will continue our work
 EOM
-                }
                 if ($!{ESRCH}) {
-                        warn "I: eidxq_lock is stale ($cur), clobbering\n";
+                        warn "# eidxq_lock is stale ($cur), clobbering\n";
                         return _eidxq_take($self);
                 }
                 warn "E: kill(0, $pid) failed: $!\n"; # fall-through:
@@ -837,7 +835,7 @@ sub reindex_unseen ($$$$) {
                 xnum => $xsmsg->{num},
                 # {mids} and {chash} will be filled in at _reindex_unseen
         };
-        warn "I: reindex_unseen ${\$ibx->eidx_key}:$req->{xnum}:$req->{oid}\n";
+        warn "# reindex_unseen ${\$ibx->eidx_key}:$req->{xnum}:$req->{oid}\n";
         $self->git->cat_async($xsmsg->{blob}, \&_reindex_unseen, $req);
 }
 
diff --git a/lib/PublicInbox/FakeInotify.pm b/lib/PublicInbox/FakeInotify.pm
index 6d269601..45b80f50 100644
--- a/lib/PublicInbox/FakeInotify.pm
+++ b/lib/PublicInbox/FakeInotify.pm
@@ -1,11 +1,10 @@
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # for systems lacking Linux::Inotify2 or IO::KQueue, just emulates
 # enough of Linux::Inotify2
 package PublicInbox::FakeInotify;
-use strict;
-use v5.10.1;
+use v5.12;
 use parent qw(Exporter);
 use Time::HiRes qw(stat);
 use PublicInbox::DS qw(add_timer);
@@ -119,7 +118,7 @@ sub poll_once {
 }
 
 package PublicInbox::FakeInotify::Watch;
-use strict;
+use v5.12;
 
 sub cancel {
         my ($self) = @_;
@@ -132,7 +131,7 @@ sub name {
 }
 
 package PublicInbox::FakeInotify::Event;
-use strict;
+use v5.12;
 
 sub fullname { ${$_[0]} }
 
@@ -141,14 +140,14 @@ sub IN_MOVED_FROM { 0 }
 sub IN_DELETE_SELF { 0 }
 
 package PublicInbox::FakeInotify::GoneEvent;
-use strict;
+use v5.12;
 our @ISA = qw(PublicInbox::FakeInotify::Event);
 
 sub IN_DELETE { 1 }
 sub IN_MOVED_FROM { 0 }
 
 package PublicInbox::FakeInotify::SelfGoneEvent;
-use strict;
+use v5.12;
 our @ISA = qw(PublicInbox::FakeInotify::GoneEvent);
 
 sub IN_DELETE_SELF { 1 }
diff --git a/lib/PublicInbox/Feed.pm b/lib/PublicInbox/Feed.pm
index ee579f6d..de1e7dfe 100644
--- a/lib/PublicInbox/Feed.pm
+++ b/lib/PublicInbox/Feed.pm
@@ -19,14 +19,14 @@ sub generate {
         my ($ctx) = @_;
         my $msgs = $ctx->{msgs} = recent_msgs($ctx);
         return _no_thread() unless @$msgs;
-        PublicInbox::WwwAtomStream->response($ctx, 200, \&generate_i);
+        PublicInbox::WwwAtomStream->response($ctx, \&generate_i);
 }
 
 sub generate_thread_atom {
         my ($ctx) = @_;
         my $msgs = $ctx->{msgs} = $ctx->{ibx}->over->get_thread($ctx->{mid});
         return _no_thread() unless @$msgs;
-        PublicInbox::WwwAtomStream->response($ctx, 200, \&generate_i);
+        PublicInbox::WwwAtomStream->response($ctx, \&generate_i);
 }
 
 sub generate_html_index {
@@ -49,12 +49,15 @@ sub generate_html_index {
 
 sub new_html_i {
         my ($ctx, $eml) = @_;
-        $ctx->zmore($ctx->html_top) if exists $ctx->{-html_tip};
+        print { $ctx->zfh } $ctx->html_top if exists $ctx->{-html_tip};
 
-        $eml and return PublicInbox::View::eml_entry($ctx, $eml);
+        if ($eml) {
+                $ctx->{smsg}->populate($eml) if !$ctx->{ibx}->{over};
+                return PublicInbox::View::eml_entry($ctx, $eml);
+        }
         my $smsg = shift @{$ctx->{msgs}} or
-                $ctx->zmore(PublicInbox::View::pagination_footer(
-                                                $ctx, './new.html'));
+                print { $ctx->zfh } PublicInbox::View::pagination_footer(
+                                                $ctx, './new.html');
         $smsg;
 }
 
@@ -67,8 +70,9 @@ sub new_html {
         }
         $ctx->{-html_tip} = '<pre>';
         $ctx->{-upfx} = '';
+        $ctx->{-spfx} = '' if $ctx->{ibx}->{coderepo};
         $ctx->{-hr} = 1;
-        PublicInbox::WwwStream::aresponse($ctx, 200, \&new_html_i);
+        PublicInbox::WwwStream::aresponse($ctx, \&new_html_i);
 }
 
 # private subs
diff --git a/lib/PublicInbox/Fetch.pm b/lib/PublicInbox/Fetch.pm
index 5261cad1..198e2a60 100644
--- a/lib/PublicInbox/Fetch.pm
+++ b/lib/PublicInbox/Fetch.pm
@@ -2,8 +2,7 @@
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 # Wrapper to "git fetch" remote public-inboxes
 package PublicInbox::Fetch;
-use strict;
-use v5.10.1;
+use v5.12;
 use parent qw(PublicInbox::IPC);
 use URI ();
 use PublicInbox::Spawn qw(popen_rd run_die spawn);
@@ -12,24 +11,9 @@ use PublicInbox::LEI;
 use PublicInbox::LeiCurl;
 use PublicInbox::LeiMirror;
 use File::Temp ();
-use PublicInbox::Config;
-use IO::Compress::Gzip qw(gzip $GzipError);
 
 sub new { bless {}, __PACKAGE__ }
 
-sub fetch_args ($$) {
-        my ($lei, $opt) = @_;
-        my @cmd; # (git --git-dir=...) to be added by caller
-        $opt->{$_} = $lei->{$_} for (0..2);
-        # we support "-c $key=$val" for arbitrary git config options
-        # e.g.: git -c http.proxy=socks5h://127.0.0.1:9050
-        push(@cmd, '-c', $_) for @{$lei->{opt}->{c} // []};
-        push @cmd, 'fetch';
-        push @cmd, '-q' if $lei->{opt}->{quiet};
-        push @cmd, '-v' if $lei->{opt}->{verbose};
-        @cmd;
-}
-
 sub remote_url ($$) {
         my ($lei, $dir) = @_;
         my $rn = $lei->{opt}->{'try-remote'} // [ 'origin', '_grokmirror' ];
@@ -44,12 +28,26 @@ sub remote_url ($$) {
         undef
 }
 
+# PSGI mount prefixes and manifest.js.gz prefixes don't always align...
+# TODO: remove, handle multi-inbox fetch
+sub deduce_epochs ($$) {
+        my ($m, $path) = @_;
+        my ($v1_ent, @v2_epochs);
+        my $path_pfx = '';
+        $path =~ s!/+\z!!;
+        do {
+                $v1_ent = $m->{$path};
+                @v2_epochs = grep(m!\A\Q$path\E/git/[0-9]+\.git\z!, keys %$m);
+        } while (!defined($v1_ent) && !@v2_epochs &&
+                $path =~ s!\A(/[^/]+)/!/! and $path_pfx .= $1);
+        ($path_pfx, $v1_ent ? $path : undef, @v2_epochs);
+}
+
 sub do_manifest ($$$) {
         my ($lei, $dir, $ibx_uri) = @_;
         my $muri = URI->new("$ibx_uri/manifest.js.gz");
         my $ft = File::Temp->new(TEMPLATE => 'm-XXXX',
                                 UNLINK => 1, DIR => $dir, SUFFIX => '.tmp');
-        my $fn = $ft->filename;
         my $mf = "$dir/manifest.js.gz";
         my $m0; # current manifest.js.gz contents
         if (open my $fh, '<', $mf) {
@@ -58,7 +56,7 @@ sub do_manifest ($$$) {
                 };
                 warn($@) if $@;
         }
-        my ($bn) = ($fn =~ m!/([^/]+)\z!);
+        my ($bn) = ($ft->filename =~ m!/([^/]+)\z!);
         my $curl_cmd = $lei->{curl}->for_uri($lei, $muri, qw(-R -o), $bn);
         my $opt = { -C => $dir };
         $opt->{$_} = $lei->{$_} for (0..2);
@@ -69,7 +67,7 @@ sub do_manifest ($$$) {
                 return;
         }
         my $m1 = eval {
-                PublicInbox::LeiMirror::decode_manifest($ft, $fn, $muri);
+                PublicInbox::LeiMirror::decode_manifest($ft, $ft, $muri);
         } or return [ 404, $muri ];
         my $mdiff = { %$m1 };
 
@@ -88,7 +86,7 @@ sub do_manifest ($$$) {
                 return;
         }
         my (undef, $v1_path, @v2_epochs) =
-                PublicInbox::LeiMirror::deduce_epochs($mdiff, $ibx_uri->path);
+                deduce_epochs($mdiff, $ibx_uri->path);
         [ 200, $muri, $v1_path, \@v2_epochs, $ft, $mf, $m1 ];
 }
 
@@ -192,7 +190,7 @@ EOM
                 if (-d $d) {
                         $fp2->[0] = get_fingerprint2($d) if $fp2;
                         $cmd = [ @$torsocks, 'git', "--git-dir=$d",
-                                fetch_args($lei, $opt) ];
+                               PublicInbox::LeiMirror::fetch_args($lei, $opt)];
                 } else {
                         my $e_uri = $ibx_uri->clone;
                         my ($epath) = ($d =~ m!(/git/[0-9]+\.git)\z!);
@@ -218,11 +216,7 @@ EOM
         }
         for my $i (@new_epoch) { $mg->epoch_cfg_set($i) }
         if ($ft) {
-                if ($mculled) {
-                        my $json = PublicInbox::Config->json->encode($m1);
-                        my $fn = $ft->filename;
-                        gzip(\$json => $fn) or die "gzip: $GzipError";
-                }
+                PublicInbox::LeiMirror::dump_manifest($m1 => $ft) if $mculled;
                 PublicInbox::LeiMirror::ft_rename($ft, $mf, 0666);
         }
         $lei->child_error($xit << 8) if $fp2 && $xit;
diff --git a/lib/PublicInbox/Filter/RubyLang.pm b/lib/PublicInbox/Filter/RubyLang.pm
index 09aa6aa8..57ebbe78 100644
--- a/lib/PublicInbox/Filter/RubyLang.pm
+++ b/lib/PublicInbox/Filter/RubyLang.pm
@@ -1,11 +1,10 @@
-# Copyright (C) 2017-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # Filter for lists.ruby-lang.org trailers
 package PublicInbox::Filter::RubyLang;
-use base qw(PublicInbox::Filter::Base);
-use strict;
-use warnings;
+use v5.10.1;
+use parent qw(PublicInbox::Filter::Base);
 use PublicInbox::MID qw(mids);
 
 my $l1 = qr/Unsubscribe:\s
@@ -56,16 +55,22 @@ sub scrub {
                 my $hdr = $mime->header_obj;
                 my $mids = mids($hdr);
                 return $self->REJECT('Message-ID missing') unless (@$mids);
-                my @v = $hdr->header_raw('X-Mail-Count');
                 my $n;
-                foreach (@v) {
-                        /\A\s*([0-9]+)\s*\z/ or next;
-                        $n = $1;
-                        last;
-                }
-                unless (defined $n) {
-                        return $self->REJECT('X-Mail-Count not numeric');
+                my @v = $hdr->header_raw('X-Mail-Count'); # old host only
+                if (@v) {
+                        for (@v) {
+                                /\A\s*([0-9]+)\s*\z/ or next;
+                                $n = $1;
+                                last;
+                        }
+                } else { # new host: nue.mailmanlists.eu
+                        for ($hdr->header_str('Subject')) {
+                                /\A\[ruby-[^:]+:([0-9]+)\]/ or next;
+                                $n = $1;
+                                last;
+                        }
                 }
+                $n // return $self->REJECT('could not get count not numeric');
                 foreach my $mid (@$mids) {
                         my $r = $altid->mm_alt->mid_set($n, $mid);
                         next if $r == 0;
diff --git a/lib/PublicInbox/Gcf2.pm b/lib/PublicInbox/Gcf2.pm
index 41ee0715..54b3d6aa 100644
--- a/lib/PublicInbox/Gcf2.pm
+++ b/lib/PublicInbox/Gcf2.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # backend for a git-cat-file-workalike based on libgit2,
@@ -10,12 +10,16 @@ use PublicInbox::Spawn qw(which popen_rd); # may set PERL_INLINE_DIRECTORY
 use Fcntl qw(LOCK_EX SEEK_SET);
 use Time::HiRes qw(clock_gettime CLOCK_MONOTONIC);
 use IO::Handle; # autoflush
+use File::Path qw(make_path);
+use PublicInbox::Git;
+
 BEGIN {
         my (%CFG, $c_src);
         # PublicInbox::Spawn will set PERL_INLINE_DIRECTORY
         # to ~/.cache/public-inbox/inline-c if it exists
         my $inline_dir = $ENV{PERL_INLINE_DIRECTORY} //
                 die 'PERL_INLINE_DIRECTORY not defined';
+        make_path($inline_dir);
         my $f = "$inline_dir/.public-inbox.lock";
         open my $fh, '+>', $f or die "open($f): $!";
 
@@ -78,7 +82,8 @@ EOM
 }
 
 sub add_alt ($$) {
-        my ($gcf2, $objdir) = @_;
+        my ($gcf2, $git_dir) = @_;
+        my $objdir = PublicInbox::Git->new($git_dir)->git_path('objects');
 
         # libgit2 (tested 0.27.7+dfsg.1-0.2 and 0.28.3+dfsg.1-1~bpo10+1
         # in Debian) doesn't handle relative epochs properly when nested
@@ -118,19 +123,19 @@ sub loop (;$) {
         while (<STDIN>) {
                 chomp;
                 my ($oid, $git_dir) = split(/ /, $_, 2);
-                $seen{$git_dir} //= add_alt($gcf2, "$git_dir/objects");
+                $seen{$git_dir} //= add_alt($gcf2, $git_dir);
                 if (!$gcf2->cat_oid(1, $oid)) {
                         # retry once if missing.  We only get unabbreviated OIDs
                         # from SQLite or Xapian DBs, here, so malicious clients
                         # can't trigger excessive retries:
-                        warn "I: $$ $oid missing, retrying in $git_dir\n";
+                        warn "# $$ $oid missing, retrying in $git_dir\n";
 
                         $gcf2 = new();
-                        %seen = ($git_dir => add_alt($gcf2,"$git_dir/objects"));
+                        %seen = ($git_dir => add_alt($gcf2, $git_dir));
                         $check_at = clock_gettime(CLOCK_MONOTONIC) + $exp;
 
                         if ($gcf2->cat_oid(1, $oid)) {
-                                warn "I: $$ $oid found after retry\n";
+                                warn "# $$ $oid found after retry\n";
                         } else {
                                 warn "W: $$ $oid missing after retry\n";
                                 print "$oid missing\n"; # mimic git-cat-file
diff --git a/lib/PublicInbox/Git.pm b/lib/PublicInbox/Git.pm
index b2ae75c8..882a9a4a 100644
--- a/lib/PublicInbox/Git.pm
+++ b/lib/PublicInbox/Git.pm
@@ -66,7 +66,7 @@ sub new {
         $git_dir =~ tr!/!/!s;
         $git_dir =~ s!/*\z!!s;
         # may contain {-tmp} field for File::Temp::Dir
-        bless { git_dir => $git_dir, alt_st => '', -git_path => {} }, $class
+        bless { git_dir => $git_dir }, $class
 }
 
 sub git_path ($$) {
@@ -90,7 +90,7 @@ sub alternates_changed {
 
         # can't rely on 'q' on some 32-bit builds, but `d' works
         my $st = pack('dd', $st[10], $st[7]); # 10: ctime, 7: size
-        return 0 if $self->{alt_st} eq $st;
+        return 0 if ($self->{alt_st} // '') eq $st;
         $self->{alt_st} = $st; # always a true value
 }
 
@@ -426,6 +426,7 @@ sub cleanup {
                                 scalar(@{$self->{inflight} // []}));
         local $in_cleanup = 1;
         delete $self->{async_cat};
+        delete $self->{async_chk};
         async_wait_all($self);
         delete $self->{inflight};
         delete $self->{inflight_c};
@@ -451,7 +452,8 @@ sub DESTROY { cleanup(@_) }
 
 sub local_nick ($) {
         # don't show full FS path, basename should be OK:
-        $_[0]->{git_dir} =~ m!/([^/]+?)(?:/*\.git/*)?\z! ? "$1.git" : '???';
+        $_[0]->{nick} // ($_[0]->{git_dir} =~ m!/([^/]+?)(?:/*\.git/*)?\z! ?
+                        "$1.git" : undef);
 }
 
 sub host_prefix_url ($$) {
@@ -463,12 +465,22 @@ sub host_prefix_url ($$) {
         "$scheme://$host_port". ($env->{SCRIPT_NAME} || '/') . $url;
 }
 
+sub base_url { # for coderepos, PSGI-only
+        my ($self, $env) = @_; # env - PSGI env
+        my $url = host_prefix_url($env, '');
+        # for mount in Plack::Builder
+        $url .= '/' if substr($url, -1, 1) ne '/';
+        $url . $self->{nick} . '/';
+}
+
+sub isrch {} # TODO
+
 sub pub_urls {
         my ($self, $env) = @_;
         if (my $urls = $self->{cgit_url}) {
                 return map { host_prefix_url($env, $_) } @$urls;
         }
-        (local_nick($self));
+        (local_nick($self) // '???');
 }
 
 sub cat_async_begin {
@@ -498,6 +510,33 @@ sub modified ($) {
         (split(/ /, <$fh> // time))[0] + 0; # integerize for JSON
 }
 
+sub try_cat {
+        my ($path) = @_;
+        open(my $fh, '<', $path) or return '';
+        local $/;
+        <$fh> // '';
+}
+
+sub cat_desc ($) {
+        my $desc = try_cat($_[0]);
+        chomp $desc;
+        utf8::decode($desc);
+        $desc =~ s/\s+/ /smg;
+        $desc eq '' ? undef : $desc;
+}
+
+sub description {
+        cat_desc("$_[0]->{git_dir}/description") // 'Unnamed repository';
+}
+
+sub cloneurl {
+        my ($self, $env) = @_;
+        $self->{cloneurl} // do {
+                my @urls = split(/\s+/s, try_cat("$self->{git_dir}/cloneurl"));
+                scalar(@urls) ? ($self->{cloneurl} = \@urls) : undef;
+        } // [ substr(base_url($self, $env), 0, -1) ];
+}
+
 # for grokmirror, which doesn't read gitweb.description
 # templates/hooks--update.sample and git-multimail in git.git
 # only match "Unnamed repository", not the full contents of
@@ -520,14 +559,8 @@ sub manifest_entry {
         chomp(my $owner = $self->qx('config', 'gitweb.owner'));
         utf8::decode($owner);
         $ent->{owner} = $owner eq '' ? undef : $owner;
-        my $desc = '';
-        if (open($fh, '<', "$git_dir/description")) {
-                local $/ = "\n";
-                chomp($desc = <$fh>);
-                utf8::decode($desc);
-        }
-        $desc = 'Unnamed repository' if $desc eq '';
-        if (defined $epoch && $desc =~ /\AUnnamed repository/) {
+        my $desc = description($self);
+        if (defined $epoch && index($desc, 'Unnamed repository') == 0) {
                 $desc = "$default_desc [epoch $epoch]";
         }
         $ent->{description} = $desc;
diff --git a/lib/PublicInbox/GitAsyncCat.pm b/lib/PublicInbox/GitAsyncCat.pm
index cea3f539..2e0725a6 100644
--- a/lib/PublicInbox/GitAsyncCat.pm
+++ b/lib/PublicInbox/GitAsyncCat.pm
@@ -1,14 +1,14 @@
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 #
 # internal class used by PublicInbox::Git + PublicInbox::DS
 # This parses the output pipe of "git cat-file --batch"
 package PublicInbox::GitAsyncCat;
-use strict;
+use v5.12;
 use parent qw(PublicInbox::DS Exporter);
 use POSIX qw(WNOHANG);
 use PublicInbox::Syscall qw(EPOLLIN EPOLLET);
-our @EXPORT = qw(ibx_async_cat ibx_async_prefetch);
+our @EXPORT = qw(ibx_async_cat ibx_async_prefetch async_check);
 use PublicInbox::Git ();
 
 our $GCF2C; # singleton PublicInbox::Gcf2Client
@@ -45,13 +45,24 @@ sub event_step {
         }
 }
 
+sub watch_cat {
+        my ($git) = @_;
+        $git->{async_cat} //= do {
+                my $self = bless { git => $git }, __PACKAGE__;
+                $git->{in}->blocking(0);
+                $self->SUPER::new($git->{in}, EPOLLIN|EPOLLET);
+                \undef; # this is a true ref()
+        };
+}
+
 sub ibx_async_cat ($$$$) {
         my ($ibx, $oid, $cb, $arg) = @_;
-        my $git = $ibx->git;
+        my $git = $ibx->{git} // $ibx->git;
         # {topdir} means ExtSearch (likely [extindex "all"]) with potentially
         # 100K alternates.  git(1) has a proposed patch for 100K alternates:
         # <https://lore.kernel.org/git/20210624005806.12079-1-e@80x24.org/>
-        if (!defined($ibx->{topdir}) && ($GCF2C //= eval {
+        if (!defined($ibx->{topdir}) && !defined($git->{-tmp}) &&
+                ($GCF2C //= eval {
                 require PublicInbox::Gcf2Client;
                 PublicInbox::Gcf2Client::new();
         } // 0)) { # 0: do not retry if libgit2 or Inline::C are missing
@@ -59,29 +70,35 @@ sub ibx_async_cat ($$$$) {
                 \undef;
         } else { # read-only end of git-cat-file pipe
                 $git->cat_async($oid, $cb, $arg);
-                $git->{async_cat} //= do {
-                        my $self = bless { git => $git }, __PACKAGE__;
-                        $git->{in}->blocking(0);
-                        $self->SUPER::new($git->{in}, EPOLLIN|EPOLLET);
-                        \undef; # this is a true ref()
-                };
+                watch_cat($git);
         }
 }
 
+sub async_check ($$$$) {
+        my ($ibx, $oidish, $cb, $arg) = @_;
+        my $git = $ibx->{git} // $ibx->git;
+        $git->check_async($oidish, $cb, $arg);
+        $git->{async_chk} //= do {
+                my $self = bless { git => $git }, 'PublicInbox::GitAsyncCheck';
+                $git->{in_c}->blocking(0);
+                $self->SUPER::new($git->{in_c}, EPOLLIN|EPOLLET);
+                \undef; # this is a true ref()
+        };
+}
+
 # this is safe to call inside $cb, but not guaranteed to enqueue
-# returns true if successful, undef if not.
+# returns true if successful, undef if not.  For fairness, we only
+# prefetch if there's no in-flight requests.
 sub ibx_async_prefetch {
         my ($ibx, $oid, $cb, $arg) = @_;
         my $git = $ibx->git;
         if (!defined($ibx->{topdir}) && $GCF2C) {
-                if (!$GCF2C->{wbuf}) {
+                if (!@{$GCF2C->{inflight} // []}) {
                         $oid .= " $git->{git_dir}\n";
                         return $GCF2C->gcf2_async(\$oid, $cb, $arg); # true
                 }
         } elsif ($git->{async_cat} && (my $inflight = $git->{inflight})) {
-                # we could use MAX_INFLIGHT here w/o the halving,
-                # but lets not allow one client to monopolize a git process
-                if (@$inflight < int(PublicInbox::Git::MAX_INFLIGHT/2)) {
+                if (!@$inflight) {
                         print { $git->{out} } $oid, "\n" or
                                                 $git->fail("write error: $!");
                         return push(@$inflight, $oid, $cb, $arg);
@@ -91,3 +108,34 @@ sub ibx_async_prefetch {
 }
 
 1;
+package PublicInbox::GitAsyncCheck;
+use v5.12;
+our @ISA = qw(PublicInbox::GitAsyncCat);
+use POSIX qw(WNOHANG);
+use PublicInbox::Syscall qw(EPOLLIN EPOLLET);
+
+sub event_step {
+        my ($self) = @_;
+        my $git = $self->{git} or return;
+        return $self->close if ($git->{in_c} // 0) != ($self->{sock} // 1);
+        my $inflight = $git->{inflight_c};
+        if ($inflight && @$inflight) {
+                $git->check_async_step($inflight);
+
+                # child death?
+                if (($git->{in_c} // 0) != ($self->{sock} // 1)) {
+                        $self->close;
+                } elsif (@$inflight || exists $git->{rbuf_c}) {
+                        # ok, more to do, requeue for fairness
+                        $self->requeue;
+                }
+        } elsif ((my $pid = waitpid($git->{pid_c}, WNOHANG)) > 0) {
+                # May happen if the child process is killed by a BOFH
+                # (or segfaults)
+                delete $git->{pid_c};
+                warn "E: git $pid exited with \$?=$?\n";
+                $self->close;
+        }
+}
+
+1;
diff --git a/lib/PublicInbox/GitHTTPBackend.pm b/lib/PublicInbox/GitHTTPBackend.pm
index ba3a8f20..1eb51f27 100644
--- a/lib/PublicInbox/GitHTTPBackend.pm
+++ b/lib/PublicInbox/GitHTTPBackend.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2016-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # when no endpoints match, fallback to this and serve a static file
@@ -23,13 +23,10 @@ my @text = qw[HEAD info/refs info/attributes
         objects/info/(?:http-alternates|alternates|packs)
         cloneurl description];
 
-my @binary = qw!
-        objects/[a-f0-9]{2}/[a-f0-9]{38}
-        objects/pack/pack-[a-f0-9]{40}\.(?:pack|idx)
-        !;
+my @binary = ('objects/[a-f0-9]{2}/[a-f0-9]{38,62}',
+        'objects/pack/pack-[a-f0-9]{40,64}\.(?:pack|idx)');
 
 our $ANY = join('|', @binary, @text, 'git-upload-pack');
-my $BIN = join('|', @binary);
 my $TEXT = join('|', @text);
 
 sub serve {
@@ -62,13 +59,13 @@ sub serve_dumb {
 
         my $h = [];
         my $type;
-        if ($path =~ m!\Aobjects/[a-f0-9]{2}/[a-f0-9]{38}\z!) {
+        if ($path =~ m!\Aobjects/[a-f0-9]{2}/[a-f0-9]{38,62}\z!) {
                 $type = 'application/x-git-loose-object';
                 cache_one_year($h);
-        } elsif ($path =~ m!\Aobjects/pack/pack-[a-f0-9]{40}\.pack\z!) {
+        } elsif ($path =~ m!\Aobjects/pack/pack-[a-f0-9]{40,64}\.pack\z!) {
                 $type = 'application/x-git-packed-objects';
                 cache_one_year($h);
-        } elsif ($path =~ m!\Aobjects/pack/pack-[a-f0-9]{40}\.idx\z!) {
+        } elsif ($path =~ m!\Aobjects/pack/pack-[a-f0-9]{40,64}\.idx\z!) {
                 $type = 'application/x-git-packed-objects-toc';
                 cache_one_year($h);
         } elsif ($path =~ /\A(?:$TEXT)\z/o) {
@@ -132,7 +129,7 @@ sub input_prepare {
 }
 
 sub parse_cgi_headers {
-        my ($r, $bref) = @_;
+        my ($r, $bref, $ctx) = @_;
         return r(500) unless defined $r && $r >= 0;
         $$bref =~ s/\A(.*?)\r?\n\r?\n//s or return $r == 0 ? r(500) : undef;
         my $h = $1;
@@ -146,7 +143,24 @@ sub parse_cgi_headers {
                         push @h, $k, $v;
                 }
         }
-        [ $code, \@h ]
+
+        # fallback to WwwCoderepo if cgit 404s.  Duplicating $ctx prevents
+        # ->finalize from the current Qspawn from using qspawn.wcb
+        if ($code == 404 && $ctx->{www} && !$ctx->{_coderepo_tried}++) {
+                my %ctx = %$ctx;
+                $ctx{env} = +{ %{$ctx->{env}} };
+                delete $ctx->{env}->{'qspawn.wcb'};
+                $ctx->{env}->{'plack.skip-deflater'} = 1; # prevent 2x gzip
+                my $res = $ctx->{www}->coderepo->srv(\%ctx);
+                if (ref($res) eq 'CODE') {
+                        $res->(delete $ctx{env}->{'qspawn.wcb'});
+                } else { # ref($res) eq 'ARRAY'
+                        $ctx->{env}->{'qspawn.wcb'} = $ctx{env}->{'qspawn.wcb'};
+                }
+                $res; # non ARRAY ref for ->psgi_return_init_cb
+        } else {
+                [ $code, \@h ]
+        }
 }
 
 1;
diff --git a/lib/PublicInbox/GzipFilter.pm b/lib/PublicInbox/GzipFilter.pm
index bdd313f5..bd72afff 100644
--- a/lib/PublicInbox/GzipFilter.pm
+++ b/lib/PublicInbox/GzipFilter.pm
@@ -94,25 +94,18 @@ sub gone { # what: search/over/mm
 
 # for GetlineBody (via Qspawn) when NOT using $env->{'pi-httpd.async'}
 # Also used for ->getline callbacks
-sub translate ($$) {
-        my $self = $_[0]; # $_[1] => input
+sub translate {
+        my $self = shift; # $_[1] => input
 
         # allocate the zlib context lazily here, instead of in ->new.
         # Deflate contexts are memory-intensive and this object may
         # be sitting in the Qspawn limiter queue for a while.
-        my $gz = $self->{gz} //= gzip_or_die();
-        my $zbuf = delete($self->{zbuf});
-        if (defined $_[1]) { # my $buf = $_[1];
-                my $err = $gz->deflate($_[1], $zbuf);
-                die "gzip->deflate: $err" if $err != Z_OK;
-                return $zbuf if length($zbuf) >= 8192;
-
-                $self->{zbuf} = $zbuf;
-                '';
+        $self->{gz} //= gzip_or_die();
+        if (defined $_[0]) { # my $buf = $_[1];
+                zmore($self, @_);
+                length($self->{zbuf}) >= 8192 ? delete($self->{zbuf}) : '';
         } else { # undef == EOF
-                my $err = $gz->flush($zbuf);
-                die "gzip->flush: $err" if $err != Z_OK;
-                $zbuf;
+                zflush($self);
         }
 }
 
@@ -131,32 +124,42 @@ sub http_out ($) {
 
 sub write {
         # my $ret = bytes::length($_[1]); # XXX does anybody care?
-        http_out($_[0])->write(translate($_[0], $_[1]));
+        http_out($_[0])->write(translate(@_));
+}
+
+sub zfh {
+        $_[0]->{zfh} // do {
+                open($_[0]->{zfh}, '>>', \($_[0]->{pbuf} //= '')) or
+                        die "open: $!";
+                $_[0]->{zfh}
+        };
 }
 
 # similar to ->translate; use this when we're sure we know we have
 # more data to buffer after this
 sub zmore {
-        my $self = $_[0]; # $_[1] => input
+        my $self = shift;
+        my $zfh = delete $self->{zfh};
+        if (@_ > 1 || $zfh) {
+                print { $zfh // zfh($self) } @_;
+                @_ = (delete $self->{pbuf});
+                delete $self->{zfh};
+        };
         http_out($self);
-        my $err = $self->{gz}->deflate($_[1], $self->{zbuf});
-        die "gzip->deflate: $err" if $err != Z_OK;
-        undef;
+        my $err;
+        ($err = $self->{gz}->deflate($_[0], $self->{zbuf})) == Z_OK or
+                die "gzip->deflate: $err";
 }
 
 # flushes and returns the final bit of gzipped data
-sub zflush ($;$) {
-        my $self = $_[0]; # $_[1] => final input (optional)
-        my $zbuf = delete $self->{zbuf};
-        my $gz = delete $self->{gz};
+sub zflush ($;@) {
+        my $self = shift; # $_[1..Inf] => final input (optional)
+        zmore($self, @_) if scalar(@_) || $self->{zfh};
+        # not a bug, recursing on DS->write failure
+        my $gz = delete $self->{gz} // return '';
         my $err;
-        if (defined $_[1]) { # it's a bug iff $gz is undef w/ $_[1]
-                $err = $gz->deflate($_[1], $zbuf);
-                die "gzip->deflate: $err" if $err != Z_OK;
-        }
-        $gz // return ''; # not a bug, recursing on DS->write failure
-        $err = $gz->flush($zbuf);
-        die "gzip->flush: $err" if $err != Z_OK;
+        my $zbuf = delete $self->{zbuf};
+        ($err = $gz->flush($zbuf)) == Z_OK or die "gzip->flush: $err";
         $zbuf;
 }
 
diff --git a/lib/PublicInbox/HTTP.pm b/lib/PublicInbox/HTTP.pm
index 3d4e3499..0dba425d 100644
--- a/lib/PublicInbox/HTTP.pm
+++ b/lib/PublicInbox/HTTP.pm
@@ -69,7 +69,7 @@ sub new ($$$) {
 
 sub event_step { # called by PublicInbox::DS
         my ($self) = @_;
-
+        local $SIG{__WARN__} = $self->{srv_env}->{'pi-httpd.warn_cb'};
         return unless $self->flush_write && $self->{sock};
 
         # only read more requests if we've drained the write buffer,
diff --git a/lib/PublicInbox/HTTPD.pm b/lib/PublicInbox/HTTPD.pm
index e531ee70..bae7281b 100644
--- a/lib/PublicInbox/HTTPD.pm
+++ b/lib/PublicInbox/HTTPD.pm
@@ -47,6 +47,7 @@ sub env_for ($$$) {
                 # detect when to use async paths for slow blobs
                 'pi-httpd.async' => \&pi_httpd_async,
                 'pi-httpd.app' => $self->{app},
+                'pi-httpd.warn_cb' => $self->{warn_cb},
         }
 }
 
diff --git a/lib/PublicInbox/HTTPD/Async.pm b/lib/PublicInbox/HTTPD/Async.pm
index 1651da88..cb76cfab 100644
--- a/lib/PublicInbox/HTTPD/Async.pm
+++ b/lib/PublicInbox/HTTPD/Async.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2016-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 #
 # XXX This is a totally unstable API for public-inbox internal use only
@@ -77,8 +77,11 @@ sub async_pass {
         # will automatically close this ($self) object.
         $http->{forward} = $self;
 
-        # write anything we overread when we were reading headers
-        $fh->write($$bref); # PublicInbox:HTTP::{chunked,identity}_wcb
+        # write anything we overread when we were reading headers.
+        # This is typically PublicInbox:HTTP::{chunked,identity}_wcb,
+        # but may be PublicInbox::GzipFilter::write.  PSGI requires
+        # *_wcb methods respond to ->write (and ->close), not ->print
+        $fh->write($$bref);
 
         # we're done with this, free this memory up ASAP since the
         # calls after this may use much memory:
diff --git a/lib/PublicInbox/IMAP.pm b/lib/PublicInbox/IMAP.pm
index bed633e5..37317948 100644
--- a/lib/PublicInbox/IMAP.pm
+++ b/lib/PublicInbox/IMAP.pm
@@ -138,6 +138,7 @@ sub login_success ($$) {
 sub auth_challenge_ok ($) {
         my ($self) = @_;
         my $tag = delete($self->{-login_tag}) or return;
+        $self->{anon} = 1;
         login_success($self, $tag);
 }
 
@@ -425,8 +426,10 @@ sub _esc ($) {
         if (!defined($v)) {
                 'NIL';
         } elsif ($v =~ /[{"\r\n%*\\\[]/) { # literal string
+                utf8::encode($v);
                 '{' . length($v) . "}\r\n" . $v;
         } else { # quoted string
+                utf8::encode($v);
                 qq{"$v"}
         }
 }
@@ -574,6 +577,16 @@ sub fetch_run_ops {
         $self->msg_more(")\r\n");
 }
 
+sub requeue { # overrides PublicInbox::DS::requeue
+        my ($self) = @_;
+        if ($self->{anon}) { # AUTH=ANONYMOUS gets high priority
+                $self->SUPER::requeue;
+        } else { # low priority
+                push(@{$self->{imapd}->{-authed_q}}, $self) == 1 and
+                        PublicInbox::DS::requeue($self->{imapd});
+        }
+}
+
 sub fetch_blob_cb { # called by git->cat_async via ibx_async_cat
         my ($bref, $oid, $type, $size, $fetch_arg) = @_;
         my ($self, undef, $msgs, $range_info, $ops, $partial) = @$fetch_arg;
@@ -588,10 +601,9 @@ sub fetch_blob_cb { # called by git->cat_async via ibx_async_cat
                 $smsg->{blob} eq $oid or die "BUG: $smsg->{blob} != $oid";
         }
         my $pre;
-        if (!$self->{wbuf} && (my $nxt = $msgs->[0])) {
-                $pre = ibx_async_prefetch($ibx, $nxt->{blob},
+        ($self->{anon} && !$self->{wbuf} && $msgs->[0]) and
+                $pre = ibx_async_prefetch($ibx, $msgs->[0]->{blob},
                                         \&fetch_blob_cb, $fetch_arg);
-        }
         fetch_run_ops($self, $smsg, $bref, $ops, $partial);
         $pre ? $self->dflush : $self->requeue_once;
 }
@@ -995,7 +1007,7 @@ sub fetch_compile ($) {
         # stabilize partial order for consistency and ease-of-debugging:
         if (scalar keys %partial) {
                 $need |= NEED_BLOB;
-                $r[2] = [ map { [ $_, @{$partial{$_}} ] } sort keys %partial ];
+                @{$r[2]} = map { [ $_, @{$partial{$_}} ] } sort keys %partial;
         }
 
         push @op, $OP_EML_NEW if ($need & (EML_HDR|EML_BDY));
@@ -1018,7 +1030,7 @@ sub fetch_compile ($) {
 
         # r[1] = [ $key1, $cb1, $key2, $cb2, ... ]
         use sort 'stable'; # makes output more consistent
-        $r[1] = [ map { ($_->[2], $_->[1]) } sort { $a->[0] <=> $b->[0] } @op ];
+        @{$r[1]} = map { ($_->[2], $_->[1]) } sort { $a->[0] <=> $b->[0] } @op;
         @r;
 }
 
@@ -1155,17 +1167,11 @@ sub process_line ($$) {
         my $err = $@;
         if ($err && $self->{sock}) {
                 $l =~ s/\r?\n//s;
-                err($self, 'error from: %s (%s)', $l, $err);
+                warn("error from: $l ($err)\n");
                 $tag //= '*';
-                $res = "$tag BAD program fault - command not performed\r\n";
+                $res = \"$tag BAD program fault - command not performed\r\n";
         }
-        return 0 unless defined $res;
-        $self->write($res);
-}
-
-sub err ($$;@) {
-        my ($self, $fmt, @args) = @_;
-        printf { $self->{imapd}->{err} } $fmt."\n", @args;
+        defined($res) ? $self->write($res) : 0;
 }
 
 sub out ($$;@) {
@@ -1176,7 +1182,7 @@ sub out ($$;@) {
 # callback used by PublicInbox::DS for any (e)poll (in/out/hup/err)
 sub event_step {
         my ($self) = @_;
-
+        local $SIG{__WARN__} = $self->{imapd}->{warn_cb};
         return unless $self->flush_write && $self->{sock} && !$self->{long_cb};
 
         # only read more requests if we've drained the write buffer,
diff --git a/lib/PublicInbox/IMAPD.pm b/lib/PublicInbox/IMAPD.pm
index 5368ff04..78323e57 100644
--- a/lib/PublicInbox/IMAPD.pm
+++ b/lib/PublicInbox/IMAPD.pm
@@ -23,7 +23,7 @@ sub new {
         }, $class;
 }
 
-sub imapd_refresh_ibx { # pi_cfg->each_inbox cb
+sub _refresh_ibx { # pi_cfg->each_inbox cb
         my ($ibx, $imapd, $cache, $dummies) = @_;
         my $ngname = $ibx->{newsgroup} // return;
 
@@ -44,7 +44,6 @@ sub imapd_refresh_ibx { # pi_cfg->each_inbox cb
                 PublicInbox::IMAP::ensure_slices_exist($imapd, $ibx);
                 # preload to avoid fragmentation:
                 $ibx->description;
-                $ibx->base_url;
                 # ensure dummies are selectable:
                 do {
                         $dummies->{$ngname} = $dummy;
@@ -56,27 +55,32 @@ sub imapd_refresh_ibx { # pi_cfg->each_inbox cb
 sub refresh_groups {
         my ($self, $sig) = @_;
         my $pi_cfg = PublicInbox::Config->new;
-        my $mailboxes = $self->{mailboxes} = {};
-        my $cache = eval { $pi_cfg->ALL->misc->nntpd_cache_load } // {};
-        my $dummies = {};
-        $pi_cfg->each_inbox(\&imapd_refresh_ibx, $self, $cache, $dummies);
-        %$dummies = (%$dummies, %$mailboxes);
-        $mailboxes = $self->{mailboxes} = $dummies;
-        @{$self->{mailboxlist}} = map { $_->[2] }
-                sort { $a->[0] cmp $b->[0] || $a->[1] <=> $b->[1] }
-                map {
-                        my $u = $_; # capitalize "INBOX" for user-familiarity
-                        $u =~ s/\Ainbox(\.|\z)/INBOX$1/i;
-                        if ($mailboxes->{$_} == $dummy) {
-                                [ $u, -1,
-                                  qq[* LIST (\\HasChildren) "." $u\r\n]]
-                        } else {
-                                $u =~ /\A(.+)\.([0-9]+)\z/ or
-                                        die "BUG: `$u' has no slice digit(s)";
-                                [ $1, $2 + 0,
-                                  qq[* LIST (\\HasNoChildren) "." $u\r\n] ]
-                        }
-                } keys %$mailboxes;
+        $self->{mailboxes} = $pi_cfg->{-imap_mailboxes} // do {
+                my $mailboxes = $self->{mailboxes} = {};
+                my $cache = eval { $pi_cfg->ALL->misc->nntpd_cache_load } // {};
+                my $dummies = {};
+                $pi_cfg->each_inbox(\&_refresh_ibx, $self, $cache, $dummies);
+                %$mailboxes = (%$dummies, %$mailboxes);
+                @{$pi_cfg->{-imap_mailboxlist}} = map { $_->[2] }
+                        sort { $a->[0] cmp $b->[0] || $a->[1] <=> $b->[1] }
+                        map {
+                                # capitalize "INBOX" for user-familiarity
+                                my $u = $_;
+                                $u =~ s/\Ainbox(\.|\z)/INBOX$1/i;
+                                if ($mailboxes->{$_} == $dummy) {
+                                        [ $u, -1,
+                                          qq[* LIST (\\HasChildren) "." $u\r\n]]
+                                } else {
+                                        $u =~ /\A(.+)\.([0-9]+)\z/ or die
+"BUG: `$u' has no slice digit(s)";
+                                        [ $1, $2 + 0, '* LIST '.
+                                          qq[(\\HasNoChildren) "." $u\r\n] ]
+                                }
+                        } keys %$mailboxes;
+                $pi_cfg->{-imap_mailboxes} = $mailboxes;
+        };
+        $self->{mailboxlist} = $pi_cfg->{-imap_mailboxlist} //
+                        die 'BUG: no mailboxlist';
         $self->{pi_cfg} = $pi_cfg;
         if (my $idler = $self->{idler}) {
                 $idler->refresh($pi_cfg);
@@ -87,4 +91,11 @@ sub idler_start {
         $_[0]->{idler} //= PublicInbox::InboxIdle->new($_[0]->{pi_cfg});
 }
 
+sub event_step { # called vai requeue for low-priority IMAP clients
+        my ($self) = @_;
+        my $imap = shift(@{$self->{-authed_q}}) // return;
+        PublicInbox::DS::requeue($self) if scalar(@{$self->{-authed_q}});
+        $imap->event_step; # PublicInbox::IMAP::event_step
+}
+
 1;
diff --git a/lib/PublicInbox/Import.pm b/lib/PublicInbox/Import.pm
index 60ce7b66..04192174 100644
--- a/lib/PublicInbox/Import.pm
+++ b/lib/PublicInbox/Import.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2016-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 #
 # git fast-import-based ssoma-mda MDA replacement
@@ -103,7 +103,7 @@ sub _check_path ($$$$) {
         return if $tip eq '';
         print $w "ls $tip $path\n" or wfail;
         local $/ = "\n";
-        defined(my $info = <$r>) or die "EOF from fast-import: $!";
+        my $info = <$r> // die "EOF from fast-import: $!";
         $info =~ /\Amissing / ? undef : $info;
 }
 
@@ -111,22 +111,21 @@ sub _cat_blob ($$$) {
         my ($r, $w, $oid) = @_;
         print $w "cat-blob $oid\n" or wfail;
         local $/ = "\n";
-        my $info = <$r>;
-        defined $info or die "EOF from fast-import / cat-blob: $!";
+        my $info = <$r> // die "EOF from fast-import / cat-blob: $!";
         $info =~ /\A[a-f0-9]{40,} blob ([0-9]+)\n\z/ or return;
         my $left = $1;
         my $offset = 0;
         my $buf = '';
         my $n;
         while ($left > 0) {
-                $n = read($r, $buf, $left, $offset);
-                defined($n) or die "read cat-blob failed: $!";
+                $n = read($r, $buf, $left, $offset) //
+                        die "read cat-blob failed: $!";
                 $n == 0 and die 'fast-export (cat-blob) died';
                 $left -= $n;
                 $offset += $n;
         }
-        $n = read($r, my $lf, 1);
-        defined($n) or die "read final byte of cat-blob failed: $!";
+        $n = read($r, my $lf, 1) //
+                die "read final byte of cat-blob failed: $!";
         die "bad read on final byte: <$lf>" if $lf ne "\n";
 
         # fixup some bugginess in old versions:
@@ -148,10 +147,8 @@ sub check_remove_v1 {
         my $oid = $1;
         my $msg = _cat_blob($r, $w, $oid) or die "BUG: cat-blob $1 failed";
         my $cur = PublicInbox::Eml->new($msg);
-        my $cur_s = $cur->header('Subject');
-        $cur_s = '' unless defined $cur_s;
-        my $cur_m = $mime->header('Subject');
-        $cur_m = '' unless defined $cur_m;
+        my $cur_s = $cur->header('Subject') // '';
+        my $cur_m = $mime->header('Subject') // '';
         if ($cur_s ne $cur_m || norm_body($cur) ne norm_body($mime)) {
                 return ('MISMATCH', $cur);
         }
@@ -185,8 +182,8 @@ sub _update_git_info ($$) {
                 my $env = { GIT_INDEX_FILE => $index };
                 run_die([@cmd, qw(read-tree -m -v -i), $self->{ref}], $env);
         }
-        eval { run_die([@cmd, 'update-server-info']) };
         my $ibx = $self->{ibx};
+        eval { run_die([@cmd, 'update-server-info']) } if $ibx;
         if ($ibx && $ibx->version == 1 && -d "$ibx->{inboxdir}/public-inbox" &&
                                 eval { require PublicInbox::SearchIdx }) {
                 eval {
@@ -195,7 +192,10 @@ sub _update_git_info ($$) {
                 };
                 warn "$ibx->{inboxdir} index failed: $@\n" if $@;
         }
-        eval { run_die([@cmd, qw(gc --auto)]) } if $do_gc;
+        if ($do_gc) {
+                my @quiet = (-t STDERR ? () : '-q');
+                eval { run_die([@cmd, qw(gc --auto), @quiet]) }
+        }
 }
 
 sub barrier {
@@ -219,7 +219,7 @@ sub get_mark {
         die "not active\n" unless $self->{in};
         my ($r, $w) = $self->gfi_start;
         print $w "get-mark $mark\n" or wfail;
-        defined(my $oid = <$r>) or die "get-mark failed, need git 2.6.0+\n";
+        my $oid = <$r> // die "get-mark failed, need git 2.6.0+\n";
         chomp($oid);
         $oid;
 }
diff --git a/lib/PublicInbox/In2Tie.pm b/lib/PublicInbox/In2Tie.pm
index ffe26a44..3689432b 100644
--- a/lib/PublicInbox/In2Tie.pm
+++ b/lib/PublicInbox/In2Tie.pm
@@ -1,10 +1,10 @@
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # used to ensure PublicInbox::DS can call fileno() as a function
 # on Linux::Inotify2 objects
 package PublicInbox::In2Tie;
-use strict;
+use v5.12;
 use Symbol qw(gensym);
 
 sub io {
diff --git a/lib/PublicInbox/Inbox.pm b/lib/PublicInbox/Inbox.pm
index 3f70e69d..cb98d2ad 100644
--- a/lib/PublicInbox/Inbox.pm
+++ b/lib/PublicInbox/Inbox.pm
@@ -181,33 +181,18 @@ sub over {
         } // ($req ? croak("E: $@") : undef);
 }
 
-sub try_cat {
-        my ($path) = @_;
-        open(my $fh, '<', $path) or return '';
-        local $/;
-        <$fh> // '';
-}
-
-sub cat_desc ($) {
-        my $desc = try_cat($_[0]);
-        local $/ = "\n";
-        chomp $desc;
-        utf8::decode($desc);
-        $desc =~ s/\s+/ /smg;
-        $desc eq '' ? undef : $desc;
-}
-
 sub description {
         my ($self) = @_;
-        ($self->{description} //= cat_desc("$self->{inboxdir}/description")) //
+        ($self->{description} //=
+                PublicInbox::Git::cat_desc("$self->{inboxdir}/description")) //
                 '($INBOX_DIR/description missing)';
 }
 
 sub cloneurl {
         my ($self) = @_;
         $self->{cloneurl} // do {
-                my $s = try_cat("$self->{inboxdir}/cloneurl");
-                my @urls = split(/\s+/s, $s);
+                my @urls = split(/\s+/s,
+                  PublicInbox::Git::try_cat("$self->{inboxdir}/cloneurl"));
                 scalar(@urls) ? ($self->{cloneurl} = \@urls) : undef;
         } // [];
 }
@@ -220,7 +205,8 @@ sub base_url {
                 $url .= '/' if $url !~ m!/\z!;
                 return $url .= $self->{name} . '/';
         }
-        # called from a non-PSGI environment (e.g. NNTP/POP3):
+        # called from a non-PSGI environment or cross-inbox environment
+        # where multiple inboxes can have different domains
         my $url = $self->{url} // return undef;
         $url = $url->[0] // return undef;
         # expand protocol-relative URLs to HTTPS if we're
@@ -285,9 +271,9 @@ sub pop3_url {
                 my $group = $self->{newsgroup};
                 my @urls;
                 ($ps && $group) and
-                        @urls = map { m!\Apops?://! ? $_ : "pop://$_" } @$ps;
+                        @urls = map { m!\Apop3?s?://! ? $_ : "pop3://$_" } @$ps;
                 if (my $mi = $self->{'pop3mirror'}) {
-                        my @m = map { m!\Apops?://! ? $_ : "pop://$_" } @$mi;
+                        my @m = map { m!\Apop3?s?://! ? $_ : "pop3://$_" } @$mi;
                         my %seen; # List::Util::uniq requires Perl 5.26+
                         @urls = grep { !$seen{$_}++ } (@urls, @m);
                 }
@@ -351,11 +337,6 @@ sub msg_by_mid ($$) {
         $smsg ? msg_by_smsg($self, $smsg) : msg_by_path($self, mid2path($mid));
 }
 
-sub recent {
-        my ($self, $opts, $after, $before) = @_;
-        $self->over->recent($opts, $after, $before);
-}
-
 sub modified {
         my ($self) = @_;
         if (my $over = $self->over) {
@@ -431,4 +412,6 @@ sub mailboxid { # rfc 8474, 8620, 8621
                 sprintf('-%x', uidvalidity($self) // 0)
 }
 
+sub thing_type { 'public inbox' }
+
 1;
diff --git a/lib/PublicInbox/InboxIdle.pm b/lib/PublicInbox/InboxIdle.pm
index 2781b3e1..f0d8a972 100644
--- a/lib/PublicInbox/InboxIdle.pm
+++ b/lib/PublicInbox/InboxIdle.pm
@@ -1,11 +1,11 @@
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # fields:
 # inot: Linux::Inotify2-like object
 # pathmap => { inboxdir => [ ibx, watch1, watch2, watch3... ] } mapping
 package PublicInbox::InboxIdle;
-use strict;
+use v5.12;
 use parent qw(PublicInbox::DS);
 use PublicInbox::Syscall qw(EPOLLIN);
 my $IN_MODIFY = 0x02; # match Linux inotify
@@ -30,9 +30,9 @@ sub in2_arm ($$) { # PublicInbox::Config::each_inbox callback
         my $old_ibx = $cur->[0];
         $cur->[0] = $ibx;
         if ($old_ibx) {
-                $ibx->{unlock_subs} and
-                        die "BUG: $dir->{unlock_subs} should not exist";
+                my $u = $ibx->{unlock_subs};
                 $ibx->{unlock_subs} = $old_ibx->{unlock_subs};
+                %{$ibx->{unlock_subs}} = (%$u, %{$ibx->{unlock_subs}}) if $u;
 
                 # Linux::Inotify2::Watch::name matches if watches are the
                 # same, no point in replacing a watch of the same name
@@ -48,11 +48,9 @@ sub in2_arm ($$) { # PublicInbox::Config::each_inbox callback
                 $self->{on_unlock}->{$w->name} = $ibx;
         } else {
                 warn "E: ".ref($inot)."->watch($lock, IN_MODIFY) failed: $!\n";
-                if ($!{ENOSPC} && $^O eq 'linux') {
-                        warn <<"";
-I: consider increasing /proc/sys/fs/inotify/max_user_watches
+                warn <<"" if $!{ENOSPC} && $^O eq 'linux';
+# consider increasing /proc/sys/fs/inotify/max_user_watches
 
-                }
         }
 
         # TODO: detect deleted packs (and possibly other files)
diff --git a/lib/PublicInbox/KQNotify.pm b/lib/PublicInbox/KQNotify.pm
index 7efb8b60..381711fa 100644
--- a/lib/PublicInbox/KQNotify.pm
+++ b/lib/PublicInbox/KQNotify.pm
@@ -1,11 +1,10 @@
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # implements the small subset of Linux::Inotify2 functionality we use
 # using IO::KQueue on *BSD systems.
 package PublicInbox::KQNotify;
-use strict;
-use v5.10.1;
+use v5.12;
 use IO::KQueue;
 use PublicInbox::DSKQXS; # wraps IO::KQueue for fork-safe DESTROY
 use PublicInbox::FakeInotify qw(fill_dirlist on_dir_change);
@@ -29,8 +28,7 @@ sub watch {
                         'PublicInbox::KQNotify::Watchdir';
         } else {
                 open($fh, '<', $path) or return;
-                $watch = bless [ $fh, $path ],
-                        'PublicInbox::KQNotify::Watch';
+                $watch = bless [ $fh, $path ], 'PublicInbox::KQNotify::Watch';
         }
         my $ident = fileno($fh);
         $self->{dskq}->{kq}->EV_SET($ident, # ident (fd)
@@ -100,14 +98,14 @@ sub read {
 }
 
 package PublicInbox::KQNotify::Watch;
-use strict;
+use v5.12;
 
 sub name { $_[0]->[1] }
 
 sub cancel { close $_[0]->[0] or die "close: $!" }
 
 package PublicInbox::KQNotify::Watchdir;
-use strict;
+use v5.12;
 
 sub name { $_[0]->[1] }
 
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index d81ca296..b78d70de 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -253,6 +253,8 @@ our %CMD = ( # sorted in order of importance/use:
 'forget-watch' => [ '{WATCH_NUMBER|--prune}', 'stop and forget a watch',
         qw(prune), @c_opt ],
 
+'reindex' => [ '', 'reindex all locally-indexed messages', @c_opt ],
+
 'index' => [ 'LOCATION...', 'one-time index from URL or filesystem',
         qw(in-format|F=s kw! offset=i recursive|r exclude=s include|I=s
         verbose|v+ incremental!), @net_opt, # mainly for --proxy=
@@ -397,8 +399,10 @@ my %OPTDESC = (
                 'include specified external(s) in search' ],
 'only|O=s@        q' => [ 'LOCATION',
                 'only use specified external(s) for search' ],
-'jobs=s        q' => [ '[SEARCH_JOBS][,WRITER_JOBS]',
-                'control number of search and writer jobs' ],
+'jobs|j=s' => [ 'JOBSPEC',
+                'control number of query and writer jobs' .
+                "integers delimited by `,', either of which may be omitted"
+                ],
 'jobs|j=i        add-external' => 'set parallelism when indexing after --mirror',
 
 'in-format|F=s' => $stdin_formats,
@@ -540,12 +544,11 @@ sub child_error { # passes non-fatal curl exit codes to user
         local $current_lei = $self;
         $child_error ||= 1 << 8;
         warn(substr($msg, -1, 1) eq "\n" ? $msg : "$msg\n") if defined $msg;
+        $self->{child_error} ||= $child_error;
         if ($self->{pkt_op_p}) { # to top lei-daemon
                 $self->{pkt_op_p}->pkt_do('child_error', $child_error);
         } elsif ($self->{sock}) { # to lei(1) client
                 send($self->{sock}, "child_error $child_error", MSG_EOR);
-        } else { # non-lei admin command
-                $self->{child_error} ||= $child_error;
         } # else noop if client disconnected
 }
 
@@ -783,7 +786,7 @@ EOM
         }
 }
 
-sub lazy_cb ($$$) {
+sub lazy_cb ($$$) { # $pfx is _complete_ or lei_
         my ($self, $cmd, $pfx) = @_;
         my $ucmd = $cmd;
         $ucmd =~ tr/-/_/;
@@ -1520,13 +1523,10 @@ sub sto_done_request {
         return unless $lei->{sto};
         local $current_lei = $lei;
         my $sock = $wq ? $wq->{lei_sock} : undef;
-        eval {
-                if ($sock //= $lei->{sock}) { # issue, async wait
-                        $lei->{sto}->wq_io_do('done', [ $sock ]);
-                } else { # forcibly wait
-                        my $wait = $lei->{sto}->wq_do('done');
-                }
-        };
+        $sock //= $lei->{sock};
+        my @io;
+        push(@io, $sock) if $sock; # async wait iff possible
+        eval { $lei->{sto}->wq_io_do('done', \@io) };
         warn($@) if $@;
 }
 
diff --git a/lib/PublicInbox/LeiBlob.pm b/lib/PublicInbox/LeiBlob.pm
index 004b156c..1692289c 100644
--- a/lib/PublicInbox/LeiBlob.pm
+++ b/lib/PublicInbox/LeiBlob.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # "lei blob $OID" command
@@ -70,7 +70,7 @@ sub do_solve_blob { # via wq_do
                         } @$git_dirs ],
                 user_cb => \&solver_user_cb,
                 uarg => $self,
-                # -cur_di, -qsp, -msg => temporary fields for Qspawn callbacks
+                # -cur_di, -msg => temporary fields for Qspawn callbacks
                 inboxes => [ $self->{lxs}->locals, @rmt ],
         }, 'PublicInbox::SolverGit';
         local $PublicInbox::DS::in_loop = 0; # waitpid synchronously
diff --git a/lib/PublicInbox/LeiCurl.pm b/lib/PublicInbox/LeiCurl.pm
index 5ffade99..48c66ee9 100644
--- a/lib/PublicInbox/LeiCurl.pm
+++ b/lib/PublicInbox/LeiCurl.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # common option and torsocks(1) wrapping for curl(1)
@@ -7,8 +7,7 @@
 # n.b. curl may support a daemon/client model like lei someday:
 #   https://github.com/curl/curl/wiki/curl-tool-master-client
 package PublicInbox::LeiCurl;
-use strict;
-use v5.10.1;
+use v5.12;
 use PublicInbox::Spawn qw(which);
 use PublicInbox::Config;
 
@@ -27,7 +26,7 @@ sub new {
         my ($cls, $lei, $curl) = @_;
         $curl //= which('curl') // return $lei->fail('curl not found');
         my $opt = $lei->{opt};
-        my @cmd = ($curl, qw(-Sf));
+        my @cmd = ($curl, qw(-gSf));
         $cmd[-1] .= 's' if $opt->{quiet}; # already the default for "lei q"
         $cmd[-1] .= 'v' if $opt->{verbose}; # we use ourselves, too
         for my $o ($lei->curl_opt) {
@@ -77,8 +76,8 @@ sub for_uri {
         my $pfx = torsocks($self, $lei, $uri) or return; # error
         if ($uri->scheme =~ /\Ahttps?\z/i) {
                 my $cfg = $lei->_lei_cfg;
-                my $p = $cfg ? $cfg->urlmatch('http.Proxy', $$uri) : undef;
-                push(@opt, "--proxy=$p") if defined($p);
+                my $p = $cfg ? $cfg->urlmatch('http.Proxy', $$uri, 1) : undef;
+                push(@opt, '--proxy', $p) if defined($p);
         }
         bless [ @$pfx, @$self, @opt, $uri->as_string ], ref($self);
 }
diff --git a/lib/PublicInbox/LeiExternal.pm b/lib/PublicInbox/LeiExternal.pm
index 30bb1a45..a6562e7f 100644
--- a/lib/PublicInbox/LeiExternal.pm
+++ b/lib/PublicInbox/LeiExternal.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # *-external commands of lei
@@ -88,7 +88,7 @@ sub get_externals {
         my @cur = externals_each($self);
         my $do_glob = !$self->{opt}->{globoff}; # glob by default
         if ($do_glob && (my $re = glob2re($loc))) {
-                @m = grep(m!$re!, @cur);
+                @m = grep(m!$re/?\z!, @cur);
                 return @m if scalar(@m);
         } elsif (index($loc, '/') < 0) { # exact basename match:
                 @m = grep(m!/\Q$loc\E/?\z!, @cur);
diff --git a/lib/PublicInbox/LeiInspect.pm b/lib/PublicInbox/LeiInspect.pm
index d7775d4b..d1dca4ef 100644
--- a/lib/PublicInbox/LeiInspect.pm
+++ b/lib/PublicInbox/LeiInspect.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # "lei inspect" general purpose inspector for stuff in SQLite and
@@ -235,7 +235,8 @@ sub inspect_argv { # via wq_do
         $lei->{1}->autoflush(0);
         $lei->out('[') if $multi;
         while (defined(my $x = shift @$argv)) {
-                inspect1($lei, $x, scalar(@$argv)) or return;
+                eval { inspect1($lei, $x, scalar(@$argv)) or return };
+                warn "E: $@\n" if $@;
         }
         $lei->out(']') if $multi;
 }
diff --git a/lib/PublicInbox/LeiLsExternal.pm b/lib/PublicInbox/LeiLsExternal.pm
index dd2eb2e7..e624cbd4 100644
--- a/lib/PublicInbox/LeiLsExternal.pm
+++ b/lib/PublicInbox/LeiLsExternal.pm
@@ -13,6 +13,7 @@ sub lei_ls_external {
         my ($OFS, $ORS) = $lei->{opt}->{z} ? ("\0", "\0\0") : (" ", "\n");
         $filter //= '*';
         my $re = $do_glob ? $lei->glob2re($filter) : undef;
+        $re .= '/?\\z' if defined $re;
         $re //= index($filter, '/') < 0 ?
                         qr!/\Q$filter\E/?\z! : # exact basename match
                         qr/\Q$filter\E/; # grep -F semantics
diff --git a/lib/PublicInbox/LeiLsMailSync.pm b/lib/PublicInbox/LeiLsMailSync.pm
index 2b167b1d..8da0c284 100644
--- a/lib/PublicInbox/LeiLsMailSync.pm
+++ b/lib/PublicInbox/LeiLsMailSync.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # front-end for the "lei ls-mail-sync" sub-command
@@ -12,7 +12,10 @@ sub lei_ls_mail_sync {
         my $lms = $lei->lms or return;
         my $opt = $lei->{opt};
         my $re = $opt->{globoff} ? undef : $lei->glob2re($filter // '*');
-        $re //= qr/\Q$filter\E/;
+        $re .= '/?\\z' if defined $re;
+        $re //= index($filter, '/') < 0 ?
+                        qr!/\Q$filter\E/?\z! : # exact basename match
+                        qr/\Q$filter\E/; # grep -F semantics
         my @f = $lms->folders;
         @f = $opt->{'invert-match'} ? grep(!/$re/, @f) : grep(/$re/, @f);
         if ($opt->{'local'} && !$opt->{remote}) {
diff --git a/lib/PublicInbox/LeiMirror.pm b/lib/PublicInbox/LeiMirror.pm
index e20d30b4..33cf55ab 100644
--- a/lib/PublicInbox/LeiMirror.pm
+++ b/lib/PublicInbox/LeiMirror.pm
@@ -1,32 +1,45 @@
-# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # "lei add-external --mirror" support (also "public-inbox-clone");
 package PublicInbox::LeiMirror;
-use strict;
-use v5.10.1;
+use v5.12;
 use parent qw(PublicInbox::IPC);
-use PublicInbox::Config;
-use PublicInbox::AutoReap;
 use IO::Uncompress::Gunzip qw(gunzip $GunzipError);
 use IO::Compress::Gzip qw(gzip $GzipError);
-use PublicInbox::Spawn qw(popen_rd spawn);
+use PublicInbox::Spawn qw(popen_rd spawn run_die);
+use File::Path ();
 use File::Temp ();
+use File::Spec ();
 use Fcntl qw(SEEK_SET O_CREAT O_EXCL O_WRONLY);
 use Carp qw(croak);
+use URI;
+use PublicInbox::Config;
+use PublicInbox::Inbox;
+use PublicInbox::LeiCurl;
+use PublicInbox::OnDestroy;
+use Digest::SHA qw(sha256_hex sha1_hex);
+use POSIX qw(strftime);
+
+our $LIVE; # pid => callback
+our $FGRP_TODO; # objstore -> [ fgrp mirror objects ]
+our $TODO; # reference => [ non-fgrp mirror objects ]
+
+sub keep_going ($) {
+        $LIVE && (!$_[0]->{lei}->{child_error} ||
+                $_[0]->{lei}->{opt}->{'keep-going'});
+}
 
 sub _wq_done_wait { # dwaitpid callback (via wq_eof)
         my ($arg, $pid) = @_;
         my ($mrr, $lei) = @$arg;
-        my $f = "$mrr->{dst}/mirror.done";
         if ($?) {
                 $lei->child_error($?);
-        } elsif (!unlink($f)) {
-                warn("unlink($f): $!\n") unless $!{ENOENT};
-        } else {
-                if ($lei->{cmd} ne 'public-inbox-clone') {
-                        $lei->lazy_cb('add-external', '_finish_'
-                                        )->($lei, $mrr->{dst});
+        } elsif (!$lei->{child_error}) {
+                if (!$mrr->{dry_run} && $lei->{cmd} ne 'public-inbox-clone') {
+                        require PublicInbox::LeiAddExternal;
+                        PublicInbox::LeiAddExternal::_finish_add_external(
+                                                        $lei, $mrr->{dst});
                 }
                 $lei->qerr("# mirrored $mrr->{src} => $mrr->{dst}");
         }
@@ -35,7 +48,7 @@ sub _wq_done_wait { # dwaitpid callback (via wq_eof)
 
 # for old installations without manifest.js.gz
 sub try_scrape {
-        my ($self) = @_;
+        my ($self, $fallback_manifest) = @_;
         my $uri = URI->new($self->{src});
         my $lei = $self->{lei};
         my $curl = $self->{curl} //= PublicInbox::LeiCurl->new($lei) or return;
@@ -46,9 +59,17 @@ sub try_scrape {
         close($fh) or return $lei->child_error($?, "@$cmd failed");
 
         # we grep with URL below, we don't want Subject/From headers
-        # making us clone random URLs
+        # making us clone random URLs.  This assumes remote instances
+        # prior to public-inbox 1.7.0
+        # 5b96edcb1e0d8252 (www: move mirror instructions to /text/, 2021-08-28)
         my @html = split(/<hr>/, $html);
         my @urls = ($html[-1] =~ m!\bgit clone --mirror ([a-z\+]+://\S+)!g);
+        if (!@urls && $fallback_manifest) {
+                warn <<EOM;
+W: failed to extract URLs from $uri, trying manifest.js.gz...
+EOM
+                return start_clone_url($self);
+        }
         my $url = $uri->as_string;
         chop($url) eq '/' or die "BUG: $uri not canonicalized";
 
@@ -57,9 +78,12 @@ sub try_scrape {
         if (my @v2_urls = grep(m!\A\Q$url\E/[0-9]+\z!, @urls)) {
                 my %v2_epochs = map {
                         my ($n) = (m!/([0-9]+)\z!);
-                        $n => URI->new($_)
+                        $n => [ URI->new($_), '' ]
                 } @v2_urls; # uniq
-                return clone_v2($self, \%v2_epochs);
+                clone_v2_prep($self, \%v2_epochs);
+                delete local $lei->{opt}->{epoch};
+                clone_all($self);
+                return;
         }
 
         # filter out common URLs served by WWW (e.g /$MSGID/T/)
@@ -84,54 +108,90 @@ sub clone_cmd {
         # e.g.: git -c http.proxy=socks5h://127.0.0.1:9050
         push(@cmd, '-c', $_) for @{$lei->{opt}->{c} // []};
         push @cmd, qw(clone --mirror);
-        push @cmd, '-q' if $lei->{opt}->{quiet};
+        push @cmd, '-q' if $lei->{opt}->{quiet} ||
+                        ($lei->{opt}->{jobs} // 1) > 1;
         push @cmd, '-v' if $lei->{opt}->{verbose};
         # XXX any other options to support?
-        # --reference is tricky with multiple epochs...
+        # --reference is tricky with multiple epochs, but handled
+        # automatically if using manifest.js.gz
         @cmd;
 }
 
-sub ft_rename ($$$) {
-        my ($ft, $dst, $open_mode) = @_;
-        my $fn = $ft->filename;
-        my @st = stat($dst);
+sub ft_rename ($$$;$) {
+        my ($ft, $dst, $open_mode, $fh) = @_;
+        my @st = stat($fh // $dst);
         my $mode = @st ? ($st[2] & 07777) : ($open_mode & ~umask);
-        chmod($mode, $ft) or croak "E: chmod $fn: $!";
-        rename($fn, $dst) or croak "E: rename($fn => $ft): $!";
+        chmod($mode, $ft) or croak "E: chmod($ft): $!";
+        require File::Copy;
+        File::Copy::mv($ft->filename, $dst) or croak "E: mv($ft => $dst): $!";
         $ft->unlink_on_destroy(0);
 }
 
-sub _get_txt { # non-fatal
-        my ($self, $endpoint, $file, $mode) = @_;
-        my $uri = URI->new($self->{src});
+sub do_reap ($;$) {
+        my ($self, $jobs) = @_;
+        $jobs //= $self->{-jobs} //= $self->{lei}->{opt}->{jobs} // 1;
+        $jobs = 1 if $jobs < 1;
+        while (keys(%$LIVE) >= $jobs) {
+                my $pid = waitpid(-1, 0) // die "waitpid(-1): $!";
+                if (my $x = delete $LIVE->{$pid}) {
+                        my $cb = shift @$x;
+                        $cb->(@$x) if $cb;
+                } else {
+                        warn "reaped unknown PID=$pid ($?)\n";
+                }
+        }
+}
+
+sub _get_txt_start { # non-fatal
+        my ($self, $endpoint, $fini) = @_;
+        my $uri = URI->new($self->{cur_src} // $self->{src});
         my $lei = $self->{lei};
         my $path = $uri->path;
         chop($path) eq '/' or die "BUG: $uri not canonicalized";
         $uri->path("$path/$endpoint");
-        my $ft = File::Temp->new(TEMPLATE => "$file-XXXX", DIR => $self->{dst});
+        my $f = (split(m!/!, $endpoint))[-1];
+        my $ft = File::Temp->new(TEMPLATE => "$f-XXXX", TMPDIR => 1);
         my $opt = { 0 => $lei->{0}, 1 => $lei->{1}, 2 => $lei->{2} };
-        my $cmd = $self->{curl}->for_uri($lei, $uri,
-                                        qw(--compressed -R -o), $ft->filename);
-        my $cerr = run_reap($lei, $cmd, $opt);
-        return "$uri missing" if ($cerr >> 8) == 22;
-        return "# @$cmd failed (non-fatal)" if $cerr;
-        ft_rename($ft, "$self->{dst}/$file", $mode);
+        my $cmd = $self->{curl}->for_uri($lei, $uri, qw(--compressed -R -o),
+                                        $ft->filename);
+        do_reap($self);
+        $lei->qerr("# @$cmd");
+        return if $self->{dry_run};
+        $self->{"-get_txt.$endpoint"} = [ $ft, $cmd, $uri ];
+        $LIVE->{spawn($cmd, undef, $opt)} =
+                        [ \&_get_txt_done, $self, $endpoint, $fini ];
+}
+
+sub _get_txt_done { # returns true on error (non-fatal), undef on success
+        my ($self, $endpoint) = @_;
+        my ($fh, $cmd, $uri) = @{delete $self->{"-get_txt.$endpoint"}};
+        my $cerr = $?;
+        $? = 0; # don't influence normal lei exit
+        return warn("$uri missing\n") if ($cerr >> 8) == 22;
+        return warn("# @$cmd failed (non-fatal)\n") if $cerr;
+        seek($fh, SEEK_SET, 0) or die "seek: $!";
+        $self->{"mtime.$endpoint"} = (stat($fh))[9];
+        local $/;
+        $self->{"txt.$endpoint"} = <$fh>;
         undef; # success
 }
 
-# tries the relatively new /$INBOX/_/text/config/raw endpoint
-sub _try_config {
+sub _write_inbox_config {
         my ($self) = @_;
-        my $dst = $self->{dst};
-        if (!-d $dst || !mkdir($dst)) {
-                require File::Path;
-                File::Path::mkpath($dst);
-                -d $dst or die "mkpath($dst): $!\n";
-        }
-        my $err = _get_txt($self,
-                        qw(_/text/config/raw inbox.config.example), 0444);
-        return warn($err, "\n") if $err;
-        my $f = "$self->{dst}/inbox.config.example";
+        my $buf = delete($self->{'txt._/text/config/raw'}) // return;
+        my $dst = $self->{cur_dst} // $self->{dst};
+        my $f = "$dst/inbox.config.example";
+        my $mtime = delete $self->{'mtime._/text/config/raw'};
+        if (sysopen(my $fh, $f, O_CREAT|O_EXCL|O_WRONLY)) {
+                print $fh $buf or die "print: $!";
+                chmod(0444 & ~umask, $fh) or die "chmod($f): $!";
+                $fh->flush or die "flush($f): $!";
+                if (defined $mtime) {
+                        utime($mtime, $mtime, $fh) or die "utime($f): $!";
+                }
+        } elsif (!$!{EEXIST}) {
+                die "open($f): $!";
+        }
         my $cfg = PublicInbox::Config->git_config_dump($f, $self->{lei}->{2});
         my $ibx = $self->{ibx} = {};
         for my $sec (grep(/\Apublicinbox\./, @{$cfg->{-section_order}})) {
@@ -143,35 +203,31 @@ sub _try_config {
 
 sub set_description ($) {
         my ($self) = @_;
-        my $f = "$self->{dst}/description";
-        open my $fh, '+>>', $f or die "open($f): $!";
-        seek($fh, 0, SEEK_SET) or die "seek($f): $!";
-        chomp(my $d = do { local $/; <$fh> } // die "read($f): $!");
-        if ($d eq '($INBOX_DIR/description missing)' ||
-                        $d =~ /^Unnamed repository/ || $d !~ /\S/) {
-                seek($fh, 0, SEEK_SET) or die "seek($f): $!";
-                truncate($fh, 0) or die "truncate($f): $!";
-                print $fh "mirror of $self->{src}\n" or die "print($f): $!";
-                close $fh or die "close($f): $!";
+        my $dst = $self->{cur_dst} // $self->{dst};
+        chomp(my $orig = PublicInbox::Git::try_cat("$dst/description"));
+        my $d = $orig;
+        while (defined($d) && ($d =~ m!^\(\$INBOX_DIR/description missing\)! ||
+                        $d =~ /^Unnamed repository/ || $d !~ /\S/)) {
+                $d = delete($self->{'txt.description'});
         }
+        $d //= 'mirror of '.($self->{cur_src} // $self->{src});
+        atomic_write($dst, 'description', $d."\n") if $d ne $orig;
 }
 
 sub index_cloned_inbox {
         my ($self, $iv) = @_;
         my $lei = $self->{lei};
-        my $err = _get_txt($self, qw(description description), 0666);
-        warn($err, "\n") if $err; # non fatal
-        eval { set_description($self) };
-        warn $@ if $@;
 
         # n.b. public-inbox-clone works w/o (SQLite || Xapian)
         # lei is useless without Xapian + SQLite
         if ($lei->{cmd} ne 'public-inbox-clone') {
+                require PublicInbox::InboxWritable;
+                require PublicInbox::Admin;
                 my $ibx = delete($self->{ibx}) // {
                         address => [ 'lei@example.com' ],
                         version => $iv,
                 };
-                $ibx->{inboxdir} = $self->{dst};
+                $ibx->{inboxdir} = $self->{cur_dst} // $self->{dst};
                 PublicInbox::Inbox->new($ibx);
                 PublicInbox::InboxWritable->new($ibx);
                 my $opt = {};
@@ -187,38 +243,336 @@ sub index_cloned_inbox {
                 PublicInbox::Admin::progress_prepare($opt, $lei->{2});
                 PublicInbox::Admin::index_inbox($ibx, undef, $opt);
         }
-        open my $x, '>', "$self->{dst}/mirror.done"; # for _wq_done_wait
+        return if defined $self->{cur_dst}; # one of many repos to clone
 }
 
 sub run_reap {
         my ($lei, $cmd, $opt) = @_;
         $lei->qerr("# @$cmd");
-        my $ar = PublicInbox::AutoReap->new(spawn($cmd, undef, $opt));
-        $ar->join;
+        waitpid(spawn($cmd, undef, $opt), 0) // die "waitpid: $!";
         my $ret = $?;
         $? = 0; # don't let it influence normal exit
         $ret;
 }
 
-sub clone_v1 {
+sub start_cmd {
+        my ($self, $cmd, $opt, $fini) = @_;
+        do_reap($self);
+        $self->{lei}->qerr("# @$cmd");
+        return if $self->{dry_run};
+        $LIVE->{spawn($cmd, undef, $opt)} = [ \&reap_cmd, $self, $cmd, $fini ]
+}
+
+sub fetch_args ($$) {
+        my ($lei, $opt) = @_;
+        my @cmd; # (git --git-dir=...) to be added by caller
+        $opt->{$_} = $lei->{$_} for (0..2);
+        # we support "-c $key=$val" for arbitrary git config options
+        # e.g.: git -c http.proxy=socks5h://127.0.0.1:9050
+        push(@cmd, '-c', $_) for @{$lei->{opt}->{c} // []};
+        push @cmd, 'fetch';
+        push @cmd, '-q' if $lei->{opt}->{quiet} ||
+                        ($lei->{opt}->{jobs} // 1) > 1;
+        push @cmd, '-v' if $lei->{opt}->{verbose};
+        push(@cmd, '-p') if $lei->{opt}->{prune};
+        @cmd;
+}
+
+sub upr { # feed `git update-ref --stdin -z' verbosely
+        my ($lei, $w, $op, @rest) = @_; # ($ref, $oid) = @rest
+        $lei->qerr("# $op @rest") if $lei->{opt}->{verbose};
+        print $w "$op ", join("\0", @rest, '') or die "print(w): $!";
+}
+
+sub fgrp_update {
+        my ($fgrp) = @_;
+        return if !keep_going($fgrp);
+        my $srcfh = delete $fgrp->{srcfh} or return;
+        my $dstfh = delete $fgrp->{dstfh} or return;
+        seek($srcfh, SEEK_SET, 0) or die "seek(src): $!";
+        seek($dstfh, SEEK_SET, 0) or die "seek(dst): $!";
+        my %src = map { chomp; split(/\0/) } (<$srcfh>);
+        close $srcfh;
+        my %dst = map { chomp; split(/\0/) } (<$dstfh>);
+        close $dstfh;
+        pipe(my ($r, $w)) or die "pipe: $!";
+        my $cmd = [ 'git', "--git-dir=$fgrp->{cur_dst}",
+                qw(update-ref --stdin -z) ];
+        my $lei = $fgrp->{lei};
+        my $pack = PublicInbox::OnDestroy->new($$, \&pack_dst, $fgrp);
+        start_cmd($fgrp, $cmd, { 0 => $r, 2 => $lei->{2} }, $pack);
+        close $r or die "close(r): $!";
+        return if $fgrp->{dry_run};
+        for my $ref (keys %dst) {
+                my $new = delete $src{$ref};
+                my $old = $dst{$ref};
+                if (defined $new) {
+                        $new eq $old or
+                                upr($lei, $w, 'update', $ref, $new, $old);
+                } else {
+                        upr($lei, $w, 'delete', $ref, $old);
+                }
+        }
+        while (my ($ref, $oid) = each %src) {
+                upr($lei, $w, 'create', $ref, $oid);
+        }
+        close($w) or warn "E: close(update-ref --stdin): $! (need git 1.8.5+)\n";
+}
+
+sub pack_dst { # packs lightweight satellite repos
+        my ($fgrp) = @_;
+        pack_refs($fgrp, $fgrp->{cur_dst});
+}
+
+sub pack_refs {
+        my ($self, $git_dir) = @_;
+        my $cmd = [ 'git', "--git-dir=$git_dir", qw(pack-refs --all --prune) ];
+        start_cmd($self, $cmd, { 2 => $self->{lei}->{2} });
+}
+
+sub fgrpv_done {
+        my ($fgrpv) = @_;
+        return if !$LIVE;
+        my $first = $fgrpv->[0] // die 'BUG: no fgrpv->[0]';
+        return if !keep_going($first);
+        pack_refs($first, $first->{-osdir}); # objstore refs always packed
+        for my $fgrp (@$fgrpv) {
+                my $rn = $fgrp->{-remote};
+                my %opt = ( 2 => $fgrp->{lei}->{2} );
+
+                my $update_ref = PublicInbox::OnDestroy->new($$,
+                                                        \&fgrp_update, $fgrp);
+
+                my $src = [ 'git', "--git-dir=$fgrp->{-osdir}", 'for-each-ref',
+                        "--format=refs/%(refname:lstrip=3)%00%(objectname)",
+                        "refs/remotes/$rn/" ];
+                open(my $sfh, '+>', undef) or die "open(src): $!";
+                $fgrp->{srcfh} = $sfh;
+                start_cmd($fgrp, $src, { %opt, 1 => $sfh }, $update_ref);
+                my $dst = [ 'git', "--git-dir=$fgrp->{cur_dst}", 'for-each-ref',
+                        '--format=%(refname)%00%(objectname)' ];
+                open(my $dfh, '+>', undef) or die "open(dst): $!";
+                $fgrp->{dstfh} = $dfh;
+                start_cmd($fgrp, $dst, { %opt, 1 => $dfh }, $update_ref);
+        }
+}
+
+sub fgrp_fetch_all {
         my ($self) = @_;
+        my $todo = $FGRP_TODO;
+        $FGRP_TODO = \'BUG on further use';
+        keys(%$todo) or return;
+
+        # Rely on the fgrptmp remote groups in the config file rather
+        # than listing all remotes since the remote name list may exceed
+        # system argv limits:
+        my $grp = 'fgrptmp';
+
+        my @git = (@{$self->{-torsocks}}, 'git');
+        my $j = $self->{lei}->{opt}->{jobs};
+        my $opt = {};
+        my @fetch = do {
+                local $self->{lei}->{opt}->{jobs} = 1;
+                (fetch_args($self->{lei}, $opt),
+                        qw(--no-tags --multiple));
+        };
+        push(@fetch, "-j$j") if $j;
+        while (my ($osdir, $fgrpv) = each %$todo) {
+                my $f = "$osdir/config";
+                return if !keep_going($self);
+
+                # clobber group from previous run atomically
+                my $cmd = ['git', "--git-dir=$osdir", qw(config -f),
+                                $f, '--unset-all', "remotes.$grp"];
+                $self->{lei}->qerr("# @$cmd");
+                if (!$self->{dry_run}) {
+                        my $pid = spawn($cmd, undef, { 2 => $self->{lei}->{2} });
+                        waitpid($pid, 0) // die "waitpid: $!";
+                        die "E: @$cmd: \$?=$?" if ($? && ($? >> 8) != 5);
+
+                        # update the config atomically via O_APPEND while
+                        # respecting git-config locking
+                        sysopen(my $lk, "$f.lock", O_CREAT|O_EXCL|O_WRONLY)
+                                or die "open($f.lock): $!";
+                        open my $fh, '>>', $f or die "open(>>$f): $!";
+                        $fh->autoflush(1);
+                        my $buf = join('', "[remotes]\n",
+                                map { "\t$grp = $_->{-remote}\n" } @$fgrpv);
+                        print $fh $buf or die "print($f): $!";
+                        close $fh or die "close($f): $!";
+                        unlink("$f.lock") or die "unlink($f.lock): $!";
+                }
+
+                $cmd = [ @git, "--git-dir=$osdir", @fetch, $grp ];
+                my $end = PublicInbox::OnDestroy->new($$, \&fgrpv_done, $fgrpv);
+                start_cmd($self, $cmd, $opt, $end);
+        }
+}
+
+# keep this idempotent for future use by public-inbox-fetch
+sub forkgroup_prep {
+        my ($self, $uri) = @_;
+        $self->{-ent} // return;
+        my $os = $self->{-objstore} // return;
+        my $fg = $self->{-ent}->{forkgroup} // return;
+        my $dir = "$os/$fg.git";
+        if (!-d $dir && !$self->{dry_run}) {
+                PublicInbox::Import::init_bare($dir);
+                my @cmd = ('git', "--git-dir=$dir", 'config');
+                my $opt = { 2 => $self->{lei}->{2} };
+                for ('repack.useDeltaIslands=true',
+                                'pack.island=refs/remotes/([^/]+)/') {
+                        run_die([@cmd, split(/=/, $_, 2)], undef, $opt);
+                }
+        }
+        my $key = $self->{-key} // die 'BUG: no -key';
+        my $rn = substr(sha256_hex($key), 0, 16);
+        if (!-d $self->{cur_dst} && !$self->{dry_run}) {
+                PublicInbox::Import::init_bare($self->{cur_dst});
+                my $f = "$self->{cur_dst}/config";
+                open my $fh, '+>>', $f or die "open:($f): $!";
+                print $fh <<EOM or die "print($f): $!";
+; rely on the "$rn" remote in the
+; $fg fork group for fetches
+; only uncomment the following iff you detach from fork groups
+; [remote "origin"]
+;        url = $uri
+;        fetch = +refs/*:refs/*
+;        mirror = true
+EOM
+                close $fh or die "close($f): $!";
+        }
+        if (!$self->{dry_run}) {
+                my $alt = File::Spec->rel2abs("$dir/objects");
+                my $o = "$self->{cur_dst}/objects";
+                my $f = "$o/info/alternates";
+                my $l = File::Spec->abs2rel($alt, File::Spec->rel2abs($o));
+                open my $fh, '+>>', $f or die "open($f): $!";
+                seek($fh, SEEK_SET, 0) or die "seek($f): $!";
+                chomp(my @cur = <$fh>);
+                if (!grep(/\A\Q$l\E\z/, @cur)) {
+                        say $fh $l or die "say($f): $!";
+                }
+                close $fh or die "close($f): $!";
+        }
+        bless {
+                %$self, -osdir => $dir, -remote => $rn, -uri => $uri
+        }, __PACKAGE__;
+}
+
+sub fp_done {
+        my ($self, $cb, @arg) = @_;
+        return if !keep_going($self);
+        my $fh = delete $self->{-show_ref} // die 'BUG: no show-ref output';
+        seek($fh, SEEK_SET, 0) or die "seek(show_ref): $!";
+        $self->{-ent} // die 'BUG: no -ent';
+        my $A = $self->{-ent}->{fingerprint} // die 'BUG: no fingerprint';
+        my $B = sha1_hex(do { local $/; <$fh> } // die("read(show_ref): $!"));
+        return $cb->($self, @arg) if $A ne $B;
+        $self->{lei}->qerr("# $self->{-key} up-to-date");
+}
+
+sub cmp_fp_do {
+        my ($self, $cb, @arg) = @_;
+        # $cb is either resume_fetch or fgrp_enqueue
+        $self->{-ent} // return $cb->($self, @arg);
+        my $new = $self->{-ent}->{fingerprint} // return $cb->($self, @arg);
+        my $key = $self->{-key} // die 'BUG: no -key';
+        if (my $cur_ent = $self->{-local_manifest}->{$key}) {
+                # runs go_fetch->DESTROY run if eq
+                return if $cur_ent->{fingerprint} eq $new;
+        }
+        my $dst = $self->{cur_dst} // $self->{dst};
+        my $cmd = ['git', "--git-dir=$dst", 'show-ref'];
+        my $opt = { 2 => $self->{lei}->{2} };
+        open($opt->{1}, '+>', undef) or die "open(tmp): $!";
+        $self->{-show_ref} = $opt->{1};
+        my $done = PublicInbox::OnDestroy->new($$, \&fp_done, $self, $cb, @arg);
+        start_cmd($self, $cmd, $opt, $done);
+}
+
+sub resume_fetch {
+        my ($self, $uri, $fini) = @_;
+        return if !keep_going($self);
+        my $dst = $self->{cur_dst} // $self->{dst};
+        my @git = ('git', "--git-dir=$dst");
+        my $opt = { 2 => $self->{lei}->{2} };
+        my $rn = 'origin'; # configurable?
+        for ("url=$uri", "fetch=+refs/*:refs/*", 'mirror=true') {
+                my @kv = split(/=/, $_, 2);
+                $kv[0] = "remote.$rn.$kv[0]";
+                next if $self->{dry_run};
+                run_die([@git, 'config', @kv], undef, $opt);
+        }
+        my $cmd = [ @{$self->{-torsocks}}, @git,
+                        fetch_args($self->{lei}, $opt), $rn ];
+        push @$cmd, '-P' if $self->{lei}->{prune}; # --prune-tags implied
+        start_cmd($self, $cmd, $opt, $fini);
+}
+
+sub fgrp_enqueue {
+        my ($fgrp, $end) = @_; # $end calls fgrp_fetch_all
+        return if !keep_going($fgrp);
+        my $opt = { 2 => $fgrp->{lei}->{2} };
+        # --no-tags is required to avoid conflicts
+        my $u = $fgrp->{-uri} // die 'BUG: no {-uri}';
+        my $rn = $fgrp->{-remote} // die 'BUG: no {-remote}';
+        my @cmd = ('git', "--git-dir=$fgrp->{-osdir}", 'config');
+        for ("url=$u", "fetch=+refs/*:refs/remotes/$rn/*", 'tagopt=--no-tags') {
+                my @kv = split(/=/, $_, 2);
+                $kv[0] = "remote.$rn.$kv[0]";
+                $fgrp->{dry_run} ? $fgrp->{lei}->qerr("# @cmd @kv") :
+                                run_die([@cmd, @kv], undef, $opt);
+        }
+        push @{$FGRP_TODO->{$fgrp->{-osdir}}}, $fgrp;
+}
+
+sub clone_v1 {
+        my ($self, $end) = @_;
         my $lei = $self->{lei};
         my $curl = $self->{curl} //= PublicInbox::LeiCurl->new($lei) or return;
-        my $uri = URI->new($self->{src});
+        my $uri = URI->new($self->{cur_src} // $self->{src});
+        my $path = $uri->path;
+        $path =~ s!/*\z!! and $uri->path($path);
         defined($lei->{opt}->{epoch}) and
                 die "$uri is a v1 inbox, --epoch is not supported\n";
-        my $pfx = $curl->torsocks($lei, $uri) or return;
-        my $cmd = [ @$pfx, clone_cmd($lei, my $opt = {}),
-                        $uri->as_string, $self->{dst} ];
-        my $cerr = run_reap($lei, $cmd, $opt);
-        return $lei->child_error($cerr, "@$cmd failed") if $cerr;
-        _try_config($self);
-        write_makefile($self->{dst}, 1);
-        index_cloned_inbox($self, 1);
+        $self->{-torsocks} //= $curl->torsocks($lei, $uri) or return;
+        my $dst = $self->{cur_dst} // $self->{dst};
+        my $fini = PublicInbox::OnDestroy->new($$, \&v1_done, $self);
+        my $resume = -d $dst;
+        if (my $fgrp = forkgroup_prep($self, $uri)) {
+                $fgrp->{-fini} = $fini;
+                $resume ? cmp_fp_do($fgrp, \&fgrp_enqueue, $end)
+                        : fgrp_enqueue($fgrp, $end);
+        } elsif ($resume) {
+                cmp_fp_do($self, \&resume_fetch, $uri, $fini);
+        } else { # normal clone
+                my $cmd = [ @{$self->{-torsocks}},
+                                clone_cmd($lei, my $opt = {}), "$uri", $dst ];
+                if (defined($self->{-ent})) {
+                        if (defined(my $ref = $self->{-ent}->{reference})) {
+                                -e "$self->{dst}$ref" and
+                                        push @$cmd, '--reference',
+                                                "$self->{dst}$ref";
+                        }
+                }
+                start_cmd($self, $cmd, $opt, $fini);
+        }
+        if (!$self->{-is_epoch} && $lei->{opt}->{'inbox-config'} =~
+                                /\A(?:always|v1)\z/s) {
+                _get_txt_start($self, '_/text/config/raw', $fini);
+        }
+
+        my $d = $self->{-ent} ? $self->{-ent}->{description} : undef;
+        $self->{'txt.description'} = $d if defined $d;
+        (!defined($d) && !$end) and
+                _get_txt_start($self, 'description', $fini);
+
+        $end or do_reap($self, 1); # for non-manifest clone
 }
 
 sub parse_epochs ($$) {
-        my ($opt_epochs, $v2_epochs) = @_; # $epcohs "LOW..HIGH"
+        my ($opt_epochs, $v2_epochs) = @_; # $epochs "LOW..HIGH"
         $opt_epochs // return; # undef => all epochs
         my ($lo, $dotdot, $hi, @extra) = split(/(\.\.)/, $opt_epochs);
         undef($lo) if ($lo // '') eq '';
@@ -260,8 +614,8 @@ EOM
         $want
 }
 
-sub init_placeholder ($$) {
-        my ($src, $edst) = @_;
+sub init_placeholder ($$$) {
+        my ($src, $edst, $ent) = @_;
         PublicInbox::Import::init_bare($edst);
         my $f = "$edst/config";
         open my $fh, '>>', $f or die "open($f): $!";
@@ -273,20 +627,186 @@ sub init_placeholder ($$) {
 
 ; This git epoch was created read-only and "public-inbox-fetch"
 ; will not fetch updates for it unless write permission is added.
+; Hint: chmod +w $edst
+EOM
+        if (defined($ent->{owner})) {
+                print $fh <<EOM or die "print($f): $!";
+[gitweb]
+        owner = $ent->{owner}
 EOM
-        close $fh or die "close:($f): $!";
+        }
+        close $fh or die "close($f): $!";
+        my %map = (head => 'HEAD', description => undef);
+        while (my ($key, $fn) = each %map) {
+                my $val = $ent->{$key} // next;
+                $fn //= $key;
+                $fn = "$edst/$fn";
+                open $fh, '>', $fn or die "open($fn): $!";
+                print $fh $val, "\n" or die "print($fn): $!";
+                close $fh or die "close($fn): $!";
+        }
+}
+
+sub reap_cmd { # async, called via SIGCHLD
+        my ($self, $cmd) = @_;
+        my $cerr = $?;
+        $? = 0; # don't let it influence normal exit
+        $self->{lei}->child_error($cerr, "@$cmd failed (\$?=$cerr)") if $cerr;
+}
+
+sub up_fp_done {
+        my ($self) = @_;
+        return if !keep_going($self);
+        my $fh = delete $self->{-show_ref_up} // die 'BUG: no show-ref output';
+        seek($fh, SEEK_SET, 0) or die "seek(show_ref): $!";
+        $self->{-ent} // die 'BUG: no -ent';
+        my $A = $self->{-ent}->{fingerprint} // die 'BUG: no fingerprint';
+        my $B = sha1_hex(do { local $/; <$fh> } // die("read(show_ref): $!"));
+        return if $A eq $B;
+        $self->{-ent}->{fingerprint} = $B;
+        push @{$self->{chg}->{fp_mismatch}}, $self->{-key};
+}
+
+sub atomic_write ($$$) {
+        my ($dn, $bn, $raw) = @_;
+        my $ft = File::Temp->new(DIR => $dn, TEMPLATE => "$bn-XXXX");
+        print $ft $raw or die "print($ft): $!";
+        $ft->flush or die "flush($ft): $!";
+        ft_rename($ft, "$dn/$bn", 0666);
+}
+
+# modifies the to-be-written manifest entry, and sets values from it, too
+sub update_ent {
+        my ($self) = @_;
+        my $key = $self->{-key} // die 'BUG: no -key';
+        my $new = $self->{-ent}->{fingerprint};
+        my $cur = $self->{-local_manifest}->{$key}->{fingerprint} // "\0";
+        my $dst = $self->{cur_dst} // $self->{dst};
+        if (defined($new) && $new ne $cur) {
+                my $cmd = ['git', "--git-dir=$dst", 'show-ref'];
+                my $opt = { 2 => $self->{lei}->{2} };
+                open($opt->{1}, '+>', undef) or die "open(tmp): $!";
+                $self->{-show_ref_up} = $opt->{1};
+                my $done = PublicInbox::OnDestroy->new($$, \&up_fp_done, $self);
+                start_cmd($self, $cmd, $opt, $done);
+        }
+        $new = $self->{-ent}->{head};
+        $cur = $self->{-local_manifest}->{$key}->{head} // "\0";
+        if (defined($new) && $new ne $cur) {
+                # n.b. grokmirror writes raw contents to $dst/HEAD w/o locking
+                my $cmd = [ 'git', "--git-dir=$dst" ];
+                if ($new =~ s/\Aref: //) {
+                        push @$cmd, qw(symbolic-ref HEAD), $new;
+                } elsif ($new =~ /\A[a-f0-9]{40,}\z/) {
+                        push @$cmd, qw(update-ref --no-deref HEAD), $new;
+                } else {
+                        undef $cmd;
+                        warn "W: $key: {head} => `$new' not understood\n";
+                }
+                start_cmd($self, $cmd, { 2 => $self->{lei}->{2} }) if $cmd;
+        }
+        if (my $symlinks = $self->{-ent}->{symlinks}) {
+                my $top = File::Spec->rel2abs($self->{dst});
+                for my $p (@$symlinks) {
+                        my $ln = "$top/$p";
+                        $ln =~ tr!/!/!s;
+                        my (undef, $dn, $bn) = File::Spec->splitpath($ln);
+                        File::Path::mkpath($dn);
+                        my $tgt = "$top/$key";
+                        $tgt = File::Spec->abs2rel($tgt, $dn);
+                        if (lstat($ln)) {
+                                if (-l _) {
+                                        next if readlink($ln) eq $tgt;
+                                        unlink($ln) or die "unlink($ln): $!";
+                                } else {
+                                        push @{$self->{chg}->{badlink}}, $p;
+                                }
+                        }
+                        symlink($tgt, $ln) or die "symlink($tgt, $ln): $!";
+                }
+        }
+        if (defined(my $t = $self->{-ent}->{modified})) {
+                my ($dn, $bn) = ("$dst/info/web", 'last-modified');
+                my $orig = PublicInbox::Git::try_cat("$dn/$bn");
+                $t = strftime('%F %T', gmtime($t))." +0000\n";
+                File::Path::mkpath($dn);
+                atomic_write($dn, $bn, $t) if $orig ne $t;
+        }
+
+        $new = $self->{-ent}->{owner} // return;
+        $cur = $self->{-local_manifest}->{$key}->{owner} // "\0";
+        return if $cur eq $new;
+        my $cmd = [ qw(git config -f), "$dst/config", 'gitweb.owner', $new ];
+        start_cmd($self, $cmd, { 2 => $self->{lei}->{2} });
 }
 
-sub clone_v2 ($$;$) {
+sub v1_done { # called via OnDestroy
+        my ($self) = @_;
+        return if $self->{dry_run} || !keep_going($self);
+        _write_inbox_config($self);
+        my $dst = $self->{cur_dst} // $self->{dst};
+        update_ent($self) if $self->{-ent};
+        my $o = "$dst/objects";
+        if (open(my $fh, '<', my $fn = "$o/info/alternates")) {;
+                my $base = File::Spec->rel2abs($o);
+                my @l = <$fh>;
+                my $ft;
+                for (@l) {
+                        next unless m!\A/!;
+                        $_ = File::Spec->abs2rel($_, $base);
+                        $ft //= File::Temp->new(TEMPLATE => '.XXXX',
+                                                DIR => "$o/info");
+                }
+                if ($ft) {
+                        print $ft @l or die "print($ft): $!";
+                        $ft->flush or die "flush($ft): $!";
+                        ft_rename($ft, $fn, 0666, $fh);
+                }
+        }
+        eval { set_description($self) };
+        warn $@ if $@;
+        return if ($self->{-is_epoch} ||
+                $self->{lei}->{opt}->{'inbox-config'} ne 'always');
+        write_makefile($dst, 1);
+        index_cloned_inbox($self, 1);
+}
+
+sub v2_done { # called via OnDestroy
+        my ($self) = @_;
+        return if $self->{dry_run} || !keep_going($self);
+        my $dst = $self->{cur_dst} // $self->{dst};
+        require PublicInbox::Lock;
+        my $lk = bless { lock_path => "$dst/inbox.lock" }, 'PublicInbox::Lock';
+        my $lck = $lk->lock_for_scope($$);
+        _write_inbox_config($self);
+        require PublicInbox::MultiGit;
+        my $mg = PublicInbox::MultiGit->new($dst, 'all.git', 'git');
+        $mg->fill_alternates;
+        for my $i ($mg->git_epochs) { $mg->epoch_cfg_set($i) }
+        for my $edst (@{delete($self->{-read_only}) // []}) {
+                my @st = stat($edst) or die "stat($edst): $!";
+                chmod($st[2] & 0555, $edst) or die "chmod(a-w, $edst): $!";
+        }
+        write_makefile($dst, 2);
+        undef $lck; # unlock
+        eval { set_description($self) };
+        warn $@ if $@;
+        index_cloned_inbox($self, 2);
+}
+
+sub clone_v2_prep ($$;$) {
         my ($self, $v2_epochs, $m) = @_; # $m => manifest.js.gz hashref
         my $lei = $self->{lei};
         my $curl = $self->{curl} //= PublicInbox::LeiCurl->new($lei) or return;
-        my $pfx = $curl->torsocks($lei, (values %$v2_epochs)[0]) or return;
-        my $dst = $self->{dst};
+        my $first_uri = (map { $_->[0] } values %$v2_epochs)[0];
+        $self->{-torsocks} //= $curl->torsocks($lei, $first_uri) or return;
+        my $dst = $self->{cur_dst} // $self->{dst};
         my $want = parse_epochs($lei->{opt}->{epoch}, $v2_epochs);
-        my (@src_edst, @read_only, @skip_nr);
+        my $task = $m ? bless { %$self }, __PACKAGE__ : $self;
+        my (@skip, $desc);
+        my $fini = PublicInbox::OnDestroy->new($$, \&v2_done, $task);
         for my $nr (sort { $a <=> $b } keys %$v2_epochs) {
-                my $uri = $v2_epochs->{$nr};
+                my ($uri, $key) = @{$v2_epochs->{$nr}};
                 my $src = $uri->as_string;
                 my $edst = $dst;
                 $src =~ m!/([0-9]+)(?:\.git)?\z! or die <<"";
@@ -294,54 +814,40 @@ failed to extract epoch number from $src
 
                 $1 + 0 == $nr or die "BUG: <$uri> miskeyed $1 != $nr";
                 $edst .= "/git/$nr.git";
+                my $ent;
+                if ($m) {
+                        $ent = $m->{$key} //
+                                die("BUG: `$key' not in manifest.js.gz");
+                        if (defined(my $d = $ent->{description})) {
+                                $d =~ s/ \[epoch [0-9]+\]\z//s;
+                                $desc = $d;
+                        }
+                }
                 if (!$want || $want->{$nr}) {
-                        push @src_edst, $src, $edst;
+                        my $etask = bless { %$task, -key => $key }, __PACKAGE__;
+                        $etask->{-ent} = $ent; # may have {reference}
+                        $etask->{cur_src} = $src;
+                        $etask->{cur_dst} = $edst;
+                        $etask->{-is_epoch} = $fini;
+                        my $ref = $ent->{reference} // '';
+                        push @{$TODO->{$ref}}, $etask;
+                        $self->{any_want}->{$key} = 1;
                 } else { # create a placeholder so users only need to chmod +w
-                        init_placeholder($src, $edst);
-                        push @read_only, $edst;
-                        push @skip_nr, $nr;
+                        init_placeholder($src, $edst, $ent);
+                        push @{$task->{-read_only}}, $edst;
+                        push @skip, $key;
                 }
         }
-        if (@skip_nr) { # filter out the epochs we skipped
-                my $re = join('|', @skip_nr);
-                my @del = grep(m!/git/$re\.git\z!, keys %$m);
-                delete @$m{@del};
-                $self->{-culled_manifest} = 1;
-        }
-        my $lk = bless { lock_path => "$dst/inbox.lock" }, 'PublicInbox::Lock';
-        _try_config($self);
-        my $on_destroy = $lk->lock_for_scope($$);
-        my @cmd = clone_cmd($lei, my $opt = {});
-        while (my ($src, $edst) = splice(@src_edst, 0, 2)) {
-                my $cmd = [ @$pfx, @cmd, $src, $edst ];
-                my $cerr = run_reap($lei, $cmd, $opt);
-                return $lei->child_error($cerr, "@$cmd failed") if $cerr;
-        }
-        require PublicInbox::MultiGit;
-        my $mg = PublicInbox::MultiGit->new($dst, 'all.git', 'git');
-        $mg->fill_alternates;
-        for my $i ($mg->git_epochs) { $mg->epoch_cfg_set($i) }
-        for my $edst (@read_only) {
-                my @st = stat($edst) or die "stat($edst): $!";
-                chmod($st[2] & 0555, $edst) or die "chmod(a-w, $edst): $!";
-        }
-        write_makefile($self->{dst}, 2);
-        undef $on_destroy; # unlock
-        index_cloned_inbox($self, 2);
-}
+        # filter out the epochs we skipped
+        $self->{chg}->{manifest} = 1 if $m && delete(@$m{@skip});
 
-# PSGI mount prefixes and manifest.js.gz prefixes don't always align...
-sub deduce_epochs ($$) {
-        my ($m, $path) = @_;
-        my ($v1_ent, @v2_epochs);
-        my $path_pfx = '';
-        $path =~ s!/+\z!!;
-        do {
-                $v1_ent = $m->{$path};
-                @v2_epochs = grep(m!\A\Q$path\E/git/[0-9]+\.git\z!, keys %$m);
-        } while (!defined($v1_ent) && !@v2_epochs &&
-                $path =~ s!\A(/[^/]+)/!/! and $path_pfx .= $1);
-        ($path_pfx, $v1_ent ? $path : undef, @v2_epochs);
+        (!$self->{dry_run} && !-d $dst) and File::Path::mkpath($dst);
+
+        $lei->{opt}->{'inbox-config'} =~ /\A(?:always|v2)\z/s and
+                _get_txt_start($task, '_/text/config/raw', $fini);
+
+        defined($desc) ? ($task->{'txt.description'} = $desc) :
+                _get_txt_start($task, 'description', $fini);
 }
 
 sub decode_manifest ($$$) {
@@ -356,61 +862,280 @@ sub decode_manifest ($$$) {
         $m;
 }
 
+sub load_current_manifest ($) {
+        my ($self) = @_;
+        my $fn = $self->{-manifest} // return;
+        if (open(my $fh, '<', $fn)) {
+                decode_manifest($fh, $fn, $fn);
+        } elsif ($!{ENOENT}) { # non-fatal, we can just do it slowly
+                warn "open($fn): $!\n" if !$self->{-initial_clone};
+                undef;
+        } else {
+                die "open($fn): $!\n";
+        }
+}
+
+sub multi_inbox ($$$) {
+        my ($self, $path, $m) = @_;
+        my $incl = $self->{lei}->{opt}->{include};
+        my $excl = $self->{lei}->{opt}->{exclude};
+
+        # assuming everything not v2 is v1, for now
+        my @v1 = sort grep(!m!.+/git/[0-9]+\.git\z!, keys %$m);
+        my @v2_epochs = sort grep(m!.+/git/[0-9]+\.git\z!, keys %$m);
+        my $v2 = {};
+
+        for (@v2_epochs) {
+                m!\A(/.+)/git/[0-9]+\.git\z! or die "BUG: $_";
+                push @{$v2->{$1}}, $_;
+        }
+        my $n = scalar(keys %$v2) + scalar(@v1);
+        my @orig = defined($incl // $excl) ? (keys %$v2, @v1) : ();
+        if (defined $incl) {
+                my $re = '(?:'.join('\\z|', map {
+                                $self->{lei}->glob2re($_) // qr/\A\Q$_\E/
+                        } @$incl).'\\z)';
+                my @gone = delete @$v2{grep(!/$re/, keys %$v2)};
+                delete @$m{map { @$_ } @gone} and $self->{chg}->{manifest} = 1;
+                delete @$m{grep(!/$re/, @v1)} and $self->{chg}->{manifest} = 1;
+                @v1 = grep(/$re/, @v1);
+        }
+        if (defined $excl) {
+                my $re = '(?:'.join('\\z|', map {
+                                $self->{lei}->glob2re($_) // qr/\A\Q$_\E/
+                        } @$excl).'\\z)';
+                my @gone = delete @$v2{grep(/$re/, keys %$v2)};
+                delete @$m{map { @$_ } @gone} and $self->{chg}->{manifest} = 1;
+                delete @$m{grep(/$re/, @v1)} and $self->{chg}->{manifest} = 1;
+                @v1 = grep(!/$re/, @v1);
+        }
+        my $ret; # { v1 => [ ... ], v2 => { "/$inbox_name" => [ epochs ] }}
+        $ret->{v1} = \@v1 if @v1;
+        $ret->{v2} = $v2 if keys %$v2;
+        $ret //= @orig ? "Nothing to clone, available repositories:\n\t".
+                                join("\n\t", sort @orig)
+                        : "Nothing available to clone\n";
+        my $path_pfx = '';
+
+        # PSGI mount prefixes and manifest.js.gz prefixes don't always align...
+        if (@v2_epochs) {
+                until (grep(m!\A\Q$$path\E/git/[0-9]+\.git\z!,
+                                @v2_epochs) == @v2_epochs) {
+                        $$path =~ s!\A(/[^/]+)/!/! or last;
+                        $path_pfx .= $1;
+                }
+        } elsif (@v1) {
+                while (!defined($m->{$$path}) && $$path =~ s!\A(/[^/]+)/!/!) {
+                        $path_pfx .= $1;
+                }
+        }
+        ($path_pfx, $n, $ret);
+}
+
+sub clone_all {
+        my ($self, $m) = @_;
+        my $todo = $TODO;
+        $TODO = \'BUG on further use';
+        my $end = PublicInbox::OnDestroy->new($$, \&fgrp_fetch_all, $self);
+        {
+                my $nodep = delete $todo->{''};
+
+                # do not download unwanted deps
+                my $any_want = delete $self->{any_want};
+                my @unwanted = grep { !$any_want->{$_} } keys %$todo;
+                my @nodep = delete(@$todo{@unwanted});
+                push(@$nodep, @$_) for @nodep;
+
+                # handle no-dependency repos, first
+                for (@$nodep) {
+                        clone_v1($_, $end);
+                        return if !keep_going($self);
+                }
+        }
+        # resolve references, deepest, first:
+        while (scalar keys %$todo) {
+                for my $x (keys %$todo) {
+                        my ($nr, $nxt);
+                        # resolve multi-level references
+                        while ($m && defined($nxt = $m->{$x}->{reference})) {
+                                exists($todo->{$nxt}) or last;
+                                die <<EOM if ++$nr > 1000;
+E: dependency loop detected (`$x' => `$nxt')
+EOM
+                                $x = $nxt;
+                        }
+                        my $y = delete $todo->{$x} // next; # already done
+                        for (@$y) {
+                                clone_v1($_, $end);
+                                return if !keep_going($self);
+                        }
+                        last; # restart %$todo iteration
+                }
+        }
+
+        # $end->DESTROY will call fgrp_fetch_all once all references
+        # in $LIVE are gone, and do_reap will eventually drain $LIVE
+        $end = undef;
+        do_reap($self, 1);
+}
+
+sub dump_manifest ($$) {
+        my ($m, $ft) = @_;
+        # write the smaller manifest if epochs were skipped so
+        # users won't have to delete manifest if they +w an
+        # epoch they no longer want to skip
+        my $json = PublicInbox::Config->json->encode($m);
+        my $mtime = (stat($ft))[9];
+        seek($ft, SEEK_SET, 0) or die "seek($ft): $!";
+        truncate($ft, 0) or die "truncate($ft): $!";
+        gzip(\$json => $ft) or die "gzip($ft): $GzipError";
+        $ft->flush or die "flush($ft): $!";
+        utime($mtime, $mtime, "$ft") or die "utime(..., $ft): $!";
+}
+
+sub dump_project_list ($$) {
+        my ($self, $m) = @_;
+        my $f = $self->{'-project-list'} // return;
+        my $old = PublicInbox::Git::try_cat($f);
+        my %new;
+
+        open my $dh, '<', '.' or die "open(.): $!";
+        chdir($self->{dst}) or die "chdir($self->{dst}): $!";
+        my @local = grep { -e $_ ? ($new{$_} = undef) : 1 } split(/\n/s, $old);
+        chdir($dh) or die "chdir(restore): $!";
+
+        $new{substr($_, 1)} = 1 for keys %$m; # drop leading '/'
+        my @list = sort keys %new;
+        my @remote = grep { !defined($new{$_}) } @list;
+
+        warn <<EOM if @remote;
+The following local repositories are ignored/gone from $self->{src}:
+EOM
+        warn "\t", $_, "\n" for @remote;
+        warn <<EOM if @local;
+The following repos in $f no longer exist on the filesystem:
+EOM
+        warn "\t", $_, "\n" for @local;
+
+        my (undef, $dn, $bn) = File::Spec->splitpath($f);
+        atomic_write($dn, $bn, join("\n", @list, ''));
+}
+
+# FIXME: this gets confused by single inbox instance w/ global manifest.js.gz
 sub try_manifest {
         my ($self) = @_;
         my $uri = URI->new($self->{src});
         my $lei = $self->{lei};
         my $curl = $self->{curl} //= PublicInbox::LeiCurl->new($lei) or return;
+        $self->{-torsocks} //= $curl->torsocks($lei, $uri) or return;
         my $path = $uri->path;
         chop($path) eq '/' or die "BUG: $uri not canonicalized";
         $uri->path($path . '/manifest.js.gz');
-        my $pdir = $lei->rel2abs($self->{dst});
-        $pdir =~ s!/[^/]+/?\z!!;
-        my $ft = File::Temp->new(TEMPLATE => 'm-XXXX',
-                                UNLINK => 1, DIR => $pdir, SUFFIX => '.tmp');
-        my $fn = $ft->filename;
-        my ($bn) = ($fn =~ m!/([^/]+)\z!);
-        my $cmd = $curl->for_uri($lei, $uri, '-R', '-o', $bn);
-        my $opt = { -C => $pdir };
-        $opt->{$_} = $lei->{$_} for (0..2);
-        my $cerr = run_reap($lei, $cmd, $opt);
+        my $manifest = $self->{-manifest} // "$self->{dst}/manifest.js.gz";
+        my %opt = (UNLINK => 1, SUFFIX => '.tmp', TMPDIR => 1);
+        if (!$self->{dry_run} && $manifest =~ m!\A(.+?)/[^/]+\z! and -d $1) {
+                $opt{DIR} = $1; # allows fast rename(2) w/o EXDEV
+                delete $opt{TMPDIR};
+        }
+        my $ft = File::Temp->new(TEMPLATE => '.manifest-XXXX', %opt);
+        my $cmd = $curl->for_uri($lei, $uri, qw(-R -o), $ft->filename);
+        push(@$cmd, '-z', $manifest) if -f $manifest;
+        my $mf_url = "$uri";
+        %opt = map { $_ => $lei->{$_} } (0..2);
+        my $cerr = run_reap($lei, $cmd, \%opt);
         if ($cerr) {
                 return try_scrape($self) if ($cerr >> 8) == 22; # 404 missing
                 return $lei->child_error($cerr, "@$cmd failed");
         }
-        my $m = eval { decode_manifest($ft, $fn, $uri) };
+
+        # bail out if curl -z/--timecond hit 304 Not Modified, $ft will be empty
+        return $lei->qerr("# $manifest unchanged") if -f $manifest && !-s $ft;
+
+        my $m = eval { decode_manifest($ft, $ft, $uri) };
         if ($@) {
                 warn $@;
                 return try_scrape($self);
         }
-        my ($path_pfx, $v1_path, @v2_epochs) = deduce_epochs($m, $path);
-        if (@v2_epochs) {
-                # It may be possible to have v1 + v2 in parallel someday:
-                warn(<<EOM) if defined $v1_path;
-# `$v1_path' appears to be a v1 inbox while v2 epochs exist:
-# @v2_epochs
-# ignoring $v1_path (use --inbox-version=1 to force v1 instead)
+        local $self->{chg} = {};
+        local $self->{-local_manifest} = load_current_manifest($self);
+        my ($path_pfx, $n, $multi) = multi_inbox($self, \$path, $m);
+        return $lei->child_error(1, $multi) if !ref($multi);
+        my $v2 = delete $multi->{v2};
+        if ($v2) {
+                for my $name (sort keys %$v2) {
+                        my $epochs = delete $v2->{$name};
+                        my %v2_epochs = map {
+                                $uri->path($n > 1 ? $path_pfx.$path.$_
+                                                : $path_pfx.$_);
+                                my ($e) = ("$uri" =~ m!/([0-9]+)\.git\z!);
+                                $e // die "no [0-9]+\.git in `$uri'";
+                                $e => [ $uri->clone, $_ ];
+                        } @$epochs;
+                        ("$uri" =~ m!\A(.+/)git/[0-9]+\.git\z!) or
+                                die "BUG: `$uri' !~ m!/git/[0-9]+.git!";
+                        local $self->{cur_src} = $1;
+                        local $self->{cur_dst} = $self->{dst};
+                        if ($n > 1 && $uri->path =~ m!\A\Q$path_pfx$path\E/(.+)/
+                                                        git/[0-9]+\.git\z!x) {
+                                $self->{cur_dst} .= "/$1";
+                        }
+                        index($self->{cur_dst}, "\n") >= 0 and die <<EOM;
+E: `$self->{cur_dst}' must not contain newline
 EOM
-                my %v2_epochs = map {
-                        $uri->path($path_pfx.$_);
-                        my ($n) = ("$uri" =~ m!/([0-9]+)\.git\z!);
-                        $n => $uri->clone
-                } @v2_epochs;
-                clone_v2($self, \%v2_epochs, $m);
-        } elsif (defined $v1_path) {
-                clone_v1($self);
-        } else {
-                die "E: confused by <$uri>, possible matches:\n\t",
-                        join(', ', sort keys %$m), "\n";
+                        clone_v2_prep($self, \%v2_epochs, $m);
+                        return if !keep_going($self);
+                }
+        }
+        if (my $v1 = delete $multi->{v1}) {
+                my $p = $path_pfx.$path;
+                chop($p) if substr($p, -1, 1) eq '/';
+                $uri->path($p);
+                for my $name (@$v1) {
+                        my $task = bless { %$self }, __PACKAGE__;
+                        $task->{-ent} = $m->{$name} //
+                                        die("BUG: no `$name' in manifest");
+                        $task->{cur_src} = "$uri";
+                        $task->{cur_dst} = $task->{dst};
+                        $task->{-key} = $name;
+                        if ($n > 1) {
+                                $task->{cur_dst} .= $name;
+                                $task->{cur_src} .= $name;
+                        }
+                        index($task->{cur_dst}, "\n") >= 0 and die <<EOM;
+E: `$task->{cur_dst}' must not contain newline
+EOM
+                        $task->{cur_src} .= '/';
+                        my $dep = $task->{-ent}->{reference} // '';
+                        push @{$TODO->{$dep}}, $task; # for clone_all
+                        $self->{any_want}->{$name} = 1;
+                }
         }
-        if (delete $self->{-culled_manifest}) { # set by clone_v2
-                # write the smaller manifest if epochs were skipped so
-                # users won't have to delete manifest if they +w an
-                # epoch they no longer want to skip
-                my $json = PublicInbox::Config->json->encode($m);
-                gzip(\$json => $fn) or die "gzip: $GzipError";
+        delete local $lei->{opt}->{epoch} if defined($v2);
+        clone_all($self, $m);
+        return if $self->{dry_run} || !keep_going($self);
+
+        # set by clone_v2_prep/-I/--exclude
+        my $mis = delete $self->{chg}->{fp_mismatch};
+        if ($mis) {
+                my $t = (stat($ft))[9];
+                $t = strftime('%F %k:%M:%S %z', localtime($t));
+                warn <<EOM;
+W: Fingerprints for the following repositories do not match
+W: $mf_url @ $t:
+W: These repositories may have updated since $t:
+EOM
+                warn "\t", $_, "\n" for @$mis;
+                warn <<EOM if !$self->{lei}->{opt}->{prune};
+W: The above fingerprints may never match without --prune
+EOM
         }
-        ft_rename($ft, "$self->{dst}/manifest.js.gz", 0666);
+        dump_manifest($m => $ft) if delete($self->{chg}->{manifest}) || $mis;
+        my $bad = delete $self->{chg}->{badlink};
+        warn(<<EOM, map { ("\t", $_, "\n") } @$bad) if $bad;
+W: The following exist and have not been converted to symlinks
+EOM
+        dump_project_list($self, $m);
+        ft_rename($ft, $manifest, 0666);
 }
 
 sub start_clone_url {
@@ -419,19 +1144,40 @@ sub start_clone_url {
         die "TODO: non-HTTP/HTTPS clone of $self->{src} not supported, yet";
 }
 
-sub do_mirror { # via wq_io_do
+sub do_mirror { # via wq_io_do or public-inbox-clone
         my ($self) = @_;
         my $lei = $self->{lei};
+        $self->{dry_run} = 1 if $lei->{opt}->{'dry-run'};
         umask($lei->{client_umask}) if defined $lei->{client_umask};
+        $self->{-initial_clone} = 1 if !-d $self->{dst};
         eval {
-                my $iv = $lei->{opt}->{'inbox-version'};
-                if (defined $iv) {
-                        return clone_v1($self) if $iv == 1;
-                        return try_scrape($self) if $iv == 2;
-                        die "bad --inbox-version=$iv\n";
+                my $ic = $lei->{opt}->{'inbox-config'} //= 'always';
+                $ic =~ /\A(?:v1|v2|always|never)\z/s or die <<"";
+--inbox-config must be one of `always', `v2', `v1', or `never'
+
+                # we support these switches with '' (empty string).
+                # defaults match example conf distributed with grokmirror
+                my @pairs = qw(objstore objstore manifest manifest.js.gz
+                                project-list projects.list);
+                while (@pairs) {
+                        my ($k, $default) = splice(@pairs, 0, 2);
+                        my $v = $lei->{opt}->{$k} // next;
+                        $v = $default if $v eq '';
+                        $v = "$self->{dst}/$v" if $v !~ m!\A\.{0,2}/!;
+                        $self->{"-$k"} = $v;
                 }
-                return start_clone_url($self) if $self->{src} =~ m!://!;
-                die "TODO: cloning local directories not supported, yet";
+
+                local $LIVE = {};
+                local $TODO = {};
+                local $FGRP_TODO = {};
+                my $iv = $lei->{opt}->{'inbox-version'} //
+                        return start_clone_url($self);
+                return clone_v1($self) if $iv == 1;
+                die "bad --inbox-version=$iv\n" if $iv != 2;
+                die <<EOM if $self->{src} !~ m!://!;
+cloning local v2 inboxes not supported
+EOM
+                try_scrape($self, 1);
         };
         $lei->fail($@) if $@;
 }
@@ -439,14 +1185,6 @@ sub do_mirror { # via wq_io_do
 sub start {
         my ($cls, $lei, $src, $dst) = @_;
         my $self = bless { src => $src, dst => $dst }, $cls;
-        if ($src =~ m!https?://!) {
-                require URI;
-                require PublicInbox::LeiCurl;
-        }
-        require PublicInbox::Lock;
-        require PublicInbox::Inbox;
-        require PublicInbox::Admin;
-        require PublicInbox::InboxWritable;
         $lei->request_umask;
         my ($op_c, $ops) = $lei->workers_start($self, 1);
         $lei->{wq1} = $self;
@@ -489,6 +1227,7 @@ help :
         @echo Rarely needed targets:
         @echo '    make reindex      - may be needed for new features/bugfixes'
         @echo '    make compact      - rewrite Xapian storage to save space'
+        @echo '    make index        - initial index after clone
 
 fetch :
         public-inbox-fetch
@@ -505,12 +1244,14 @@ update :
                 echo 'public-inbox index not initialized'; \
                 echo 'see public-inbox-index(1) man page'; \
         fi
+index :
+        public-inbox-index
 reindex :
         public-inbox-index --reindex
 compact :
         public-inbox-compact
 
-.PHONY : help fetch update reindex compact
+.PHONY : help fetch update index reindex compact
 EOM
                 close $fh or die "close($f): $!";
         } else {
diff --git a/lib/PublicInbox/LeiQuery.pm b/lib/PublicInbox/LeiQuery.pm
index c998e5c0..358574ea 100644
--- a/lib/PublicInbox/LeiQuery.pm
+++ b/lib/PublicInbox/LeiQuery.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # handles "lei q" command and provides internals for
@@ -6,6 +6,7 @@
 package PublicInbox::LeiQuery;
 use strict;
 use v5.10.1;
+use PublicInbox::OverIdx;
 
 sub prep_ext { # externals_each callback
         my ($lxs, $exclude, $loc) = @_;
@@ -17,6 +18,7 @@ sub _start_query { # used by "lei q" and "lei up"
         require PublicInbox::LeiOverview;
         PublicInbox::LeiOverview->new($self) or return;
         my $opt = $self->{opt};
+        PublicInbox::OverIdx::fork_ok($opt);
         my ($xj, $mj) = split(/,/, $opt->{jobs} // '');
         (defined($xj) && $xj ne '' && $xj !~ /\A[1-9][0-9]*\z/) and
                 die "`$xj' search jobs must be >= 1\n";
@@ -37,8 +39,11 @@ sub _start_query { # used by "lei q" and "lei up"
                         $lms->lms_write_prepare->lms_pause; # just create
                 }
         }
-        $l2m and $l2m->{-wq_nr_workers} //= $mj //
-                int($nproc * 0.75 + 0.5); # keep some CPU for git
+        $l2m and $l2m->{-wq_nr_workers} //= $mj // do {
+                # keep some CPU for git, and don't overload IMAP destinations
+                my $n = int($nproc * 0.75 + 0.5);
+                $self->{net} && $n > 4 ? 4 : $n;
+        };
 
         # descending docid order is cheapest, MUA controls sorting order
         $self->{mset_opt}->{relevance} //= -2 if $l2m || $opt->{threads};
@@ -69,6 +74,12 @@ sub qstr_add { # PublicInbox::InputPipe::consume callback for --stdin
         $lei->fail($@) if $@;
 }
 
+# make the URI||PublicInbox::{Inbox,ExtSearch} a config-file friendly string
+sub cfg_ext ($) {
+        my ($x) = @_;
+        $x->isa('URI') ? "$x" : ($x->{inboxdir} // $x->{topdir});
+}
+
 sub lxs_prepare {
         my ($self) = @_;
         require PublicInbox::LeiXSearch;
@@ -84,21 +95,32 @@ sub lxs_prepare {
                 $lxs->prepare_external($self->{lse});
         }
         if (@only) {
+                my $only;
                 for my $loc (@only) {
                         my @loc = $self->get_externals($loc) or return;
-                        $lxs->prepare_external($_) for @loc;
+                        for (@loc) {
+                                my $x = $lxs->prepare_external($_);
+                                push(@$only, cfg_ext($x)) if $x;
+                        }
                 }
+                $opt->{only} = $only if $only;
         } else {
-                my (@ilocals, @iremotes);
+                my (@ilocals, @iremotes, $incl);
                 for my $loc (@{$opt->{include} // []}) {
                         my @loc = $self->get_externals($loc) or return;
-                        $lxs->prepare_external($_) for @loc;
+                        for (@loc) {
+                                my $x = $lxs->prepare_external($_);
+                                push(@$incl, cfg_ext($x)) if $x;
+                        }
                         @ilocals = @{$lxs->{locals} // []};
                         @iremotes = @{$lxs->{remotes} // []};
                 }
+                $opt->{include} = $incl if $incl;
                 # --external is enabled by default, but allow --no-external
                 if ($opt->{external} //= 1) {
                         my $ex = $self->canonicalize_excludes($opt->{exclude});
+                        my @excl = keys %$ex;
+                        $opt->{exclude} = \@excl if scalar(@excl);
                         $self->externals_each(\&prep_ext, $lxs, $ex);
                         $opt->{remote} //= !($lxs->locals - $opt->{'local'});
                         $lxs->{locals} = \@ilocals if !$opt->{'local'};
diff --git a/lib/PublicInbox/LeiReindex.pm b/lib/PublicInbox/LeiReindex.pm
new file mode 100644
index 00000000..3f109f33
--- /dev/null
+++ b/lib/PublicInbox/LeiReindex.pm
@@ -0,0 +1,49 @@
+# Copyright all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# "lei reindex" command to reindex everything in lei/store
+package PublicInbox::LeiReindex;
+use v5.12;
+use parent qw(PublicInbox::IPC);
+
+sub reindex_full {
+        my ($lei) = @_;
+        my $sto = $lei->{sto};
+        my $max = $sto->search->over(1)->max;
+        $lei->qerr("# reindexing 1..$max");
+        $sto->wq_do('reindex_art', $_) for (1..$max);
+}
+
+sub reindex_store { # via wq_do
+        my ($self) = @_;
+        my ($lei, $argv) = delete @$self{qw(lei argv)};
+        if (!@$argv) {
+                reindex_full($lei);
+        }
+}
+
+sub lei_reindex {
+        my ($lei, @argv) = @_;
+        my $sto = $lei->_lei_store or return $lei->fail('nothing indexed');
+        $sto->write_prepare($lei);
+        my $self = bless { lei => $lei, argv => \@argv }, __PACKAGE__;
+        my ($op_c, $ops) = $lei->workers_start($self, 1);
+        $lei->{wq1} = $self;
+        $lei->wait_wq_events($op_c, $ops);
+        $self->wq_do('reindex_store');
+        $self->wq_close;
+}
+
+sub _lei_wq_eof { # EOF callback for main lei daemon
+        my ($lei) = @_;
+        $lei->{sto}->wq_do('reindex_done');
+        $lei->wq_eof;
+}
+
+sub ipc_atfork_child {
+        my ($self) = @_;
+        $self->{lei}->_lei_atfork_child;
+        $self->SUPER::ipc_atfork_child;
+}
+
+1;
diff --git a/lib/PublicInbox/LeiSavedSearch.pm b/lib/PublicInbox/LeiSavedSearch.pm
index 1d13aef6..ed92bfd1 100644
--- a/lib/PublicInbox/LeiSavedSearch.pm
+++ b/lib/PublicInbox/LeiSavedSearch.pm
@@ -299,7 +299,6 @@ no warnings 'once';
 *smsg_by_mid = \&PublicInbox::Inbox::smsg_by_mid;
 *msg_by_mid = \&PublicInbox::Inbox::msg_by_mid;
 *modified = \&PublicInbox::Inbox::modified;
-*recent = \&PublicInbox::Inbox::recent;
 *max_git_epoch = *nntp_usable = *msg_by_path = \&mm; # undef
 *isrch = *search = \&mm; # TODO
 *DESTROY = \&pause_dedupe;
diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm
index 66049dfe..57f0e013 100644
--- a/lib/PublicInbox/LeiStore.pm
+++ b/lib/PublicInbox/LeiStore.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 #
 # Local storage (cache/memo) for lei(1), suitable for personal/private
@@ -255,13 +255,13 @@ sub remove_eml_vmd { # remove just the VMD
 
 sub _lms_rw ($) { # it is important to have eidx processes open before lms
         my ($self) = @_;
-        my ($eidx, $tl) = eidx_init($self);
-        $self->{lms} //= do {
+        $self->{lms} // do {
                 require PublicInbox::LeiMailSync;
+                my ($eidx, $tl) = eidx_init($self);
                 my $f = "$self->{priv_eidx}->{topdir}/mail_sync.sqlite3";
                 my $lms = PublicInbox::LeiMailSync->new($f);
                 $lms->lms_write_prepare;
-                $lms;
+                $self->{lms} = $lms;
         };
 }
 
@@ -335,6 +335,46 @@ sub _docids_and_maybe_kw ($$) {
         ($docids, [ sort keys %$kw ]);
 }
 
+sub _reindex_1 { # git->cat_async callback
+        my ($bref, $hex, $type, $size, $smsg) = @_;
+        my $self = delete $smsg->{-sto};
+        my ($eidx, $tl) = eidx_init($self);
+        $bref //= _lms_rw($self)->local_blob($hex, 1);
+        if ($bref) {
+                my $eml = PublicInbox::Eml->new($bref);
+                $smsg->{-merge_vmd} = 1; # preserve existing keywords
+                $eidx->idx_shard($smsg->{num})->index_eml($eml, $smsg);
+        } elsif ($type eq 'missing') {
+                # pre-release/buggy lei may've indexed external-only msgs,
+                # try to correct that, here
+                warn("E: missing $hex, culling (ancient lei artifact?)\n");
+                $smsg->{to} = $smsg->{cc} = $smsg->{from} = '';
+                $smsg->{bytes} = 0;
+                $eidx->{oidx}->update_blob($smsg, '');
+                my $eml = PublicInbox::Eml->new("\r\n\r\n");
+                $eidx->idx_shard($smsg->{num})->index_eml($eml, $smsg);
+        } else {
+                warn("E: $type $hex\n");
+        }
+}
+
+sub reindex_art {
+        my ($self, $art) = @_;
+        my ($eidx, $tl) = eidx_init($self);
+        my $smsg = $eidx->{oidx}->get_art($art) // return;
+        return if $smsg->{bytes} == 0; # external-only message
+        $smsg->{-sto} = $self;
+        $eidx->git->cat_async($smsg->{blob} // die("no blob (#$art)"),
+                                \&_reindex_1, $smsg);
+}
+
+sub reindex_done {
+        my ($self) = @_;
+        my ($eidx, $tl) = eidx_init($self);
+        $eidx->git->async_wait_all;
+        # ->done to be called via sto_done_request
+}
+
 sub add_eml {
         my ($self, $eml, $vmd, $xoids) = @_;
         my $im = $self->{-fake_im} // $self->importer; # may create new epoch
diff --git a/lib/PublicInbox/LeiStoreErr.pm b/lib/PublicInbox/LeiStoreErr.pm
index cc085fdc..47fa2277 100644
--- a/lib/PublicInbox/LeiStoreErr.pm
+++ b/lib/PublicInbox/LeiStoreErr.pm
@@ -1,13 +1,12 @@
-# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # forwards stderr from lei/store process to any lei clients using
 # the same store, falls back to syslog if no matching clients exist.
 package PublicInbox::LeiStoreErr;
-use strict;
-use v5.10.1;
+use v5.12;
 use parent qw(PublicInbox::DS);
-use PublicInbox::Syscall qw(EPOLLIN EPOLLONESHOT);
+use PublicInbox::Syscall qw(EPOLLIN);
 use Sys::Syslog qw(openlog syslog closelog);
 use IO::Handle (); # ->blocking
 
@@ -15,24 +14,24 @@ sub new {
         my ($cls, $rd, $lei) = @_;
         my $self = bless { sock => $rd, store_path => $lei->store_path }, $cls;
         $rd->blocking(0);
-        $self->SUPER::new($rd, EPOLLIN | EPOLLONESHOT);
+        $self->SUPER::new($rd, EPOLLIN); # level-trigger
 }
 
 sub event_step {
         my ($self) = @_;
-        my $rbuf = $self->{rbuf} // \(my $x = '');
-        $self->do_read($rbuf, 8192, length($$rbuf)) or return;
-        my $cb;
+        my $n = sysread($self->{sock}, my $buf, 8192);
+        return ($!{EAGAIN} ? 0 : $self->close) if !defined($n);
+        return $self->close if !$n;
         my $printed;
         for my $lei (values %PublicInbox::DS::DescriptorMap) {
-                $cb = $lei->can('store_path') // next;
+                my $cb = $lei->can('store_path') // next;
                 next if $cb->($lei) ne $self->{store_path};
                 my $err = $lei->{2} // next;
-                print $err $$rbuf and $printed = 1;
+                print $err $buf and $printed = 1;
         }
         if (!$printed) {
                 openlog('lei/store', 'pid,nowait,nofatal,ndelay', 'user');
-                for my $l (split(/\n/, $$rbuf)) { syslog('warning', '%s', $l) }
+                for my $l (split(/\n/, $buf)) { syslog('warning', '%s', $l) }
                 closelog(); # don't share across fork
         }
 }
diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm
index 2aa3977e..b58e2652 100644
--- a/lib/PublicInbox/LeiToMail.pm
+++ b/lib/PublicInbox/LeiToMail.pm
@@ -132,19 +132,22 @@ sub eml2mboxcl2 {
 }
 
 sub git_to_mail { # git->cat_async callback
-        my ($bref, $oid, $type, $size, $arg) = @_;
+        my ($bref, $oid, $type, $size, $smsg) = @_;
+        my $self = delete $smsg->{l2m} // die "BUG: no l2m";
         $type // return; # called by git->async_abort
-        my ($write_cb, $smsg) = @$arg;
-        if ($type eq 'missing' && $smsg->{-lms_rw}) {
-                if ($bref = $smsg->{-lms_rw}->local_blob($oid, 1)) {
+        eval {
+                if ($type eq 'missing' &&
+                          ($bref = $self->{-lms_rw}->local_blob($oid, 1))) {
                         $type = 'blob';
                         $size = length($$bref);
                 }
-        }
-        return warn("W: $oid is $type (!= blob)\n") if $type ne 'blob';
-        return warn("E: $oid is empty\n") unless $size;
-        die "BUG: expected=$smsg->{blob} got=$oid" if $smsg->{blob} ne $oid;
-        $write_cb->($bref, $smsg);
+                $type eq 'blob' or return $self->{lei}->child_error(1,
+                                                "W: $oid is $type (!= blob)");
+                $size or return $self->{lei}->child_error(1,"E: $oid is empty");
+                $smsg->{blob} eq $oid or die "BUG: expected=$smsg->{blob}";
+                $self->{wcb}->($bref, $smsg);
+        };
+        $self->{lei}->fail("$@ (oid=$oid)") if $@;
 }
 
 sub reap_compress { # dwaitpid callback
@@ -310,8 +313,11 @@ sub _imap_write_cb ($$) {
         my $dedupe = $lei->{dedupe};
         $dedupe->prepare_dedupe if $dedupe;
         my $append = $lei->{net}->can('imap_append');
-        my $uri = $self->{uri};
-        my $mic = $lei->{net}->mic_get($uri);
+        my $uri = $self->{uri} // die 'BUG: no {uri}';
+        my $mic = $lei->{net}->mic_get($uri) // die <<EOM;
+E: $uri connection failed.
+E: Consider using `--jobs ,1' to limit IMAP connections
+EOM
         my $folder = $uri->mailbox;
         $uri->uidvalidity($mic->uidvalidity($folder));
         my $lse = $lei->{lse}; # may be undef
@@ -749,7 +755,8 @@ sub do_post_auth {
                 $au_peers->[1] = undef;
                 sysread($au_peers->[0], my $barrier1, 1);
         }
-        $self->{wcb} = $self->write_cb($lei);
+        eval { $self->{wcb} = $self->write_cb($lei) };
+        $lei->fail($@) if $@;
         if ($au_peers) { # wait for peer l2m to set write_cb
                 $au_peers->[3] = undef;
                 sysread($au_peers->[2], my $barrier2, 1);
@@ -786,19 +793,22 @@ sub poke_dst {
 
 sub write_mail { # via ->wq_io_do
         my ($self, $smsg, $eml) = @_;
-        return $self->{wcb}->(undef, $smsg, $eml) if $eml;
-        $smsg->{-lms_rw} = $self->{-lms_rw};
-        $self->{git}->cat_async($smsg->{blob}, \&git_to_mail,
-                                [$self->{wcb}, $smsg]);
+        if ($eml) {
+                eval { $self->{wcb}->(undef, $smsg, $eml) };
+                $self->{lei}->fail("blob=$smsg->{blob} $@") if $@;
+        } else {
+                $smsg->{l2m} = $self;
+                $self->{git}->cat_async($smsg->{blob}, \&git_to_mail, $smsg);
+        }
 }
 
 sub wq_atexit_child {
         my ($self) = @_;
         local $PublicInbox::DS::in_loop = 0; # waitpid synchronously
         my $lei = $self->{lei};
-        delete $self->{wcb};
         $lei->{ale}->git->async_wait_all;
         my ($nr_w, $nr_s) = delete(@$lei{qw(-nr_write -nr_seen)});
+        delete $self->{wcb};
         $nr_s or return;
         return if $lei->{early_mua} || !$lei->{-progress} || !$lei->{pkt_op_p};
         $lei->{pkt_op_p}->pkt_do('l2m_progress', $nr_w, $nr_s);
diff --git a/lib/PublicInbox/LeiUp.pm b/lib/PublicInbox/LeiUp.pm
index b8a98360..49917339 100644
--- a/lib/PublicInbox/LeiUp.pm
+++ b/lib/PublicInbox/LeiUp.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # "lei up" - updates the result of "lei q --save"
@@ -7,7 +7,7 @@ use strict;
 use v5.10.1;
 # n.b. we use LeiInput to setup IMAP auth
 use parent qw(PublicInbox::IPC PublicInbox::LeiInput);
-use PublicInbox::LeiSavedSearch;
+use PublicInbox::LeiSavedSearch; # OverIdx
 use PublicInbox::DS;
 use PublicInbox::PktOp;
 use PublicInbox::LeiFinmsg;
@@ -32,8 +32,10 @@ sub up1 ($$) {
         my $rawstr = $lss->{-cfg}->{'lei.internal.rawstr'} //
                 (scalar(@$q) == 1 && substr($q->[0], -1) eq "\n");
         if ($rawstr) {
-                scalar(@$q) > 1 and
-                        die "$f: lei.q has multiple values (@$q) (out=$out)\n";
+                die <<EOM if scalar(@$q) > 1;
+$f: lei.q has multiple values (@$q) (out=$out)
+$f: while lei.internal.rawstr is set
+EOM
                 $lse->query_approxidate($lse->git, $mset_opt->{qstr} = $q->[0]);
         } else {
                 $mset_opt->{qstr} = $lse->query_argv_to_string($lse->git, $q);
@@ -75,6 +77,7 @@ sub redispatch_all ($$) {
         my $upq = [ (@{$self->{o_local} // []}, @{$self->{o_remote} // []}) ];
         return up1($lei, $upq->[0]) if @$upq == 1; # just one, may start MUA
 
+        PublicInbox::OverIdx::fork_ok($lei->{opt});
         # FIXME: this is also used per-query, see lei->_start_query
         my $j = $lei->{opt}->{jobs} || do {
                 my $n = $self->detect_nproc // 1;
diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index 6f877019..730df1f7 100644
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -103,13 +103,6 @@ sub smsg_for {
         $smsg;
 }
 
-sub recent {
-        my ($self, $qstr, $opt) = @_;
-        $opt //= {};
-        $opt->{relevance} //= -2;
-        $self->mset($qstr //= 'z:1..', $opt);
-}
-
 sub over {}
 
 sub _check_mset_limit ($$$) {
@@ -612,34 +605,40 @@ sub add_uri {
                 require IO::Uncompress::Gunzip;
                 require PublicInbox::LeiCurl;
                 push @{$self->{remotes}}, $uri;
+                $uri;
         } else {
                 warn "curl missing, ignoring $uri\n";
+                undef;
         }
 }
 
+# returns URI or PublicInbox::Inbox-like object
 sub prepare_external {
         my ($self, $loc, $boost) = @_; # n.b. already ordered by boost
         if (ref $loc) { # already a URI, or PublicInbox::Inbox-like object
                 return add_uri($self, $loc) if $loc->can('scheme');
+                # fall-through on Inbox-like objects
         } elsif ($loc =~ m!\Ahttps?://!) {
                 require URI;
                 return add_uri($self, URI->new($loc));
-        } elsif (-f "$loc/ei.lock") {
+        } elsif (-f "$loc/ei.lock" && -d "$loc/ALL.git/objects") {
                 require PublicInbox::ExtSearch;
                 die "`\\n' not allowed in `$loc'\n" if index($loc, "\n") >= 0;
                 $loc = PublicInbox::ExtSearch->new($loc);
-        } elsif (-f "$loc/inbox.lock" || -d "$loc/public-inbox") {
+        } elsif ((-f "$loc/inbox.lock" && -d "$loc/all.git/objects") ||
+                        (-d "$loc/public-inbox" && -d "$loc/objects")) {
                 die "`\\n' not allowed in `$loc'\n" if index($loc, "\n") >= 0;
                 require PublicInbox::Inbox; # v2, v1
                 $loc = bless { inboxdir => $loc }, 'PublicInbox::Inbox';
         } elsif (!-e $loc) {
                 warn "W: $loc gone, perhaps run: lei forget-external $loc\n";
-                return;
+                return undef;
         } else {
                 warn "W: $loc ignored, unable to determine external type\n";
-                return;
+                return undef;
         }
         push @{$self->{locals}}, $loc;
+        $loc;
 }
 
 sub _lcat_i { # LeiMailSync->each_src iterator callback
diff --git a/lib/PublicInbox/Linkify.pm b/lib/PublicInbox/Linkify.pm
index 2ac74e2a..9fc3128f 100644
--- a/lib/PublicInbox/Linkify.pm
+++ b/lib/PublicInbox/Linkify.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2014-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # two-step linkification.
@@ -11,7 +11,7 @@
 # Maybe this could be done more efficiently...
 package PublicInbox::Linkify;
 use strict;
-use warnings;
+use v5.10.1;
 use Digest::SHA qw/sha1_hex/;
 use PublicInbox::Hval qw(ascii_html mid_href);
 use PublicInbox::MID qw($MID_EXTRACT);
@@ -68,23 +68,22 @@ sub linkify_1 {
                 # salt this, as this could be exploited to show
                 # links in the HTML which don't show up in the raw mail.
                 my $key = sha1_hex($url . $SALT);
-
+                $key =~ tr/0-9/A-J/; # no digits for YAML highlight
                 $_[0]->{$key} = $url;
-                $beg . 'PI-LINK-'. $key . $end;
+                $beg . 'LINKIFY' . $key . $end;
         ^geo;
         $_[1];
 }
 
 sub linkify_2 {
-        # Added "PI-LINK-" prefix to avoid false-positives on git commits
-        $_[1] =~ s!\bPI-LINK-([a-f0-9]{40})\b!
+        # Added "LINKIFY" prefix to avoid false-positives on git commits
+        $_[1] =~ s!\bLINKIFY([a-fA-J]{40})\b!
                 my $key = $1;
                 my $url = $_[0]->{$key};
                 if (defined $url) {
                         "<a\nhref=\"$url\">$url</a>";
-                } else {
-                        # false positive or somebody tried to mess with us
-                        $key;
+                } else { # false positive or somebody tried to mess with us
+                        'LINKIFY'.$key;
                 }
         !ge;
         $_[1];
@@ -102,20 +101,20 @@ sub linkify_mids {
                 # salt this, as this could be exploited to show
                 # links in the HTML which don't show up in the raw mail.
                 my $key = sha1_hex($html . $SALT);
+                $key =~ tr/0-9/A-J/;
                 my $repl = qq(&lt;<a\nhref="$pfx/$href/">$html</a>&gt;);
                 $repl .= qq{ (<a\nhref="$pfx/$href/raw">raw</a>)} if $raw;
                 $self->{$key} = $repl;
-                'PI-LINK-'. $key;
+                'LINKIFY'.$key;
                 !ge;
         $$str = ascii_html($$str);
-        $$str =~ s!\bPI-LINK-([a-f0-9]{40})\b!
+        $$str =~ s!\bLINKIFY([a-fA-J]{40})\b!
                 my $key = $1;
                 my $repl = $_[0]->{$key};
                 if (defined $repl) {
                         $repl;
-                } else {
-                        # false positive or somebody tried to mess with us
-                        $key;
+                } else { # false positive or somebody tried to mess with us
+                        'LINKIFY'.$key;
                 }
         !ge;
 }
diff --git a/lib/PublicInbox/ManifestJsGz.pm b/lib/PublicInbox/ManifestJsGz.pm
index d5048a96..1f739baa 100644
--- a/lib/PublicInbox/ManifestJsGz.pm
+++ b/lib/PublicInbox/ManifestJsGz.pm
@@ -1,10 +1,10 @@
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
-# generates manifest.js.gz for grokmirror(1)
+# generates manifest.js.gz for grokmirror(1) via PublicInbox::WWW
+# This doesn't parse manifest.js.gz (that happens in LeiMirror)
 package PublicInbox::ManifestJsGz;
-use strict;
-use v5.10.1;
+use v5.12;
 use parent qw(PublicInbox::WwwListing);
 use PublicInbox::Config;
 use IO::Compress::Gzip qw(gzip);
diff --git a/lib/PublicInbox/Mbox.pm b/lib/PublicInbox/Mbox.pm
index e65f38f0..18db9d38 100644
--- a/lib/PublicInbox/Mbox.pm
+++ b/lib/PublicInbox/Mbox.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2015-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # Streaming interface for mboxrd HTTP responses
@@ -19,12 +19,10 @@ sub getline {
         my $smsg = $ctx->{smsg} or return;
         my $ibx = $ctx->{ibx};
         my $eml = delete($ctx->{eml}) // $ibx->smsg_eml($smsg) // return;
-        my $n = $ctx->{smsg} = $ibx->over->next_by_mid(@{$ctx->{next_arg}});
-        $ctx->zmore(msg_hdr($ctx, $eml));
-        if ($n) {
-                $ctx->translate(msg_body($eml));
+        if (($ctx->{smsg} = $ibx->over->next_by_mid(@{$ctx->{next_arg}}))) {
+                $ctx->translate(msg_hdr($ctx, $eml), msg_body($eml));
         } else { # last message
-                $ctx->zflush(msg_body($eml));
+                $ctx->zflush(msg_hdr($ctx, $eml), msg_body($eml));
         }
 }
 
@@ -45,8 +43,7 @@ sub async_eml { # for async_blob_cb
         # next message
         $ctx->{smsg} = $ctx->{ibx}->over->next_by_mid(@{$ctx->{next_arg}});
         local $ctx->{eml} = $eml; # for mbox_hdr
-        $ctx->zmore(msg_hdr($ctx, $eml));
-        $ctx->write(msg_body($eml));
+        $ctx->write(msg_hdr($ctx, $eml), msg_body($eml));
 }
 
 sub mbox_hdr ($) {
@@ -82,7 +79,6 @@ sub no_over_raw ($) {
 # /$INBOX/$MESSAGE_ID/raw
 sub emit_raw {
         my ($ctx) = @_;
-        $ctx->{base_url} = $ctx->{ibx}->base_url($ctx->{env});
         my $over = $ctx->{ibx}->over or return no_over_raw($ctx);
         my ($id, $prev);
         my $mip = $ctx->{next_arg} = [ $ctx->{mid}, \$id, \$prev ];
diff --git a/lib/PublicInbox/MboxGz.pm b/lib/PublicInbox/MboxGz.pm
index 3ed33867..533d2ff1 100644
--- a/lib/PublicInbox/MboxGz.pm
+++ b/lib/PublicInbox/MboxGz.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2015-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 package PublicInbox::MboxGz;
 use strict;
@@ -22,7 +22,6 @@ sub async_next ($) {
 sub mbox_gz {
         my ($self, $cb, $fn) = @_;
         $self->{cb} = $cb;
-        $self->{base_url} = $self->{ibx}->base_url($self->{env});
         $self->{gz} = PublicInbox::GzipFilter::gzip_or_die();
         $fn = to_filename($fn // '') // 'no-subject';
         # http://www.iana.org/assignments/media-types/application/gzip
@@ -38,8 +37,7 @@ sub getline {
         my $cb = $self->{cb} or return;
         while (my $smsg = $cb->($self)) {
                 my $eml = $self->{ibx}->smsg_eml($smsg) or next;
-                $self->zmore(msg_hdr($self, $eml));
-                return $self->translate(msg_body($eml));
+                return $self->translate(msg_hdr($self, $eml), msg_body($eml));
         }
         # signal that we're done and can return undef next call:
         delete $self->{cb};
diff --git a/lib/PublicInbox/MiscIdx.pm b/lib/PublicInbox/MiscIdx.pm
index 76b33b16..19200b92 100644
--- a/lib/PublicInbox/MiscIdx.pm
+++ b/lib/PublicInbox/MiscIdx.pm
@@ -72,7 +72,7 @@ sub remove_eidx_key {
         }
         for my $docid (@docids) {
                 $xdb->delete_document($docid);
-                warn "I: remove inbox docid #$docid ($eidx_key)\n";
+                warn "# remove inbox docid #$docid ($eidx_key)\n";
         }
 }
 
diff --git a/lib/PublicInbox/NNTP.pm b/lib/PublicInbox/NNTP.pm
index 524784cb..dd33a232 100644
--- a/lib/PublicInbox/NNTP.pm
+++ b/lib/PublicInbox/NNTP.pm
@@ -72,9 +72,8 @@ sub process_line ($$) {
         my $res = eval { $req->($self, @args) };
         my $err = $@;
         if ($err && $self->{sock}) {
-                local $/ = "\n";
-                chomp($l);
-                err($self, 'error from: %s (%s)', $l, $err);
+                $l =~ s/\r?\n//s;
+                warn("error from: $l ($err)\n");
                 $res = \"503 program fault - command not performed\r\n";
         }
         defined($res) ? $self->write($res) : 0;
@@ -189,7 +188,7 @@ sub listgroup_range_i {
         my ($self, $beg, $end) = @_;
         my $r = $self->{ibx}->mm(1)->msg_range($beg, $end, 'num');
         scalar(@$r) or return;
-        $self->msg_more(join("\r\n", @$r, ''));
+        $self->msg_more(join('', map { "$_->[0]\r\n" } @$r));
         1;
 }
 
@@ -945,11 +944,6 @@ sub cmd_xpath ($$) {
         '223 '.join(' ', sort(@paths))."\r\n";
 }
 
-sub err ($$;@) {
-        my ($self, $fmt, @args) = @_;
-        printf { $self->{nntpd}->{err} } $fmt."\n", @args;
-}
-
 sub out ($$;@) {
         my ($self, $fmt, @args) = @_;
         printf { $self->{nntpd}->{out} } $fmt."\n", @args;
@@ -958,7 +952,7 @@ sub out ($$;@) {
 # callback used by PublicInbox::DS for any (e)poll (in/out/hup/err)
 sub event_step {
         my ($self) = @_;
-
+        local $SIG{__WARN__} = $self->{nntpd}->{warn_cb};
         return unless $self->flush_write && $self->{sock} && !$self->{long_cb};
 
         # only read more requests if we've drained the write buffer,
diff --git a/lib/PublicInbox/NNTPD.pm b/lib/PublicInbox/NNTPD.pm
index 4f550bb0..4401a29b 100644
--- a/lib/PublicInbox/NNTPD.pm
+++ b/lib/PublicInbox/NNTPD.pm
@@ -45,7 +45,6 @@ sub refresh_groups {
                         # only valid if msgmap and over works
                         # preload to avoid fragmentation:
                         $ibx->description;
-                        $ibx->base_url;
                 } else {
                         delete $groups->{$ngname};
                         # Note: don't be tempted to delete more for memory
diff --git a/lib/PublicInbox/NetReader.pm b/lib/PublicInbox/NetReader.pm
index c1af03a3..4de2583e 100644
--- a/lib/PublicInbox/NetReader.pm
+++ b/lib/PublicInbox/NetReader.pm
@@ -685,7 +685,13 @@ sub mic_get {
         }
         my $mic = mic_new($self, $mic_arg, $sec, $uri);
         $cached //= {}; # invalid placeholder if no cache enabled
-        $mic && $mic->IsConnected ? ($cached->{$sec} = $mic) : undef;
+        if ($mic && $mic->IsConnected) {
+                $cached->{$sec} = $mic;
+        } else {
+                warn 'IMAP LastError: ',$mic->LastError, "\n" if $mic;
+                warn "IMAP errno: $!\n" if $!;
+                undef;
+        }
 }
 
 sub imap_each {
diff --git a/lib/PublicInbox/OnDestroy.pm b/lib/PublicInbox/OnDestroy.pm
index 615bc450..d9a6cd24 100644
--- a/lib/PublicInbox/OnDestroy.pm
+++ b/lib/PublicInbox/OnDestroy.pm
@@ -1,13 +1,16 @@
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 package PublicInbox::OnDestroy;
+use v5.12;
 
 sub new {
         shift; # ($class, $cb, @args)
         bless [ @_ ], __PACKAGE__;
 }
 
+sub cancel { @{$_[0]} = () }
+
 sub DESTROY {
         my ($cb, @args) = @{$_[0]};
         if (!ref($cb) && $cb) {
diff --git a/lib/PublicInbox/OverIdx.pm b/lib/PublicInbox/OverIdx.pm
index e7c96e14..6cc86d5d 100644
--- a/lib/PublicInbox/OverIdx.pm
+++ b/lib/PublicInbox/OverIdx.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2018-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # for XOVER, OVER in NNTP, and feeds/homepage/threads in PSGI
@@ -509,12 +509,12 @@ EOF
                                 next;
                         }
                         $pr->(<<EOM) if $pr;
-I: ghost $r->{num} <$mid> THREADID=$r->{tid} culled
+# ghost $r->{num} <$mid> THREADID=$r->{tid} culled
 EOM
                 }
                 delete_by_num($self, $r->{num});
         }
-        $pr->("I: rethread culled $total ghosts\n") if $pr && $total;
+        $pr->("# rethread culled $total ghosts\n") if $pr && $total;
 }
 
 # used for cross-inbox search
@@ -670,4 +670,16 @@ sub vivify_xvmd {
         $smsg->{-vivify_xvmd} = \@vivify_xvmd;
 }
 
+sub fork_ok {
+        return 1 if $DBD::SQLite::sqlite_version >= 3008003;
+        my ($opt) = @_;
+        my @j = split(/,/, $opt->{jobs} // '');
+        state $warned;
+        grep { $_ > 1 } @j and $warned //= warn('DBD::SQLite version is ',
+                 $DBD::SQLite::sqlite_version,
+                ", need >= 3008003 (3.8.3) for --jobs > 1\n");
+        $opt->{jobs} = '1,1';
+        undef;
+}
+
 1;
diff --git a/lib/PublicInbox/POP3.pm b/lib/PublicInbox/POP3.pm
index 7469922b..5f992e14 100644
--- a/lib/PublicInbox/POP3.pm
+++ b/lib/PublicInbox/POP3.pm
@@ -45,11 +45,6 @@ use constant {
 
 # XXX FIXME: duplicated stuff from NNTP.pm and IMAP.pm
 
-sub err ($$;@) {
-        my ($self, $fmt, @args) = @_;
-        printf { $self->{pop3d}->{err} } $fmt."\n", @args;
-}
-
 sub out ($$;@) {
         my ($self, $fmt, @args) = @_;
         printf { $self->{pop3d}->{out} } $fmt."\n", @args;
@@ -154,32 +149,32 @@ SELECT num,ddd FROM over WHERE num >= ? AND num <= ?
 ORDER BY num ASC
 
         $sth->execute($beg, $end);
-        do {
-                $m = $sth->fetchall_arrayref({}, 1000);
+        my $tot = 0;
+        while (defined($m = $sth->fetchall_arrayref({}, 1000))) {
                 for my $x (@$m) {
                         PublicInbox::Over::load_from_row($x);
                         push(@cache, $x->{num}, $x->{bytes} + 0, $x->{blob});
                         undef $x; # saves ~1.5M memory w/ 50k messages
+                        $tot += $cache[-2];
                 }
-        } while (scalar(@$m) && ($beg = $cache[-3] + 1));
-        \@cache;
+        }
+        $self->{total_bytes} = $tot;
+        $self->{cache} = \@cache;
 }
 
 sub cmd_stat {
         my ($self) = @_;
         my $err; $err = need_txn($self) and return $err;
-        my $cache = $self->{cache} //= _stat_cache($self);
-        my $tot = 0;
-        for (my $i = 1; $i < scalar(@$cache); $i += 3) { $tot += $cache->[$i] }
+        my $cache = $self->{cache} // _stat_cache($self);
         my $nr = @$cache / 3 - ($self->{nr_dele} // 0);
-        "+OK $nr $tot\r\n";
+        "+OK $nr $self->{total_bytes}\r\n";
 }
 
 # for LIST and UIDL
 sub _list {
         my ($desc, $idx, $self, $msn) = @_;
         my $err; $err = need_txn($self) and return $err;
-        my $cache = $self->{cache} //= _stat_cache($self);
+        my $cache = $self->{cache} // _stat_cache($self);
         if (defined $msn) {
                 my $base_off = ($msn - 1) * 3;
                 my $val = $cache->[$base_off + $idx] //
@@ -209,8 +204,9 @@ sub mark_dele ($$) {
         my $old = $self->{txn_max_uid} //= $uid;
         $self->{txn_max_uid} = $uid if $uid > $old;
 
+        $self->{total_bytes} -= $cache->[$base_off + 1];
         $cache->[$base_off] = undef; # clobber UID
-        $cache->[$base_off + 1] = 0; # zero bytes (simplifies cmd_stat)
+        $cache->[$base_off + 1] = undef; # clobber bytes
         $cache->[$base_off + 2] = undef; # clobber oidhex
         ++$self->{nr_dele};
 }
@@ -238,7 +234,7 @@ sub retr_cb { # called by git->cat_async via ibx_async_cat
                 my @tmp = split(/^/m, $bdy);
                 $hdr .= join('', splice(@tmp, 0, $top_nr));
         } elsif (exists $self->{expire}) {
-                $self->{expire} .= pack('S', $off + 1);
+                $self->{expire} .= pack('S', $off);
         }
         $$bref =~ s/^\./../gms;
         $$bref .= substr($$bref, -2, 2) eq "\r\n" ? ".\r\n" : "\r\n.\r\n";
@@ -252,7 +248,7 @@ sub cmd_retr {
         return \"-ERR lines must be a non-negative number\r\n" if
                         (defined($top_nr) && $top_nr !~ /\A[0-9]+\z/);
         my $err; $err = need_txn($self) and return $err;
-        my $cache = $self->{cache} //= _stat_cache($self);
+        my $cache = $self->{cache} // _stat_cache($self);
         my $off = $msn - 1;
         my $hex = $cache->[$off * 3 + 2] // return \"-ERR no such message\r\n";
         ${ibx_async_cat($self->{ibx}, $hex, \&retr_cb,
@@ -272,7 +268,7 @@ sub cmd_rset {
 sub cmd_dele {
         my ($self, $msn) = @_;
         my $err; $err = need_txn($self) and return $err;
-        $self->{cache} //= _stat_cache($self);
+        $self->{cache} // _stat_cache($self);
         $msn =~ /\A[1-9][0-9]*\z/ or return \"-ERR no such message\r\n";
         mark_dele($self, $msn - 1) ? \"+OK\r\n" : \"-ERR no such message\r\n";
 }
@@ -301,6 +297,27 @@ sub close {
         $self->SUPER::close;
 }
 
+# must be called inside a state_dbh transaction with flock held
+sub __cleanup_state {
+        my ($self, $txn_id) = @_;
+        my $user_id = $self->{user_id} // die 'BUG: no {user_id}';
+        $self->{pop3d}->{-state_dbh}->prepare_cached(<<'')->execute($txn_id);
+DELETE FROM deletes WHERE txn_id = ? AND uid_dele = -1
+
+        my $sth = $self->{pop3d}->{-state_dbh}->prepare_cached(<<'', undef, 1);
+SELECT COUNT(*) FROM deletes WHERE user_id = ?
+
+        $sth->execute($user_id);
+        my $nr = $sth->fetchrow_array;
+        if ($nr == 0) {
+                $sth = $self->{pop3d}->{-state_dbh}->prepare_cached(<<'');
+DELETE FROM users WHERE user_id = ?
+
+                $sth->execute($user_id);
+        }
+        $nr;
+}
+
 sub cmd_quit {
         my ($self) = @_;
         if (defined(my $txn_id = $self->{txn_id})) {
@@ -308,23 +325,25 @@ sub cmd_quit {
                 if (my $exp = delete $self->{expire}) {
                         mark_dele($self, $_) for unpack('S*', $exp);
                 }
+                my $keep = 1;
                 my $dbh = $self->{pop3d}->{-state_dbh};
                 my $lk = $self->{pop3d}->lock_for_scope;
-                my $sth;
                 $dbh->begin_work;
 
-                if (defined $self->{txn_max_uid}) {
-                        $sth = $dbh->prepare_cached(<<'');
+                if (defined(my $max = $self->{txn_max_uid})) {
+                        $dbh->prepare_cached(<<'')->execute($max, $txn_id, $max)
 UPDATE deletes SET uid_dele = ? WHERE txn_id = ? AND uid_dele < ?
 
-                        $sth->execute($self->{txn_max_uid}, $txn_id,
-                                        $self->{txn_max_uid});
+                } else {
+                        $keep = $self->__cleanup_state($txn_id);
                 }
-                $sth = $dbh->prepare_cached(<<'');
+                $dbh->prepare_cached(<<'')->execute(time, $user_id) if $keep;
 UPDATE users SET last_seen = ? WHERE user_id = ?
 
-                $sth->execute(time, $user_id);
                 $dbh->commit;
+                # we MUST do txn_id F_UNLCK here inside ->lock_for_scope:
+                $self->{did_quit} = 1;
+                $self->{pop3d}->unlock_mailbox($self);
         }
         $self->write(\"+OK public-inbox POP3 server signing off\r\n");
         $self->close;
@@ -341,8 +360,8 @@ sub process_line ($$) {
                 \"-ERR command not recognized\r\n";
         my $err = $@;
         if ($err && $self->{sock}) {
-                chomp($l);
-                err($self, 'error from: %s (%s)', $l, $err);
+                $l =~ s/\r?\n//s;
+                warn("error from: $l ($err)\n");
                 $res = \"-ERR program fault - command not performed\r\n";
         }
         defined($res) ? $self->write($res) : 0;
@@ -351,6 +370,7 @@ sub process_line ($$) {
 # callback used by PublicInbox::DS for any (e)poll (in/out/hup/err)
 sub event_step {
         my ($self) = @_;
+        local $SIG{__WARN__} = $self->{pop3d}->{warn_cb};
         return unless $self->flush_write && $self->{sock} && !$self->{long_cb};
 
         # only read more requests if we've drained the write buffer,
diff --git a/lib/PublicInbox/POP3D.pm b/lib/PublicInbox/POP3D.pm
index 764f9ffe..3fc85efc 100644
--- a/lib/PublicInbox/POP3D.pm
+++ b/lib/PublicInbox/POP3D.pm
@@ -14,7 +14,7 @@ use PublicInbox::Syscall;
 use File::Temp 0.19 (); # 0.19 for ->newdir
 use Fcntl qw(F_SETLK F_UNLCK F_WRLCK SEEK_SET);
 my @FLOCK;
-if ($^O eq 'linux' || $^O eq 'freebsd') {
+if ($^O eq 'linux' || $^O =~ /bsd/) {
         require Config;
         my $off_t;
         my $sz = $Config::Config{lseeksize};
@@ -27,7 +27,7 @@ if ($^O eq 'linux' || $^O eq 'freebsd') {
                 if ($^O eq 'linux') {
                         @FLOCK = ("ss\@8$off_t$off_t\@32",
                                 qw(l_type l_whence l_start l_len));
-                } elsif ($^O eq 'freebsd') {
+                } elsif ($^O =~ /bsd/) {
                         @FLOCK = ("${off_t}${off_t}lss\@256",
                                 qw(l_start l_len l_pid l_type l_whence));
                 }
@@ -245,6 +245,12 @@ SELECT txn_id,uid_dele FROM deletes WHERE user_id = ? AND mailbox_id = ?
 sub unlock_mailbox {
         my ($self, $pop3) = @_;
         my $txn_id = delete($pop3->{txn_id}) // return;
+        if (!$pop3->{did_quit}) { # deal with QUIT-less disconnects
+                my $lk = $self->lock_for_scope;
+                $self->{-state_dbh}->begin_work;
+                $pop3->__cleanup_state($txn_id);
+                $self->{-state_dbh}->commit;
+        }
         delete $self->{txn_locks}->{$txn_id}; # same worker
 
         # other workers
diff --git a/lib/PublicInbox/Qspawn.pm b/lib/PublicInbox/Qspawn.pm
index 53d0ad55..ef9db43e 100644
--- a/lib/PublicInbox/Qspawn.pm
+++ b/lib/PublicInbox/Qspawn.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2016-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # Like most Perl modules in public-inbox, this is internal and
@@ -26,6 +26,7 @@
 
 package PublicInbox::Qspawn;
 use strict;
+use v5.10.1;
 use PublicInbox::Spawn qw(popen_rd);
 use PublicInbox::GzipFilter;
 
@@ -38,6 +39,7 @@ my $def_limiter;
 # $cmd is the command to spawn
 # $cmd_env is the environ for the child process (not PSGI env)
 # $opt can include redirects and perhaps other process spawning options
+# {qsp_err} is an optional error buffer callers may access themselves
 sub new ($$$;) {
         my ($class, $cmd, $cmd_env, $opt) = @_;
         bless { args => [ $cmd, $cmd_env, $opt ] }, $class;
@@ -93,18 +95,18 @@ sub finalize ($$) {
         }
 
         if ($err) {
-                if (defined $self->{err}) {
-                        $self->{err} .= "; $err";
-                } else {
-                        $self->{err} = $err;
-                }
-                if ($env && $self->{cmd}) {
-                        warn join(' ', @{$self->{cmd}}) . ": $err";
+                utf8::decode($err);
+                if (my $dst = $self->{qsp_err}) {
+                        $$dst .= $$dst ? " $err" : "; $err";
                 }
+                warn "@{$self->{cmd}}: $err" if $self->{cmd};
         }
         if ($qx_cb) {
                 eval { $qx_cb->($qx_buf, $qx_arg) };
-        } elsif (my $wcb = delete $env->{'qspawn.wcb'}) {
+                return unless $@;
+                warn "E: $@"; # hope qspawn.wcb can handle it
+        }
+        if (my $wcb = delete $env->{'qspawn.wcb'}) {
                 # have we started writing, yet?
                 require PublicInbox::WwwStatic;
                 $wcb->(PublicInbox::WwwStatic::r(500));
@@ -132,7 +134,7 @@ sub start ($$$) {
 
 sub psgi_qx_init_cb {
         my ($self) = @_;
-        my $async = delete $self->{async};
+        my $async = delete $self->{async}; # PublicInbox::HTTPD::Async
         my ($r, $buf);
         my $qx_fh = $self->{qx_fh};
 reread:
@@ -223,20 +225,19 @@ sub psgi_return_init_cb {
         my ($self) = @_;
         my $r = rd_hdr($self) or return;
         my $env = $self->{psgi_env};
-        my $filter = delete $env->{'qspawn.filter'} //
-                PublicInbox::GzipFilter::qsp_maybe($r->[1], $env);
+        my $filter = delete($env->{'qspawn.filter'}) // (ref($r) eq 'ARRAY' ?
+                PublicInbox::GzipFilter::qsp_maybe($r->[1], $env) : undef);
 
         my $wcb = delete $env->{'qspawn.wcb'};
-        my $async = delete $self->{async};
-        if (scalar(@$r) == 3) { # error
-                if ($async) {
-                        # calls rpipe->close && ->event_step
-                        $async->close;
+        my $async = delete $self->{async}; # PublicInbox::HTTPD::Async
+        if (ref($r) ne 'ARRAY' || scalar(@$r) == 3) { # error
+                if ($async) { # calls rpipe->close && ->event_step
+                        $async->close; # PublicInbox::HTTPD::Async::close
                 } else {
                         $self->{rpipe}->close;
                         event_step($self);
                 }
-                $wcb->($r);
+                $wcb->($r) if ref($r) eq 'ARRAY';
         } elsif ($async) {
                 # done reading headers, handoff to read body
                 my $fh = $wcb->($r); # scalar @$r == 2
@@ -255,9 +256,9 @@ sub psgi_return_init_cb {
 
 sub psgi_return_start { # may run later, much later...
         my ($self) = @_;
-        if (my $async = $self->{psgi_env}->{'pi-httpd.async'}) {
+        if (my $cb = $self->{psgi_env}->{'pi-httpd.async'}) {
                 # PublicInbox::HTTPD::Async->new(rpipe, $cb, $cb_arg, $end_obj)
-                $self->{async} = $async->($self->{rpipe},
+                $self->{async} = $cb->($self->{rpipe},
                                         \&psgi_return_init_cb, $self, $self);
         } else { # generic PSGI
                 psgi_return_init_cb($self) while $self->{parse_hdr};
diff --git a/lib/PublicInbox/RepoSnapshot.pm b/lib/PublicInbox/RepoSnapshot.pm
new file mode 100644
index 00000000..826392a8
--- /dev/null
+++ b/lib/PublicInbox/RepoSnapshot.pm
@@ -0,0 +1,97 @@
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# cgit-compatible /snapshot/ endpoint for WWW coderepos
+package PublicInbox::RepoSnapshot;
+use v5.12;
+use PublicInbox::Git;
+use PublicInbox::Qspawn;
+use PublicInbox::GitAsyncCat;
+use PublicInbox::WwwStatic qw(r);
+
+# Not using standard mime types since the compressed tarballs are
+# special or do not match my /etc/mime.types.  Choose what gitweb
+# and cgit agree on for compatibility.
+our %FMT_TYPES = (
+        'tar' => 'application/x-tar',
+        'tar.gz' => 'application/x-gzip',
+        'tar.bz2' => 'application/x-bzip2',
+        'tar.xz' => 'application/x-xz',
+        'zip' => 'application/x-zip',
+);
+
+our %FMT_CFG = (
+        'tar.xz' => 'xz -c',
+        'tar.bz2' => 'bzip2 -c',
+        # not supporting lz nor zstd for now to avoid format proliferation
+        # and increased cache overhead required to handle extra formats.
+);
+
+my $SUFFIX = join('|', map { quotemeta } keys %FMT_TYPES);
+
+# TODO deal with tagged blobs
+
+sub archive_hdr { # parse_hdr for Qspawn
+        my ($r, $bref, $ctx) = @_;
+        $r or return [500, [qw(Content-Type text/plain Content-Length 0)], []];
+        my $fn = "$ctx->{snap_pfx}.$ctx->{snap_fmt}";
+        my $type = $FMT_TYPES{$ctx->{snap_fmt}} //
+                                die "BUG: bad fmt: $ctx->{snap_fmt}";
+        [ 200, [ 'Content-Type', "$type; charset=UTF-8",
+                'Content-Disposition', qq(inline; filename="$fn"),
+                'ETag', qq("$ctx->{etag}") ] ];
+}
+
+sub archive_cb {
+        my ($ctx) = @_;
+        my @cfg;
+        if (my $cmd = $FMT_CFG{$ctx->{snap_fmt}}) {
+                @cfg = ('-c', "tar.$ctx->{snap_fmt}.command=$cmd");
+        }
+        my $qsp = PublicInbox::Qspawn->new(['git', @cfg,
+                        "--git-dir=$ctx->{git}->{git_dir}", 'archive',
+                        "--prefix=$ctx->{snap_pfx}/",
+                        "--format=$ctx->{snap_fmt}", $ctx->{treeish}]);
+        $qsp->psgi_return($ctx->{env}, undef, \&archive_hdr, $ctx);
+}
+
+sub ver_check { # git->check_async callback
+        my ($oid, $type, $size, $ctx) = @_;
+        if ($type eq 'missing') { # try 'v' and 'V' prefixes
+                my $pfx = shift @{$ctx->{try_pfx}} or return
+                        delete($ctx->{env}->{'qspawn.wcb'})->(r(404));
+                my $v = $ctx->{treeish} = $pfx.$ctx->{snap_ver};
+                return $ctx->{env}->{'pi-httpd.async'} ?
+                        async_check($ctx, $v, \&ver_check, $ctx) :
+                        $ctx->{git}->check_async($v, \&ver_check, $ctx);
+        }
+        $ctx->{etag} = $oid;
+        archive_cb($ctx);
+}
+
+sub srv {
+        my ($ctx, $fn) = @_;
+        return if $fn =~ /["\s]/s;
+        my $fmt = $ctx->{wcr}->{snapshots}; # TODO per-repo snapshots
+        $fn =~ s/\.($SUFFIX)\z//o and $fmt->{$1} or return;
+        $ctx->{snap_fmt} = $1;
+        my $pfx = $ctx->{git}->local_nick // return;
+        $pfx =~ s/(?:\.git)?\z/-/;
+        ($pfx) = ($pfx =~ m!([^/]+)\z!);
+        substr($fn, 0, length($pfx)) eq $pfx or return;
+        $ctx->{snap_pfx} = $fn;
+        my $v = $ctx->{snap_ver} = substr($fn, length($pfx), length($fn));
+        $ctx->{treeish} = $v; # try without [vV] prefix, first
+        @{$ctx->{try_pfx}} = qw(v V); # cf. cgit:ui-snapshot.c
+        sub {
+                $ctx->{env}->{'qspawn.wcb'} = $_[0];
+                if ($ctx->{env}->{'pi-httpd.async'}) {
+                        async_check($ctx, $v, \&ver_check, $ctx);
+                } else {
+                        $ctx->{git}->check_async($v, \&ver_check, $ctx);
+                        $ctx->{git}->check_async_wait;
+                }
+        }
+}
+
+1;
diff --git a/lib/PublicInbox/SaPlugin/ListMirror.pm b/lib/PublicInbox/SaPlugin/ListMirror.pm
index 9acf86c0..06903cad 100644
--- a/lib/PublicInbox/SaPlugin/ListMirror.pm
+++ b/lib/PublicInbox/SaPlugin/ListMirror.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2016-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # SpamAssassin rules useful for running a mailing list mirror.  We want to:
@@ -39,7 +39,11 @@ sub check_list_mirror_received {
                 my $v = $pms->get($hdr) or next;
                 local $/ = "\n";
                 chomp $v;
-                next if $v ne $hval;
+                if (ref($hval)) {
+                        next if $v !~ $hval;
+                } else {
+                        next if $v ne $hval;
+                }
                 return 1 if $recvd !~ $host_re;
         }
 
@@ -91,6 +95,8 @@ sub config_list_mirror {
         $host_glob =~ s!(.)!$patmap{$1} || "\Q$1"!ge;
         my $host_re = qr/\A\s*from\s+$host_glob(?:\s|$)/si;
 
+        (lc($hdr) eq 'list-id' && $hval =~ /<([^>]+)>/) and
+                $hval = qr/\A<\Q$1\E>\z/;
         push @{$self->{list_mirror_check}}, [ $hdr, $hval, $host_re, $addr ];
 }
 
diff --git a/lib/PublicInbox/SaPlugin/ListMirror.pod b/lib/PublicInbox/SaPlugin/ListMirror.pod
index d931d762..e6a6c2ad 100644
--- a/lib/PublicInbox/SaPlugin/ListMirror.pod
+++ b/lib/PublicInbox/SaPlugin/ListMirror.pod
@@ -6,11 +6,11 @@ PublicInbox::SaPlugin::ListMirror - SpamAssassin plugin for mailing list mirrors
 
   loadplugin PublicInbox::SaPlugin::ListMirror
 
-Declare some mailing lists based on the expected List-Id value,
+Declare some mailing lists based on the expected List-ID value,
 expected servers, and mailing list address:
 
-  list_mirror List-Id <foo.example.com> *.example.com foo@example.com
-  list_mirror List-Id <bar.example.com> *.example.com bar@example.com
+  list_mirror List-ID <foo.example.com> *.example.com foo@example.com
+  list_mirror List-ID <bar.example.com> *.example.com bar@example.com
 
 Bump the score for messages which come from unexpected servers:
 
@@ -43,14 +43,25 @@ C<allow_user_rules 1>
 
 =item list_mirror HEADER HEADER_VALUE HOSTNAME_GLOB [LIST_ADDRESS]
 
-Declare a list based on an expected C<HEADER> matching C<HEADER_NAME>
-exactly coming from C<HOSTNAME_GLOB>.  C<LIST_ADDRESS> is optional,
+Declare a list based on an expected C<HEADER> matching C<HEADER_VALUE>
+coming from C<HOSTNAME_GLOB>.  C<LIST_ADDRESS> is optional,
 but may specify the address of the mailing list being mirrored.
 
-C<List-Id> or C<X-Mailing-List> are common values of C<HEADER>
+C<List-ID> is the recommended value of C<HEADER> as most
+mailing lists support it.
 
 An example of C<HEADER_VALUE> is C<E<lt>foo.example.orgE<gt>>
-if C<HEADER> is C<List-Id>.
+if C<HEADER> is C<List-ID>.
+
+As of public-inbox 2.0, using C<List-ID> as the C<HEADER> and a
+C<HEADER_VALUE> contained by angle brackets (E<lt>list-idE<gt>),
+matching is done in accordance with
+L<RFC 2919|https://tools.ietf.org/html/rfc2919>.  That is,
+C<HEADER_VALUE> will be a case-insensitive substring match
+and ignore the optional description C<phrase> as documented
+in RFC 2919.
+
+All other C<HEADER> values use exact matches for backwards-compatibility.
 
 C<HOSTNAME_GLOB> may be a wildcard match for machines where mail can
 come from or an exact match.
@@ -105,7 +116,7 @@ and L<http://4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/meta
 
 =head1 COPYRIGHT
 
-Copyright (C) 2016-2021 all contributors L<mailto:meta@public-inbox.org>
+Copyright (C) all contributors L<mailto:meta@public-inbox.org>
 
 License: AGPL-3.0+ L<http://www.gnu.org/licenses/agpl-3.0.txt>
 
diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm
index bdb84fc7..257b83a5 100644
--- a/lib/PublicInbox/SearchIdx.pm
+++ b/lib/PublicInbox/SearchIdx.pm
@@ -453,7 +453,7 @@ sub eml2doc ($$$;$) {
         index_ids($self, $doc, $eml, $mids);
 
         # by default, we maintain compatibility with v1.5.0 and earlier
-        # by writing to docdata.glass, users who never exect to downgrade can
+        # by writing to docdata.glass, users who never expect to downgrade can
         # use --skip-docdata
         if (!$self->{-skip_docdata}) {
                 # WWW doesn't need {to} or {cc}, only NNTP
diff --git a/lib/PublicInbox/SearchView.pm b/lib/PublicInbox/SearchView.pm
index b025ec96..8932c73d 100644
--- a/lib/PublicInbox/SearchView.pm
+++ b/lib/PublicInbox/SearchView.pm
@@ -325,16 +325,16 @@ sub mset_thread {
 
         @$msgs = reverse @$msgs if $r;
         $ctx->{msgs} = $msgs;
-        PublicInbox::WwwStream::aresponse($ctx, 200, \&mset_thread_i);
+        PublicInbox::WwwStream::aresponse($ctx, \&mset_thread_i);
 }
 
 # callback for PublicInbox::WwwStream::getline
 sub mset_thread_i {
         my ($ctx, $eml) = @_;
-        $ctx->zmore($ctx->html_top) if exists $ctx->{-html_tip};
+        print { $ctx->zfh } $ctx->html_top if exists $ctx->{-html_tip};
         $eml and return PublicInbox::View::eml_entry($ctx, $eml);
         my $smsg = shift @{$ctx->{msgs}} or
-                $ctx->zmore(${delete($ctx->{skel})});
+                print { $ctx->zfh } ${delete($ctx->{skel})};
         $smsg;
 }
 
@@ -359,7 +359,7 @@ sub adump {
         my ($cb, $mset, $q, $ctx) = @_;
         $ctx->{ids} = $ctx->{ibx}->isrch->mset_to_artnums($mset);
         $ctx->{search_query} = $q; # used by WwwAtomStream::atom_header
-        PublicInbox::WwwAtomStream->response($ctx, 200, \&adump_i);
+        PublicInbox::WwwAtomStream->response($ctx, \&adump_i);
 }
 
 # callback for PublicInbox::WwwAtomStream::getline
diff --git a/lib/PublicInbox/Sigfd.pm b/lib/PublicInbox/Sigfd.pm
index 81e5a1b1..3d964be3 100644
--- a/lib/PublicInbox/Sigfd.pm
+++ b/lib/PublicInbox/Sigfd.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2019-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # Wraps a signalfd (or similar) for PublicInbox::DS
@@ -6,7 +6,7 @@
 package PublicInbox::Sigfd;
 use strict;
 use parent qw(PublicInbox::DS);
-use PublicInbox::Syscall qw(signalfd EPOLLIN EPOLLET);
+use PublicInbox::Syscall qw(signalfd EPOLLIN EPOLLET %SIGNUM);
 use POSIX ();
 
 # returns a coderef to unblock signals if neither signalfd or kqueue
@@ -14,13 +14,8 @@ use POSIX ();
 sub new {
         my ($class, $sig, $nonblock) = @_;
         my %signo = map {;
-                my $cb = $sig->{$_};
-                # SIGWINCH is 28 on FreeBSD, NetBSD, OpenBSD
-                my $num = ($_ eq 'WINCH' && $^O =~ /linux|bsd/i) ? 28 : do {
-                        my $m = "SIG$_";
-                        POSIX->$m;
-                };
-                $num => $cb;
+                # $num => $cb;
+                ($SIGNUM{$_} // POSIX->can("SIG$_")->()) => $sig->{$_}
         } keys %$sig;
         my $self = bless { sig => \%signo }, $class;
         my $io;
diff --git a/lib/PublicInbox/Smsg.pm b/lib/PublicInbox/Smsg.pm
index 260ce6bb..b132381b 100644
--- a/lib/PublicInbox/Smsg.pm
+++ b/lib/PublicInbox/Smsg.pm
@@ -99,9 +99,6 @@ sub populate {
                 # to protect git and NNTP clients
                 $val =~ tr/\0\t\n/   /;
 
-                # rare: in case headers have wide chars (not RFC2047-encoded)
-                utf8::decode($val);
-
                 # lower-case fields for read-only stuff
                 $self->{lc($f)} = $val;
 
@@ -115,8 +112,10 @@ sub populate {
                 $self->{$f} = $val if $val ne '';
         }
         $sync //= {};
-        $self->{-ds} = [ my @ds = msg_datestamp($hdr, $sync->{autime}) ];
-        $self->{-ts} = [ my @ts = msg_timestamp($hdr, $sync->{cotime}) ];
+        my @ds = msg_datestamp($hdr, $sync->{autime} // $self->{ds});
+        my @ts = msg_timestamp($hdr, $sync->{cotime} // $self->{ts});
+        $self->{-ds} = \@ds;
+        $self->{-ts} = \@ts;
         $self->{ds} //= $ds[0]; # no zone
         $self->{ts} //= $ts[0];
         $self->{mid} //= mids($hdr)->[0];
diff --git a/lib/PublicInbox/SolverGit.pm b/lib/PublicInbox/SolverGit.pm
index d3567aa2..80bb0a17 100644
--- a/lib/PublicInbox/SolverGit.pm
+++ b/lib/PublicInbox/SolverGit.pm
@@ -81,9 +81,8 @@ sub solve_existing ($$) {
         my $oid_b = $want->{oid_b};
         my ($oid_full, $type, $size) = $git->check($oid_b);
 
-        # other than {oid_b, try_gits, try_ibxs}
-        my $have_hints = scalar keys %$want > 3;
-        if (defined($type) && (!$have_hints || $type eq 'blob')) {
+        if ($oid_b eq ($oid_full // '') || (defined($type) &&
+-                                (!$self->{have_hints} || $type eq 'blob'))) {
                 delete $want->{try_gits};
                 return [ $git, $oid_full, $type, int($size) ]; # done, success
         }
@@ -106,6 +105,11 @@ sub solve_existing ($$) {
         scalar(@$try);
 }
 
+sub _tmp {
+        $_[0]->{tmp} //=
+                File::Temp->newdir("solver.$_[0]->{oid_want}-XXXX", TMPDIR => 1);
+}
+
 sub extract_diff ($$) {
         my ($p, $arg) = @_;
         my ($self, $want, $smsg) = @$arg;
@@ -193,8 +197,8 @@ sub extract_diff ($$) {
 
         my $path = ++$self->{tot};
         $di->{n} = $path;
-        open(my $tmp, '>:utf8', $self->{tmp}->dirname . "/$path") or
-                die "open(tmp): $!";
+        my $f = _tmp($self)->dirname."/$path";
+        open(my $tmp, '>:utf8', $f) or die "open($f): $!";
         print $tmp $di->{hdr_lines}, $patch or die "print(tmp): $!";
         close $tmp or die "close(tmp): $!";
 
@@ -242,10 +246,8 @@ sub find_smsgs ($$$) {
 
 sub update_index_result ($$) {
         my ($bref, $self) = @_;
-        my ($qsp, $msg) = delete @$self{qw(-qsp -msg)};
-        if (my $err = $qsp->{err}) {
-                ERR($self, "git update-index error: $err");
-        }
+        my ($qsp_err, $msg) = delete @$self{qw(-qsp_err -msg)};
+        ERR($self, "git update-index error:$qsp_err") if $qsp_err;
         dbg($self, $msg);
         next_step($self); # onto do_git_apply
 }
@@ -278,7 +280,7 @@ sub prepare_index ($) {
         my $cmd = [ qw(git update-index -z --index-info) ];
         my $qsp = PublicInbox::Qspawn->new($cmd, $self->{git_env}, $rdr);
         $path_a = git_quote($path_a);
-        $self->{-qsp} = $qsp;
+        $qsp->{qsp_err} = \($self->{-qsp_err} = '');
         $self->{-msg} = "index prepared:\n$mode_a $oid_full\t$path_a";
         $qsp->psgi_qx($self->{psgi_env}, undef, \&update_index_result, $self);
 }
@@ -286,8 +288,7 @@ sub prepare_index ($) {
 # pure Perl "git init"
 sub do_git_init ($) {
         my ($self) = @_;
-        my $dir = $self->{tmp}->dirname;
-        my $git_dir = "$dir/git";
+        my $git_dir = _tmp($self)->dirname.'/git';
 
         foreach ('', qw(objects refs objects/info refs/heads)) {
                 mkdir("$git_dir/$_") or die "mkdir $_: $!";
@@ -405,21 +406,18 @@ sub mark_found ($$$) {
 
 sub parse_ls_files ($$) {
         my ($self, $bref) = @_;
-        my ($qsp, $di) = delete @$self{qw(-qsp -cur_di)};
-        if (my $err = $qsp->{err}) {
-                die "git ls-files error: $err";
-        }
+        my ($qsp_err, $di) = delete @$self{qw(-qsp_err -cur_di)};
+        die "git ls-files -s -z error:$qsp_err" if $qsp_err;
 
-        my ($line, @extra) = split(/\0/, $$bref);
+        my @ls = split(/\0/, $$bref);
+        my ($line, @extra) = grep(/\t\Q$di->{path_b}\E\z/, @ls);
         scalar(@extra) and die "BUG: extra files in index: <",
-                                join('> <', @extra), ">";
-
+                                join('> <', $line, @extra), ">";
+        $line // die "no \Q$di->{path_b}\E in <",join('> <', @ls), '>';
         my ($info, $file) = split(/\t/, $line, 2);
         my ($mode_b, $oid_b_full, $stage) = split(/ /, $info);
-        if ($file ne $di->{path_b}) {
-                die
+        $file eq $di->{path_b} or die
 "BUG: index mismatch: file=$file != path_b=$di->{path_b}";
-        }
 
         my $tmp_git = $self->{tmp_git} or die 'no git working tree';
         my (undef, undef, $size) = $tmp_git->check($oid_b_full);
@@ -456,11 +454,11 @@ sub skip_identical ($$$) {
 
 sub apply_result ($$) {
         my ($bref, $self) = @_;
-        my ($qsp, $di) = delete @$self{qw(-qsp -cur_di)};
+        my ($qsp_err, $di) = delete @$self{qw(-qsp_err -cur_di)};
         dbg($self, $$bref);
         my $patches = $self->{patches};
-        if (my $err = $qsp->{err}) {
-                my $msg = "git apply error: $err";
+        if ($qsp_err) {
+                my $msg = "git apply error:$qsp_err";
                 my $nxt = $patches->[0];
                 if ($nxt && oids_same_ish($nxt->{oid_b}, $di->{oid_b})) {
                         dbg($self, $msg);
@@ -474,30 +472,28 @@ sub apply_result ($$) {
         }
 
         my @cmd = qw(git ls-files -s -z);
-        $qsp = PublicInbox::Qspawn->new(\@cmd, $self->{git_env});
+        my $qsp = PublicInbox::Qspawn->new(\@cmd, $self->{git_env});
         $self->{-cur_di} = $di;
-        $self->{-qsp} = $qsp;
+        $qsp->{qsp_err} = \($self->{-qsp_err} = '');
         $qsp->psgi_qx($self->{psgi_env}, undef, \&ls_files_result, $self);
 }
 
 sub do_git_apply ($) {
         my ($self) = @_;
-        my $dn = $self->{tmp}->dirname;
         my $patches = $self->{patches};
 
         # we need --ignore-whitespace because some patches are CRLF
         my @cmd = (qw(git apply --cached --ignore-whitespace
                         --unidiff-zero --whitespace=warn --verbose));
         my $len = length(join(' ', @cmd));
-        my $total = $self->{tot};
         my $di; # keep track of the last one for "git ls-files"
         my $prv_oid_b;
 
         do {
                 my $i = ++$self->{nr};
                 $di = shift @$patches;
-                dbg($self, "\napplying [$i/$total] " . di_url($self, $di) .
-                        "\n" . $di->{hdr_lines});
+                dbg($self, "\napplying [$i/$self->{nr_p}] " .
+                        di_url($self, $di) . "\n" . $di->{hdr_lines});
                 my $path = $di->{n};
                 $len += length($path) + 1;
                 push @cmd, $path;
@@ -505,10 +501,10 @@ sub do_git_apply ($) {
         } while (@$patches && $len < $ARG_SIZE_MAX &&
                  !oids_same_ish($patches->[0]->{oid_b}, $prv_oid_b));
 
-        my $opt = { 2 => 1, -C => $dn, quiet => 1 };
+        my $opt = { 2 => 1, -C => _tmp($self)->dirname, quiet => 1 };
         my $qsp = PublicInbox::Qspawn->new(\@cmd, $self->{git_env}, $opt);
         $self->{-cur_di} = $di;
-        $self->{-qsp} = $qsp;
+        $qsp->{qsp_err} = \($self->{-qsp_err} = '');
         $qsp->psgi_qx($self->{psgi_env}, undef, \&apply_result, $self);
 }
 
@@ -558,8 +554,10 @@ sub extract_diffs_done {
         my $diffs = delete $self->{tmp_diffs};
         if (scalar @$diffs) {
                 unshift @{$self->{patches}}, @$diffs;
-                dbg($self, "found $want->{oid_b} in " .  join(" ||\n\t",
-                        map { di_url($self, $_) } @$diffs));
+                my %seen; # List::Util::uniq requires Perl 5.26+ :<
+                my @u = grep { !$seen{$_}++ } map { di_url($self, $_) } @$diffs;
+                dbg($self, "found $want->{oid_b} in " .  join(" ||\n\t", @u));
+                ++$self->{nr_p};
 
                 # good, we can find a path to the oid we $want, now
                 # lets see if we need to apply more patches:
@@ -641,7 +639,7 @@ sub resolve_patch ($$) {
 
         # scan through inboxes to look for emails which results in
         # the oid we want:
-        my $ibx = shift(@{$want->{try_ibxs}}) or die 'BUG: {try_ibxs} empty';
+        my $ibx = shift(@{$want->{try_ibxs}}) or return done($self, undef);
         if (my $msgs = find_smsgs($self, $ibx, $want)) {
                 $want->{try_smsgs} = $msgs;
                 $want->{cur_ibx} = $ibx;
@@ -656,14 +654,14 @@ sub resolve_patch ($$) {
 sub new {
         my ($class, $ibx, $user_cb, $uarg) = @_;
 
-        bless {
-                gits => $ibx->{-repo_objs},
+        bless { # $ibx is undef if coderepo only (see WwwCoderepo)
+                gits => $ibx ? $ibx->{-repo_objs} : undef,
                 user_cb => $user_cb,
                 uarg => $uarg,
-                # -cur_di, -qsp, -msg => temporary fields for Qspawn callbacks
+                # -cur_di, -qsp_err, -msg => temp fields for Qspawn callbacks
 
                 # TODO: config option for searching related inboxes
-                inboxes => [ $ibx ],
+                inboxes => $ibx ? [ $ibx ] : [],
         }, $class;
 }
 
@@ -682,12 +680,12 @@ sub solve ($$$$$) {
         $self->{oid_want} = $oid_want;
         $self->{out} = $out;
         $self->{seen_oid} = {};
-        $self->{tot} = 0;
+        $self->{tot} = $self->{nr_p} = 0;
         $self->{psgi_env} = $env;
+        $self->{have_hints} = 1 if scalar keys %$hints;
         $self->{todo} = [ { %$hints, oid_b => $oid_want } ];
         $self->{patches} = []; # [ $di, $di, ... ]
         $self->{found} = {}; # { abbr => [ ::Git, oid, type, size, $di ] }
-        $self->{tmp} = File::Temp->newdir("solver.$oid_want-XXXX", TMPDIR => 1);
 
         dbg($self, "solving $oid_want ...");
         if (my $async = $env->{'pi-httpd.async'}) {
diff --git a/lib/PublicInbox/Syscall.pm b/lib/PublicInbox/Syscall.pm
index 777c44d0..ee4c6107 100644
--- a/lib/PublicInbox/Syscall.pm
+++ b/lib/PublicInbox/Syscall.pm
@@ -21,13 +21,14 @@ use parent qw(Exporter);
 use POSIX qw(ENOENT ENOSYS EINVAL O_NONBLOCK);
 use Socket qw(SOL_SOCKET SCM_RIGHTS);
 use Config;
+our %SIGNUM = (WINCH => 28); # most Linux, {Free,Net,Open}BSD, *Darwin
 
 # $VERSION = '0.25'; # Sys::Syscall version
 our @EXPORT_OK = qw(epoll_ctl epoll_create epoll_wait
                   EPOLLIN EPOLLOUT EPOLLET
                   EPOLL_CTL_ADD EPOLL_CTL_DEL EPOLL_CTL_MOD
                   EPOLLONESHOT EPOLLEXCLUSIVE
-                  signalfd rename_noreplace);
+                  signalfd rename_noreplace %SIGNUM);
 our %EXPORT_TAGS = (epoll => [qw(epoll_ctl epoll_create epoll_wait
                              EPOLLIN EPOLLOUT
                              EPOLL_CTL_ADD EPOLL_CTL_DEL EPOLL_CTL_MOD
@@ -97,15 +98,14 @@ if ($^O eq "linux") {
     # boundaries.
     my $u64_mod_8 = 0;
 
-    # if we're running on an x86_64 kernel, but a 32-bit process,
-    # we need to use the x32 or i386 syscall numbers.
-    if ($machine eq "x86_64" && $Config{ptrsize} == 4) {
-        $machine = $Config{cppsymbols} =~ /\b__ILP32__=1\b/ ? 'x32' : 'i386';
-    }
-
-    # Similarly for mips64 vs mips
-    if ($machine eq "mips64" && $Config{ptrsize} == 4) {
-        $machine = "mips";
+    if ($Config{ptrsize} == 4) {
+        # if we're running on an x86_64 kernel, but a 32-bit process,
+        # we need to use the x32 or i386 syscall numbers.
+        if ($machine eq 'x86_64') {
+            $machine = $Config{cppsymbols} =~ /\b__ILP32__=1\b/ ? 'x32' : 'i386'
+        } elsif ($machine eq 'mips64') { # similarly for mips64 vs mips
+            $machine = 'mips';
+        }
     }
 
     if ($machine =~ m/^i[3456]86$/) {
@@ -160,6 +160,7 @@ if ($^O eq "linux") {
         $SYS_epoll_wait   = 226;
         $u64_mod_8        = 1;
         $SYS_signalfd4 = 309;
+        $SIGNUM{WINCH} = 23;
     } elsif ($machine =~ m/^ppc64/) {
         $SYS_epoll_create = 236;
         $SYS_epoll_ctl    = 237;
@@ -206,7 +207,7 @@ if ($^O eq "linux") {
         $u64_mod_8        = 1;
         $SYS_signalfd4 = 484;
         $SFD_CLOEXEC = 010000000;
-    } elsif ($machine eq 'aarch64' || $machine eq 'loongarch64') {
+    } elsif ($machine =~ /\A(?:loong)?aarch64\z/ || $machine eq 'riscv64') {
         $SYS_epoll_create = 20;  # (sys_epoll_create1)
         $SYS_epoll_ctl    = 21;
         $SYS_epoll_wait   = 22;  # (sys_epoll_pwait)
@@ -253,6 +254,7 @@ if ($^O eq "linux") {
         $SYS_recvmsg = 4177;
         $FS_IOC_GETFLAGS = 0x40046601;
         $FS_IOC_SETFLAGS = 0x80046602;
+        $SIGNUM{WINCH} = 20;
     } else {
         # as a last resort, try using the *.ph files which may not
         # exist or may be wrong
@@ -454,7 +456,7 @@ no warnings 'once';
 
 *recv_cmd4 = sub ($$$) {
         my ($sock, undef, $len) = @_;
-        vec($_[1], ($len + 1) * 8, 1) = 0;
+        vec($_[1] //= '', ($len + 1) * 8, 1) = 0;
         my $cmsghdr = "\0" x msg_controllen; # 10 * sizeof(int)
         my $iov = pack('P'.TMPL_size_t, $_[1], $len);
         my $mh = pack('PL' . # msg_name, msg_namelen (socklen_t (U32))
diff --git a/lib/PublicInbox/TestCommon.pm b/lib/PublicInbox/TestCommon.pm
index ecf7a261..888c1f1e 100644
--- a/lib/PublicInbox/TestCommon.pm
+++ b/lib/PublicInbox/TestCommon.pm
@@ -14,6 +14,9 @@ our @EXPORT;
 my $lei_loud = $ENV{TEST_LEI_ERR_LOUD};
 my $tail_cmd = $ENV{TAIL};
 our ($lei_opt, $lei_out, $lei_err, $lei_cwdfh);
+
+$_ = File::Spec->rel2abs($_) for (grep(!m!^/!, @INC));
+
 BEGIN {
         @EXPORT = qw(tmpdir tcp_server tcp_connect require_git require_mods
                 run_script start_script key2sub xsys xsys_e xqx eml_load tick
@@ -117,6 +120,12 @@ sub require_git ($;$) {
         1;
 }
 
+my %IPv6_VERSION = (
+        'Net::NNTP' => 3.00,
+        'Mail::IMAPClient' => 3.40,
+        'HTTP::Tiny' => 0.042,
+);
+
 sub require_mods {
         my @mods = @_;
         my $maybe = pop @mods if $mods[-1] =~ /\A[0-9]+\z/;
@@ -136,7 +145,7 @@ sub require_mods {
                         push @mods, qw(Parse::RecDescent DBD::SQLite
                                         Email::Address::XS||Mail::Address);
                         next;
-                } elsif ($mod eq '-nntpd') {
+                } elsif ($mod eq '-nntpd' || $mod eq 'v2') {
                         push @mods, qw(DBD::SQLite);
                         next;
                 }
@@ -167,6 +176,9 @@ sub require_mods {
                                 !eval{ IO::Socket::SSL->VERSION(2.007); 1 }) {
                         push @need, $@;
                 }
+                if (defined(my $v = $IPv6_VERSION{$mod})) {
+                        $ENV{TEST_IPV4_ONLY} = 1 if !eval { $mod->VERSION($v) };
+                }
         }
         return unless @need;
         my $m = join(', ', @need)." missing for $0";
@@ -279,6 +291,7 @@ sub run_script ($;$$) {
         my ($cmd, $env, $opt) = @_;
         my ($key, @argv) = @$cmd;
         my $run_mode = $ENV{TEST_RUN_MODE} // $opt->{run_mode} // 1;
+        $run_mode = 0 if $key eq '-clone'; # relies on SIGCHLD + waitpid(-1)
         my $sub = $run_mode == 0 ? undef : key2sub($key);
         my $fhref = [];
         my $spawn_opt = {};
@@ -734,20 +747,28 @@ sub create_inbox ($$;@) {
         $ibx;
 }
 
-sub test_httpd ($$;$) {
-        my ($env, $client, $skip) = @_;
-        for (qw(PI_CONFIG TMPDIR)) {
-                $env->{$_} or BAIL_OUT "$_ unset";
-        }
+sub test_httpd ($$;$$) {
+        my ($env, $client, $skip, $cb) = @_;
+        my ($tmpdir, $for_destroy);
+        $env->{TMPDIR} //= do {
+                ($tmpdir, $for_destroy) = tmpdir();
+                $tmpdir;
+        };
+        for (qw(PI_CONFIG)) { $env->{$_} or BAIL_OUT "$_ unset" }
         SKIP: {
-                require_mods(qw(Plack::Test::ExternalServer), $skip // 1);
+                require_mods(qw(Plack::Test::ExternalServer LWP::UserAgent),
+                                $skip // 1);
                 my $sock = tcp_server() or die;
                 my ($out, $err) = map { "$env->{TMPDIR}/std$_.log" } qw(out err);
                 my $cmd = [ qw(-httpd -W0), "--stdout=$out", "--stderr=$err" ];
                 my $td = start_script($cmd, $env, { 3 => $sock });
                 my ($h, $p) = tcp_host_port($sock);
                 local $ENV{PLACK_TEST_EXTERNALSERVER_URI} = "http://$h:$p";
-                Plack::Test::ExternalServer::test_psgi(client => $client);
+                my $ua = LWP::UserAgent->new;
+                $ua->max_redirect(0);
+                Plack::Test::ExternalServer::test_psgi(client => $client,
+                                                        ua => $ua);
+                $cb->() if $cb;
                 $td->join('TERM');
                 open my $fh, '<', $err or BAIL_OUT $!;
                 my $e = do { local $/; <$fh> };
diff --git a/lib/PublicInbox/View.pm b/lib/PublicInbox/View.pm
index 26094082..071a2093 100644
--- a/lib/PublicInbox/View.pm
+++ b/lib/PublicInbox/View.pm
@@ -7,6 +7,7 @@ package PublicInbox::View;
 use strict;
 use v5.10.1;
 use List::Util qw(max);
+use Text::Wrap qw(wrap); # stdlib, we need Perl 5.6+ for $huge
 use PublicInbox::MsgTime qw(msg_datestamp);
 use PublicInbox::Hval qw(ascii_html obfuscate_addrs prurl mid_href
                         ts2str fmt_ts);
@@ -19,6 +20,7 @@ use PublicInbox::WwwStream qw(html_oneshot);
 use PublicInbox::Reply;
 use PublicInbox::ViewDiff qw(flush_diff);
 use PublicInbox::Eml;
+use POSIX qw(strftime);
 use Time::Local qw(timegm);
 use PublicInbox::Smsg qw(subject_normalized);
 use PublicInbox::ContentHash qw(content_hash);
@@ -36,14 +38,12 @@ sub msg_page_i {
                                 : $ctx->gone('over');
                 $ctx->{mhref} = ($ctx->{nr} || $ctx->{smsg}) ?
                                 "../${\mid_href($smsg->{mid})}/" : '';
-                my $obuf = $ctx->{obuf} = _msg_page_prepare_obuf($eml, $ctx);
-                if (length($$obuf)) {
-                        multipart_text_as_html($eml, $ctx);
-                        $$obuf .= '</pre><hr>';
+                if (_msg_page_prepare($eml, $ctx)) {
+                        $eml->each_part(\&add_text_body, $ctx, 1);
+                        print { $ctx->{zfh} } '</pre><hr>';
                 }
-                delete $ctx->{obuf};
-                $$obuf .= html_footer($ctx, $ctx->{first_hdr}) if !$ctx->{smsg};
-                $$obuf;
+                html_footer($ctx, $ctx->{first_hdr}) if !$ctx->{smsg};
+                ''; # XXX TODO cleanup
         } else { # called by WwwStream::async_next or getline
                 $ctx->{smsg}; # may be undef
         }
@@ -56,14 +56,12 @@ sub no_over_html ($) {
         my $eml = PublicInbox::Eml->new($bref);
         $ctx->{mhref} = '';
         PublicInbox::WwwStream::init($ctx);
-        my $obuf = $ctx->{obuf} = _msg_page_prepare_obuf($eml, $ctx);
-        if (length($$obuf)) {
-                multipart_text_as_html($eml, $ctx);
-                $$obuf .= '</pre><hr>';
+        if (_msg_page_prepare($eml, $ctx)) { # sets {-title_html}
+                $eml->each_part(\&add_text_body, $ctx, 1);
+                print { $ctx->{zfh} } '</pre><hr>';
         }
-        delete $ctx->{obuf};
-        eval { $$obuf .= html_footer($ctx, $eml) };
-        html_oneshot($ctx, 200, $obuf);
+        html_footer($ctx, $eml);
+        $ctx->html_done;
 }
 
 # public functions: (unstable)
@@ -82,7 +80,8 @@ sub msg_page {
         # allow user to easily browse the range around this message if
         # they have ->over
         $ctx->{-t_max} = $smsg->{ts};
-        PublicInbox::WwwStream::aresponse($ctx, 200, \&msg_page_i);
+        $ctx->{-spfx} = '../' if $ibx->{coderepo};
+        PublicInbox::WwwStream::aresponse($ctx, \&msg_page_i);
 }
 
 # /$INBOX/$MESSAGE_ID/#R
@@ -242,20 +241,22 @@ sub eml_entry {
                 my $html = ascii_html($irt);
                 $rv .= qq(In-Reply-To: &lt;<a\nhref="$href">$html</a>&gt;\n)
         }
-        $rv .= "\n";
+        say { $ctx->zfh } $rv;
 
         # scan through all parts, looking for displayable text
         $ctx->{mhref} = $mhref;
-        $ctx->{obuf} = \$rv;
-        $eml->each_part(\&add_text_body, $ctx, 1);
-        delete $ctx->{obuf};
+        $ctx->{changed_href} = "#e$id"; # for diffstat "files? changed,"
+        $eml->each_part(\&add_text_body, $ctx, 1); # expensive
 
         # add the footer
-        $rv .= "\n<a\nhref=#$id_m\nid=e$id>^</a> ".
+        $rv = "\n<a\nhref=#$id_m\nid=e$id>^</a> ".
                 "<a\nhref=\"$mhref\">permalink</a>" .
                 " <a\nhref=\"${mhref}raw\">raw</a>" .
                 " <a\nhref=\"${mhref}#R\">reply</a>";
 
+        delete($ctx->{-qry}) and
+                $rv .= qq[ <a\nhref="${mhref}#related">related</a>];
+
         my $hr;
         if (defined(my $pct = $smsg->{pct})) { # used by SearchView.pm
                 $rv .= "\t[relevance $pct%]";
@@ -300,8 +301,7 @@ sub _th_index_lite {
         my $rv = '';
         my $mapping = $ctx->{mapping} or return $rv;
         my $pad = '  ';
-        my $mid_map = $mapping->{$mid_raw};
-        defined $mid_map or
+        my $mid_map = $mapping->{$mid_raw} //
                 return 'public-inbox BUG: '.ascii_html($mid_raw).' not mapped';
         my ($attr, $node, $idx, $level) = @$mid_map;
         my $children = $node->{children};
@@ -333,10 +333,10 @@ sub _th_index_lite {
         }
         my $s_s = nr_to_s($nr_s, 'sibling', 'siblings');
         my $s_c = nr_to_s($nr_c, 'reply', 'replies');
-        $attr =~ s!\n\z!</b>\n!s;
+        chop $attr; # remove "\n"
         $attr =~ s!<a\nhref.*</a> (?:&#34; )?!!s; # no point in dup subject
         $attr =~ s!<a\nhref=[^>]+>([^<]+)</a>!$1!s; # no point linking to self
-        $rv .= "<b>@ $attr";
+        $rv .= "<b>@ $attr</b>\n";
         if ($nr_c) {
                 my $cmid = $children->[0] ? $children->[0]->{mid} : undef;
                 $rv .= $pad . _skel_hdr($mapping, $cmid);
@@ -386,7 +386,9 @@ sub pre_thread  { # walk_thread callback
 sub thread_eml_entry {
         my ($ctx, $eml) = @_;
         my ($beg, $end) = thread_adj_level($ctx, $ctx->{level});
-        $beg . '<pre>' . eml_entry($ctx, $eml) . '</pre>' . $end;
+        print { $ctx->zfh } $beg, '<pre>';
+        print { $ctx->{zfh} } eml_entry($ctx, $eml), '</pre>';
+        $end;
 }
 
 sub next_in_queue ($$) {
@@ -413,15 +415,15 @@ sub stream_thread_i { # PublicInbox::WwwStream::getline callback
                                 if (!$ghost_ok) { # first non-ghost
                                         $ctx->{-title_html} =
                                                 ascii_html($smsg->{subject});
-                                        $ctx->zmore($ctx->html_top);
+                                        print { $ctx->zfh } $ctx->html_top;
                                 }
                                 return $smsg;
                         }
                         # buffer the ghost entry and loop
-                        $ctx->zmore(ghost_index_entry($ctx, $lvl, $smsg));
+                        print { $ctx->zfh } ghost_index_entry($ctx, $lvl, $smsg)
                 } else { # all done
-                        $ctx->zmore(join('', thread_adj_level($ctx, 0)));
-                        $ctx->zmore(${delete($ctx->{skel})});
+                        print { $ctx->zfh } thread_adj_level($ctx, 0),
+                                                ${delete($ctx->{skel})};
                         return;
                 }
         }
@@ -430,7 +432,7 @@ sub stream_thread_i { # PublicInbox::WwwStream::getline callback
 sub stream_thread ($$) {
         my ($rootset, $ctx) = @_;
         @{$ctx->{-queue}} = map { (0, $_) } @$rootset;
-        PublicInbox::WwwStream::aresponse($ctx, 200, \&stream_thread_i);
+        PublicInbox::WwwStream::aresponse($ctx, \&stream_thread_i);
 }
 
 # /$INBOX/$MSGID/t/ and /$INBOX/$MSGID/T/
@@ -441,6 +443,7 @@ sub thread_html {
         my $ibx = $ctx->{ibx};
         my ($nr, $msgs) = $ibx->over->get_thread($mid);
         return missing_thread($ctx) if $nr == 0;
+        $ctx->{-spfx} = '../../' if $ibx->{coderepo};
 
         # link $INBOX_DIR/description text to "index_topics" view around
         # the newest message in this thread
@@ -481,7 +484,7 @@ EOF
         # flat display: lazy load the full message from smsg
         $ctx->{msgs} = $msgs;
         $ctx->{-html_tip} = '<pre>';
-        PublicInbox::WwwStream::aresponse($ctx, 200, \&thread_html_i);
+        PublicInbox::WwwStream::aresponse($ctx, \&thread_html_i);
 }
 
 sub thread_html_i { # PublicInbox::WwwStream::getline callback
@@ -490,7 +493,7 @@ sub thread_html_i { # PublicInbox::WwwStream::getline callback
                 my $smsg = $ctx->{smsg};
                 if (exists $ctx->{-html_tip}) {
                         $ctx->{-title_html} = ascii_html($smsg->{subject});
-                        $ctx->zmore($ctx->html_top);
+                        print { $ctx->zfh } $ctx->html_top;
                 }
                 return eml_entry($ctx, $eml);
         } else {
@@ -498,31 +501,19 @@ sub thread_html_i { # PublicInbox::WwwStream::getline callback
                         return $smsg if exists($smsg->{blob});
                 }
                 my $skel = delete($ctx->{skel}) or return; # all done
-                $ctx->zmore($$skel);
+                print { $ctx->zfh } $$skel;
                 undef;
         }
 }
 
-sub multipart_text_as_html {
-        # ($mime, $ctx) = @_; # each_part may do "$_[0] = undef"
-
-        # scan through all parts, looking for displayable text
-        $_[0]->each_part(\&add_text_body, $_[1], 1);
-}
-
 sub submsg_hdr ($$) {
         my ($ctx, $eml) = @_;
-        my $obfs_ibx = $ctx->{-obfs_ibx};
-        my $rv = $ctx->{obuf};
-        $$rv .= "\n";
+        my $s = "\n";
         for my $h (qw(From To Cc Subject Date Message-ID X-Alt-Message-ID)) {
-                my @v = $eml->header($h);
-                for my $v (@v) {
-                        obfuscate_addrs($obfs_ibx, $v) if $obfs_ibx;
-                        $v = ascii_html($v);
-                        $$rv .= "$h: $v\n";
-                }
+                $s .= "$h: $_\n" for $eml->header($h);
         }
+        obfuscate_addrs($ctx->{-obfs_ibx}, $s) if $ctx->{-obfs_ibx};
+        ascii_html($s);
 }
 
 sub attach_link ($$$$;$) {
@@ -533,7 +524,6 @@ sub attach_link ($$$$;$) {
         # downloads for 0-byte multipart attachments
         return unless $part->{bdy};
 
-        my $nl = $idx eq '1' ? '' : "\n"; # like join("\n", ...)
         my $size = length($part->body);
         delete $part->{bdy}; # save memory
 
@@ -549,23 +539,17 @@ sub attach_link ($$$$;$) {
         } else {
                 $sfn = 'a.bin';
         }
-        my $rv = $ctx->{obuf};
-        $$rv .= qq($nl<a\nhref="$ctx->{mhref}$idx-$sfn">);
-        if ($err) {
-                $$rv .= <<EOF;
+        my $rv = $idx eq '1' ? '' : "\n"; # like join("\n", ...)
+        $rv .= qq(<a\nhref="$ctx->{mhref}$idx-$sfn">);
+        $rv .= <<EOF if $err;
 [-- Warning: decoded text below may be mangled, UTF-8 assumed --]
 EOF
-        }
-        $$rv .= "[-- Attachment #$idx: ";
-        my $ts = "Type: $ct, Size: $size bytes";
+        $rv .= "[-- Attachment #$idx: ";
         my $desc = $part->header('Content-Description') // $fn // '';
-        $desc = ascii_html($desc);
-        $$rv .= ($desc eq '') ? "$ts --]" : "$desc --]\n[-- $ts --]";
-        $$rv .= "</a>\n";
-
-        submsg_hdr($ctx, $part) if $part->{is_submsg};
-
-        undef;
+        $rv .= ascii_html($desc)." --]\n[-- " if $desc ne '';
+        $rv .= "Type: $ct, Size: $size bytes --]</a>\n";
+        $rv .= submsg_hdr($ctx, $part) if $part->{is_submsg};
+        $rv;
 }
 
 sub add_text_body { # callback for each_part
@@ -578,13 +562,9 @@ sub add_text_body { # callback for each_part
         my $ct = $part->content_type || 'text/plain';
         my $fn = $part->filename;
         my ($s, $err) = msg_part_text($part, $ct);
-        return attach_link($ctx, $ct, $p, $fn) unless defined $s;
-
-        my $rv = $ctx->{obuf};
-        if ($part->{is_submsg}) {
-                submsg_hdr($ctx, $part);
-                $$rv .= "\n";
-        }
+        my $zfh = $ctx->zfh;
+        $s // return print $zfh (attach_link($ctx, $ct, $p, $fn) // '');
+        say $zfh submsg_hdr($ctx, $part) if $part->{is_submsg};
 
         # makes no difference to browsers, and don't screw up filename
         # link generation in diffs with the extra '%0D'
@@ -607,24 +587,6 @@ sub add_text_body { # callback for each_part
                 $ctx->{-anchors} = {} if $s =~ /^diff --git /sm;
                 $diff = 1;
                 delete $ctx->{-long_path};
-                my $spfx;
-                # absolute URL (Atom feeds)
-                if ($ibx->{coderepo}) {
-                        if (index($upfx, '//') >= 0) {
-                                $spfx = $upfx;
-                                $spfx =~ s!/([^/]*)/\z!/!;
-                        } else {
-                                my $n_slash = $upfx =~ tr!/!/!;
-                                if ($n_slash == 0) {
-                                        $spfx = '../';
-                                } elsif ($n_slash == 1) {
-                                        $spfx = '';
-                                } else { # nslash == 2
-                                        $spfx = '../../';
-                                }
-                        }
-                }
-                $ctx->{-spfx} = $spfx;
         };
 
         # split off quoted and unquoted blocks:
@@ -632,110 +594,114 @@ sub add_text_body { # callback for each_part
         undef $s; # free memory
         if (defined($fn) || ($depth > 0 && !$part->{is_submsg}) || $err) {
                 # badly-encoded message with $err? tell the world about it!
-                attach_link($ctx, $ct, $p, $fn, $err);
-                $$rv .= "\n";
+                say $zfh attach_link($ctx, $ct, $p, $fn, $err);
         }
         delete $part->{bdy}; # save memory
-        foreach my $cur (@sections) {
+        for my $cur (@sections) { # $cur may be huge
                 if ($cur =~ /\A>/) {
                         # we use a <span> here to allow users to specify
                         # their own color for quoted text
-                        $$rv .= qq(<span\nclass="q">);
-                        $$rv .= $l->to_html($cur);
-                        $$rv .= '</span>';
+                        print $zfh qq(<span\nclass="q">),
+                                        $l->to_html($cur), '</span>';
                 } elsif ($diff) {
                         flush_diff($ctx, \$cur);
-                } else {
-                        # regular lines, OK
-                        $$rv .= $l->to_html($cur);
+                } else { # regular lines, OK
+                        print $zfh $l->to_html($cur);
                 }
                 undef $cur; # free memory
         }
 }
 
-sub _msg_page_prepare_obuf {
+sub _msg_page_prepare {
         my ($eml, $ctx) = @_;
-        my $over = $ctx->{ibx}->over;
-        my $obfs_ibx = $ctx->{-obfs_ibx};
-        my $rv = '';
+        my $have_over = !!$ctx->{ibx}->over;
         my $mids = mids_for_index($eml);
         my $nr = $ctx->{nr}++;
         if ($nr) { # unlikely
                 if ($ctx->{chash} eq content_hash($eml)) {
                         warn "W: BUG? @$mids not deduplicated properly\n";
-                        return \$rv;
+                        return;
                 }
-                $rv .=
-"<pre>WARNING: multiple messages have this Message-ID\n</pre>";
-                $rv .= '<pre>';
+                $ctx->{-html_tip} =
+"<pre>WARNING: multiple messages have this Message-ID\n</pre><pre>";
         } else {
                 $ctx->{first_hdr} = $eml->header_obj;
                 $ctx->{chash} = content_hash($eml) if $ctx->{smsg}; # reused MID
-                $rv .= "<pre\nid=b>"; # anchor for body start
+                $ctx->{-html_tip} = "<pre\nid=b>"; # anchor for body start
         }
-        $ctx->{-upfx} = '../' if $over;
+        $ctx->{-upfx} = '../';
         my @title; # (Subject[0], From[0])
+        my $hbuf = '';
         for my $v ($eml->header('From')) {
                 my @n = PublicInbox::Address::names($v);
-                $v = ascii_html($v);
-                $title[1] //= ascii_html(join(', ', @n));
-                if ($obfs_ibx) {
-                        obfuscate_addrs($obfs_ibx, $v);
-                        obfuscate_addrs($obfs_ibx, $title[1]);
-                }
-                $rv .= "From: $v\n" if $v ne '';
+                $title[1] //= join(', ', @n);
+                $hbuf .= "From: $v\n" if $v ne '';
         }
-        foreach my $h (qw(To Cc)) {
+        for my $h (qw(To Cc)) {
                 for my $v ($eml->header($h)) {
                         fold_addresses($v);
-                        $v = ascii_html($v);
-                        obfuscate_addrs($obfs_ibx, $v) if $obfs_ibx;
-                        $rv .= "$h: $v\n" if $v ne '';
+                        $hbuf .= "$h: $v\n" if $v ne '';
                 }
         }
         my @subj = $eml->header('Subject');
-        if (@subj) {
-                my $v = ascii_html(shift @subj);
-                obfuscate_addrs($obfs_ibx, $v) if $obfs_ibx;
-                $rv .= 'Subject: ';
-                $rv .= $over ? qq(<a\nhref="#r"\nid=t>$v</a>\n) : "$v\n";
-                $title[0] = $v;
-                for $v (@subj) { # multi-Subject message :<
-                        $v = ascii_html($v);
-                        obfuscate_addrs($obfs_ibx, $v) if $obfs_ibx;
-                        $rv .= "Subject: $v\n";
-                }
-        } else { # dummy anchor for thread skeleton at bottom of page
-                $rv .= qq(<a\nhref="#r"\nid=t></a>) if $over;
-                $title[0] = '(no subject)';
+        $hbuf .= "Subject: $_\n" for @subj;
+        $title[0] = $subj[0] // '(no subject)';
+        $hbuf .= "Date: $_\n" for $eml->header('Date');
+        $hbuf = ascii_html($hbuf);
+        $ctx->{-title_html} = ascii_html(join(' - ', @title));
+        if (my $obfs_ibx = $ctx->{-obfs_ibx}) {
+                obfuscate_addrs($obfs_ibx, $hbuf);
+                obfuscate_addrs($obfs_ibx, $ctx->{-title_html});
         }
-        for my $v ($eml->header('Date')) {
-                $v = ascii_html($v);
-                obfuscate_addrs($obfs_ibx, $v) if $obfs_ibx; # possible :P
-                $rv .= qq{Date: $v\t<a\nhref="#r">[thread overview]</a>\n};
-        }
-        if (!$nr) { # first (and only) message, common case
-                $ctx->{-title_html} = join(' - ', @title);
-                $rv = $ctx->html_top . $rv;
+
+        # [thread overview] link is typically added after Date,
+        # but added after Subject, or even nothing.
+        if ($have_over) {
+                chop $hbuf; # drop "\n", or noop if $rv eq ''
+                $hbuf .= qq{\t<a\nhref="#r">[thread overview]</a>\n};
+                $hbuf =~ s!^Subject:\x20(.*?)(\n[A-Z]|\z)
+                                !Subject: <a\nhref="#r"\nid=t>$1</a>$2!msx or
+                        $hbuf .= qq(<a\nhref="#r\nid=t></a>);
         }
         if (scalar(@$mids) == 1) { # common case
-                my $mhtml = ascii_html($mids->[0]);
-                $rv .= "Message-ID: &lt;$mhtml&gt; ";
-                $rv .= "(<a\nhref=\"raw\">raw</a>)\n";
+                my $x = ascii_html($mids->[0]);
+                $hbuf .= qq[Message-ID: &lt;$x&gt; (<a href="raw">raw</a>)\n];
+        }
+        if (!$nr) { # first (and only) message, common case
+                print { $ctx->zfh } $ctx->html_top, $hbuf;
         } else {
+                delete $ctx->{-title_html};
+                print { $ctx->zfh } $ctx->{-html_tip}, $hbuf;
+        }
+        $ctx->{-linkify} //= PublicInbox::Linkify->new;
+        $hbuf = '';
+        if (scalar(@$mids) != 1) { # unlikely, but it happens :<
                 # X-Alt-Message-ID can happen if a message is injected from
                 # public-inbox-nntpd because of multiple Message-ID headers.
-                my $lnk = PublicInbox::Linkify->new;
-                my $s = '';
                 for my $h (qw(Message-ID X-Alt-Message-ID)) {
-                        $s .= "$h: $_\n" for ($eml->header_raw($h));
+                        $hbuf .= "$h: $_\n" for ($eml->header_raw($h));
                 }
-                $lnk->linkify_mids('..', \$s, 1);
-                $rv .= $s;
+                $ctx->{-linkify}->linkify_mids('..', \$hbuf, 1); # escapes HTML
+                print { $ctx->{zfh} } $hbuf;
+                $hbuf = '';
+        }
+        my @irt = $eml->header_raw('In-Reply-To');
+        my $refs;
+        if (!@irt) {
+                $refs = references($eml);
+                $irt[0] = pop(@$refs) if scalar @$refs;
+        }
+        $hbuf .= "In-Reply-To: $_\n" for @irt;
+
+        # do not display References: if search is present,
+        # we show the thread skeleton at the bottom, instead.
+        if (!$have_over) {
+                $refs //= references($eml);
+                $hbuf .= 'References: <'.join(">\n\t<", @$refs).">\n" if @$refs;
         }
-        $rv .= _parent_headers($eml, $over);
-        $rv .= "\n";
-        \$rv;
+        $ctx->{-linkify}->linkify_mids('..', \$hbuf); # escapes HTML
+        say { $ctx->{zfh} } $hbuf;
+        1;
 }
 
 sub SKEL_EXPAND () {
@@ -772,7 +738,6 @@ sub thread_skel ($$$) {
         # when multiple Subject: headers are present, so we follow suit:
         my $subj = $hdr->header('Subject') // '';
         $subj = '(no subject)' if $subj eq '';
-        $ctx->{prev_subj} = [ split(/ /, subject_normalized($subj)) ];
         $ctx->{cur} = $mid;
         $ctx->{prev_attr} = '';
         $ctx->{prev_level} = 0;
@@ -785,54 +750,44 @@ sub thread_skel ($$$) {
         $ctx->{parent_msg} = $parent;
 }
 
-sub _parent_headers {
-        my ($hdr, $over) = @_;
-        my $rv = '';
-        my @irt = $hdr->header_raw('In-Reply-To');
-        my $refs;
-        if (@irt) {
-                my $lnk = PublicInbox::Linkify->new;
-                $rv .= "In-Reply-To: $_\n" for @irt;
-                $lnk->linkify_mids('..', \$rv);
-        } else {
-                $refs = references($hdr);
-                my $irt = pop @$refs;
-                if (defined $irt) {
-                        my $html = ascii_html($irt);
-                        my $href = mid_href($irt);
-                        $rv .= "In-Reply-To: &lt;";
-                        $rv .= "<a\nhref=\"../$href/\">$html</a>&gt;\n";
-                }
-        }
-
-        # do not display References: if search is present,
-        # we show the thread skeleton at the bottom, instead.
-        return $rv if $over;
-
-        $refs //= references($hdr);
-        if (@$refs) {
-                @$refs = map { linkify_ref_no_over($_) } @$refs;
-                $rv .= 'References: '. join("\n\t", @$refs) . "\n";
-        }
-        $rv;
-}
-
-# returns a string buffer
+# writes to zbuf
 sub html_footer {
         my ($ctx, $hdr) = @_;
-        my $ibx = $ctx->{ibx};
         my $upfx = '../';
-        my $skel;
-        my $rv = '<pre>';
-        if ($ibx->over) {
+        my (@related, $skel);
+        my $foot = '<pre>';
+        my $qry = delete $ctx->{-qry};
+        if ($qry && $ctx->{ibx}->isrch) {
+                my $q = ''; # search for either ancestor or descendent patches
+                for (@{$qry->{dfpre}}, @{$qry->{dfpost}}) {
+                        chop if length > 7; # include 1 abbrev "older" patches
+                        $q .= "dfblob:$_ ";
+                }
+                chop $q; # omit trailing SP
+                local $Text::Wrap::columns = COLS;
+                local $Text::Wrap::huge = 'overflow';
+                $q = wrap('', '', $q);
+                my $rows = ($q =~ tr/\n/\n/) + 1;
+                $q = ascii_html($q);
+                $related[0] = <<EOM;
+<form id=related
+action=$upfx
+><pre>find likely ancestor, descendant, or conflicting patches for <a
+href=#t>this message</a>:
+<textarea name=q cols=${\COLS} rows=$rows>$q</textarea>
+<input type=submit value=search
+/>\t(<a href=${upfx}_/text/help/#search>help</a>)</pre></form>
+EOM
+        }
+        if ($ctx->{ibx}->over) {
                 my $t = ts2str($ctx->{-t_max});
                 my $t_fmt = fmt_ts($ctx->{-t_max});
-                $skel .= <<EOF;
-        other threads:[<a
+                my $fallback = @related ? "\t" : "<a id=related>\t</a>";
+                $skel = <<EOF;
+${fallback}other threads:[<a
 href="$upfx?t=$t">~$t_fmt UTC</a>|<a
 href="$upfx">newest</a>]
 EOF
-
                 thread_skel(\$skel, $ctx, $hdr);
                 my ($next, $prev);
                 my $parent = '       ';
@@ -840,43 +795,32 @@ EOF
 
                 if (my $n = $ctx->{next_msg}) {
                         $n = mid_href($n);
-                        $next = "<a\nhref=\"$upfx$n/\"\nrel=next>next</a>";
+                        $next = qq(<a\nhref="$upfx$n/"\nrel=next>next</a>);
                 }
-                my $u;
                 my $par = $ctx->{parent_msg};
-                if ($par) {
-                        $u = mid_href($par);
-                        $u = "$upfx$u/";
-                }
+                my $u = $par ? $upfx.mid_href($par).'/' : undef;
                 if (my $p = $ctx->{prev_msg}) {
                         $prev = mid_href($p);
                         if ($p && $par && $p eq $par) {
-                                $prev = "<a\nhref=\"$upfx$prev/\"\n" .
+                                $prev = qq(<a\nhref="$upfx$prev/"\n) .
                                         'rel=prev>prev parent</a>';
                                 $parent = '';
                         } else {
-                                $prev = "<a\nhref=\"$upfx$prev/\"\n" .
+                                $prev = qq(<a\nhref="$upfx$prev/"\n) .
                                         'rel=prev>prev</a>';
-                                $parent = " <a\nhref=\"$u\">parent</a>" if $u;
+                                $parent = qq( <a\nhref="$u">parent</a>) if $u;
                         }
                 } elsif ($u) { # unlikely
-                        $parent = " <a\nhref=\"$u\"\nrel=prev>parent</a>";
+                        $parent = qq( <a\nhref="$u"\nrel=prev>parent</a>);
                 }
-                $rv .= "$next $prev$parent ";
+                $foot .= "$next $prev$parent ";
         } else { # unindexed inboxes w/o over
                 $skel = qq( <a\nhref="$upfx">latest</a>);
         }
-        $rv .= qq(<a\nhref="#R">reply</a>);
-        $rv .= $skel;
-        $rv .= '</pre>';
-        $rv .= msg_reply($ctx, $hdr);
-}
-
-sub linkify_ref_no_over {
-        my ($mid) = @_;
-        my $href = mid_href($mid);
-        my $html = ascii_html($mid);
-        "&lt;<a\nhref=\"../$href/\">$html</a>&gt;";
+        # $skel may be big for big threads, don't append it to $foot
+        print { $ctx->zfh } $foot, qq(<a\nhref="#R">reply</a>),
+                                $skel, '</pre>', @related,
+                                msg_reply($ctx, $hdr);
 }
 
 sub ghost_parent {
@@ -1131,9 +1075,10 @@ sub dump_topics {
         }
 
         my @out;
-        my $ibx = $ctx->{ibx};
-        my $obfs_ibx = $ibx->{obfuscate} ? $ibx : undef;
-
+        my $obfs_ibx = $ctx->{ibx}->{obfuscate} ? $ctx->{ibx} : undef;
+        if (my $note = delete $ctx->{t_note}) {
+                push @out, $note; # "messages from ... to ..."
+        }
         # sort by recency, this allows new posts to "bump" old topics...
         foreach my $topic (sort { $b->[0] <=> $a->[0] } @$order) {
                 my ($ds, $n, $seen, $top_subj, @extra) = @$topic;
@@ -1158,9 +1103,9 @@ sub dump_topics {
 
                 my $s = "<a\nhref=\"$href/T/$anchor\">$top_subj</a>\n" .
                         " $ds UTC $n\n";
-                for (my $i = 0; $i < scalar(@extra); $i += 2) {
-                        my $level = $extra[$i];
-                        my $subj = $extra[$i + 1]; # already normalized
+                while (@extra) {
+                        my $level = shift @extra;
+                        my $subj = shift @extra; # already normalized
                         $mid = delete $seen->{$subj};
                         my @subj = split(/ /, $subj);
                         my @next_prev = @subj; # full copy
@@ -1192,7 +1137,7 @@ sub pagination_footer ($$) {
                 $next = $next ? "$next | " : '             | ';
                 $prev .= qq[ | <a\nhref="$latest">latest</a>];
         }
-        "<hr><pre>page: $next$prev</pre>";
+        ($next || $prev) ? "<hr><pre id=nav>page: $next$prev</pre>" : '';
 }
 
 sub paginate_recent ($$) {
@@ -1207,23 +1152,30 @@ sub paginate_recent ($$) {
         $t =~ s/\A([0-9]{8,14})-// and $after = str2ts($1);
         $t =~ /\A([0-9]{8,14})\z/ and $before = str2ts($1);
 
-        my $ibx = $ctx->{ibx};
-        my $msgs = $ibx->recent($opts, $after, $before);
-        my $nr = scalar @$msgs;
-        if ($nr < $lim && defined($after)) {
+        my $msgs = $ctx->{ibx}->over->recent($opts, $after, $before);
+        if (defined($after) && scalar(@$msgs) < $lim) {
                 $after = $before = undef;
-                $msgs = $ibx->recent($opts);
-                $nr = scalar @$msgs;
+                $msgs = $ctx->{ibx}->over->recent($opts);
         }
-        my $more = $nr == $lim;
+        my $more = scalar(@$msgs) == $lim;
         my ($newest, $oldest);
-        if ($nr) {
+        if (@$msgs) {
                 $newest = $msgs->[0]->{ts};
                 $oldest = $msgs->[-1]->{ts};
                 # if we only had $after, our SQL query in ->recent ordered
                 if ($newest < $oldest) {
                         ($oldest, $newest) = ($newest, $oldest);
-                        $more = 0 if defined($after) && $after < $oldest;
+                        $more = undef if defined($after) && $after < $oldest;
+                }
+                if (defined($after // $before)) {
+                        my $n = strftime('%Y-%m-%d %H:%M:%S', gmtime($newest));
+                        my $o = strftime('%Y-%m-%d %H:%M:%S', gmtime($oldest));
+                        $ctx->{t_note} = <<EOM;
+ messages from $o to $n UTC [<a href="#nav">more...</a>]
+EOM
+                        my $s = ts2str($newest);
+                        $ctx->{prev_page} = qq[<a\nhref="?t=$s-"\nrel=prev>] .
+                                                'prev (newer)</a>';
                 }
         }
         if (defined($oldest) && $more) {
@@ -1231,11 +1183,6 @@ sub paginate_recent ($$) {
                 $ctx->{next_page} = qq[<a\nhref="?t=$s"\nrel=next>] .
                                         'next (older)</a>';
         }
-        if (defined($newest) && (defined($before) || defined($after))) {
-                my $s = ts2str($newest);
-                $ctx->{prev_page} = qq[<a\nhref="?t=$s-"\nrel=prev>] .
-                                        'prev (newer)</a>';
-        }
         $msgs;
 }
 
@@ -1243,11 +1190,8 @@ sub paginate_recent ($$) {
 sub index_topics {
         my ($ctx) = @_;
         my $msgs = paginate_recent($ctx, 200); # 200 is our window
-        if (@$msgs) {
-                walk_thread(thread_results($ctx, $msgs), $ctx, \&acc_topic);
-        }
-        html_oneshot($ctx, dump_topics($ctx), \pagination_footer($ctx, '.'));
-
+        walk_thread(thread_results($ctx, $msgs), $ctx, \&acc_topic) if @$msgs;
+        html_oneshot($ctx, dump_topics($ctx), pagination_footer($ctx, '.'));
 }
 
 sub thread_adj_level {
diff --git a/lib/PublicInbox/ViewDiff.pm b/lib/PublicInbox/ViewDiff.pm
index fb394b7c..124a723a 100644
--- a/lib/PublicInbox/ViewDiff.pm
+++ b/lib/PublicInbox/ViewDiff.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2019-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 #
 # used by PublicInbox::View
@@ -7,15 +7,13 @@
 # (or reconstruct) blobs.
 
 package PublicInbox::ViewDiff;
-use strict;
-use v5.10.1;
+use v5.12;
 use parent qw(Exporter);
-our @EXPORT_OK = qw(flush_diff);
+our @EXPORT_OK = qw(flush_diff uri_escape_path);
 use URI::Escape qw(uri_escape_utf8);
 use PublicInbox::Hval qw(ascii_html to_attr);
 use PublicInbox::Git qw(git_unquote);
 
-my $UNSAFE = "^A-Za-z0-9\-\._~/"; # '/' + $URI::Escape::Unsafe{RFC3986}
 my $OID_NULL = '0{7,}';
 my $OID_BLOB = '[a-f0-9]{7,}';
 my $LF = qr!\n!;
@@ -41,22 +39,24 @@ our $EXTRACT_DIFFS = qr/(
                 ^\+{3}\x20($FN)$LF)/msx;
 our $IS_OID = qr/\A$OID_BLOB\z/s;
 
+sub uri_escape_path {
+        # '/' + $URI::Escape::Unsafe{RFC3986}
+        uri_escape_utf8($_[0], "^A-Za-z0-9\-\._~/");
+}
+
 # link to line numbers in blobs
-sub diff_hunk ($$$$) {
-        my ($dst, $dctx, $ca, $cb) = @_;
+sub diff_hunk ($$$) {
+        my ($dctx, $ca, $cb) = @_;
         my ($oid_a, $oid_b, $spfx) = @$dctx{qw(oid_a oid_b spfx)};
 
         if (defined($spfx) && defined($oid_a) && defined($oid_b)) {
-                my ($n) = ($ca =~ /^-([0-9]+)/);
-                $n = defined($n) ? "#n$n" : '';
-
-                $$dst .= qq(@@ <a\nhref="$spfx$oid_a/s/$dctx->{Q}$n">$ca</a>);
+                my $n = ($ca =~ /^-([0-9]+)/) ? "#n$1" : '';
+                my $x = qq(@@ <a\nhref="$spfx$oid_a/s/$dctx->{Q}$n">$ca</a>);
 
-                ($n) = ($cb =~ /^\+([0-9]+)/);
-                $n = defined($n) ? "#n$n" : '';
-                $$dst .= qq( <a\nhref="$spfx$oid_b/s/$dctx->{Q}$n">$cb</a> @@);
+                $n = ($cb =~ /^\+([0-9]+)/) ? "#n$1" : '';
+                $x .= qq( <a\nhref="$spfx$oid_b/s/$dctx->{Q}$n">$cb</a> @@);
         } else {
-                $$dst .= "@@ $ca $cb @@";
+                "@@ $ca $cb @@";
         }
 }
 
@@ -66,8 +66,8 @@ sub oid ($$$) {
 }
 
 # returns true if diffstat anchor written, false otherwise
-sub anchor0 ($$$$) {
-        my ($dst, $ctx, $fn, $rest) = @_;
+sub anchor0 ($$$) {
+        my ($ctx, $fn, $rest) = @_;
 
         my $orig = $fn;
 
@@ -83,15 +83,12 @@ sub anchor0 ($$$$) {
         # long filenames will require us to check in anchor1()
         push(@{$ctx->{-long_path}}, $fn) if $fn =~ s!\A\.\.\./?!!;
 
-        if (defined(my $attr = to_attr($ctx->{-apfx}.$fn))) {
-                $ctx->{-anchors}->{$attr} = 1;
-                my $spaces = ($orig =~ s/( +)\z//) ? $1 : '';
-                $$dst .= " <a\nid=i$attr\nhref=#$attr>" .
-                        ascii_html($orig) . '</a>' . $spaces .
+        my $attr = to_attr($ctx->{-apfx}.$fn) // return;
+        $ctx->{-anchors}->{$attr} = 1;
+        my $spaces = ($orig =~ s/( +)\z//) ? $1 : '';
+        print { $ctx->{zfh} } " <a\nid=i$attr\nhref=#$attr>",
+                        ascii_html($orig), '</a>', $spaces,
                         $ctx->{-linkify}->to_html($rest);
-                return 1;
-        }
-        undef;
 }
 
 # returns "diff --git" anchor destination, undef otherwise
@@ -123,15 +120,11 @@ sub diff_header ($$$) {
         $pa = (split(m'/', git_unquote($pa), 2))[1] if $pa ne '/dev/null';
         $pb = (split(m'/', git_unquote($pb), 2))[1] if $pb ne '/dev/null';
         if ($pa eq $pb && $pb ne '/dev/null') {
-                $dctx->{Q} = "?b=".uri_escape_utf8($pb, $UNSAFE);
+                $dctx->{Q} = '?b='.uri_escape_path($pb);
         } else {
                 my @q;
-                if ($pb ne '/dev/null') {
-                        push @q, 'b='.uri_escape_utf8($pb, $UNSAFE);
-                }
-                if ($pa ne '/dev/null') {
-                        push @q, 'a='.uri_escape_utf8($pa, $UNSAFE);
-                }
+                push @q, 'b='.uri_escape_path($pb) if $pb ne '/dev/null';
+                push @q, 'a='.uri_escape_path($pa) if $pa ne '/dev/null';
                 $dctx->{Q} = '?'.join('&amp;', @q);
         }
 
@@ -141,44 +134,48 @@ sub diff_header ($$$) {
 
         # no need to capture oid_a and oid_b on add/delete,
         # we just linkify OIDs directly via s///e in conditional
-        if (($$x =~ s/$NULL_TO_BLOB/$1 . oid($dctx, $spfx, $2)/e) ||
-                ($$x =~ s/$BLOB_TO_NULL/
-                        'index ' . oid($dctx, $spfx, $1) . $2/e)) {
+        if ($$x =~ s/$NULL_TO_BLOB/$1 . oid($dctx, $spfx, $2)/e) {
+                push @{$ctx->{-qry}->{dfpost}}, $2;
+        } elsif ($$x =~ s/$BLOB_TO_NULL/'index '.oid($dctx, $spfx, $1).$2/e) {
+                push @{$ctx->{-qry}->{dfpre}}, $1;
         } elsif ($$x =~ $BLOB_TO_BLOB) {
                 # modification-only, not add/delete:
                 # linkify hunk headers later using oid_a and oid_b
                 @$dctx{qw(oid_a oid_b)} = ($1, $2);
+                push @{$ctx->{-qry}->{dfpre}}, $1;
+                push @{$ctx->{-qry}->{dfpost}}, $2;
         } else {
                 warn "BUG? <$$x> had no ^index line";
         }
         $$x =~ s!^diff --git!anchor1($ctx, $pb) // 'diff --git'!ems;
-        my $dst = $ctx->{obuf};
-        $$dst .= qq(<span\nclass="head">);
-        $$dst .= $$x;
-        $$dst .= '</span>';
+        print { $ctx->{zfh} } qq(<span\nclass="head">), $$x, '</span>';
         $dctx;
 }
 
 sub diff_before_or_after ($$) {
         my ($ctx, $x) = @_;
-        my $linkify = $ctx->{-linkify};
-        my $dst = $ctx->{obuf};
-        my $anchors = exists($ctx->{-anchors}) ? 1 : 0;
-        for my $y (split(/(^---\n)/sm, $$x)) {
-                if ($y =~ /\A---\n\z/s) {
-                        $$dst .= "---\n"; # all HTML is "\r\n" => "\n"
-                        $anchors |= 2;
-                } elsif ($anchors == 3 && $y =~ /^ [0-9]+ files? changed, /sm) {
-                        # ok, looks like a diffstat, go line-by-line:
-                        for my $l (split(/^/m, $y)) {
-                                if ($l =~ /^ (.+)( +\| .*\z)/s) {
-                                        anchor0($dst, $ctx, $1, $2) and next;
-                                }
-                                $$dst .= $linkify->to_html($l);
-                        }
-                } else { # commit message, notes, etc
-                        $$dst .= $linkify->to_html($y);
+        if (exists $ctx->{-anchors} && $$x =~ # diffstat lines:
+                        /((?:^\x20(?:[^\n]+?)(?:\x20+\|\x20[^\n]*\n))+)
+                        (\x20[0-9]+\x20files?\x20)changed,/msx) {
+                my $pre = substr($$x, 0, $-[0]); # (likely) short prefix
+                substr($$x, 0, $+[0], ''); # sv_chop on $$x ($$x may be long)
+                my @x = ($2, $1);
+                my $lnk = $ctx->{-linkify};
+                my $zfh = $ctx->{zfh};
+                # uninteresting prefix
+                print $zfh $lnk->to_html($pre);
+                for my $l (split(/^/m, pop(@x))) { # $2 per-file stat lines
+                        $l =~ /^ (.+)( +\| .*\z)/s and
+                                anchor0($ctx, $1, $2) and next;
+                         print $zfh $lnk->to_html($l);
                 }
+                my $ch = $ctx->{changed_href} // '#related';
+                print $zfh pop(@x), # $3 /^ \d+ files? /
+                        qq(<a href="$ch">changed</a>,),
+                        # insertions/deletions, notes, commit message, etc:
+                        $lnk->to_html($$x);
+        } else {
+                print { $ctx->{zfh} } $ctx->{-linkify}->to_html($$x);
         }
 }
 
@@ -189,9 +186,9 @@ sub flush_diff ($$) {
         my @top = split($EXTRACT_DIFFS, $$cur);
         undef $$cur; # free memory
 
-        my $linkify = $ctx->{-linkify};
-        my $dst = $ctx->{obuf};
+        my $lnk = $ctx->{-linkify};
         my $dctx; # {}, keys: Q, oid_a, oid_b
+        my $zfh = $ctx->zfh;
 
         while (defined(my $x = shift @top)) {
                 if (scalar(@top) >= 4 &&
@@ -199,7 +196,8 @@ sub flush_diff ($$) {
                                 $top[0] =~ $IS_OID) {
                         $dctx = diff_header(\$x, $ctx, \@top);
                 } elsif ($dctx) {
-                        my $after = '';
+                        open(my $afh, '>>:utf8', \(my $after='')) or
+                                die "open: $!";
 
                         # Quiet "Complex regular subexpression recursion limit"
                         # warning.  Perl will truncate matches upon hitting
@@ -214,29 +212,33 @@ sub flush_diff ($$) {
                         for my $s (split(/((?:(?:^\+[^\n]*\n)+)|
                                         (?:(?:^-[^\n]*\n)+)|
                                         (?:^@@ [^\n]+\n))/xsm, $x)) {
+                                undef $x;
                                 if (!defined($dctx)) {
-                                        $after .= $s;
+                                        print $afh $s;
                                 } elsif ($s =~ s/\A@@ (\S+) (\S+) @@//) {
-                                        $$dst .= qq(<span\nclass="hunk">);
-                                        diff_hunk($dst, $dctx, $1, $2);
-                                        $$dst .= $linkify->to_html($s);
-                                        $$dst .= '</span>';
-                                } elsif ($s =~ /\A\+/) {
-                                        $$dst .= qq(<span\nclass="add">);
-                                        $$dst .= $linkify->to_html($s);
-                                        $$dst .= '</span>';
+                                        print $zfh qq(<span\nclass="hunk">),
+                                                diff_hunk($dctx, $1, $2),
+                                                $lnk->to_html($s),
+                                                '</span>';
+                                } elsif ($s =~ /\A\+/) { # $s may be huge
+                                        print $zfh qq(<span\nclass="add">),
+                                                        $lnk->to_html($s),
+                                                        '</span>';
                                 } elsif ($s =~ /\A-- $/sm) { # email sig starts
                                         $dctx = undef;
-                                        $after .= $s;
-                                } elsif ($s =~ /\A-/) {
-                                        $$dst .= qq(<span\nclass="del">);
-                                        $$dst .= $linkify->to_html($s);
-                                        $$dst .= '</span>';
-                                } else {
-                                        $$dst .= $linkify->to_html($s);
+                                        print $afh $s;
+                                } elsif ($s =~ /\A-/) { # $s may be huge
+                                        print $zfh qq(<span\nclass="del">),
+                                                        $lnk->to_html($s),
+                                                        '</span>';
+                                } else { # $s may be huge
+                                        print $zfh $lnk->to_html($s);
                                 }
                         }
-                        diff_before_or_after($ctx, \$after) unless $dctx;
+                        if (!$dctx) {
+                                utf8::decode($after);
+                                diff_before_or_after($ctx, \$after);
+                        }
                 } else {
                         diff_before_or_after($ctx, \$x);
                 }
diff --git a/lib/PublicInbox/ViewVCS.pm b/lib/PublicInbox/ViewVCS.pm
index 3cbc363b..02e98768 100644
--- a/lib/PublicInbox/ViewVCS.pm
+++ b/lib/PublicInbox/ViewVCS.pm
@@ -1,8 +1,7 @@
-# Copyright (C) 2019-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # show any VCS object, similar to "git show"
-# FIXME: we only show blobs for now
 #
 # This can use a "solver" to reconstruct blobs based on git
 # patches (with abbreviated OIDs in the header).  However, the
@@ -16,11 +15,18 @@
 package PublicInbox::ViewVCS;
 use strict;
 use v5.10.1;
+use File::Temp 0.19 (); # newdir
 use PublicInbox::SolverGit;
+use PublicInbox::GitAsyncCat;
 use PublicInbox::WwwStream qw(html_oneshot);
 use PublicInbox::Linkify;
 use PublicInbox::Tmpfile;
-use PublicInbox::Hval qw(ascii_html to_filename);
+use PublicInbox::ViewDiff qw(flush_diff uri_escape_path);
+use PublicInbox::View;
+use PublicInbox::Eml;
+use Text::Wrap qw(wrap);
+use PublicInbox::Hval qw(ascii_html to_filename prurl);
+use POSIX qw(strftime);
 my $hl = eval {
         require PublicInbox::HlMod;
         PublicInbox::HlMod->new;
@@ -29,22 +35,47 @@ my $hl = eval {
 my %QP_MAP = ( A => 'oid_a', a => 'path_a', b => 'path_b' );
 our $MAX_SIZE = 1024 * 1024; # TODO: configurable
 my $BIN_DETECT = 8000; # same as git
+my $SHOW_FMT = '--pretty=format:'.join('%n', '%P', '%p', '%H', '%T', '%s', '%f',
+        '%an <%ae>  %ai', '%cn <%ce>  %ci', '%b%x00');
 
-sub html_page ($$$) {
-        my ($ctx, $code, $strref) = @_;
+my %GIT_MODE = (
+        '100644' => ' ', # blob
+        '100755' => 'x', # executable blob
+        '040000' => 'd', # tree
+        '120000' => 'l', # symlink
+        '160000' => 'g', # commit (gitlink)
+);
+
+sub html_page ($$;@) {
+        my ($ctx, $code) = @_[0, 1];
         my $wcb = delete $ctx->{-wcb};
         $ctx->{-upfx} = '../../'; # from "/$INBOX/$OID/s/"
-        my $res = html_oneshot($ctx, $code, $strref);
+        my $res = html_oneshot($ctx, $code, @_[2..$#_]);
         $wcb ? $wcb->($res) : $res;
 }
 
+sub dbg_log ($) {
+        my ($ctx) = @_;
+        my $log = delete $ctx->{lh} // die 'BUG: already captured debug log';
+        if (!seek($log, 0, 0)) {
+                warn "seek(log): $!";
+                return '<pre>debug log seek error</pre>';
+        }
+        $log = do { local $/; <$log> } // do {
+                warn "readline(log): $!";
+                return '<pre>debug log read error</pre>';
+        };
+        $ctx->{-linkify} //= PublicInbox::Linkify->new;
+        "<hr><pre>debug log:\n\n".
+                $ctx->{-linkify}->to_html($log).'</pre>';
+}
+
 sub stream_blob_parse_hdr { # {parse_hdr} for Qspawn
         my ($r, $bref, $ctx) = @_;
-        my ($res, $logref) = delete @$ctx{qw(-res -logref)};
-        my ($git, $oid, $type, $size, $di) = @$res;
+        my ($git, $oid, $type, $size, $di) = @{$ctx->{-res}};
         my @cl = ('Content-Length', $size);
-        if (!defined $r) { # error
-                html_page($ctx, 500, $logref);
+        if (!defined $r) { # sysread error
+                html_page($ctx, 500, dbg_log($ctx));
         } elsif (index($$bref, "\0") >= 0) {
                 [200, [qw(Content-Type application/octet-stream), @cl] ];
         } else {
@@ -54,17 +85,16 @@ sub stream_blob_parse_hdr { # {parse_hdr} for Qspawn
                                 'text/plain; charset=UTF-8', @cl ] ];
                 }
                 if ($r == 0) {
-                        warn "premature EOF on $oid $$logref";
-                        return html_page($ctx, 500, $logref);
+                        my $log = dbg_log($ctx);
+                        warn "premature EOF on $oid $log";
+                        return html_page($ctx, 500, $log);
                 }
-                @$ctx{qw(-res -logref)} = ($res, $logref);
                 undef; # bref keeps growing
         }
 }
 
-sub stream_large_blob ($$$$) {
-        my ($ctx, $res, $logref, $fn) = @_;
-        $ctx->{-logref} = $logref;
+sub stream_large_blob ($$) {
+        my ($ctx, $res) = @_;
         $ctx->{-res} = $res;
         my ($git, $oid, $type, $size, $di) = @$res;
         my $cmd = ['git', "--git-dir=$git->{git_dir}", 'cat-file', $type, $oid];
@@ -74,94 +104,418 @@ sub stream_large_blob ($$$$) {
         $qsp->psgi_return($env, undef, \&stream_blob_parse_hdr, $ctx);
 }
 
-sub show_other_result ($$) {
+sub show_other_result ($$) { # tag
         my ($bref, $ctx) = @_;
-        my ($qsp, $logref) = delete @$ctx{qw(-qsp -logref)};
-        if (my $err = $qsp->{err}) {
-                utf8::decode($$err);
-                $$logref .= "git show error: $err";
-                return html_page($ctx, 500, $logref);
+        if (my $qsp_err = delete $ctx->{-qsp_err}) {
+                return html_page($ctx, 500, dbg_log($ctx) .
+                                "git show error:$qsp_err");
         }
         my $l = PublicInbox::Linkify->new;
         utf8::decode($$bref);
-        $$bref = '<pre>'. $l->to_html($$bref);
-        $$bref .= '</pre><hr>' . $$logref;
-        html_page($ctx, 200, $bref);
+        html_page($ctx, 200, '<pre>', $l->to_html($$bref), '</pre><hr>',
+                dbg_log($ctx));
 }
 
-sub show_other ($$$$) {
-        my ($ctx, $res, $logref, $fn) = @_;
-        my ($git, $oid, $type, $size) = @$res;
-        if ($size > $MAX_SIZE) {
-                $$logref = "$oid is too big to show\n" . $$logref;
-                return html_page($ctx, 200, $logref);
+sub cmt_title { # git->cat_async callback
+        my ($bref, $oid, $type, $size, $ctx) = @_;
+        utf8::decode($$bref);
+        my $title = $$bref =~ /\r?\n\r?\n([^\r\n]+)\r?\n?/ ? $1 : '';
+        push(@{$ctx->{-cmt_pt}} , ascii_html($title)) == @{$ctx->{-cmt_P}} and
+                cmt_finalize($ctx);
+}
+
+sub show_commit_start { # ->psgi_qx callback
+        my ($bref, $ctx) = @_;
+        if (my $qsp_err = delete $ctx->{-qsp_err}) {
+                return html_page($ctx, 500, dbg_log($ctx) .
+                                "git show/patch-id error:$qsp_err");
+        }
+        my $patchid = (split(/ /, $$bref))[0]; # ignore commit
+        $ctx->{-q_value_html} = "patchid:$patchid" if defined $patchid;
+        open my $fh, '<:utf8', "$ctx->{-tmp}/h" or
+                die "open $ctx->{-tmp}/h: $!";
+        chop(my $buf = do { local $/ = "\0"; <$fh> });
+        chomp $buf;
+        my ($P, $p);
+        ($P, $p, @{$ctx->{cmt_info}}) = split(/\n/, $buf, 9);
+        return cmt_finalize($ctx) if !$P;
+        @{$ctx->{-cmt_P}} = split(/ /, $P);
+        @{$ctx->{-cmt_p}} = split(/ /, $p); # abbreviated
+        if ($ctx->{env}->{'pi-httpd.async'}) {
+                for (@{$ctx->{-cmt_P}}) {
+                        ibx_async_cat($ctx, $_, \&cmt_title, $ctx);
+                }
+        } else { # synchronous
+                for (@{$ctx->{-cmt_P}}) {
+                        $ctx->{git}->cat_async($_, \&cmt_title, $ctx);
+                }
+                $ctx->{git}->cat_async_wait;
         }
+}
+
+sub ibx_url_for {
+        my ($ctx) = @_;
+        $ctx->{ibx} and return; # just fall back to $upfx
+        $ctx->{git} or return; # /$CODEREPO/$OID/s/ to (eidx|ibx)
+        if (my $ALL = $ctx->{www}->{pi_cfg}->ALL) {
+                $ALL->base_url // $ALL->base_url($ctx->{env});
+        } elsif (my $ibxs = $ctx->{git}->{-ibxs}) {
+                for my $ibx (@$ibxs) {
+                        if ($ibx->isrch) {
+                                return defined($ibx->{url}) ?
+                                        prurl($ctx->{env}, $ibx->{url}) :
+                                        "../../../$ibx->{name}/";
+                        }
+                }
+        } else {
+                undef;
+        }
+}
+
+sub cmt_finalize {
+        my ($ctx) = @_;
+        $ctx->{-linkify} //= PublicInbox::Linkify->new;
+        my $upfx = $ctx->{-upfx} = '../../'; # from "/$INBOX/$OID/s/"
+        my ($H, $T, $s, $f, $au, $co, $bdy) = @{delete $ctx->{cmt_info}};
+        # try to keep author and committer dates lined up
+        my $x = length($au) - length($co);
+        if ($x > 0) {
+                $x = ' ' x $x;
+                $co =~ s/>/>$x/;
+        } elsif ($x < 0) {
+                $x = ' ' x (-$x);
+                $au =~ s/>/>$x/;
+        }
+        $_ = ascii_html($_) for ($au, $co);
+        $au =~ s!(&gt; +)([0-9]{4,}-\S+ \S+)!
+                my ($gt, $t) = ($1, $2);
+                $t =~ tr/ :-//d;
+                qq($gt<a
+href="$upfx?t=$t"
+title="list contemporary emails">$2</a>)
+                !e;
+        $ctx->{-title_html} = $s = $ctx->{-linkify}->to_html($s);
+        my ($P, $p, $pt) = delete @$ctx{qw(-cmt_P -cmt_p -cmt_pt)};
+        $_ = qq(<a href="$upfx$_/s/">).shift(@$p).'</a> '.shift(@$pt) for @$P;
+        if (@$P == 1) {
+                $x = qq{ (<a
+href="$f.patch">patch</a>)\n   <a href=#parent>parent</a> $P->[0]};
+        } elsif (@$P > 1) {
+                $x = qq(\n  <a href=#parents>parents</a> $P->[0]\n);
+                shift @$P;
+                $x .= qq(          $_\n) for @$P;
+                chop $x;
+        } else {
+                $x = ' (<a href=#root_commit>root commit</a>)';
+        }
+        PublicInbox::WwwStream::html_init($ctx);
+        my $zfh = $ctx->zfh;
+        print $zfh <<EOM;
+<pre>   <a href=#commit>commit</a> $H$x
+     <a href=#tree>tree</a> <a href="$upfx$T/s/">$T</a>
+   author $au
+committer $co
+
+<b>$s</b>
+EOM
+        print $zfh "\n", $ctx->{-linkify}->to_html($bdy) if length($bdy);
+        $bdy = '';
+        open my $fh, '<:utf8', "$ctx->{-tmp}/p" or
+                die "open $ctx->{-tmp}/p: $!";
+        if (-s $fh > $MAX_SIZE) {
+                print $zfh "---\n patch is too large to show\n";
+        } else { # prepare flush_diff:
+                read($fh, $x, -s _);
+                $ctx->{-apfx} = $ctx->{-spfx} = $upfx;
+                $x =~ s/\r?\n/\n/gs;
+                $ctx->{-anchors} = {} if $x =~ /^diff --git /sm;
+                flush_diff($ctx, \$x); # undefs $x
+                # TODO: should there be another textarea which attempts to
+                # search for the exact email which was applied to make this
+                # commit?
+                if (my $qry = delete $ctx->{-qry}) {
+                        my $q = '';
+                        for (@{$qry->{dfpost}}, @{$qry->{dfpre}}) {
+                                # keep blobs as short as reasonable, emails
+                                # are going to be older than what's in git
+                                substr($_, 7, 64, '');
+                                $q .= "dfblob:$_ ";
+                        }
+                        chop $q; # no trailing SP
+                        local $Text::Wrap::columns = PublicInbox::View::COLS;
+                        local $Text::Wrap::huge = 'overflow';
+                        $q = wrap('', '', $q);
+                        my $rows = ($q =~ tr/\n/\n/) + 1;
+                        $q = ascii_html($q);
+                        my $ibx_url = ibx_url_for($ctx);
+                        my $alt;
+                        if (defined $ibx_url) {
+                                $ibx_url = ascii_html($ibx_url);
+                                $alt = ' '.$ibx_url;
+                        } else {
+                                $ibx_url = $upfx;
+                                $alt = '';
+                        }
+                        print $zfh <<EOM;
+<hr><form action="$ibx_url"
+id=related><pre>find related emails, including ancestors/descendants/conflicts
+<textarea name=q cols=${\PublicInbox::View::COLS} rows=$rows>$q</textarea>
+<input type=submit value="search$alt"
+/>\t(<a href="${ibx_url}_/text/help/">help</a>)</pre></form>
+EOM
+                }
+        }
+        chop($x = <<EOM);
+<hr><pre>glossary
+--------
+<dfn
+id=commit>Commit</dfn> objects reference one tree, and zero or more parents.
+
+Single <dfn
+id=parent>parent</dfn> commits can typically generate a patch in
+unified diff format via `git format-patch'.
+
+Multiple <dfn id=parents>parents</dfn> means the commit is a merge.
+
+<dfn id=root_commit>Root commits</dfn> have no ancestor.  Note that it is
+possible to have multiple root commits when merging independent histories.
+
+Every commit references one top-level <dfn id=tree>tree</dfn> object.</pre>
+EOM
+        delete($ctx->{env}->{'qspawn.wcb'})->($ctx->html_done($x));
+}
+
+sub stream_patch_parse_hdr { # {parse_hdr} for Qspawn
+        my ($r, $bref, $ctx) = @_;
+        if (!defined $r) { # sysread error
+                html_page($ctx, 500, dbg_log($ctx));
+        } elsif (index($$bref, "\n\n") >= 0) {
+                my $eml = bless { hdr => $bref }, 'PublicInbox::Eml';
+                my $fn = to_filename($eml->header('Subject') // '');
+                $fn = substr($fn // 'PATCH-no-subject', 6); # drop "PATCH-"
+                return [ 200, [ 'Content-Type', 'text/plain; charset=UTF-8',
+                                'Content-Disposition',
+                                qq(inline; filename=$fn.patch) ] ];
+        } elsif ($r == 0) {
+                my $log = dbg_log($ctx);
+                warn "premature EOF on $ctx->{patch_oid} $log";
+                return html_page($ctx, 500, $log);
+        } else {
+                undef; # bref keeps growing until "\n\n"
+        }
+}
+
+sub show_patch ($$) {
+        my ($ctx, $res) = @_;
+        my ($git, $oid) = @$res;
+        my @cmd = ('git', "--git-dir=$git->{git_dir}",
+                qw(format-patch -1 --stdout -C),
+                "--signature=git format-patch -1 --stdout -C $oid", $oid);
+        my $qsp = PublicInbox::Qspawn->new(\@cmd);
+        $ctx->{env}->{'qspawn.wcb'} = delete $ctx->{-wcb};
+        $ctx->{patch_oid} = $oid;
+        $qsp->psgi_return($ctx->{env}, undef, \&stream_patch_parse_hdr, $ctx);
+}
+
+sub show_commit ($$) {
+        my ($ctx, $res) = @_;
+        return show_patch($ctx, $res) if ($ctx->{fn} // '') =~ /\.patch\z/;
+        my ($git, $oid) = @$res;
+        # patch-id needs two passes, and we use the initial show to ensure
+        # a patch embedded inside the commit message body doesn't get fed
+        # to patch-id:
+        my $cmd = [ '/bin/sh', '-c',
+                "git show --encoding=UTF-8 '$SHOW_FMT'".
+                " -z --no-notes --no-patch $oid >h && ".
+                'git show --encoding=UTF-8 --pretty=format:%n -M'.
+                " --stat -p $oid >p && ".
+                "git patch-id --stable <p" ];
+        my $e = { GIT_DIR => $git->{git_dir} };
+        my $qsp = PublicInbox::Qspawn->new($cmd, $e, { -C => "$ctx->{-tmp}" });
+        $qsp->{qsp_err} = \($ctx->{-qsp_err} = '');
+        $ctx->{env}->{'qspawn.wcb'} = delete $ctx->{-wcb};
+        $ctx->{git} = $git;
+        $qsp->psgi_qx($ctx->{env}, undef, \&show_commit_start, $ctx);
+}
+
+sub show_other ($$) { # just in case...
+        my ($ctx, $res) = @_;
+        my ($git, $oid, $type, $size) = @$res;
+        $size > $MAX_SIZE and return html_page($ctx, 200,
+                ascii_html($type)." $oid is too big to show\n". dbg_log($ctx));
         my $cmd = ['git', "--git-dir=$git->{git_dir}",
                 qw(show --encoding=UTF-8 --no-color --no-abbrev), $oid ];
         my $qsp = PublicInbox::Qspawn->new($cmd);
-        my $env = $ctx->{env};
-        $ctx->{-qsp} = $qsp;
-        $ctx->{-logref} = $logref;
-        $qsp->psgi_qx($env, undef, \&show_other_result, $ctx);
+        $qsp->{qsp_err} = \($ctx->{-qsp_err} = '');
+        $qsp->psgi_qx($ctx->{env}, undef, \&show_other_result, $ctx);
 }
 
-# user_cb for SolverGit, called as: user_cb->($result_or_error, $uarg)
-sub solve_result {
-        my ($res, $ctx) = @_;
-        my ($log, $hints, $fn) = delete @$ctx{qw(log hints fn)};
-
-        unless (seek($log, 0, 0)) {
-                warn "seek(log): $!";
-                return html_page($ctx, 500, \'seek error');
+sub show_tree_result ($$) {
+        my ($bref, $ctx) = @_;
+        if (my $qsp_err = delete $ctx->{-qsp_err}) {
+                return html_page($ctx, 500, dbg_log($ctx) .
+                                "git ls-tree -z error:$qsp_err");
+        }
+        my @ent = split(/\0/, $$bref);
+        my $qp = delete $ctx->{qp};
+        my $l = $ctx->{-linkify} //= PublicInbox::Linkify->new;
+        my $pfx = $qp->{b};
+        $$bref = "<pre><a href=#tree>tree</a> $ctx->{tree_oid}";
+        if (defined $pfx) {
+                my $x = ascii_html($pfx);
+                $pfx .= '/';
+                $$bref .= qq(  <a href=#path>path</a>: $x</a>\n);
+        } else {
+                $pfx = '';
+                $$bref .= qq[  (<a href=#path>path</a> unknown)\n];
         }
-        $log = do { local $/; <$log> };
+        my ($x, $m, $t, $oid, $sz, $f, $n);
+        $$bref .= "\n        size        name";
+        for (@ent) {
+                ($x, $f) = split(/\t/, $_, 2);
+                undef $_;
+                ($m, $t, $oid, $sz) = split(/ +/, $x, 4);
+                $m = $GIT_MODE{$m} // '?';
+                utf8::decode($f);
+                $n = ascii_html($f);
+                if ($m eq 'g') { # gitlink submodule commit
+                        $$bref .= "\ng\t\t$n @ <a\nhref=#g>commit</a>$oid";
+                        next;
+                }
+                my $q = 'b='.ascii_html(uri_escape_path($pfx.$f));
+                if ($m eq 'd') { $n .= '/' }
+                elsif ($m eq 'x') { $n = "<b>$n</b>" }
+                elsif ($m eq 'l') { $n = "<i>$n</i>" }
+                $$bref .= qq(\n$m\t$sz\t<a\nhref="../../$oid/s/?$q">$n</a>);
+        }
+        $$bref .= dbg_log($ctx);
+        $$bref .= <<EOM;
+<pre>glossary
+--------
+<dfn
+id=tree>Tree</dfn> objects belong to commits or other tree objects.  Trees may
+reference blobs, sub-trees, or commits of submodules.
+
+<dfn
+id=path>Path</dfn> names are stored in tree objects, but trees do not know
+their own path name.  A tree's path name comes from their parent tree,
+or it is the root tree referenced by a commit object.  Thus, this web UI
+relies on the `b=' URI parameter as a hint to display the path name.
+
+<dfn title="submodule commit"
+id=g>Commit</dfn> objects may be stored in trees to reference submodules.</pre>
+EOM
+        chop $$bref;
+        html_page($ctx, 200, $$bref);
+}
 
-        my $ref = ref($res);
+sub show_tree ($$) {
+        my ($ctx, $res) = @_;
+        my ($git, $oid, undef, $size) = @$res;
+        $size > $MAX_SIZE and return html_page($ctx, 200,
+                        "tree $oid is too big to show\n". dbg_log($ctx));
+        my $cmd = [ 'git', "--git-dir=$git->{git_dir}",
+                qw(ls-tree -z -l --no-abbrev), $oid ];
+        my $qsp = PublicInbox::Qspawn->new($cmd);
+        $ctx->{tree_oid} = $oid;
+        $qsp->{qsp_err} = \($ctx->{-qsp_err} = '');
+        $qsp->psgi_qx($ctx->{env}, undef, \&show_tree_result, $ctx);
+}
+
+# returns seconds offset from git TZ offset
+sub tz_adj ($) {
+        my ($tz) = @_; # e.g "-0700"
+        $tz = int($tz);
+        my $mm = $tz < 0 ? -$tz : $tz;
+        $mm = int($mm / 100) * 60 + ($mm % 100);
+        $mm = $tz < 0 ? -$mm : $mm;
+        ($mm * 60);
+}
+
+sub show_tag_result { # git->cat_async callback
+        my ($bref, $oid, $type, $size, $ctx) = @_;
+        utf8::decode($$bref);
         my $l = PublicInbox::Linkify->new;
-        $log = '<pre>debug log:</pre><hr /><pre>' .
-                $l->to_html($log) . '</pre>';
+        $$bref = $l->to_html($$bref);
+        $$bref =~ s!^object ([a-f0-9]+)!object <a
+href=../../$1/s/>$1</a>!;
+
+        $$bref =~ s/^(tagger .*&gt; )([0-9]+) ([\-+]?[0-9]+)/$1.strftime(
+                '%Y-%m-%d %H:%M:%S', gmtime($2 + tz_adj($3)))." $3"/sme;
+        # TODO: download link
+        html_page($ctx, 200, '<pre>', $$bref, '</pre>', dbg_log($ctx));
+}
 
-        $res or return html_page($ctx, 404, \$log);
-        $ref eq 'ARRAY' or return html_page($ctx, 500, \$log);
+sub show_tag ($$) {
+        my ($ctx, $res) = @_;
+        my ($git, $oid) = @$res;
+        $ctx->{git} = $git;
+        if ($ctx->{env}->{'pi-httpd.async'}) {
+                ibx_async_cat($ctx, $oid, \&show_tag_result, $ctx);
+        } else { # synchronous (generic PSGI)
+                $git->cat_async($oid, \&show_tag_result, $ctx);
+                $git->cat_async_wait;
+        }
+}
+
+# user_cb for SolverGit, called as: user_cb->($result_or_error, $uarg)
+sub solve_result {
+        my ($res, $ctx) = @_;
+        my $hints = delete $ctx->{hints};
+        $res or return html_page($ctx, 404, dbg_log($ctx));
+        ref($res) eq 'ARRAY' or return html_page($ctx, 500, dbg_log($ctx));
 
         my ($git, $oid, $type, $size, $di) = @$res;
-        return show_other($ctx, $res, \$log, $fn) if $type ne 'blob';
+        return show_commit($ctx, $res) if $type eq 'commit';
+        return show_tree($ctx, $res) if $type eq 'tree';
+        return show_tag($ctx, $res) if $type eq 'tag';
+        return show_other($ctx, $res) if $type ne 'blob';
         my $path = to_filename($di->{path_b} // $hints->{path_b} // 'blob');
         my $raw_link = "(<a\nhref=$path>raw</a>)";
         if ($size > $MAX_SIZE) {
-                return stream_large_blob($ctx, $res, \$log, $fn) if defined $fn;
-                $log = "<pre><b>Too big to show, download available</b>\n" .
-                        "$oid $type $size bytes $raw_link</pre>" . $log;
-                return html_page($ctx, 200, \$log);
+                return stream_large_blob($ctx, $res) if defined $ctx->{fn};
+                return html_page($ctx, 200, <<EOM . dbg_log($ctx));
+<pre><b>Too big to show, download available</b>
+blob $oid $size bytes $raw_link</pre>
+EOM
+        }
+        @{$ctx->{-paths}} = ($path, $raw_link);
+        bless $ctx, 'PublicInbox::WwwStream'; # for DESTROY
+        $ctx->{git} = $git;
+        if ($ctx->{env}->{'pi-httpd.async'}) {
+                ibx_async_cat($ctx, $oid, \&show_blob, $ctx);
+        } else { # synchronous
+                $git->cat_async($oid, \&show_blob, $ctx);
+                $git->cat_async_wait;
         }
+}
 
-        my $blob = $git->cat_file($oid);
-        if (!$blob) { # WTF?
+sub show_blob { # git->cat_async callback
+        my ($blob, $oid, $type, $size, $ctx) = @_;
+        if (!$blob) {
                 my $e = "Failed to retrieve generated blob ($oid)";
-                warn "$e ($git->{git_dir})";
-                $log = "<pre><b>$e</b></pre>" . $log;
-                return html_page($ctx, 500, \$log);
+                warn "$e ($ctx->{git}->{git_dir}) type=$type";
+                return html_page($ctx, 500, "<pre><b>$e</b></pre>".dbg_log($ctx))
         }
 
         my $bin = index(substr($$blob, 0, $BIN_DETECT), "\0") >= 0;
-        if (defined $fn) {
+        if (defined $ctx->{fn}) {
                 my $h = [ 'Content-Length', $size, 'Content-Type' ];
                 push(@$h, ($bin ? 'application/octet-stream' : 'text/plain'));
                 return delete($ctx->{-wcb})->([200, $h, [ $$blob ]]);
         }
 
-        if ($bin) {
-                $log = "<pre>$oid $type $size bytes (binary)" .
-                        " $raw_link</pre>" . $log;
-                return html_page($ctx, 200, \$log);
-        }
+        my ($path, $raw_link) = @{delete $ctx->{-paths}};
+        $bin and return html_page($ctx, 200,
+                                "<pre>blob $oid $size bytes (binary)" .
+                                " $raw_link</pre>".dbg_log($ctx));
 
         # TODO: detect + convert to ensure validity
         utf8::decode($$blob);
         my $nl = ($$blob =~ s/\r?\n/\n/sg);
         my $pad = length($nl);
 
-        $l->linkify_1($$blob);
+        ($ctx->{-linkify} //= PublicInbox::Linkify->new)->linkify_1($$blob);
         my $ok = $hl->do_hl($blob, $path) if $hl;
         if ($ok) {
                 $blob = $ok;
@@ -170,17 +524,15 @@ sub solve_result {
         }
 
         # using some of the same CSS class names and ids as cgit
-        $log = "<pre>$oid $type $size bytes $raw_link</pre>" .
+        my $x = "<pre>blob $oid $size bytes $raw_link</pre>" .
                 "<hr /><table\nclass=blob>".
-                "<tr><td\nclass=linenumbers><pre>" . join('', map {
-                        sprintf("<a id=n$_ href=#n$_>% ${pad}u</a>\n", $_)
-                } (1..$nl)) . '</pre></td>' .
-                '<td><pre> </pre></td>'. # pad for non-CSS users
-                "<td\nclass=lines><pre\nstyle='white-space:pre'><code>" .
-                $l->linkify_2($$blob) .
-                '</code></pre></td></tr></table>' . $log;
-
-        html_page($ctx, 200, \$log);
+                "<tr><td\nclass=linenumbers><pre>";
+        # scratchpad in this loop is faster here than `printf $zfh':
+        $x .= sprintf("<a id=n$_ href=#n$_>% ${pad}u</a>\n", $_) for (1..$nl);
+        $x .= '</pre></td><td><pre> </pre></td>'. # pad for non-CSS users
+                "<td\nclass=lines><pre\nstyle='white-space:pre'><code>";
+        html_page($ctx, 200, $x, $ctx->{-linkify}->linkify_2($$blob),
+                '</code></pre></td></tr></table>'.dbg_log($ctx));
 }
 
 # GET /$INBOX/$GIT_OBJECT_ID/s/
@@ -193,15 +545,17 @@ sub show ($$;$) {
                 defined(my $v = $qp->{$from}) or next;
                 $hints->{$to} = $v if $v ne '';
         }
-
-        $ctx->{'log'} = tmpfile("solve.$oid_b") // die "tmpfile: $!";
         $ctx->{fn} = $fn;
+        $ctx->{-tmp} = File::Temp->newdir("solver.$oid_b-XXXX", TMPDIR => 1);
+        open $ctx->{lh}, '+>>', "$ctx->{-tmp}/solve.log" or die "open: $!";
         my $solver = PublicInbox::SolverGit->new($ctx->{ibx},
                                                 \&solve_result, $ctx);
+        $solver->{gits} //= [ $ctx->{git} ];
+        $solver->{tmp} = $ctx->{-tmp}; # share tmpdir
         # PSGI server will call this immediately and give us a callback (-wcb)
         sub {
                 $ctx->{-wcb} = $_[0]; # HTTP write callback
-                $solver->solve($ctx->{env}, $ctx->{log}, $oid_b, $hints);
+                $solver->solve($ctx->{env}, $ctx->{lh}, $oid_b, $hints);
         };
 }
 
diff --git a/lib/PublicInbox/WWW.pm b/lib/PublicInbox/WWW.pm
index 755d7558..f861b192 100644
--- a/lib/PublicInbox/WWW.pm
+++ b/lib/PublicInbox/WWW.pm
@@ -23,7 +23,7 @@ use PublicInbox::WwwStatic qw(r path_info_raw);
 use PublicInbox::Eml;
 
 # TODO: consider a routing tree now that we have more endpoints:
-our $INBOX_RE = qr!\A/([\w\-][\w\.\-]*)!;
+our $INBOX_RE = qr!\A/([\w\-][\w\.\-\+]*)!;
 our $MID_RE = qr!([^/]+)!;
 our $END_RE = qr!(T/|t/|t\.mbox(?:\.gz)?|t\.atom|raw|)!;
 our $ATTACH_RE = qr!([0-9][0-9\.]*)-($PublicInbox::Hval::FN)!;
@@ -194,10 +194,20 @@ sub r404 {
 
 sub news_cgit_fallback ($) {
         my ($ctx) = @_;
-        my $www = $ctx->{www};
-        my $env = $ctx->{env};
-        my $res = $www->news_www->call($env);
-        $res->[0] == 404 ? $www->cgit->call($env) : $res;
+        my $res = $ctx->{www}->news_www->call($ctx->{env});
+
+        $res->[0] == 404 and ($ctx->{www}->{cgit_fallback} //= do {
+                my $c = $ctx->{www}->{pi_cfg}->{'publicinbox.cgit'} // 'first';
+                $c ne 'first' # `fallback' and `rewrite' => true
+        } // 0) and $res = $ctx->{www}->coderepo->srv($ctx);
+
+        ref($res) eq 'ARRAY' && $res->[0] == 404 and
+                $res = $ctx->{www}->cgit->call($ctx->{env}, $ctx);
+
+        ref($res) eq 'ARRAY' && $res->[0] == 404 &&
+                        !$ctx->{www}->{cgit_fallback} and
+                $res = $ctx->{www}->coderepo->srv($ctx);
+        $res;
 }
 
 # returns undef if valid, array ref response if invalid
@@ -303,7 +313,8 @@ sub get_text {
 sub get_vcs_object ($$$;$) {
         my ($ctx, $inbox, $oid, $filename) = @_;
         my $r404 = invalid_inbox($ctx, $inbox);
-        return $r404 if $r404 || !$ctx->{www}->{pi_cfg}->repo_objs($ctx->{ibx});
+        return $r404 if $r404;
+        return r(404) if !$ctx->{www}->{pi_cfg}->repo_objs($ctx->{ibx});
         require PublicInbox::ViewVCS;
         PublicInbox::ViewVCS::show($ctx, $oid, $filename);
 }
@@ -319,7 +330,7 @@ sub get_altid_dump {
 sub need {
         my ($ctx, $extra) = @_;
         require PublicInbox::WwwStream;
-        PublicInbox::WwwStream::html_oneshot($ctx, 501, \<<EOF);
+        PublicInbox::WwwStream::html_oneshot($ctx, 501, <<EOF);
 <pre>$extra is not available for this public-inbox
 <a\nhref="../">Return to index</a></pre>
 EOF
@@ -480,16 +491,21 @@ sub news_www {
 
 sub cgit {
         my ($self) = @_;
-        $self->{cgit} //= do {
-                my $pi_cfg = $self->{pi_cfg};
-
-                if (defined($pi_cfg->{'publicinbox.cgitrc'})) {
+        $self->{cgit} //=
+                (defined($self->{pi_cfg}->{'publicinbox.cgitrc'}) ? do {
                         require PublicInbox::Cgit;
-                        PublicInbox::Cgit->new($pi_cfg);
-                } else {
+                        PublicInbox::Cgit->new($self->{pi_cfg});
+                } : undef) // do {
                         require Plack::Util;
                         Plack::Util::inline_object(call => sub { r404() });
-                }
+                };
+}
+
+sub coderepo {
+        my ($self) = @_;
+        $self->{coderepo} //= do {
+                require PublicInbox::WwwCoderepo;
+                PublicInbox::WwwCoderepo->new($self->{pi_cfg});
         }
 }
 
diff --git a/lib/PublicInbox/Watch.pm b/lib/PublicInbox/Watch.pm
index 3f6fe21b..082ecfb9 100644
--- a/lib/PublicInbox/Watch.pm
+++ b/lib/PublicInbox/Watch.pm
@@ -328,7 +328,7 @@ sub imap_idle_once ($$$$) {
         my ($self, $mic, $intvl, $uri) = @_;
         my $i = $intvl //= (29 * 60);
         my $end = now() + $intvl;
-        warn "I: $uri idling for ${intvl}s\n";
+        warn "# $uri idling for ${intvl}s\n";
         local $0 = "IDLE $0";
         return if $self->{quit};
         unless ($mic->idle) {
@@ -517,7 +517,7 @@ sub poll_fetch_reap {
         if ($?) {
                 warn "W: PID=$pid died: \$?=$?\n", map { "$_\n" } @$uris;
         }
-        warn("I: will check $_ in ${intvl}s\n") for @$uris;
+        warn("# will check $_ in ${intvl}s\n") for @$uris;
         add_timer($intvl, \&poll_fetch_fork, $self, $intvl, $uris);
 }
 
diff --git a/lib/PublicInbox/WwwAltId.pm b/lib/PublicInbox/WwwAltId.pm
index e107dfe0..47056160 100644
--- a/lib/PublicInbox/WwwAltId.pm
+++ b/lib/PublicInbox/WwwAltId.pm
@@ -33,14 +33,14 @@ sub sqldump ($$) {
         my $altid_map = $ibx->altid_map;
         my $fn = $altid_map->{$altid_pfx};
         unless (defined $fn) {
-                return html_oneshot($ctx, 404, \<<EOF);
+                return html_oneshot($ctx, 404, <<EOF);
 <pre>`$altid_pfx' is not a valid altid for this inbox</pre>
 EOF
         }
 
         if ($env->{REQUEST_METHOD} ne 'POST') {
                 my $url = $ibx->base_url($ctx->{env}) . "$altid_pfx.sql.gz";
-                return html_oneshot($ctx, 405, \<<EOF);
+                return html_oneshot($ctx, 405, <<EOF);
 <pre>A POST request is required to retrieve $altid_pfx.sql.gz
 
         curl -d '' -O $url
@@ -54,7 +54,7 @@ or
 EOF
         }
 
-        $sqlite3 //= which('sqlite3') // return html_oneshot($ctx, 501, \<<EOF);
+        $sqlite3 //= which('sqlite3') // return html_oneshot($ctx, 501, <<EOF);
 <pre>sqlite3 not available
 
 The administrator needs to install the sqlite3(1) binary
diff --git a/lib/PublicInbox/WwwAtomStream.pm b/lib/PublicInbox/WwwAtomStream.pm
index 82895db6..83a8818e 100644
--- a/lib/PublicInbox/WwwAtomStream.pm
+++ b/lib/PublicInbox/WwwAtomStream.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2016-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 #
 # Atom body stream for HTTP responses
@@ -16,6 +16,7 @@ use PublicInbox::MsgTime qw(msg_timestamp);
 sub new {
         my ($class, $ctx, $cb) = @_;
         $ctx->{feed_base_url} = $ctx->{ibx}->base_url($ctx->{env});
+        $ctx->{-spfx} = $ctx->{feed_base_url} if $ctx->{ibx}->{coderepo};
         $ctx->{cb} = $cb || \&PublicInbox::GzipFilter::close;
         $ctx->{emit_header} = 1;
         bless $ctx, $class;
@@ -38,14 +39,15 @@ sub async_next ($) {
 sub async_eml { # for async_blob_cb
         my ($ctx, $eml) = @_;
         my $smsg = delete $ctx->{smsg};
+        $smsg->{mid} // $smsg->populate($eml);
         $ctx->write(feed_entry($ctx, $smsg, $eml));
 }
 
 sub response {
-        my ($class, $ctx, $code, $cb) = @_;
+        my ($class, $ctx, $cb) = @_;
         my $res_hdr = [ 'Content-Type' => 'application/atom+xml' ];
         $class->new($ctx, $cb);
-        $ctx->psgi_response($code, $res_hdr);
+        $ctx->psgi_response(200, $res_hdr);
 }
 
 # called once for each message by PSGI server
@@ -145,19 +147,19 @@ sub feed_entry {
         my $name = ascii_html(join(', ', PublicInbox::Address::names($from)));
         $email = ascii_html($email // $ctx->{ibx}->{-primary_address});
 
-        my $s = delete($ctx->{emit_header}) ? atom_header($ctx, $title) : '';
-        $s .= "<entry><author><name>$name</name><email>$email</email>" .
+        print { $ctx->zfh }
+                (delete($ctx->{emit_header}) ? atom_header($ctx, $title) : ''),
+                "<entry><author><name>$name</name><email>$email</email>" .
                 "</author>$title$updated" .
-                qq(<link\nhref="$href"/>).
+                qq(<link\nhref="$href"/>) .
                 "<id>$uuid</id>$irt" .
                 qq{<content\ntype="xhtml">} .
                 qq{<div\nxmlns="http://www.w3.org/1999/xhtml">} .
                 qq(<pre\nstyle="white-space:pre-wrap">);
-        $ctx->{obuf} = \$s;
         $ctx->{mhref} = $href;
-        PublicInbox::View::multipart_text_as_html($eml, $ctx);
-        delete $ctx->{obuf};
-        $s .= '</pre></div></content></entry>';
+        $ctx->{changed_href} = "${href}#related";
+        $eml->each_part(\&PublicInbox::View::add_text_body, $ctx, 1);
+        '</pre></div></content></entry>';
 }
 
 sub feed_updated {
diff --git a/lib/PublicInbox/WwwCoderepo.pm b/lib/PublicInbox/WwwCoderepo.pm
new file mode 100644
index 00000000..99df39ef
--- /dev/null
+++ b/lib/PublicInbox/WwwCoderepo.pm
@@ -0,0 +1,243 @@
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+#
+# Standalone code repository viewer for users w/o cgit.
+# This isn't intended to replicate all of cgit, but merely to be a
+# "good enough" viewer with search support and some UI hints to encourage
+# cloning + command-line usage.
+package PublicInbox::WwwCoderepo;
+use v5.12;
+use File::Temp 0.19 (); # newdir
+use PublicInbox::ViewVCS;
+use PublicInbox::WwwStatic qw(r);
+use PublicInbox::GitHTTPBackend;
+use PublicInbox::Git;
+use PublicInbox::GitAsyncCat;
+use PublicInbox::WwwStream;
+use PublicInbox::Hval qw(ascii_html);
+use PublicInbox::RepoSnapshot;
+
+my $EACH_REF = "git for-each-ref --sort=-creatordate --format='%(HEAD)%00".
+        join('%00', map { "%($_)" }
+        qw(objectname refname:short subject creatordate:short))."'";
+
+# shared with PublicInbox::Cgit
+sub prepare_coderepos {
+        my ($self) = @_;
+        my $pi_cfg = $self->{pi_cfg};
+
+        # TODO: support gitweb and other repository viewers?
+        defined($pi_cfg->{'publicinbox.cgitrc'}) and
+                $pi_cfg->parse_cgitrc(undef, 0);
+
+        my $code_repos = $pi_cfg->{-code_repos};
+        for my $k (grep(/\Acoderepo\.(?:.+)\.dir\z/, keys %$pi_cfg)) {
+                $k = substr($k, length('coderepo.'), -length('.dir'));
+                $code_repos->{$k} //= $pi_cfg->fill_code_repo($k);
+        }
+
+        # associate inboxes and extindices with coderepos for search:
+        for my $k (grep(/\Apublicinbox\.(?:.+)\.coderepo\z/, keys %$pi_cfg)) {
+                $k = substr($k, length('publicinbox.'), -length('.coderepo'));
+                my $ibx = $pi_cfg->lookup_name($k) // next;
+                $pi_cfg->repo_objs($ibx);
+                push @{$self->{-strong}}, $ibx; # strengthen {-ibxs} weakref
+        }
+        for my $k (grep(/\Aextindex\.(?:.+)\.coderepo\z/, keys %$pi_cfg)) {
+                $k = substr($k, length('extindex.'), -length('.coderepo'));
+                my $eidx = $pi_cfg->lookup_ei($k) // next;
+                $pi_cfg->repo_objs($eidx);
+                push @{$self->{-strong}}, $eidx; # strengthen {-ibxs} weakref
+        }
+        while (my ($nick, $repo) = each %$code_repos) {
+                $self->{"\0$nick"} = $repo;
+        }
+}
+
+sub new {
+        my ($cls, $pi_cfg) = @_;
+        my $self = bless { pi_cfg => $pi_cfg }, $cls;
+        prepare_coderepos($self);
+        $self->{snapshots} = do {
+                my $s = $pi_cfg->{'coderepo.snapshots'} // '';
+                $s eq 'all' ? \%PublicInbox::RepoSnapshot::FMT_TYPES :
+                        +{ map { $_ => 1 } split(/\s+/, $s) };
+        };
+        $self->{$_} = 10 for qw(summary_branches summary_tags);
+        $self->{$_} = 10 for qw(summary_log);
+        $self;
+}
+
+sub summary_finish {
+        my ($ctx) = @_;
+        my $wcb = delete($ctx->{env}->{'qspawn.wcb'}) or return; # already done
+        my @x = split(/\n\n/sm, delete($ctx->{-each_refs}));
+        PublicInbox::WwwStream::html_init($ctx);
+        my $zfh = $ctx->zfh;
+
+        # git log
+        my @r = split(/\n/s, pop(@x) // '');
+        my $last = pop(@r) if scalar(@r) > $ctx->{wcr}->{summary_log};
+        print $zfh <<EOM;
+<pre>
+<a
+href='#readme'>about</a> <a
+href='#heads'>heads</a> <a
+href='#tags'>tags</a>
+
+<a
+id=log>\$</a> git log --pretty=format:'%h %s (%cs)%d'
+EOM
+        for (@r) {
+                my $d; # decorations
+                s/^ \(([^\)]+)\)// and $d = $1;
+                substr($_, 0, 1, '');
+                my ($H, $h, $cs, $s) = split(/ /, $_, 4);
+                print $zfh "<a\nhref=./$H/s/>$h</a> ", ascii_html($s),
+                        " (", $cs, ")\n";
+                print $zfh "\t(", ascii_html($d), ")\n" if $d;
+        }
+        print $zfh "# no commits, yet\n" if !@r;
+        print $zfh "...\n" if $last;
+
+        # README
+        my ($bref, $oid, $ref_path) = @{delete $ctx->{-readme}};
+        if ($bref) {
+                my $l = PublicInbox::Linkify->new;
+                $$bref =~ s/\s*\z//sm;
+                print $zfh "\n<a id=readme>\$</a> " .
+                        "git cat-file blob <a href=./$oid/s/>",
+                        ascii_html($ref_path), "</a>\n",
+                        $l->to_html($$bref), '</pre><hr><pre>';
+        }
+
+        # refs/heads
+        print $zfh "<a id=heads># heads (aka `branches'):</a>\n\$ " .
+                "git for-each-ref --sort=-creatordate refs/heads" .
+                " \\\n\t--format='%(HEAD) ". # no space for %(align:) hint
+                "%(refname:short) %(subject) (%(creatordate:short))'\n";
+        @r = split(/^/sm, shift(@x) // '');
+        $last = pop(@r) if scalar(@r) > $ctx->{wcr}->{summary_branches};
+        for (@r) {
+                my ($pfx, $oid, $ref, $s, $cd) = split(/\0/);
+                utf8::decode($_) for ($ref, $s);
+                chomp $cd;
+                my $align = length($ref) < 12 ? ' ' x (12 - length($ref)) : '';
+                print $zfh "$pfx <a\nhref=./$oid/s/>", ascii_html($ref),
+                        "</a>$align ", ascii_html($s), " ($cd)\n";
+        }
+        print $zfh "# no heads (branches) yet...\n" if !@r;
+        print $zfh "...\n" if $last;
+        print $zfh "\n<a id=tags># tags:</a>\n\$ " .
+                "git for-each-ref --sort=-creatordate refs/tags" .
+                " \\\n\t--format='". # no space for %(align:) hint
+                "%(refname:short) %(subject) (%(creatordate:short))'\n";
+        @r = split(/^/sm, shift(@x) // '');
+        $last = pop(@r) if scalar(@r) > $ctx->{wcr}->{summary_tags};
+        my @s = sort keys %{$ctx->{wcr}->{snapshots}};
+        my $n;
+        if (@s) {
+                $n = $ctx->{git}->local_nick // die "BUG: $ctx->{git_dir} nick";
+                $n =~ s/\.git\z/-/;
+                ($n) = ($n =~ m!([^/]+)\z!);
+                $n = ascii_html($n);
+        }
+        for (@r) {
+                my (undef, $oid, $ref, $s, $cd) = split(/\0/);
+                utf8::decode($_) for ($ref, $s);
+                chomp $cd;
+                my $align = length($ref) < 12 ? ' ' x (12 - length($ref)) : '';
+                print $zfh "<a\nhref=./$oid/s/>", ascii_html($ref),
+                        "</a>$align ", ascii_html($s), " ($cd)";
+                if (@s) {
+                        my $v = $ref;
+                        $v =~ s/\A[vV]//;
+                        print $zfh "\t",  join(' ', map {
+                                qq{<a href="snapshot/$n$v.$_">$_</a>} } @s);
+                }
+                print $zfh "\n";
+        }
+        print $zfh "# no tags yet...\n" if !@r;
+        print $zfh "...\n" if $last;
+        $wcb->($ctx->html_done('</pre>'));
+}
+
+sub capture_refs ($$) { # psgi_qx callback to capture git-for-each-ref + git-log
+        my ($bref, $ctx) = @_;
+        my $qsp_err = delete $ctx->{-qsp_err};
+        $ctx->{-each_refs} = $$bref;
+        summary_finish($ctx) if $ctx->{-readme};
+}
+
+sub set_readme { # git->cat_async callback
+        my ($bref, $oid, $type, $size, $ctx) = @_;
+        my $ref_path = shift @{$ctx->{-nr_readme_tries}}; # e.g. HEAD:README
+        if ($type eq 'blob' && !$ctx->{-readme}) {
+                $ctx->{-readme} = [ $bref, $oid, $ref_path ];
+        } elsif (scalar @{$ctx->{-nr_readme_tries}} == 0) {
+                $ctx->{-readme} //= []; # nothing left to try
+        } # or try another README...
+        summary_finish($ctx) if $ctx->{-each_refs} && $ctx->{-readme};
+}
+
+sub summary {
+        my ($self, $ctx) = @_;
+        $ctx->{wcr} = $self;
+        my $nb = $self->{summary_branches} + 1;
+        my $nt = $self->{summary_tags} + 1;
+        my $nl = $self->{summary_log} + 1;
+        my $qsp = PublicInbox::Qspawn->new([qw(/bin/sh -c),
+                "$EACH_REF --count=$nb refs/heads; echo && " .
+                "$EACH_REF --count=$nt refs/tags; echo && " .
+                "git log -$nl --pretty=format:'%d %H %h %cs %s' --" ],
+                { GIT_DIR => $ctx->{git}->{git_dir} });
+        $qsp->{qsp_err} = \($ctx->{-qsp_err} = '');
+        my @try = qw(HEAD:README HEAD:README.md); # TODO: configurable
+        $ctx->{-nr_readme_tries} = [ @try ];
+        $ctx->{git}->cat_async($_, \&set_readme, $ctx) for @try;
+        if ($ctx->{env}->{'pi-httpd.async'}) {
+                PublicInbox::GitAsyncCat::watch_cat($ctx->{git});
+        } else { # synchronous
+                $ctx->{git}->cat_async_wait;
+        }
+        sub { # $_[0] => PublicInbox::HTTP::{Identity,Chunked}
+                $ctx->{env}->{'qspawn.wcb'} = $_[0];
+                $qsp->psgi_qx($ctx->{env}, undef, \&capture_refs, $ctx);
+        }
+}
+
+sub srv { # endpoint called by PublicInbox::WWW
+        my ($self, $ctx) = @_;
+        my $path_info = $ctx->{env}->{PATH_INFO};
+        my $git;
+        # handle clone requests
+        if ($path_info =~ m!\A/(.+?)/($PublicInbox::GitHTTPBackend::ANY)\z!x) {
+                $git = $self->{"\0$1"} and return
+                        PublicInbox::GitHTTPBackend::serve($ctx->{env},$git,$2);
+        }
+        $path_info =~ m!\A/(.+?)/\z! and
+                ($ctx->{git} = $self->{"\0$1"}) and return summary($self, $ctx);
+        $path_info =~ m!\A/(.+?)/([a-f0-9]+)/s/\z! and
+                        ($ctx->{git} = $self->{"\0$1"}) and
+                return PublicInbox::ViewVCS::show($ctx, $2);
+
+        # snapshots:
+        if ($path_info =~ m!\A/(.+?)/snapshot/([^/]+)\z! and
+                        ($ctx->{git} = $self->{"\0$1"})) {
+                $ctx->{wcr} = $self;
+                return PublicInbox::RepoSnapshot::srv($ctx, $2) // r(404);
+        }
+
+        # enforce trailing slash:
+        if ($path_info =~ m!\A/(.+?)\z! and ($git = $self->{"\0$1"})) {
+                my $qs = $ctx->{env}->{QUERY_STRING};
+                my $url = $git->base_url($ctx->{env});
+                $url .= "?$qs" if $qs ne '';
+                [ 301, [ Location => $url, 'Content-Type' => 'text/plain' ],
+                        [ "Redirecting to $url\n" ] ];
+        } else {
+                r(404);
+        }
+}
+
+1;
diff --git a/lib/PublicInbox/WwwListing.pm b/lib/PublicInbox/WwwListing.pm
index 79c0a8ec..72c940dd 100644
--- a/lib/PublicInbox/WwwListing.pm
+++ b/lib/PublicInbox/WwwListing.pm
@@ -169,17 +169,15 @@ sub mset_nav_top {
         my ($ctx, $mset) = @_;
         my $q = $ctx->{-sq};
         my $qh = $q->{'q'} // '';
-        utf8::decode($qh);
-        $qh = ascii_html($qh);
-        $qh = qq[\nvalue="$qh"] if $qh ne '';
-        my $rv = <<EOM;
-<form
-action="./"><pre><input name=q type=text$qh
-/><input type=submit value="locate inbox"
-/><input type=submit name=a value="search all inboxes"
-/></pre></form><pre>
+        if ($qh ne '') {
+                utf8::decode($qh);
+                $qh = qq[\nvalue="].ascii_html($qh).'"';
+        }
+        chop(my $rv = <<EOM);
+<form action="./"><pre><input name=q type=text$qh/><input
+type=submit value="locate inbox"/><input type=submit name=a
+value="search all inboxes"/></pre></form><pre>
 EOM
-        chomp $rv;
         if (defined($q->{'q'})) {
                 my $initial_q = $ctx->{-uxs_retried};
                 if (defined $initial_q) {
@@ -210,28 +208,28 @@ sub psgi_triple {
         my $h = [ 'Content-Type', 'text/html; charset=UTF-8',
                         'Content-Length', undef ];
         my $gzf = gzf_maybe($h, $ctx->{env});
-        $gzf->zmore('<html><head><title>public-inbox listing</title>' .
-                        $ctx->{www}->style('+/') .
-                        '</head><body>');
+        my $zfh = $gzf->zfh;
+        print $zfh '<html><head><title>public-inbox listing</title>',
+                        $ctx->{www}->style('+/'),
+                        '</head><body>';
         my $code = 404;
         if (my $list = delete $ctx->{-list}) {
                 my $mset = delete $ctx->{-mset};
                 $code = 200;
                 if ($mset) { # already sorted, so search bar:
-                        $gzf->zmore(mset_nav_top($ctx, $mset));
+                        print $zfh mset_nav_top($ctx, $mset);
                 } else { # sort config dump by ->modified
                         @$list = map { $_->[1] }
                                 sort { $b->[0] <=> $a->[0] } @$list;
                 }
-                $gzf->zmore('<pre>');
-                $gzf->zmore(join("\n", @$list));
-                $gzf->zmore(mset_footer($ctx, $mset)) if $mset;
+                print $zfh '<pre>', join("\n", @$list); # big
+                print $zfh mset_footer($ctx, $mset) if $mset;
         } elsif (my $mset = delete $ctx->{-mset}) {
-                $gzf->zmore(mset_nav_top($ctx, $mset));
-                $gzf->zmore('<pre>no matching inboxes');
-                $gzf->zmore(mset_footer($ctx, $mset));
+                print $zfh mset_nav_top($ctx, $mset),
+                                '<pre>no matching inboxes',
+                                mset_footer($ctx, $mset);
         } else {
-                $gzf->zmore('<pre>no inboxes, yet');
+                print $zfh '<pre>no inboxes, yet';
         }
         my $out = $gzf->zflush('</pre><hr><pre>'.
 qq(This is a listing of public inboxes, see the `mirror' link of each inbox
diff --git a/lib/PublicInbox/WwwStatic.pm b/lib/PublicInbox/WwwStatic.pm
index eeb5e565..1c1a3d38 100644
--- a/lib/PublicInbox/WwwStatic.pm
+++ b/lib/PublicInbox/WwwStatic.pm
@@ -275,12 +275,11 @@ sub dir_response ($$$) {
         my $path_info = $env->{PATH_INFO};
         push @entries, '..' if $path_info ne '/';
         for my $base (@entries) {
+                my @st = stat($fs_path . $base) or next; # unlikely
                 my $href = ascii_html(uri_escape_utf8($base));
                 my $name = ascii_html($base);
-                my @st = stat($fs_path . $base) or next; # unlikely
-                my ($gzipped, $uncompressed, $hsize);
-                my $entry = '';
                 my $mtime = $st[9];
+                my ($entry, $hsize);
                 if (-d _) {
                         $href .= '/';
                         $name .= '/';
@@ -296,12 +295,12 @@ sub dir_response ($$$) {
                         next;
                 }
                 # 54 = 80 - (SP length(strftime(%Y-%m-%d %k:%M)) SP human_size)
-                $hsize = sprintf('% 8s', $hsize);
                 my $pad = 54 - length($name);
                 $pad = 1 if $pad <= 0;
-                $entry .= qq(<a\nhref="$href">$name</a>) . (' ' x $pad);
-                $mtime = strftime('%Y-%m-%d %k:%M', gmtime($mtime));
-                $entry .= $mtime . $hsize;
+                $entry = qq(\n<a\nhref="$href">$name</a>) .
+                                (' ' x $pad) .
+                                strftime('%Y-%m-%d %k:%M', gmtime($mtime)) .
+                                sprintf('% 8s', $hsize);
         }
 
         # filter out '.gz' files as long as the mtime matches the
@@ -309,17 +308,16 @@ sub dir_response ($$$) {
         delete(@other{keys %want_gz});
         @entries = ((map { ${$dirs{$_}} } sort keys %dirs),
                         (map { ${$other{$_}} } sort keys %other));
-
         my $path_info_html = ascii_html($path_info);
-        my $h = [qw(Content-Type text/html Content-Length), undef];
-        my $gzf = gzf_maybe($h, $env);
-        $gzf->zmore("<html><head><title>Index of $path_info_html</title>" .
-                ${$self->{style}} .
-                "</head><body><pre>Index of $path_info_html</pre><hr><pre>\n");
-        $gzf->zmore(join("\n", @entries));
-        my $out = $gzf->zflush("</pre><hr></body></html>\n");
-        $h->[3] = length($out);
-        [ 200, $h, [ $out ] ]
+        my @h = qw(Content-Type text/html);
+        my $gzf = gzf_maybe(\@h, $env);
+        print { $gzf->zfh } '<html><head><title>Index of ', $path_info_html,
+                '</title>', ${$self->{style}}, '</head><body><pre>Index of ',
+                $path_info_html, '</pre><hr><pre>', @entries,
+                '</pre><hr></body></html>';
+        my $out = $gzf->zflush;
+        push @h, 'Content-Length', length($out);
+        [ 200, \@h, [ $out ] ]
 }
 
 sub call { # PSGI app endpoint
diff --git a/lib/PublicInbox/WwwStream.pm b/lib/PublicInbox/WwwStream.pm
index aee78170..f5b4df9f 100644
--- a/lib/PublicInbox/WwwStream.pm
+++ b/lib/PublicInbox/WwwStream.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2016-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 #
 # HTML body stream for which yields getline+close methods for
@@ -18,7 +18,7 @@ https://public-inbox.org/public-inbox.git) ];
 
 sub base_url ($) {
         my $ctx = shift;
-        my $base_url = $ctx->{ibx}->base_url($ctx->{env});
+        my $base_url = ($ctx->{ibx} // $ctx->{git})->base_url($ctx->{env});
         chop $base_url; # no trailing slash for clone
         $base_url;
 }
@@ -27,6 +27,9 @@ sub init {
         my ($ctx, $cb) = @_;
         $ctx->{cb} = $cb;
         $ctx->{base_url} = base_url($ctx);
+        $ctx->{-res_hdr} = [ 'Content-Type' => 'text/html; charset=UTF-8' ];
+        $ctx->{gz} = PublicInbox::GzipFilter::gz_or_noop($ctx->{-res_hdr},
+                                                        $ctx->{env});
         bless $ctx, __PACKAGE__;
 }
 
@@ -37,7 +40,7 @@ sub async_eml { # for async_blob_cb
 
 sub html_top ($) {
         my ($ctx) = @_;
-        my $ibx = $ctx->{ibx};
+        my $ibx = $ctx->{ibx} // $ctx->{git};
         my $desc = ascii_html($ibx->description);
         my $title = delete($ctx->{-title_html}) // $desc;
         my $upfx = $ctx->{-upfx} || '';
@@ -81,27 +84,27 @@ sub html_top ($) {
                 '</head><body>'. $top . (delete($ctx->{-html_tip}) // '');
 }
 
+sub inboxes { () } # TODO
+
 sub coderepos ($) {
         my ($ctx) = @_;
+        $ctx->{ibx} // return inboxes($ctx);
         my $cr = $ctx->{ibx}->{coderepo} // return ();
         my $cfg = $ctx->{www}->{pi_cfg};
         my $upfx = ($ctx->{-upfx} // ''). '../';
-        my @ret;
-        for my $cr_name (@$cr) {
-                $ret[0] //= <<EOF;
-<a id=code>Code repositories for project(s) associated with this inbox:
-EOF
-                my $urls = $cfg->get_all("coderepo.$cr_name.cgiturl");
-                if ($urls) {
-                        for (@$urls) {
-                                # relative or absolute URL?, prefix relative
-                                # "foo.git" with appropriate number of "../"
-                                my $u = m!\A(?:[a-z\+]+:)?//! ? $_ : $upfx.$_;
-                                $u = ascii_html(prurl($ctx->{env}, $u));
-                                $ret[0] .= qq(\n\t<a\nhref="$u">$u</a>);
-                        }
-                } else {
-                        $ret[0] .= qq[\n\t$cr_name.git (no URL configured)];
+        my $pfx = $ctx->{base_url} //= $ctx->base_url;
+        my $up = $upfx =~ tr!/!/!;
+        $pfx =~ s!/[^/]+\z!/! for (1..$up);
+        my @ret = ('<a id=code>' .
+                'Code repositories for project(s) associated with this '.
+                $ctx->{ibx}->thing_type . "\n");
+        my $objs = $cfg->repo_objs($ctx->{ibx});
+        for my $git (@$objs) {
+                my @urls = $git->pub_urls($ctx->{env});
+                for (@urls) {
+                        my $u = m!\A(?:[a-z\+]+:)?//! ? $_ : $pfx.$_;
+                        $u = ascii_html(prurl($ctx->{env}, $u));
+                        $ret[0] .= qq(\n\t<a\nhref="$u">$u</a>);
                 }
         }
         @ret; # may be empty, this sub is called as an arg for join()
@@ -111,8 +114,8 @@ sub _html_end {
         my ($ctx) = @_;
         my $upfx = $ctx->{-upfx} || '';
         my $m = "${upfx}_/text/mirror/";
-        my $x;
-        if ($ctx->{ibx}->can('cloneurl')) {
+        my $x = '';
+        if ($ctx->{ibx} && $ctx->{ibx}->can('cloneurl')) {
                 $x = <<EOF;
 This is a public inbox, see <a
 href="$m">mirroring instructions</a>
@@ -136,12 +139,15 @@ as well as URLs for IMAP folder(s).
 EOM
                         }
                 }
-        } else {
+        } elsif ($ctx->{ibx}) { # extindex
                 $x = <<EOF;
 This is an external index of several public inboxes,
 see <a href="$m">mirroring instructions</a> on how to clone and mirror
 all data and code used by this external index.
 EOF
+        } elsif ($ctx->{git}) { # coderepo
+                $x = join('', map { "git clone $_\n" }
+                        @{$ctx->{git}->cloneurl($ctx->{env})});
         }
         chomp $x;
         '<hr><pre>'.join("\n\n", coderepos($ctx), $x).'</pre></body></html>'
@@ -164,18 +170,26 @@ sub getline {
         $ctx->zflush(_html_end($ctx));
 }
 
-sub html_oneshot ($$;$) {
-        my ($ctx, $code, $sref) = @_;
+sub html_done ($;@) {
+        my $ctx = $_[0];
+        my $bdy = $ctx->zflush(@_[1..$#_], _html_end($ctx));
+        my $res_hdr = delete $ctx->{-res_hdr};
+        push @$res_hdr, 'Content-Length', length($bdy);
+        [ 200, $res_hdr, [ $bdy ] ]
+}
+
+sub html_oneshot ($$;@) {
+        my ($ctx, $code) = @_[0, 1];
         my $res_hdr = [ 'Content-Type' => 'text/html; charset=UTF-8',
                 'Content-Length' => undef ];
         bless $ctx, __PACKAGE__;
         $ctx->{gz} = PublicInbox::GzipFilter::gz_or_noop($res_hdr, $ctx->{env});
+        my @top;
         $ctx->{base_url} // do {
-                $ctx->zmore(html_top($ctx));
+                @top = html_top($ctx);
                 $ctx->{base_url} = base_url($ctx);
         };
-        $ctx->zmore($$sref) if $sref;
-        my $bdy = $ctx->zflush(_html_end($ctx));
+        my $bdy = $ctx->zflush(@top, @_[2..$#_], _html_end($ctx));
         $res_hdr->[3] = length($bdy);
         [ $code, $res_hdr, [ $bdy ] ]
 }
@@ -195,10 +209,23 @@ sub async_next ($) {
 }
 
 sub aresponse {
-        my ($ctx, $code, $cb) = @_;
-        my $res_hdr = [ 'Content-Type' => 'text/html; charset=UTF-8' ];
+        my ($ctx, $cb) = @_;
         init($ctx, $cb);
-        $ctx->psgi_response($code, $res_hdr);
+        $ctx->psgi_response(200, delete $ctx->{-res_hdr});
+}
+
+sub html_init {
+        my ($ctx) = @_;
+        $ctx->{base_url} = base_url($ctx);
+        my $h = $ctx->{-res_hdr} = ['Content-Type', 'text/html; charset=UTF-8'];
+        $ctx->{gz} = PublicInbox::GzipFilter::gz_or_noop($h, $ctx->{env});
+        bless $ctx, __PACKAGE__;
+        print { $ctx->zfh } html_top($ctx);
+}
+
+sub DESTROY {
+        my ($ctx) = @_;
+        $ctx->{git}->cleanup if $ctx->{git} && $ctx->{git}->{-tmp};
 }
 
 1;
diff --git a/lib/PublicInbox/WwwText.pm b/lib/PublicInbox/WwwText.pm
index 369328ee..224fed5c 100644
--- a/lib/PublicInbox/WwwText.pm
+++ b/lib/PublicInbox/WwwText.pm
@@ -31,16 +31,17 @@ sub get_text {
         my $have_tslash = ($key =~ s!/\z!!) if !$raw;<