new public-inbox-{clone,fetch} commands

Setting up and maintaining git-only mirrors of v2 inboxes is
complex since multiple commands are required to clone and fetch
into epochs.

Unlike grokmirror, these commands do not require any
configuration.  Instead, they rely on existing git config files
and work like "git clone --mirror" and "git fetch",
respectively.

Like grokmirror, they use manifest.js.gz, but only on a
per-inbox basis so users won't have to clone every inbox of a
large instance nor edit config files to include/exclude inboxes
they're interested in.

10 files changed, 475 insertions, 40 deletions

diff --git a/Documentation/public-inbox-clone.pod b/Documentation/public-inbox-clone.pod
new file mode 100644
index 00000000..fdb57663
--- /dev/null
+++ b/Documentation/public-inbox-clone.pod
@@ -0,0 +1,71 @@
+=head1 NAME
+
+public-inbox-clone - "git clone --mirror" wrapper
+
+=head1 SYNOPSIS
+
+public-inbox-clone INBOX_URL [INBOX_DIR]
+
+=head1 DESCRIPTION
+
+public-inbox-clone is a wrapper around C<git clone --mirror> for
+making the initial clone of a remote HTTP(S) public-inbox.  It
+allows cloning multi-epoch v2 inboxes with a single command and
+zero configuration.
+
+It does not run L<public-inbox-init(1)> nor
+L<public-inbox-index(1)>.  Those commands must be run separately
+if serving/searching the mirror is required.  As-is,
+public-inbox-clone is suitable for creating a git-only backup.
+
+public-inbox-clone does not use nor require any extra
+configuration files (not even C<~/.public-inbox/config>).
+
+L<public-inbox-fetch(1)> may be used to keep C<INBOX_DIR>
+up-to-date.
+
+For v2 inboxes, it will create a C<$INBOX_DIR/manifest.js.gz>
+file to speed up subsequent L<public-inbox-fetch(1)>.
+
+=head1 OPTIONS
+
+=over
+
+=item -q
+
+=item --quiet
+
+Quiets down progress messages, also passed to L<git-fetch(1)>.
+
+=item -v
+
+=item --verbose
+
+Increases verbosity, also passed to L<git-fetch(1)>.
+
+=item --torsocks=auto|no|yes
+
+=item --no-torsocks
+
+Whether to wrap L<git(1)> and L<curl(1)> commands with torsocks.
+
+Default: C<auto>
+
+=back
+
+=head1 CONTACT
+
+Feedback welcome via plain-text mail to L<mailto:meta@public-inbox.org>
+
+The mail archives are hosted at L<https://public-inbox.org/meta/> and
+L<http://4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/meta/>
+
+=head1 COPYRIGHT
+
+Copyright all contributors L<mailto:meta@public-inbox.org>
+
+License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>
+
+=head1 SEE ALSO
+
+L<public-inbox-fetch(1)>, L<public-inbox-init(1)>, L<public-inbox-index(1)>
diff --git a/Documentation/public-inbox-fetch.pod b/Documentation/public-inbox-fetch.pod
new file mode 100644
index 00000000..7944fdcd
--- /dev/null
+++ b/Documentation/public-inbox-fetch.pod
@@ -0,0 +1,63 @@
+=head1 NAME
+
+public-inbox-fetch - "git fetch" wrapper for v2 inbox mirrors
+
+=head1 SYNOPSIS
+
+public-inbox-fetch -C INBOX_DIR
+
+=head1 DESCRIPTION
+
+public-inbox-fetch updates git storage of public-inbox mirrors.
+With v2 inboxes, it allows detection of new epochs and avoids
+unnecessary traffic on old epochs.
+
+public-inbox-fetch does not use nor require any configuration
+files of its own.
+
+It does not run L<public-inbox-index(1)>, making it suitable
+for maintaining git-only backups.
+
+For v2 inboxes, it will maintain C<$INBOX_DIR/manifest.js.gz>
+file to speed up future invocations.
+
+=head1 OPTIONS
+
+=over
+
+=item -q
+
+=item --quiet
+
+Quiets down progress messages, also passed to L<git-fetch(1)>.
+
+=item -v
+
+=item --verbose
+
+Increases verbosity, also passed to L<git-fetch(1)>.
+
+=item --torsocks=auto|no|yes
+
+=item --no-torsocks
+
+Whether to wrap L<git(1)> and L<curl(1)> commands with torsocks.
+
+Default: C<auto>
+
+=head1 CONTACT
+
+Feedback welcome via plain-text mail to L<mailto:meta@public-inbox.org>
+
+The mail archives are hosted at L<https://public-inbox.org/meta/> and
+L<http://4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/meta/>
+
+=head1 COPYRIGHT
+
+Copyright all contributors L<mailto:meta@public-inbox.org>
+
+License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>
+
+=head1 SEE ALSO
+
+L<public-inbox-index(1)>
diff --git a/MANIFEST b/MANIFEST
index c64f7d94..a1450880 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -51,6 +51,7 @@ Documentation/lei.pod
 Documentation/lei_design_notes.txt
 Documentation/marketing.txt
 Documentation/mknews.perl
+Documentation/public-inbox-clone.pod
 Documentation/public-inbox-compact.pod
 Documentation/public-inbox-config.pod
 Documentation/public-inbox-convert.pod
@@ -58,6 +59,7 @@ Documentation/public-inbox-daemon.pod
 Documentation/public-inbox-edit.pod
 Documentation/public-inbox-extindex-format.pod
 Documentation/public-inbox-extindex.pod
+Documentation/public-inbox-fetch.pod
 Documentation/public-inbox-glossary.pod
 Documentation/public-inbox-httpd.pod
 Documentation/public-inbox-imapd.pod
@@ -163,6 +165,7 @@ lib/PublicInbox/ExtSearchIdx.pm
 lib/PublicInbox/FakeImport.pm
 lib/PublicInbox/FakeInotify.pm
 lib/PublicInbox/Feed.pm
+lib/PublicInbox/Fetch.pm
 lib/PublicInbox/Filter/Base.pm
 lib/PublicInbox/Filter/Gmane.pm
 lib/PublicInbox/Filter/Mirror.pm
@@ -329,10 +332,12 @@ sa_config/README
 sa_config/root/etc/spamassassin/public-inbox.pre
 sa_config/user/.spamassassin/user_prefs
 script/lei
+script/public-inbox-clone
 script/public-inbox-compact
 script/public-inbox-convert
 script/public-inbox-edit
 script/public-inbox-extindex
+script/public-inbox-fetch
 script/public-inbox-httpd
 script/public-inbox-imapd
 script/public-inbox-index
diff --git a/lib/PublicInbox/Admin.pm b/lib/PublicInbox/Admin.pm
index 2534958b..9ff59bca 100644
--- a/lib/PublicInbox/Admin.pm
+++ b/lib/PublicInbox/Admin.pm
@@ -372,4 +372,12 @@ sub index_prepare ($$) {
         $env;
 }
 
+sub do_chdir ($) {
+        my $chdir = $_[0] // return;
+        for my $d (@$chdir) {
+                next if $d eq ''; # same as git(1)
+                chdir $d or die "cd $d: $!";
+        }
+}
+
 1;
diff --git a/lib/PublicInbox/Fetch.pm b/lib/PublicInbox/Fetch.pm
new file mode 100644
index 00000000..d795731c
--- /dev/null
+++ b/lib/PublicInbox/Fetch.pm
@@ -0,0 +1,145 @@
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+# Wrapper to "git fetch" remote public-inboxes
+package PublicInbox::Fetch;
+use strict;
+use v5.10.1;
+use parent qw(PublicInbox::IPC);
+use URI ();
+use PublicInbox::Spawn qw(popen_rd);
+use PublicInbox::Admin;
+use PublicInbox::LEI;
+use PublicInbox::LeiCurl;
+use PublicInbox::LeiMirror;
+use IO::Uncompress::Gunzip qw(gunzip $GunzipError);
+use File::Temp ();
+
+sub new { bless {}, __PACKAGE__ }
+
+sub fetch_cmd ($$) {
+        my ($lei, $opt) = @_;
+        my @cmd = qw(git);
+        $opt->{$_} = $lei->{$_} for (0..2);
+        # we support "-c $key=$val" for arbitrary git config options
+        # e.g.: git -c http.proxy=socks5h://127.0.0.1:9050
+        push(@cmd, '-c', $_) for @{$lei->{opt}->{c} // []};
+        push @cmd, 'fetch';
+        push @cmd, '-q' if $lei->{opt}->{quiet};
+        push @cmd, '-v' if $lei->{opt}->{verbose};
+        @cmd;
+}
+
+sub remote_url ($$) {
+        my ($lei, $dir) = @_; # TODO: support non-"origin"?
+        my $cmd = [ qw(git config remote.origin.url) ];
+        my $fh = popen_rd($cmd, undef, { -C => $dir, 2 => $lei->{2} });
+        my $url = <$fh>;
+        close $fh or return;
+        chomp $url;
+        $url;
+}
+
+sub do_fetch {
+        my ($cls, $lei, $cd) = @_;
+        my $ibx_ver;
+        my $curl = PublicInbox::LeiCurl->new($lei) or return;
+        my $dir = PublicInbox::Admin::resolve_inboxdir($cd, \$ibx_ver);
+        if ($ibx_ver == 1) {
+                my $url = remote_url($lei, $dir) //
+                        die "E: $dir missing remote.origin.url\n";
+                my $uri = URI->new($url);
+                my $torsocks = $curl->torsocks($lei, $uri);
+                my $opt = { -C => $dir };
+                my $cmd = [ @$torsocks, fetch_cmd($lei, $opt) ];
+                my $cerr = PublicInbox::LeiMirror::run_reap($lei, $cmd, $opt);
+                $lei->child_error($cerr, "@$cmd failed") if $cerr;
+                return;
+        }
+        # v2:
+        opendir my $dh, "$dir/git" or die "opendir $dir/git: $!";
+        my @epochs = sort { $b <=> $a } map { substr($_, 0, -4) + 0 }
+                                grep(/\A[0-9]+\.git\z/, readdir($dh));
+        my ($git_url, $epoch);
+        for my $nr (@epochs) { # try newest epoch, first
+                my $edir = "$dir/git/$nr.git";
+                if (defined(my $url = remote_url($lei, $edir))) {
+                        $git_url = $url;
+                        $epoch = $nr;
+                        last;
+                } else {
+                        warn "W: $edir missing remote.origin.url\n";
+                }
+        }
+        $git_url or die "Unable to determine git URL\n";
+        my $inbox_url = $git_url;
+        $inbox_url =~ s!/git/$epoch(?:\.git)?/?\z!! or
+                $inbox_url =~ s!/$epoch(?:\.git)?/?\z!! or die <<EOM;
+Unable to infer inbox URL from <$git_url>
+EOM
+        $lei->qerr("# inbox URL: $inbox_url/");
+        my $muri = URI->new("$inbox_url/manifest.js.gz");
+        my $ft = File::Temp->new(TEMPLATE => 'manifest-XXXX',
+                                UNLINK => 1, DIR => $dir);
+        my $fn = $ft->filename;
+        my @opt = (qw(-R -o), $fn);
+        my $mf = "$dir/manifest.js.gz";
+        my $m0; # current manifest.js.gz contents
+        if (open my $fh, '<', $mf) {
+                $m0 = eval {
+                        PublicInbox::LeiMirror::decode_manifest($fh, $mf, $mf)
+                };
+                $lei->err($@) if $@;
+                push @opt, '-z', $mf if defined($m0);
+        }
+        my $curl_cmd = $curl->for_uri($lei, $muri, @opt);
+        my $opt = {};
+        $opt->{$_} = $lei->{$_} for (0..2);
+        my $cerr = PublicInbox::LeiMirror::run_reap($lei, $curl_cmd, $opt);
+        return $lei->child_error($cerr, "@$curl_cmd failed") if $cerr;
+        return if !-s $ft; # 304 Not Modified via curl -z
+
+        my $m1 = PublicInbox::LeiMirror::decode_manifest($ft, $fn, $muri);
+        my $mdiff = { %$m1 };
+
+        # filter out unchanged entries
+        while (my ($k, $v0) = each %{$m0 // {}}) {
+                my $cur = $m1->{$k} // next;
+                my $f0 = $v0->{fingerprint} // next;
+                my $f1 = $cur->{fingerprint} // next;
+                my $t0 = $v0->{modified} // next;
+                my $t1 = $cur->{modified} // next;
+                delete($mdiff->{$k}) if $f0 eq $f1 && $t0 == $t1;
+        }
+        my $ibx_uri = URI->new("$inbox_url/");
+        my ($path_pfx, $v1_bare, @v2_epochs) =
+                PublicInbox::LeiMirror::deduce_epochs($mdiff, $ibx_uri->path);
+        defined($v1_bare) and die <<EOM;
+E: got v1 `$v1_bare' when expecting v2 epoch(s) in <$muri>, WTF?
+EOM
+        my @epoch_nr = sort { $a <=> $b }
+                map { my ($nr) = (m!/([0-9]+)\.git\z!g) } @v2_epochs;
+
+        # n.b. this expects all epochs are from the same host
+        my $torsocks = $curl->torsocks($lei, $muri);
+        for my $nr (@epoch_nr) {
+                my $dir = "$dir/git/$nr.git";
+                my $cmd;
+                my $opt = {};
+                if (-d $dir) {
+                        $opt->{-C} = $dir;
+                        $cmd = [ @$torsocks, fetch_cmd($lei, $opt) ];
+                } else {
+                        my $e_uri = $ibx_uri->clone;
+                        $e_uri->path($ibx_uri->path."git/$nr.git");
+                        $cmd = [ @$torsocks,
+                                PublicInbox::LeiMirror::clone_cmd($lei, $opt),
+                                $$e_uri, $dir ];
+                }
+                my $cerr = PublicInbox::LeiMirror::run_reap($lei, $cmd, $opt);
+                return $lei->child_error($cerr, "@$cmd failed") if $cerr;
+        }
+        rename($fn, $mf) or die "E: rename($fn, $mf): $!\n";
+        $ft->unlink_on_destroy(0);
+}
+
+1;
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index aff2bf19..6d5d3c03 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -468,6 +468,8 @@ sub x_it ($$) {
                 $self->{pkt_op_p}->pkt_do('x_it', $code);
         } elsif ($self->{sock}) { # to lei(1) client
                 send($self->{sock}, "x_it $code", MSG_EOR);
+        } elsif ($quit == \&CORE::exit) { # an admin command
+                exit($code >> 8);
         } # else ignore if client disconnected
 }
 
@@ -511,7 +513,7 @@ sub fail ($$;$) {
         my ($self, $buf, $exit_code) = @_;
         $self->{failed}++;
         err($self, $buf) if defined $buf;
-        # calls fail_handler:
+        # calls fail_handler
         $self->{pkt_op_p}->pkt_do('!') if $self->{pkt_op_p};
         x_it($self, ($exit_code // 1) << 8);
         undef;
@@ -536,6 +538,8 @@ sub child_error { # passes non-fatal curl exit codes to user
                 $self->{pkt_op_p}->pkt_do('child_error', $child_error);
         } elsif ($self->{sock}) { # to lei(1) client
                 send($self->{sock}, "child_error $child_error", MSG_EOR);
+        } else { # non-lei admin command
+                $self->{child_error} ||= $child_error;
         } # else noop if client disconnected
 }
 
diff --git a/lib/PublicInbox/LeiMirror.pm b/lib/PublicInbox/LeiMirror.pm
index 355813bd..c128d13d 100644
--- a/lib/PublicInbox/LeiMirror.pm
+++ b/lib/PublicInbox/LeiMirror.pm
@@ -8,6 +8,7 @@ use v5.10.1;
 use parent qw(PublicInbox::IPC);
 use IO::Uncompress::Gunzip qw(gunzip $GunzipError);
 use PublicInbox::Spawn qw(popen_rd spawn);
+use File::Temp ();
 
 sub do_finish_mirror { # dwaitpid callback
         my ($arg, $pid) = @_;
@@ -18,7 +19,9 @@ sub do_finish_mirror { # dwaitpid callback
         } elsif (!unlink($f)) {
                 $lei->err("unlink($f): $!") unless $!{ENOENT};
         } else {
-                $lei->add_external_finish($mrr->{dst});
+                if ($lei->{cmd} ne 'public-inbox-clone') {
+                        $lei->add_external_finish($mrr->{dst});
+                }
                 $lei->qerr("# mirrored $mrr->{src} => $mrr->{dst}");
         }
         $lei->dclose;
@@ -121,33 +124,38 @@ sub _try_config {
 
 sub index_cloned_inbox {
         my ($self, $iv) = @_;
-        my $ibx = delete($self->{ibx}) // {
-                address => [ 'lei@example.com' ],
-                version => $iv,
-        };
-        $ibx->{inboxdir} = $self->{dst};
-        PublicInbox::Inbox->new($ibx);
-        PublicInbox::InboxWritable->new($ibx);
-        my $opt = {};
         my $lei = $self->{lei};
-        for my $sw ($lei->index_opt) {
-                my ($k) = ($sw =~ /\A([\w-]+)/);
-                $opt->{$k} = $lei->{opt}->{$k};
+
+        # n.b. public-inbox-clone works w/o (SQLite || Xapian)
+        # lei is useless without Xapian + SQLite
+        if ($lei->{cmd} ne 'public-inbox-clone') {
+                my $ibx = delete($self->{ibx}) // {
+                        address => [ 'lei@example.com' ],
+                        version => $iv,
+                };
+                $ibx->{inboxdir} = $self->{dst};
+                PublicInbox::Inbox->new($ibx);
+                PublicInbox::InboxWritable->new($ibx);
+                my $opt = {};
+                for my $sw ($lei->index_opt) {
+                        my ($k) = ($sw =~ /\A([\w-]+)/);
+                        $opt->{$k} = $lei->{opt}->{$k};
+                }
+                # force synchronous dwaitpid for v2:
+                local $PublicInbox::DS::in_loop = 0;
+                my $cfg = PublicInbox::Config->new(undef, $lei->{2});
+                my $env = PublicInbox::Admin::index_prepare($opt, $cfg);
+                local %ENV = (%ENV, %$env) if $env;
+                PublicInbox::Admin::progress_prepare($opt, $lei->{2});
+                PublicInbox::Admin::index_inbox($ibx, undef, $opt);
         }
-        # force synchronous dwaitpid for v2:
-        local $PublicInbox::DS::in_loop = 0;
-        my $cfg = PublicInbox::Config->new(undef, $lei->{2});
-        my $env = PublicInbox::Admin::index_prepare($opt, $cfg);
-        local %ENV = (%ENV, %$env) if $env;
-        PublicInbox::Admin::progress_prepare($opt, $lei->{2});
-        PublicInbox::Admin::index_inbox($ibx, undef, $opt);
         open my $x, '>', "$self->{dst}/mirror.done"; # for do_finish_mirror
 }
 
 sub run_reap {
         my ($lei, $cmd, $opt) = @_;
         $lei->qerr("# @$cmd");
-        $opt->{pgid} = 0;
+        $opt->{pgid} = 0 if $lei->{sock};
         my $pid = spawn($cmd, undef, $opt);
         my $reap = PublicInbox::OnDestroy->new($lei->can('sigint_reap'), $pid);
         waitpid($pid, 0) == $pid or die "waitpid @$cmd: $!";
@@ -205,6 +213,7 @@ sub deduce_epochs ($$) {
         my ($m, $path) = @_;
         my ($v1_bare, @v2_epochs);
         my $path_pfx = '';
+        $path =~ s!/+\z!!;
         do {
                 $v1_bare = $m->{$path};
                 @v2_epochs = grep(m!\A\Q$path\E/git/[0-9]+\.git\z!, keys %$m);
@@ -213,6 +222,18 @@ sub deduce_epochs ($$) {
         ($path_pfx, $v1_bare, @v2_epochs);
 }
 
+sub decode_manifest ($$$) {
+        my ($fh, $fn, $uri) = @_;
+        my $js;
+        my $gz = do { local $/; <$fh> } // die "slurp($fn): $!";
+        gunzip(\$gz => \$js, MultiStream => 1) or
+                die "gunzip($uri): $GunzipError\n";
+        my $m = eval { PublicInbox::Config->json->decode($js) };
+        die "$uri: error decoding `$js': $@\n" if $@;
+        ref($m) eq 'HASH' or die "$uri unknown type: ".ref($m);
+        $m;
+}
+
 sub try_manifest {
         my ($self) = @_;
         my $uri = URI->new($self->{src});
@@ -221,26 +242,19 @@ sub try_manifest {
         my $path = $uri->path;
         chop($path) eq '/' or die "BUG: $uri not canonicalized";
         $uri->path($path . '/manifest.js.gz');
-        my $cmd = $curl->for_uri($lei, $uri);
-        $lei->qerr("# @$cmd");
-        my $opt = { 0 => $lei->{0}, 2 => $lei->{2} };
-        my ($fh, $pid) = popen_rd($cmd, undef, $opt);
-        my $reap = PublicInbox::OnDestroy->new($lei->can('sigint_reap'), $pid);
-        my $gz = do { local $/; <$fh> } // die "read(curl $uri): $!";
-        close $fh;
-        waitpid($pid, 0) == $pid or die "waitpid @$cmd: $!";
-        @$reap = ();
-        if ($?) {
-                return try_scrape($self) if ($? >> 8) == 22; # 404 missing
-                return $lei->child_error($?, "@$cmd failed");
+        my $pdir = $lei->rel2abs($self->{dst});
+        $pdir =~ s!/[^/]+/?\z!!;
+        my $ft = File::Temp->new(TEMPLATE => 'manifest-XXXX',
+                                UNLINK => 1, DIR => $pdir);
+        my $fn = $ft->filename;
+        my $cmd = $curl->for_uri($lei, $uri, '-R', '-o', $fn);
+        my $opt = { 0 => $lei->{0}, 1 => $lei->{1}, 2 => $lei->{2} };
+        my $cerr = run_reap($lei, $cmd, $opt);
+        if ($cerr) {
+                return try_scrape($self) if ($cerr >> 8) == 22; # 404 missing
+                return $lei->child_error($cerr, "@$cmd failed");
         }
-        my $js;
-        gunzip(\$gz => \$js, MultiStream => 1) or
-                die "gunzip($uri): $GunzipError";
-        my $m = eval { PublicInbox::Config->json->decode($js) };
-        die "$uri: error decoding `$js': $@" if $@;
-        ref($m) eq 'HASH' or die "$uri unknown type: ".ref($m);
-
+        my $m = decode_manifest($ft, $fn, $uri);
         my ($path_pfx, $v1_bare, @v2_epochs) = deduce_epochs($m, $path);
         if (@v2_epochs) {
                 # It may be possible to have v1 + v2 in parallel someday:
@@ -254,6 +268,9 @@ EOM
                         $uri->clone
                 } @v2_epochs;
                 clone_v2($self, \@v2_epochs);
+                my $fin = "$self->{dst}/manifest.js.gz";
+                rename($fn, $fin) or die "E: rename($fn, $fin): $!";
+                $ft->unlink_on_destroy(0);
         } elsif (defined $v1_bare) {
                 clone_v1($self);
         } else {
diff --git a/script/public-inbox-clone b/script/public-inbox-clone
new file mode 100755
index 00000000..2b18969f
--- /dev/null
+++ b/script/public-inbox-clone
@@ -0,0 +1,58 @@
+#!perl -w
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+# Wrapper to git clone remote public-inboxes
+use strict;
+use v5.10.1;
+use Getopt::Long qw(:config gnu_getopt no_ignore_case auto_abbrev);
+my $opt = {};
+my $help = <<EOF; # the following should fit w/o scrolling in 80x24 term:
+usage: public-inbox-clone INBOX_URL [DESTINATION]
+
+  clone remote public-inboxes
+
+options:
+
+  --torsocks VAL      whether or not to wrap git and curl commands with
+                      torsocks (default: `auto')
+                      Must be one of: `auto', `no' or `yes'
+  --verbose | -v      increase verbosity (may be repeated)
+    --quiet | -q      increase verbosity (may be repeated)
+    -C DIR            chdir to specified directory
+EOF
+GetOptions($opt, qw(help|h quiet|q verbose|v+ C=s@ c=s@
+                no-torsocks torsocks=s)) or die $help;
+if ($opt->{help}) { print $help; exit };
+require PublicInbox::Admin; # loads Config
+PublicInbox::Admin::do_chdir(delete $opt->{C});
+PublicInbox::Admin::setup_signals();
+$SIG{PIPE} = 'IGNORE';
+
+my ($url, $dst, $extra) = @ARGV;
+die $help if !defined($url) || defined($extra);
+defined($dst) or ($dst) = ($url =~ m!/([^/]+)/?\z!);
+index($dst, "\n") >= 0 and die "`\\n' not allowed in `$dst'";
+
+# n.b. this is still a truckload of code...
+require URI;
+require PublicInbox::LEI;
+require PublicInbox::LeiExternal;
+require PublicInbox::LeiMirror;
+require PublicInbox::LeiCurl;
+require PublicInbox::Lock;
+
+$url = PublicInbox::LeiExternal::ext_canonicalize($url);
+my $lei = bless {
+        env => \%ENV, opt => $opt, cmd => 'public-inbox-clone',
+        0 => *STDIN{GLOB}, 2 => *STDERR{GLOB},
+}, 'PublicInbox::LEI';
+open $lei->{1}, '+<&=', 1 or die "dup: $!";
+open $lei->{3}, '.' or die "open . $!";
+my $mrr = bless {
+        lei => $lei,
+        src => $url,
+        dst => $dst,
+}, 'PublicInbox::LeiMirror';
+$mrr->do_mirror;
+$mrr->can('do_finish_mirror')->([$mrr, $lei], $$);
+exit(($lei->{child_error} // 0) >> 8);
diff --git a/script/public-inbox-fetch b/script/public-inbox-fetch
new file mode 100755
index 00000000..5d303574
--- /dev/null
+++ b/script/public-inbox-fetch
@@ -0,0 +1,35 @@
+#!perl -w
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+# Wrapper to git fetch remote public-inboxes
+use strict;
+use v5.10.1;
+use Getopt::Long qw(:config gnu_getopt no_ignore_case auto_abbrev);
+my $opt = {};
+my $help = <<EOF; # the following should fit w/o scrolling in 80x24 term:
+usage: public-inbox-fetch -C DESTINATION
+
+  fetch remote public-inboxes
+
+options:
+
+  --torsocks VAL      whether or not to wrap git and curl commands with
+                      torsocks (default: `auto')
+                      Must be one of: `auto', `no' or `yes'
+  --verbose | -v      increase verbosity (may be repeated)
+    --quiet | -q      increase verbosity (may be repeated)
+    -C DIR            chdir to specified directory
+EOF
+GetOptions($opt, qw(help|h quiet|q verbose|v+ C=s@ c=s@
+        no-torsocks torsocks=s)) or die $help;
+if ($opt->{help}) { print $help; exit };
+require PublicInbox::Fetch; # loads Admin
+PublicInbox::Admin::do_chdir(delete $opt->{C});
+PublicInbox::Admin::setup_signals();
+$SIG{PIPE} = 'IGNORE';
+
+my $lei = bless {
+        env => \%ENV, opt => $opt, cmd => 'public-inbox-fetch',
+        0 => *STDIN{GLOB}, 1 => *STDOUT{GLOB}, 2 => *STDERR{GLOB},
+}, 'PublicInbox::LEI';
+PublicInbox::Fetch->do_fetch($lei, '.');
diff --git a/t/lei-mirror.t b/t/lei-mirror.t
index a61a7565..75e25b3f 100644
--- a/t/lei-mirror.t
+++ b/t/lei-mirror.t
@@ -46,6 +46,7 @@ test_lei({ tmpdir => $tmpdir }, sub {
 
         lei_ok('add-external', "$t1-pfx", '--mirror', "$http/pfx/t1/",
                         \'--mirror v1 w/ PSGI prefix');
+        ok(!-e "$t1-pfx/mirror.done", 'no leftover mirror.done');
 
         my $d = "$home/404";
         ok(!lei(qw(add-external --mirror), "$http/404", $d), 'mirror 404');
@@ -77,6 +78,34 @@ test_lei({ tmpdir => $tmpdir }, sub {
         } # for
 });
 
+SKIP: {
+        undef $sock;
+        my $d = "$tmpdir/d";
+        mkdir $d or xbail "mkdir $d $!";
+        my $opt = { -C => $d, 2 => \(my $err) };
+        ok(!run_script([qw(-clone -q), "$http/404"], undef, $opt), '404 fails');
+        ok(!-d "$d/404", 'destination not created');
+        delete $opt->{2};
+
+        ok(run_script([qw(-clone -q -C), $d, "$http/t2"], undef, $opt),
+                '-clone succeeds on v2');
+        ok(-d "$d/t2/git/0.git", 'epoch cloned');
+        ok(-f "$d/t2/manifest.js.gz", 'manifest saved');
+        ok(!-e "$d/t2/mirror.done", 'no leftover mirror.done');
+        ok(run_script([qw(-fetch -q -C), "$d/t2"], undef, $opt),
+                '-fetch succeeds w/ manifest.js.gz');
+        unlink("$d/t2/manifest.js.gz") or xbail "unlink $!";
+        ok(run_script([qw(-fetch -q -C), "$d/t2"], undef, $opt),
+                '-fetch succeeds w/o manifest.js.gz');
+
+        ok(run_script([qw(-clone -q -C), $d, "$http/t1"], undef, $opt),
+                'cloning v1 works');
+        ok(-d "$d/t1", 'v1 cloned');
+        ok(!-e "$d/t1/mirror.done", 'no leftover file');
+        ok(run_script([qw(-fetch -q -C), "$d/t1"], undef, $opt),
+                'fetching v1 works');
+}
+
 ok($td->kill, 'killed -httpd');
 $td->join;