lei: add-external --mirror support

This can be useful for users who want to clone and
mirror an existing public-inbox.  This doesn't have
update support, yet, so users will need to run
"git fetch && public-inbox-index" for now.

1 files changed, 65 insertions, 0 deletions

diff --git a/lib/PublicInbox/LeiCurl.pm b/lib/PublicInbox/LeiCurl.pm
new file mode 100644
index 00000000..c8747d4f
--- /dev/null
+++ b/lib/PublicInbox/LeiCurl.pm
@@ -0,0 +1,65 @@
+# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# common option and torsocks(1) wrapping for curl(1)
+package PublicInbox::LeiCurl;
+use strict;
+use v5.10.1;
+use PublicInbox::Spawn qw(which);
+use PublicInbox::Config;
+
+# prepares a common command for curl(1) based on $lei command
+sub new {
+        my ($cls, $lei, $curl) = @_;
+        $curl //= which('curl') // return $lei->fail('curl not found');
+        my $opt = $lei->{opt};
+        my @cmd = ($curl, qw(-Sf));
+        $cmd[-1] .= 's' if $opt->{quiet}; # already the default for "lei q"
+        $cmd[-1] .= 'v' if $opt->{verbose}; # we use ourselves, too
+        for my $o ($lei->curl_opt) {
+                $o =~ s/\|[a-z0-9]\b//i; # remove single char short option
+                if ($o =~ s/=[is]@\z//) {
+                        my $ary = $opt->{$o} or next;
+                        push @cmd, map { ("--$o", $_) } @$ary;
+                } elsif ($o =~ s/=[is]\z//) {
+                        my $val = $opt->{$o} // next;
+                        push @cmd, "--$o", $val;
+                } elsif ($opt->{$o}) {
+                        push @cmd, "--$o";
+                }
+        }
+        push @cmd, '-v' if $opt->{verbose}; # lei uses this itself
+        bless \@cmd, $cls;
+}
+
+sub torsocks { # useful for "git clone" and "git fetch", too
+        my ($self, $lei, $uri)= @_;
+        my $opt = $lei->{opt};
+        $opt->{torsocks} = 'false' if $opt->{'no-torsocks'};
+        my $torsocks = $opt->{torsocks} //= 'auto';
+        if ($torsocks eq 'auto' && substr($uri->host, -6) eq '.onion' &&
+                        (($lei->{env}->{LD_PRELOAD}//'') !~ /torsocks/)) {
+                # "auto" continues anyways if torsocks is missing;
+                # a proxy may be specified via CLI, curlrc,
+                # environment variable, or even firewall rule
+                [ ($lei->{torsocks} //= which('torsocks')) // () ]
+        } elsif (PublicInbox::Config::git_bool($torsocks)) {
+                my $x = $lei->{torsocks} //= which('torsocks');
+                $x or return $lei->fail(<<EOM);
+--torsocks=yes specified but torsocks not found in PATH=$ENV{PATH}
+EOM
+                [ $x ];
+        } else { # the common case for current Internet :<
+                [];
+        }
+}
+
+# completes the result of cmd() for $uri
+sub for_uri {
+        my ($self, $lei, $uri) = @_;
+        my $pfx = torsocks($self, $lei, $uri) or return; # error
+        [ @$pfx, @$self, substr($uri->path, -3) eq '.gz' ? () : '--compressed',
+                $uri->as_string ]
+}
+
+1;