From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on dcvr.yhbt.net X-Spam-Level: X-Spam-ASN: X-Spam-Status: No, score=-4.0 required=3.0 tests=ALL_TRUSTED,AWL,BAYES_00, DKIM_SIGNED,DKIM_VALID,DKIM_VALID_AU,DKIM_VALID_EF shortcircuit=no autolearn=ham autolearn_force=no version=3.4.2 Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id 16D6A203B8 for ; Mon, 28 Nov 2022 05:32:47 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=80x24.org; s=selector1; t=1669613567; bh=iFK0C79CyBtuvQssT3SnQuUStn9GcrLaP7SaHEVUkPY=; h=From:To:Subject:Date:In-Reply-To:References:From; b=O9eae3oVmIewbsq67zZ985jWZNqnUtswC2bMmwK2JFv1LZPHj8As5YXJo/SfdAaJa vYMgO/67zJE5vbpR1SOQ+k8zjLK+l1ACT7IC9eGzpalSVzFJKp+NScxE29/mJ6GJCQ dLFBhHzX6X9ZIQzMjXo2RPiAGGoKkDSHqpfI+o+4= From: Eric Wong To: meta@public-inbox.org Subject: [PATCH 68/95] lei_mirror: support resuming multi-repo clones Date: Mon, 28 Nov 2022 05:32:05 +0000 Message-Id: <20221128053232.291618-69-e@80x24.org> In-Reply-To: <20221128053232.291618-1-e@80x24.org> References: <20221128053232.291618-1-e@80x24.org> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit List-Id: This is actually a combination of clone and fetch, and I don't think `public-inbox-fetch' will be used to update multiple git repos (inbox or not). Our use of `git update-ref --stdin -z' was broken for incremental updates, but now fixed to properly NUL-terminate commands. --- lib/PublicInbox/LeiMirror.pm | 34 +++++++++++++++++++++++++++------- 1 file changed, 27 insertions(+), 7 deletions(-) diff --git a/lib/PublicInbox/LeiMirror.pm b/lib/PublicInbox/LeiMirror.pm index 8b276336..f2111dd2 100644 --- a/lib/PublicInbox/LeiMirror.pm +++ b/lib/PublicInbox/LeiMirror.pm @@ -256,7 +256,7 @@ sub run_reap { $ret; } -sub start_clone { +sub start_cmd { my ($self, $cmd, $opt, $fini) = @_; do_reap($self); $self->{lei}->qerr("# @$cmd"); @@ -279,9 +279,9 @@ sub fetch_args ($$) { } sub upr { # feed `git update-ref --stdin -z' verbosely - my ($lei, $w, $op, $ref, $oid) = @_; - $lei->qerr("# $op $ref $oid") if $lei->{opt}->{verbose}; - print $w "$op $ref\0$oid\0" or die "print(w): $!"; + my ($lei, $w, $op, @rest) = @_; # ($ref, $oid) = @rest + $lei->qerr("# $op @rest") if $lei->{opt}->{verbose}; + print $w "$op ", join("\0", @rest, '') or die "print(w): $!"; } sub fgrp_update { @@ -306,7 +306,8 @@ sub fgrp_update { my $new = delete $src{$ref}; my $old = $dst{$ref}; if (defined $new) { - upr($lei, $w, 'update', $ref, $new) if $new ne $old; + $new eq $old or + upr($lei, $w, 'update', $ref, $new, $old); } else { upr($lei, $w, 'delete', $ref, $old); } @@ -482,6 +483,23 @@ EOM bless { %$self, -osdir => $dir, -remote => $rn }, __PACKAGE__; } +sub resume_fetch { + my ($self, $uri, $fini) = @_; + my $dst = $self->{cur_dst} // $self->{dst}; + my @git = ('git', "--git-dir=$dst"); + my $opt = +{ map { $_ => $self->{lei}->{$_} } (0..2) }; + my $rn = 'origin'; # configurable? + for ("url=$uri", "fetch=+refs/*:refs/*", 'mirror=true') { + my @kv = split(/=/, $_, 2); + $kv[0] = "remote.$rn.$kv[0]"; + next if $self->{dry_run}; + run_die([@git, 'config', @kv], undef, $opt); + } + my $cmd = [ @{$self->{-torsocks}}, @git, + fetch_args($self->{lei}, $opt), $rn ]; + start_cmd($self, $cmd, $opt, $fini); +} + sub clone_v1 { my ($self, $nohang) = @_; my $lei = $self->{lei}; @@ -495,7 +513,9 @@ sub clone_v1 { if (my $fgrp = forkgroup_prep($self, $uri)) { $fgrp->{-fini} = $fini; push @{$self->{fgrp_todo}->{$fgrp->{-osdir}}}, $fgrp; - } else { # normal fetch + } elsif (-d $dst) { + resume_fetch($self, $uri, $fini); + } else { # normal clone my $cmd = [ @{$self->{-torsocks}}, clone_cmd($lei, my $opt = {}), "$uri", $dst ]; if (defined($self->{-ent})) { @@ -505,7 +525,7 @@ sub clone_v1 { "$self->{dst}$ref"; } } - start_clone($self, $cmd, $opt, $fini); + start_cmd($self, $cmd, $opt, $fini); } if (!$self->{-is_epoch} && $lei->{opt}->{'inbox-config'} =~ /\A(?:always|v1)\z/s) {