From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on dcvr.yhbt.net X-Spam-Level: X-Spam-ASN: X-Spam-Status: No, score=-4.1 required=3.0 tests=ALL_TRUSTED,AWL,BAYES_00, DKIM_SIGNED,DKIM_VALID,DKIM_VALID_AU,DKIM_VALID_EF shortcircuit=no autolearn=ham autolearn_force=no version=3.4.2 Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id 2A9B620178 for ; Mon, 28 Nov 2022 05:32:37 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=80x24.org; s=selector1; t=1669613557; bh=cVV64HhpMb0kGDfByRMZSHr0IzhRf6hdCiqTIs6SpRw=; h=From:To:Subject:Date:In-Reply-To:References:From; b=C55fUXiosiM4c/KL8GNn/EWXpTcB6tgYI7iZUjulohd1nVaZk/hfkVzEVW/SXIeqG 0Q9ejkd/w2TFtQzbzavE7Yuc/+aaI+F0/6kmm7iWwzZTOcU/TEPiHpn9uMjVe0nCHT JVrYShV95dyg0z/dYHK/1TWJbAU7e7Obz50M0hjM= From: Eric Wong To: meta@public-inbox.org Subject: [PATCH 22/95] lei_mirror: support {reference} for v1 manifest clones Date: Mon, 28 Nov 2022 05:31:19 +0000 Message-Id: <20221128053232.291618-23-e@80x24.org> In-Reply-To: <20221128053232.291618-1-e@80x24.org> References: <20221128053232.291618-1-e@80x24.org> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit List-Id: This will be generalized to v2, as well. --- lib/PublicInbox/LeiMirror.pm | 48 +++++++++++++++++++++++++++++++++--- 1 file changed, 45 insertions(+), 3 deletions(-) diff --git a/lib/PublicInbox/LeiMirror.pm b/lib/PublicInbox/LeiMirror.pm index e6b3e32a..b6e12b95 100644 --- a/lib/PublicInbox/LeiMirror.pm +++ b/lib/PublicInbox/LeiMirror.pm @@ -11,6 +11,7 @@ use IO::Compress::Gzip qw(gzip $GzipError); use PublicInbox::Spawn qw(popen_rd spawn run_die); use File::Path (); use File::Temp (); +use File::Spec (); use Fcntl qw(SEEK_SET O_CREAT O_EXCL O_WRONLY); use Carp qw(croak); use URI; @@ -253,6 +254,9 @@ sub clone_v1 { my $dst = $self->{cur_dst} // $self->{dst}; my $fini = PublicInbox::OnDestroy->new($$, \&v1_done, $self); my $cmd = [ @$pfx, clone_cmd($lei, my $opt = {}), "$uri", $dst ]; + my $ref = $self->{-ent} ? $self->{-ent}->{reference} : undef; + defined($ref) && -e "$self->{dst}$ref" and + push @$cmd, '--reference', "$self->{dst}$ref"; start_clone($self, $cmd, $opt, $fini); _get_txt_start($self, '_/text/config/raw', $fini); @@ -354,6 +358,17 @@ sub v1_done { # called via OnDestroy if (defined(my $o = $self->{-ent} ? $self->{-ent}->{owner} : undef)) { run_die([qw(git config -f), "$dst/config", 'gitweb.owner', $o]); } + my $o = "$dst/objects"; + if (open(my $fh, '<', "$o/info/alternates")) { + chomp(my @l = <$fh>); + for (@l) { $_ = File::Spec->abs2rel($_, $o)."\n" } + my $f = File::Temp->new(TEMPLATE => '.XXXX', DIR => "$o/info"); + print $f @l; + $f->flush or die "flush($f): $!"; + rename($f->filename, "$o/info/alternates") or + die "rename($f, $o/info/alternates): $!"; + $f->unlink_on_destroy(0); + } write_makefile($dst, 1); index_cloned_inbox($self, 1); } @@ -510,6 +525,30 @@ sub multi_inbox ($$$) { ($path_pfx, $n, $ret); } +sub clone_all { + my ($self, $todo, $m) = @_; + # handle no-dependency repos, first + for (@{delete($todo->{''}) // []}) { + clone_v1($_, 1); + return if $self->{lei}->{child_error}; + } + # resolve references, deepest, first: + while (scalar keys %$todo) { + for my $x (keys %$todo) { + # resolve multi-level references + while (defined($m->{$x}->{reference})) { + $x = $m->{$x}->{reference}; + } + my $y = delete $todo->{$x} // next; # already done + for (@$y) { + clone_v1($_, 1); + return if $self->{lei}->{child_error}; + } + last; # restart %$todo iteration + } + } +} + # FIXME: this gets confused by single inbox instance w/ global manifest.js.gz sub try_manifest { my ($self) = @_; @@ -566,9 +605,9 @@ EOM my $p = $path_pfx.$path; chop($p) if substr($p, -1, 1) eq '/'; $uri->path($p); + my $todo = {}; + my %want = map { $_ => 1 } @$v1; for my $name (@$v1) { - return if $self->{lei}->{child_error}; - my $task = bless { %$self }, __PACKAGE__; $task->{-ent} = $m->{$name} // die("BUG: no `$name' in manifest"); @@ -582,8 +621,11 @@ EOM E: `$task->{cur_dst}' must not contain newline EOM $task->{cur_src} .= '/'; - clone_v1($task, 1); + my $dep = $task->{-ent}->{reference} // ''; + $dep = '' if !$want{$dep}; + push @{$todo->{$dep}}, $task; } + clone_all($self, $todo, $m); } reap_live() while keys(%LIVE); return if $self->{lei}->{child_error} || $self->{dry_run};