From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on dcvr.yhbt.net X-Spam-Level: X-Spam-ASN: X-Spam-Status: No, score=-4.0 required=3.0 tests=ALL_TRUSTED,AWL,BAYES_00 shortcircuit=no autolearn=ham autolearn_force=no version=3.4.2 Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id 19AEE1F97E for ; Sun, 9 Jun 2019 02:51:48 +0000 (UTC) From: "Eric Wong (Contractor, The Linux Foundation)" To: meta@public-inbox.org Subject: [PATCH 03/11] import: switch to "replace_oids" interface for purge Date: Sun, 9 Jun 2019 02:51:39 +0000 Message-Id: <20190609025147.24966-4-e@80x24.org> In-Reply-To: <20190609025147.24966-1-e@80x24.org> References: <20190609025147.24966-1-e@80x24.org> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit List-Id: Continuing the work by Eric Biederman in commit a118d58a402bd31b ("Import.pm: When purging replace a purged file with a zero length file"), we can use a generic OID replacement mechanism to implement purge. --- lib/PublicInbox/Import.pm | 33 +++++++++++++++++++-------------- lib/PublicInbox/V2Writable.pm | 6 +++--- 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/lib/PublicInbox/Import.pm b/lib/PublicInbox/Import.pm index 6ee1935..2c8fe84 100644 --- a/lib/PublicInbox/Import.pm +++ b/lib/PublicInbox/Import.pm @@ -501,16 +501,16 @@ sub clean_purge_buffer { } } -sub purge_oids { - my ($self, $purge) = @_; - my $tmp = "refs/heads/purge-".((keys %$purge)[0]); +sub replace_oids { + my ($self, $replace) = @_; # oid => raw string + my $tmp = "refs/heads/replace-".((keys %$replace)[0]); my $old = $self->{'ref'}; my $git = $self->{git}; my @export = (qw(fast-export --no-data --use-done-feature), $old); my $rd = $git->popen(@export); my ($r, $w) = $self->gfi_start; my @buf; - my $npurge = 0; + my $nreplace = 0; my @oids; my ($done, $mark); my $tree = $self->{-tree}; @@ -533,10 +533,13 @@ sub purge_oids { } elsif (/^M 100644 ([a-f0-9]+) (\w+)/) { my ($oid, $path) = ($1, $2); $tree->{$path} = 1; - if ($purge->{$oid}) { + my $sref = $replace->{$oid}; + if (defined $sref) { push @oids, $oid; - my $cmd = "M 100644 inline $path\ndata 0\n\n"; - push @buf, $cmd; + my $n = length($$sref); + push @buf, "M 100644 inline $path\ndata $n\n"; + push @buf, $$sref; # hope CoW works... + push @buf, "\n"; } else { push @buf, $_; } @@ -549,7 +552,7 @@ sub purge_oids { $out =~ s/^/# /sgm; warn "purge rewriting\n", $out, "\n"; clean_purge_buffer(\@oids, \@buf); - $npurge++; + $nreplace++; } $w->print(@buf, "\n") or wfail; @buf = (); @@ -567,28 +570,30 @@ sub purge_oids { $w->print(@buf) or wfail; } die 'done\n not seen from fast-export' unless $done; - chomp(my $cmt = $self->get_mark(":$mark")) if $npurge; + chomp(my $cmt = $self->get_mark(":$mark")) if $nreplace; $self->{nchg} = 0; # prevent _update_git_info until update-ref: $self->done; my @git = ('git', "--git-dir=$git->{git_dir}"); - run_die([@git, qw(update-ref), $old, $tmp]) if $npurge; + run_die([@git, qw(update-ref), $old, $tmp]) if $nreplace; run_die([@git, qw(update-ref -d), $tmp]); - return if $npurge == 0; + return if $nreplace == 0; run_die([@git, qw(-c gc.reflogExpire=now gc --prune=all)]); + + # check that old OIDs are gone my $err = 0; - foreach my $oid (keys %$purge) { + foreach my $oid (keys %$replace) { my @info = $git->check($oid); if (@info) { - warn "$oid not purged\n"; + warn "$oid not replaced\n"; $err++; } } _update_git_info($self, 0); - die "Failed to purge $err object(s)\n" if $err; + die "Failed to replace $err object(s)\n" if $err; $cmt; } diff --git a/lib/PublicInbox/V2Writable.pm b/lib/PublicInbox/V2Writable.pm index a435814..d6f72b0 100644 --- a/lib/PublicInbox/V2Writable.pm +++ b/lib/PublicInbox/V2Writable.pm @@ -298,7 +298,7 @@ sub idx_init { } sub purge_oids ($$) { - my ($self, $purge) = @_; # $purge = { $object_id => 1, ... } + my ($self, $purge) = @_; # $purge = { $object_id => \'', ... } $self->done; my $pfx = "$self->{-inbox}->{mainrepo}/git"; my $purges = []; @@ -313,7 +313,7 @@ sub purge_oids ($$) { -d $git_dir or next; my $git = PublicInbox::Git->new($git_dir); my $im = $self->import_init($git, 0, 1); - $purges->[$i] = $im->purge_oids($purge); + $purges->[$i] = $im->replace_oids($purge); $im->done; } $purges; @@ -386,7 +386,7 @@ sub remove_internal ($$$$) { $removed = $smsg; my $oid = $smsg->{blob}; if ($purge) { - $purge->{$oid} = 1; + $purge->{$oid} = \''; } else { ($mark, undef) = $im->remove($orig, $cmt_msg); } -- EW