From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on dcvr.yhbt.net X-Spam-Level: X-Spam-ASN: X-Spam-Status: No, score=-4.0 required=3.0 tests=ALL_TRUSTED,BAYES_00 shortcircuit=no autolearn=ham autolearn_force=no version=3.4.0 Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id AF6FE1FAEA; Thu, 29 Mar 2018 10:28:30 +0000 (UTC) From: "Eric Wong (Contractor, The Linux Foundation)" To: meta@public-inbox.org Cc: "Eric Wong (Contractor, The Linux Foundation)" Subject: [PATCH 07/14] v2writable: support purging messages from git entirely Date: Thu, 29 Mar 2018 10:28:12 +0000 Message-Id: <20180329102819.15234-8-e@80x24.org> In-Reply-To: <20180329102819.15234-1-e@80x24.org> References: <20180329102819.15234-1-e@80x24.org> List-Id: Purging existing messages is fairly straightforward since we can take advantage of Xapian and lookup the git object_id with it. Unfortunately, purging an already "removed" message (which is no longer in Xapian) is not as easy and we'll need to expose ->purge_oids to purge by the git object_id (currently SHA-1). Furthermore, we expire reflogs and prune in hopes a dumb HTTP client won't get the object. --- lib/PublicInbox/Import.pm | 90 +++++++++++++++++++++++++++++++++++ lib/PublicInbox/V2Writable.pm | 39 +++++++++++++-- t/v2writable.t | 7 +++ 3 files changed, 131 insertions(+), 5 deletions(-) diff --git a/lib/PublicInbox/Import.pm b/lib/PublicInbox/Import.pm index e07edda..f00c260 100644 --- a/lib/PublicInbox/Import.pm +++ b/lib/PublicInbox/Import.pm @@ -430,6 +430,96 @@ sub digest2mid ($) { $b64 . '@localhost'; } +sub clean_purge_buffer { + my ($oid, $buf) = @_; + my $cmt_msg = "purged $oid\n"; + + foreach my $i (0..$#$buf) { + my $l = $buf->[$i]; + if ($l =~ /^author .* (\d+ [\+-]?\d+)$/) { + $buf->[$i] = "author <> $1\n"; + } elsif ($l =~ /^data (\d+)/) { + $buf->[$i++] = "data " . length($cmt_msg) . "\n"; + $buf->[$i] = $cmt_msg; + last; + } + } +} + +sub purge_oids { + my ($self, $purge) = @_; + my $tmp = "refs/heads/purge-".((keys %$purge)[0]); + my $old = $self->{'ref'}; + my $git = $self->{git}; + my @export = (qw(fast-export --no-data --use-done-feature), $old); + my ($rd, $pid) = $git->popen(@export); + my ($r, $w) = $self->gfi_start; + my @buf; + my $npurge = 0; + while (<$rd>) { + if (/^reset (?:.+)/) { + push @buf, "reset $tmp\n"; + } elsif (/^commit (?:.+)/) { + if (@buf) { + $w->print(@buf) or wfail; + @buf = (); + } + push @buf, "commit $tmp\n"; + } elsif (/^data (\d+)/) { + # only commit message, so $len is small: + my $len = $1; # + 1 for trailing "\n" + push @buf, $_; + my $n = read($rd, my $buf, $len) or die "read: $!"; + $len == $n or die "short read ($n < $len)"; + push @buf, $buf; + } elsif (/^M 100644 ([a-f0-9]+) /) { + my $oid = $1; + if ($purge->{$oid}) { + my $lf = <$rd>; + if ($lf eq "\n") { + my $out = join('', @buf); + $out =~ s/^/# /sgm; + warn "purge rewriting\n", $out, "\n"; + clean_purge_buffer($oid, \@buf); + $out = join('', @buf); + $w->print(@buf, "\n") or wfail; + @buf = (); + $npurge++; + } else { + die "expected LF: $lf\n"; + } + } else { + push @buf, $_; + } + } else { + push @buf, $_; + } + } + if (@buf) { + $w->print(@buf) or wfail; + } + $w = $r = undef; + $self->done; + my @git = ('git', "--git-dir=$git->{git_dir}"); + + run_die([@git, qw(update-ref), $old, $tmp]) if $npurge; + + run_die([@git, qw(update-ref -d), $tmp]); + + return if $npurge == 0; + + run_die([@git, qw(-c gc.reflogExpire=now gc --prune=all)]); + my $err = 0; + foreach my $oid (keys %$purge) { + my @info = $git->check($oid); + if (@info) { + warn "$oid not purged\n"; + $err++; + } + } + die "Failed to purge $err object(s)\n" if $err; +} + 1; __END__ =pod diff --git a/lib/PublicInbox/V2Writable.pm b/lib/PublicInbox/V2Writable.pm index 9b280c6..ef9867d 100644 --- a/lib/PublicInbox/V2Writable.pm +++ b/lib/PublicInbox/V2Writable.pm @@ -209,11 +209,22 @@ sub idx_init { $skel->_msgmap_init->{dbh}->begin_work; } -sub remove { - my ($self, $mime, $cmt_msg) = @_; +sub purge_oids { + my ($self, $purge) = @_; # $purge = { $object_id => 1, ... } + $self->done; + my $pfx = "$self->{-inbox}->{mainrepo}/git"; + foreach my $i (0..$self->{max_git}) { + my $git = PublicInbox::Git->new("$pfx/$i.git"); + my $im = $self->import_init($git, 0); + $im->purge_oids($purge); + } +} + +sub remove_internal { + my ($self, $mime, $cmt_msg, $purge) = @_; $self->barrier; $self->idx_init; - my $im = $self->importer; + my $im = $self->importer unless $purge; my $ibx = $self->{-inbox}; my $srch = $ibx->search; my $cid = content_id($mime); @@ -245,11 +256,15 @@ sub remove { # no bugs in our deduplication code: $removed = $smsg; $removed->{mime} = $cur; - $im->remove(\$orig, $cmt_msg); + my $oid = $smsg->{blob}; + if ($purge) { + $purge->{$oid} = 1; + } else { + $im->remove(\$orig, $cmt_msg); + } $orig = undef; $removed->num; # memoize this for callers - my $oid = $smsg->{blob}; foreach my $idx (@$parts, $skel) { $idx->remote_remove($oid, $mid); } @@ -258,9 +273,23 @@ sub remove { }); $self->barrier; } + if ($purge && scalar keys %$purge) { + purge_oids($self, $purge); + } $removed; } +sub remove { + my ($self, $mime, $cmt_msg) = @_; + remove_internal($self, $mime, $cmt_msg); +} + +sub purge { + my ($self, $mime) = @_; + remove_internal($self, $mime, undef, {}); +} + + sub done { my ($self) = @_; my $locked = defined $self->{idx_parts}; diff --git a/t/v2writable.t b/t/v2writable.t index c48f060..0eda432 100644 --- a/t/v2writable.t +++ b/t/v2writable.t @@ -231,4 +231,11 @@ EOF ok(!$@, '->done is idempotent'); } +{ + ok($im->add($mime), 'add message to be purged'); + local $SIG{__WARN__} = sub {}; + ok($im->purge($mime), 'purged message'); + $im->done; +} + done_testing(); -- EW