* [PATCH 03/11] import: switch to "replace_oids" interface for purge
2019-06-09 2:51 7% [PATCH 00/11] v2: implement message editing Eric Wong (Contractor, The Linux Foundation)
@ 2019-06-09 2:51 6% ` Eric Wong (Contractor, The Linux Foundation)
0 siblings, 0 replies; 2+ results
From: Eric Wong (Contractor, The Linux Foundation) @ 2019-06-09 2:51 UTC (permalink / raw)
To: meta
Continuing the work by Eric Biederman in commit a118d58a402bd31b
("Import.pm: When purging replace a purged file with a zero length file"),
we can use a generic OID replacement mechanism to implement
purge.
---
lib/PublicInbox/Import.pm | 33 +++++++++++++++++++--------------
lib/PublicInbox/V2Writable.pm | 6 +++---
2 files changed, 22 insertions(+), 17 deletions(-)
diff --git a/lib/PublicInbox/Import.pm b/lib/PublicInbox/Import.pm
index 6ee1935..2c8fe84 100644
--- a/lib/PublicInbox/Import.pm
+++ b/lib/PublicInbox/Import.pm
@@ -501,16 +501,16 @@ sub clean_purge_buffer {
}
}
-sub purge_oids {
- my ($self, $purge) = @_;
- my $tmp = "refs/heads/purge-".((keys %$purge)[0]);
+sub replace_oids {
+ my ($self, $replace) = @_; # oid => raw string
+ my $tmp = "refs/heads/replace-".((keys %$replace)[0]);
my $old = $self->{'ref'};
my $git = $self->{git};
my @export = (qw(fast-export --no-data --use-done-feature), $old);
my $rd = $git->popen(@export);
my ($r, $w) = $self->gfi_start;
my @buf;
- my $npurge = 0;
+ my $nreplace = 0;
my @oids;
my ($done, $mark);
my $tree = $self->{-tree};
@@ -533,10 +533,13 @@ sub purge_oids {
} elsif (/^M 100644 ([a-f0-9]+) (\w+)/) {
my ($oid, $path) = ($1, $2);
$tree->{$path} = 1;
- if ($purge->{$oid}) {
+ my $sref = $replace->{$oid};
+ if (defined $sref) {
push @oids, $oid;
- my $cmd = "M 100644 inline $path\ndata 0\n\n";
- push @buf, $cmd;
+ my $n = length($$sref);
+ push @buf, "M 100644 inline $path\ndata $n\n";
+ push @buf, $$sref; # hope CoW works...
+ push @buf, "\n";
} else {
push @buf, $_;
}
@@ -549,7 +552,7 @@ sub purge_oids {
$out =~ s/^/# /sgm;
warn "purge rewriting\n", $out, "\n";
clean_purge_buffer(\@oids, \@buf);
- $npurge++;
+ $nreplace++;
}
$w->print(@buf, "\n") or wfail;
@buf = ();
@@ -567,28 +570,30 @@ sub purge_oids {
$w->print(@buf) or wfail;
}
die 'done\n not seen from fast-export' unless $done;
- chomp(my $cmt = $self->get_mark(":$mark")) if $npurge;
+ chomp(my $cmt = $self->get_mark(":$mark")) if $nreplace;
$self->{nchg} = 0; # prevent _update_git_info until update-ref:
$self->done;
my @git = ('git', "--git-dir=$git->{git_dir}");
- run_die([@git, qw(update-ref), $old, $tmp]) if $npurge;
+ run_die([@git, qw(update-ref), $old, $tmp]) if $nreplace;
run_die([@git, qw(update-ref -d), $tmp]);
- return if $npurge == 0;
+ return if $nreplace == 0;
run_die([@git, qw(-c gc.reflogExpire=now gc --prune=all)]);
+
+ # check that old OIDs are gone
my $err = 0;
- foreach my $oid (keys %$purge) {
+ foreach my $oid (keys %$replace) {
my @info = $git->check($oid);
if (@info) {
- warn "$oid not purged\n";
+ warn "$oid not replaced\n";
$err++;
}
}
_update_git_info($self, 0);
- die "Failed to purge $err object(s)\n" if $err;
+ die "Failed to replace $err object(s)\n" if $err;
$cmt;
}
diff --git a/lib/PublicInbox/V2Writable.pm b/lib/PublicInbox/V2Writable.pm
index a435814..d6f72b0 100644
--- a/lib/PublicInbox/V2Writable.pm
+++ b/lib/PublicInbox/V2Writable.pm
@@ -298,7 +298,7 @@ sub idx_init {
}
sub purge_oids ($$) {
- my ($self, $purge) = @_; # $purge = { $object_id => 1, ... }
+ my ($self, $purge) = @_; # $purge = { $object_id => \'', ... }
$self->done;
my $pfx = "$self->{-inbox}->{mainrepo}/git";
my $purges = [];
@@ -313,7 +313,7 @@ sub purge_oids ($$) {
-d $git_dir or next;
my $git = PublicInbox::Git->new($git_dir);
my $im = $self->import_init($git, 0, 1);
- $purges->[$i] = $im->purge_oids($purge);
+ $purges->[$i] = $im->replace_oids($purge);
$im->done;
}
$purges;
@@ -386,7 +386,7 @@ sub remove_internal ($$$$) {
$removed = $smsg;
my $oid = $smsg->{blob};
if ($purge) {
- $purge->{$oid} = 1;
+ $purge->{$oid} = \'';
} else {
($mark, undef) = $im->remove($orig, $cmt_msg);
}
--
EW
^ permalink raw reply related [relevance 6%]
* [PATCH 00/11] v2: implement message editing
@ 2019-06-09 2:51 7% Eric Wong (Contractor, The Linux Foundation)
2019-06-09 2:51 6% ` [PATCH 03/11] import: switch to "replace_oids" interface for purge Eric Wong (Contractor, The Linux Foundation)
0 siblings, 1 reply; 2+ results
From: Eric Wong (Contractor, The Linux Foundation) @ 2019-06-09 2:51 UTC (permalink / raw)
To: meta
Some organizations are legally responsible for removing certain
content but prefer to edit out sensitive parts of a message
instead of purging it completely from history.
We can build off existing purge functionality. Instead of
replacing a message with an empty file; we instead replace
it with the desired content.
This ->replace method reindexes the modified message and
updates the corresponding git commit in case the subject
or authorship ident changes.
A new tool, public-inbox-edit(1) wraps the new ->replace
functionality by providing an editable mboxrd (suitable
for publicinbox.mailEditor "mutt -f").
GIT_EDITOR/VISUAL/EDITOR can be used if publicinbox.mailEditor
is not configured, but those are generally not ideal
for editing base64 or QP-encoded messages.
Eric Wong (Contractor, The Linux Foundation) (11):
v2writable: consolidate overview and indexing call
import: extract_author_info becomes extract_commit_info
import: switch to "replace_oids" interface for purge
v2writable: implement ->replace call
admin: remove warning arg for unconfigured inboxes
purge: start moving common options to AdminEdit module
admin: beef up resolve_inboxes to handle purge options
AdminEdit: move editability checks from -purge
admin: expose ->config
doc: document the --prune option for -index
edit: new tool to perform edits
Documentation/include.mk | 1 +
Documentation/public-inbox-config.pod | 4 +
Documentation/public-inbox-edit.pod | 109 ++++++++++++
Documentation/public-inbox-index.pod | 7 +
MANIFEST | 5 +
lib/PublicInbox/Admin.pm | 75 ++++++---
lib/PublicInbox/AdminEdit.pm | 50 ++++++
lib/PublicInbox/Import.pm | 101 ++++++-----
lib/PublicInbox/V2Writable.pm | 200 +++++++++++++++++-----
script/public-inbox-edit | 233 ++++++++++++++++++++++++++
script/public-inbox-purge | 103 ++----------
t/edit.t | 178 ++++++++++++++++++++
t/replace.t | 199 ++++++++++++++++++++++
13 files changed, 1065 insertions(+), 200 deletions(-)
create mode 100644 Documentation/public-inbox-edit.pod
create mode 100644 lib/PublicInbox/AdminEdit.pm
create mode 100755 script/public-inbox-edit
create mode 100644 t/edit.t
create mode 100644 t/replace.t
--
EW
^ permalink raw reply [relevance 7%]
Results 1-2 of 2 | reverse | options above
-- pct% links below jump to the message on this page, permalinks otherwise --
2019-06-09 2:51 7% [PATCH 00/11] v2: implement message editing Eric Wong (Contractor, The Linux Foundation)
2019-06-09 2:51 6% ` [PATCH 03/11] import: switch to "replace_oids" interface for purge Eric Wong (Contractor, The Linux Foundation)
Code repositories for project(s) associated with this public inbox
https://80x24.org/public-inbox.git
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).