user/dev discussion of public-inbox itself
 help / color / Atom feed
16d785294f49bcb79a62620af2a5548437ba508c blob 6772 bytes (raw)

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
 
#!/usr/bin/perl -w
# Copyright (C) 2019 all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
#
# Used for editing messages in a public-inbox.
# Supports v2 inboxes only, for now.
use strict;
use warnings;
use Getopt::Long qw(:config gnu_getopt no_ignore_case auto_abbrev);
use PublicInbox::AdminEdit;
use File::Temp qw(tempfile);
use PublicInbox::ContentId qw(content_id);
use PublicInbox::MID qw(mid_clean mids);
PublicInbox::Admin::check_require('-index');
require PublicInbox::MIME;
require PublicInbox::InboxWritable;
require PublicInbox::Import;

my $usage = "$0 -m MESSAGE_ID [--all] [INBOX_DIRS]";
my $opt = { verbose => 1, all => 0, -min_inbox_version => 2, raw => 0 };
my @opt = qw(mid|m=s file|F=s raw);
GetOptions($opt, @PublicInbox::AdminEdit::OPT, @opt) or
	die "bad command-line args\n$usage\n";

my $editor = $ENV{MAIL_EDITOR}; # e.g. "mutt -f"
unless (defined $editor) {
	my $k = 'publicinbox.mailEditor';
	if (my $cfg = PublicInbox::Admin::config()) {
		$editor = $cfg->{lc($k)};
	}
	unless (defined $editor) {
		warn "\`$k' not configured, trying \`git var GIT_EDITOR'\n";
		chomp($editor = `git var GIT_EDITOR`);
		warn "Will use $editor to edit mail\n";
	}
}

my $mid = $opt->{mid};
my $file = $opt->{file};
if (defined $mid && defined $file) {
	die "the --mid and --file options are mutually exclusive\n";
}

my @ibxs = PublicInbox::Admin::resolve_inboxes(\@ARGV, $opt);
PublicInbox::AdminEdit::check_editable(\@ibxs);

my $found = {}; # cid => [ [ibx, smsg] [, [ibx, smsg] ] ]

sub find_mid ($) {
	my ($mid) = @_;
	foreach my $ibx (@ibxs) {
		my $over = $ibx->over;
		my ($id, $prev);
		while (my $smsg = $over->next_by_mid($mid, \$id, \$prev)) {
			my $ref = $ibx->msg_by_smsg($smsg);
			my $mime = PublicInbox::MIME->new($ref);
			my $cid = content_id($mime);
			my $tuple = [ $ibx, $smsg ];
			push @{$found->{$cid} ||= []}, $tuple
		}
		delete @$ibx{qw(over mm git search)}; # cleanup
	}
	$found;
}

sub show_cmd ($$) {
	my ($ibx, $smsg) = @_;
	" GIT_DIR=$ibx->{mainrepo}/all.git \\\n    git show $smsg->{blob}\n";
}

sub show_found () {
	foreach my $to_edit (values %$found) {
		foreach my $tuple (@$to_edit) {
			my ($ibx, $smsg) = @$tuple;
			warn show_cmd($ibx, $smsg);
		}
	}
}

if (defined($mid)) {
	$mid = mid_clean($mid);
	$found = find_mid($mid);
	my $nr = scalar(keys %$found);
	die "No message found for <$mid>\n" unless $nr;
	if ($nr > 1) {
		warn <<"";
Multiple messages with different content found matching
<$mid>:

		show_found();
		die "Use --force to edit all of them\n" if !$opt->{force};
		warn "Will edit all of them\n";
	}
} else {
	open my $fh, '<', $file or die "open($file) failed: $!";
	my $orig = do { local $/; <$fh> };
	my $mime = PublicInbox::MIME->new(\$orig);
	my $mids = mids($mime->header_obj);
	find_mid($_) for (@$mids); # populates $found
	my $cid = content_id($mime);
	my $to_edit = $found->{$cid};
	unless ($to_edit) {
		my $nr = scalar(keys %$found);
		if ($nr > 0) {
			warn <<"";
$nr matches to Message-ID(s) in $file, but none matched content
Partial matches below:

			show_found();
		} elsif ($nr == 0) {
			$mids = join('', map { "  <$_>\n" } @$mids);
			warn <<"";
No matching messages found matching Message-ID(s) in $file
$mids

		}
		exit 1;
	}
	$found = { $cid => $to_edit };
}

my $tmpl = 'public-inbox-edit-XXXXXX';
foreach my $to_edit (values %$found) {
	my ($edit_fh, $edit_fn) = tempfile($tmpl, TMPDIR => 1);
	$edit_fh->autoflush(1);
	my ($ibx, $smsg) = @{$to_edit->[0]};
	my $old_raw = $ibx->msg_by_smsg($smsg);
	delete @$ibx{qw(over mm git search)}; # cleanup

	my $tmp = $$old_raw;
	if (!$opt->{raw}) {
		my $oid = $smsg->{blob};
		print $edit_fh "From mboxrd\@$oid Thu Jan  1 00:00:00 1970\n";
		$tmp =~ s/^(>*From )/>$1/gm;
	}
	print $edit_fh $tmp or
		die "failed to write tempfile for editing: $!";

	# run the editor, respecting spaces/quote
retry_edit:
	if (system(qw(sh -c), qq(eval "$editor" '"\$@"'), '--', $edit_fn)) {
		if (!(-t STDIN) && !$opt->{force}) {
			die "E: $editor failed: $?\n";
		}
		print STDERR "$editor failed, ";
		print STDERR "continuing as forced\n" if $opt->{force};
		while (!$opt->{force}) {
			print STDERR "(r)etry, (c)ontinue, (q)uit?\n";
			chomp(my $op = <STDIN> || '');
			$op = lc($op);
			goto retry_edit if $op eq 'r';
			exit $? if $op eq 'q';
			last if $op eq 'c'; # continuing
			print STDERR "\`$op' not recognized\n";
		}
	}

	# reread the edited file, not using $edit_fh since $EDITOR may
	# rename/relink $edit_fn
	open my $new_fh, '<', $edit_fn or
		die "can't read edited file ($edit_fn): $!\n";
	my $new_raw = do { local $/; <$new_fh> };

	if (!$opt->{raw}) {
		# get rid of the From we added
		$new_raw =~ s/\A[\r\n]*From [^\r\n]*\r?\n//s;

		# check if user forgot to purge (in mutt) after editing
		if ($new_raw =~ /^From /sm) {
			if (-t STDIN) {
				print STDERR <<'';
Extra "From " lines detected in new mbox.
Did you forget to purge the original message from the mbox after editing?

				while (1) {
					print STDERR <<"";
(y)es to re-edit, (n)o to continue

					chomp(my $op = <STDIN> || '');
					$op = lc($op);
					goto retry_edit if $op eq 'y';
					last if $op eq 'n'; # continuing
					print STDERR "\`$op' not recognized\n";
				}
			} else { # non-interactive path
				# unlikely to happen, as extra From lines are
				# only a common mistake (for me) with
				# interactive use
				warn <<"";
W: possible message boundary splitting error

			}
		}
		# unescape what we escaped:
		$new_raw =~ s/^>(>*From )/$1/gm;
	}

	my $new_mime = PublicInbox::MIME->new(\$new_raw);
	my $old_mime = PublicInbox::MIME->new($old_raw);

	# make sure we don't compare unwanted headers, since mutt adds
	# Content-Length, Status, and Lines headers:
	PublicInbox::Import::drop_unwanted_headers($new_mime);
	PublicInbox::Import::drop_unwanted_headers($old_mime);

	# allow changing Received: and maybe other headers which can
	# contain sensitive info.
	my $nhdr = $new_mime->header_obj;
	my $ohdr = $old_mime->header_obj;
	if (($nhdr->as_string eq $ohdr->as_string) &&
	    (content_id($new_mime) eq content_id($old_mime))) {
		warn "No change detected to:\n", show_cmd($ibx, $smsg);

		next unless $opt->{verbose};
		# should we consider this machine-parseable?
		PublicInbox::AdminEdit::show_rewrites(\*STDOUT, $ibx, []);
		next;
	}

	foreach my $tuple (@$to_edit) {
		$ibx = PublicInbox::InboxWritable->new($tuple->[0]);
		$smsg = $tuple->[1];
		my $im = $ibx->importer(0);
		my $commits = $im->replace($old_mime, $new_mime);
		$im->done;
		unless ($commits) {
			warn "Failed to replace:\n", show_cmd($ibx, $smsg);
			next;
		}
		next unless $opt->{verbose};
		# should we consider this machine-parseable?
		PublicInbox::AdminEdit::show_rewrites(\*STDOUT, $ibx, $commits);
	}
}
debug log:

solving 16d7852 ...
found 16d7852 in https://80x24.org/public-inbox.git

user/dev discussion of public-inbox itself

Archives are clonable:
	git clone --mirror https://public-inbox.org/meta
	git clone --mirror http://czquwvybam4bgbro.onion/meta
	git clone --mirror http://hjrcffqmbrq6wope.onion/meta
	git clone --mirror http://ou63pmih66umazou.onion/meta

Example config snippet for mirrors

Newsgroups are available over NNTP:
	nntp://news.public-inbox.org/inbox.comp.mail.public-inbox.meta
	nntp://ou63pmih66umazou.onion/inbox.comp.mail.public-inbox.meta
	nntp://czquwvybam4bgbro.onion/inbox.comp.mail.public-inbox.meta
	nntp://hjrcffqmbrq6wope.onion/inbox.comp.mail.public-inbox.meta
	nntp://news.gmane.org/gmane.mail.public-inbox.general

 note: .onion URLs require Tor: https://www.torproject.org/

AGPL code for this site: git clone https://public-inbox.org/public-inbox.git