From d002f24a9648d1499a16ed4dec84f05c0f849740 Mon Sep 17 00:00:00 2001 From: Eric Wong Date: Sat, 11 Nov 2023 09:04:57 +0000 Subject: mda|learn|watch: support dropUniqueUnsubscribe config List-Unsubscribe headers with unique identifiers (such as those generated by our examples/unsubscribe.milter) should not end up in public archives. Add a new config knob to strip List-Unsubscribe headers if they have the `List-Unsubscribe-Post: List-Unsubscribe=One-Click' header. Unfortunately, this breaks DKIM signatures if the signature covers either of these List-Unsubscribe* headers. However, breaking DKIM is the lesser evil compared to any archive reader being able to stop archival by an independent archivist. As much as I would like this to be the default, it probably affects few users at the moment since very few mailing lists use unique identifiers in List-Unsubscribe (but that number has grown, recently). --- Documentation/public-inbox-config.pod | 17 +++++++++++++ Documentation/public-inbox-learn.pod | 19 ++++++++++++++ Documentation/public-inbox-mda.pod | 18 ++++++++++++- Documentation/public-inbox-watch.pod | 6 ++++- lib/PublicInbox/Import.pm | 27 ++++++++++++++++++++ lib/PublicInbox/LeiToMail.pm | 6 +++++ lib/PublicInbox/Watch.pm | 1 + script/public-inbox-learn | 3 +++ script/public-inbox-mda | 4 +++ script/public-inbox-watch | 2 ++ t/lei-import.t | 48 ++++++++++++++++++++++++++++++++++- t/mda.t | 41 +++++++++++++++++++++++++++--- t/watch_maildir.t | 30 +++++++++++++++++++--- 13 files changed, 212 insertions(+), 10 deletions(-) diff --git a/Documentation/public-inbox-config.pod b/Documentation/public-inbox-config.pod index 871ac6c5..1ef7f46f 100644 --- a/Documentation/public-inbox-config.pod +++ b/Documentation/public-inbox-config.pod @@ -196,6 +196,23 @@ and the path may be "/dev/null" or any empty file. Multiple files may be specified and will be included in the order specified. +=item publicinboxImport.dropUniqueUnsubscribe + +Drop C headers if the message also includes +the C header +to signal MUAs to support an instantaneous unsubscribe. This +is strongly recommended for users creating their own public +archives of mailing lists they subscribe to, otherwise any +archive reader can unsubscribe the archivist. + +This may break DKIM signatures if the C +headers are signed, but breaking DKIM signatures is the +lesser evil compared to allowing any reader to unsubscribe +the archivist. + +This affects L, L, +and L + =item publicinboxmda.spamcheck This may be set to C to disable the use of SpamAssassin diff --git a/Documentation/public-inbox-learn.pod b/Documentation/public-inbox-learn.pod index f776df6b..b08e4bc8 100644 --- a/Documentation/public-inbox-learn.pod +++ b/Documentation/public-inbox-learn.pod @@ -73,6 +73,25 @@ Default: ~/.public-inbox/config =back +=head1 CONFIGURATION + +These configuration knobs should be used in the +L file. + +=over 8 + +=item publicinboxImport.dropUniqueUnsubscribe + +=item publicinbox..address + +=item publicinbox..listid + +=item publicinboxmda.spamcheck + +See L for descriptions of these options + +=back + =head1 CONTACT Feedback welcome via plain-text mail to L diff --git a/Documentation/public-inbox-mda.pod b/Documentation/public-inbox-mda.pod index 93cb0e9c..edc90287 100644 --- a/Documentation/public-inbox-mda.pod +++ b/Documentation/public-inbox-mda.pod @@ -68,6 +68,22 @@ Default: ~/.public-inbox/emergency/ =back +=head1 CONFIGURATION + +Various configuration knobs should be used in the +L file. + +=over 8 + +=item publicinboxImport.dropUniqueUnsubscribe + +=item publicinbox..address + +=item publicinbox..listid + +See L for descriptions of these options + +=back =head1 CONTACT @@ -78,7 +94,7 @@ L =head1 COPYRIGHT -Copyright 2013-2021 all contributors L +Copyright all contributors L License: AGPL-3.0+ L diff --git a/Documentation/public-inbox-watch.pod b/Documentation/public-inbox-watch.pod index febda0b1..7c21f7ce 100644 --- a/Documentation/public-inbox-watch.pod +++ b/Documentation/public-inbox-watch.pod @@ -66,6 +66,10 @@ L file. =over 8 +=item publicinboxImport.dropUniqueUnsubscribe + +See L + =item publicinbox..watch A location to watch. public-inbox 1.5.0 and earlier only supported @@ -201,7 +205,7 @@ L =head1 COPYRIGHT -Copyright 2016-2021 all contributors L +Copyright all contributors L License: AGPL-3.0+ L diff --git a/lib/PublicInbox/Import.pm b/lib/PublicInbox/Import.pm index 2d60db55..e4f8615e 100644 --- a/lib/PublicInbox/Import.pm +++ b/lib/PublicInbox/Import.pm @@ -321,11 +321,38 @@ sub extract_cmt_info ($;$) { # kill potentially confusing/misleading headers our @UNWANTED_HEADERS = (qw(Bytes Lines Content-Length), qw(Status X-Status)); +our $DROP_UNIQUE_UNSUB; sub drop_unwanted_headers ($) { my ($eml) = @_; for (@UNWANTED_HEADERS, @PublicInbox::MDA::BAD_HEADERS) { $eml->header_set($_); } + + # We don't want public-inbox readers to be able to unsubcribe the + # address which does archiving. WARNING: this breaks DKIM if the + # mailing list sender follows RFC 8058, section 4; but breaking DKIM + # (or have senders ignore RFC 8058 sec. 4) is preferable to having + # saboteurs unsubscribing independent archivists: + if ($DROP_UNIQUE_UNSUB && grep(/\AList-Unsubscribe=One-Click\z/, + $eml->header_raw('List-Unsubscribe-Post'))) { + for (qw(List-Unsubscribe-Post List-Unsubscribe)) { + $eml->header_set($_) + } + } +} + +sub load_config ($;$) { + my ($cfg, $do_exit) = @_; + my $v = $cfg->{lc 'publicinboxImport.dropUniqueUnsubscribe'}; + if (defined $v) { + $DROP_UNIQUE_UNSUB = $cfg->git_bool($v) // do { + warn <{-f} is not boolean +EOM + $do_exit //= \&CORE::exit; + $do_exit->(78); # EX_CONFIG + }; + } } # used by V2Writable, too diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm index b73af68a..0d2f586a 100644 --- a/lib/PublicInbox/LeiToMail.pm +++ b/lib/PublicInbox/LeiToMail.pm @@ -10,6 +10,7 @@ use PublicInbox::Eml; use PublicInbox::IO; use PublicInbox::Git; use PublicInbox::Spawn qw(spawn); +use PublicInbox::Import; use IO::Handle; # ->autoflush use Fcntl qw(SEEK_SET SEEK_END O_CREAT O_EXCL O_WRONLY); use PublicInbox::Syscall qw(rename_noreplace); @@ -672,6 +673,11 @@ sub _pre_augment_v2 { }); } PublicInbox::InboxWritable->new($ibx, @creat); + local $PublicInbox::Import::DROP_UNIQUE_UNSUB; # only for workers + PublicInbox::Import::load_config(PublicInbox::Config->new, sub { + $lei->x_it(shift); + die "E: can't write v2 inbox with broken config\n"; + }); $ibx->init_inbox if @creat; my $v2w = $ibx->importer; $v2w->wq_workers_start("lei/v2w $dir", 1, $lei->oldset, {lei => $lei}, diff --git a/lib/PublicInbox/Watch.pm b/lib/PublicInbox/Watch.pm index 1cdf12a5..5253ec94 100644 --- a/lib/PublicInbox/Watch.pm +++ b/lib/PublicInbox/Watch.pm @@ -45,6 +45,7 @@ sub new { my (%mdmap); my (%imap, %nntp); # url => [inbox objects] or 'watchspam' my (@imap, @nntp); + PublicInbox::Import::load_config($cfg); # "publicinboxwatch" is the documented namespace # "publicinboxlearn" is legacy but may be supported diff --git a/script/public-inbox-learn b/script/public-inbox-learn index 8069d919..6a1bc890 100755 --- a/script/public-inbox-learn +++ b/script/public-inbox-learn @@ -28,6 +28,7 @@ use PublicInbox::Spamcheck::Spamc; use Getopt::Long qw(:config gnu_getopt no_ignore_case auto_abbrev); my %opt = (all => 0); GetOptions(\%opt, qw(all help|h)) or die $help; +use PublicInbox::Import; my $train = shift or die $help; if ($train !~ /\A(?:ham|spam|rm)\z/) { @@ -37,6 +38,8 @@ die "--all only works with `rm'\n" if $opt{all} && $train ne 'rm'; my $spamc = PublicInbox::Spamcheck::Spamc->new; my $pi_cfg = PublicInbox::Config->new; +local $PublicInbox::Import::DROP_UNIQUE_UNSUB; +PublicInbox::Import::load_config($pi_cfg); my $err; my $mime = PublicInbox::Eml->new(do{ defined(my $data = do { local $/; }) or die "read STDIN: $!\n"; diff --git a/script/public-inbox-mda b/script/public-inbox-mda index cac819ac..04fd8aad 100755 --- a/script/public-inbox-mda +++ b/script/public-inbox-mda @@ -16,6 +16,8 @@ use strict; use Getopt::Long qw(:config gnu_getopt no_ignore_case auto_abbrev); my ($ems, $emm, $show_help); my $precheck = 1; +use PublicInbox::Import; +local $PublicInbox::Import::DROP_UNIQUE_UNSUB; # does this need a CLI switch? GetOptions('precheck!' => \$precheck, 'help|h' => \$show_help) or do { print STDERR $help; exit 1 }; @@ -47,6 +49,8 @@ my $key = 'publicinboxmda.spamcheck'; my $default = 'PublicInbox::Spamcheck::Spamc'; my $spamc = PublicInbox::Spamcheck::get($cfg, $key, $default); my $dests = []; +PublicInbox::Import::load_config($cfg, $do_exit); + my $recipient = $ENV{ORIGINAL_RECIPIENT}; if (defined $recipient) { my $ibx = $cfg->lookup($recipient); # first check diff --git a/script/public-inbox-watch b/script/public-inbox-watch index d9215de9..9bcd42ed 100755 --- a/script/public-inbox-watch +++ b/script/public-inbox-watch @@ -11,6 +11,8 @@ use strict; use Getopt::Long qw(:config gnu_getopt no_ignore_case auto_abbrev); use IO::Handle; # ->autoflush use PublicInbox::Watch; +use PublicInbox::Import; +local $PublicInbox::Import::DROP_UNIQUE_UNSUB; use PublicInbox::Config; use PublicInbox::DS; my $do_scan = 1; diff --git a/t/lei-import.t b/t/lei-import.t index 1edd607d..bd562617 100644 --- a/t/lei-import.t +++ b/t/lei-import.t @@ -3,7 +3,8 @@ # License: AGPL-3.0+ use v5.12; use PublicInbox::TestCommon; use PublicInbox::DS qw(now); -use autodie qw(open close); +use PublicInbox::IO qw(write_file); +use autodie qw(open close truncate); test_lei(sub { ok(!lei(qw(import -F bogus), 't/plack-qp.eml'), 'fails with bogus format'); like($lei_err, qr/\bis `eml', not --in-format/, 'gave error message'); @@ -180,6 +181,51 @@ SKIP: { 'EIO noted in stderr'); } +{ + local $ENV{PI_CONFIG} = "$ENV{HOME}/pi_config"; + write_file '>', $ENV{PI_CONFIG}, < +List-Unsubscribe-Post: List-Unsubscribe=One-Click +Message-ID: +Subject: unsubscribe-1 example +From: u\@example.com +To: 2\@example.com +Date: Fri, 02 Oct 1993 00:00:00 +0000 + +EOM + lei_ok [qw(import -F eml +L:unsub)], undef, { %$lei_opt, 0 => \$in }, + 'import succeeds w/ List-Unsubscribe'; + lei_ok qw(q L:unsub -f mboxrd); + like $lei_out, qr/some-UUID-here/, + 'Unsubscribe header preserved despite PI_CONFIG dropping'; + lei_ok qw(q L:unsub -o), "v2:$ENV{HOME}/v2-1"; + lei_ok qw(q s:unsubscribe -f mboxrd --only), "$ENV{HOME}/v2-1"; + unlike $lei_out, qr/some-UUID-here/, + 'Unsubscribe header dropped w/ dropUniqueUnsubscribe'; + like $lei_out, qr/Message-ID: /, + 'wrote expected message to v2 output'; + + # the default for compatibility: + truncate $ENV{PI_CONFIG}, 0; + lei_ok qw(q L:unsub -o), "v2:$ENV{HOME}/v2-2"; + lei_ok qw(q s:unsubscribe -f mboxrd --only), "$ENV{HOME}/v2-2"; + like $lei_out, qr/some-UUID-here/, + 'Unsubscribe header preserved by default :<'; + + # ensure we can fail + write_file '>', $ENV{PI_CONFIG}, <>', $pi_config or die; - print $fh <>', $pi_config, <qx(qw(diff HEAD~1..HEAD)); like($cur, qr/^-Message-ID: <2lids\@example>/sm, 'changed in git'); + + # ensure we can strip List-Unsubscribe + $in = < +List-Id: <$list_id> +Message-ID: +Subject: unsubscribe-1 +From: user +To: $addr +Date: Fri, 02 Oct 1993 00:00:00 +0000 +List-Unsubscribe: +List-Unsubscribe-Post: List-Unsubscribe=One-Click + +List-Unsubscribe should be stripped +EOF + write_file '>>', $pi_config, <qx(qw(diff HEAD~1..HEAD)))); + like $cur, qr/Message-ID: qx(qw(diff HEAD~1..HEAD)))); + like $cur, qr/Message-ID: $cfg_path }; $git->cleanup; + write_file '>>', $cfg_path, <{pid}); @@ -194,13 +199,32 @@ More majordomo info at http://vger.kernel.org/majordomo-info.html\n); $em->commit; # wake -watch up diag 'waiting for -watch to import new message'; PublicInbox::DS::event_loop(); + + my $head = $git->qx(qw(cat-file commit HEAD)); + my $subj = $eml->header('Subject'); + like($head, qr/^\Q$subj\E/sm, 'new commit made'); + + # try dropUniqueUnsubscribe + $delivered = 0; + $eml->header_set('Message-ID', ''); + $eml->header_set('List-Unsubscribe', + 'header_set('List-Unsubscribe-Post', 'List-Unsubscribe=One-Click'); + $em = PublicInbox::Emergency->new($maildir); + $em->prepare(\($eml->as_string)); + $em->commit; # wake -watch up + diag 'waiting for -watch to import dropUniqueUnsubscribe message'; + PublicInbox::DS::event_loop(); + my $cur = $git->qx(qw(diff HEAD~1..HEAD)); + like $cur, qr/Message-ID: /, + 'unsubscribe@example imported'; + unlike $cur, qr/List-Unsubscribe\b/, + 'List-Unsubscribe-* headers gone w/ dropUniqueUnsubscribe'; + $wm->kill; $wm->join; $ii->close; PublicInbox::DS->Reset; - my $head = $git->qx(qw(cat-file commit HEAD)); - my $subj = $eml->header('Subject'); - like($head, qr/^\Q$subj\E/sm, 'new commit made'); } sub is_maildir { -- cgit v1.2.3-24-ge0c7