From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on dcvr.yhbt.net X-Spam-Level: X-Spam-ASN: X-Spam-Status: No, score=-4.0 required=3.0 tests=ALL_TRUSTED,BAYES_00 shortcircuit=no autolearn=ham autolearn_force=no version=3.4.0 Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id 2B2892018E for ; Sun, 19 Jun 2016 00:28:50 +0000 (UTC) From: Eric Wong To: meta@public-inbox.org Subject: [PATCH 3/4] watch_maildir: spam removal support Date: Sun, 19 Jun 2016 00:28:46 +0000 Message-Id: <20160619002847.26685-4-e@80x24.org> In-Reply-To: <20160619002847.26685-1-e@80x24.org> References: <20160619002847.26685-1-e@80x24.org> List-Id: We can support spam removal by watching a special "spam" Maildir, too. We can run public-inbox-learn as a separate step, and that command will be improved to support auto-learning, too. --- lib/PublicInbox/WatchMaildir.pm | 134 ++++++++++++++++++++++++++++++---------- t/watch_maildir.t | 46 +++++++++++++- 2 files changed, 146 insertions(+), 34 deletions(-) diff --git a/lib/PublicInbox/WatchMaildir.pm b/lib/PublicInbox/WatchMaildir.pm index f1a21b9..cb64f89 100644 --- a/lib/PublicInbox/WatchMaildir.pm +++ b/lib/PublicInbox/WatchMaildir.pm @@ -9,11 +9,24 @@ $Email::MIME::ContentType::STRICT_PARAMS = 0; # user input is imperfect use PublicInbox::Git; use PublicInbox::Import; use PublicInbox::MDA; +use PublicInbox::Spawn qw(spawn); sub new { my ($class, $config) = @_; my (%mdmap, @mdir); - foreach my $k (keys %$config) { + my $k = 'publicinboxlearn.watchspam'; + if (my $spamdir = $config->{$k}) { + if ($spamdir =~ s/\Amaildir://) { + $spamdir =~ s!/+\z!!; + # skip "new", no MUA has seen it, yet. + my $cur = "$spamdir/cur"; + push @mdir, $cur; + $mdmap{$cur} = 'watchspam'; + } else { + warn "unsupported $k=$spamdir\n"; + } + } + foreach $k (keys %$config) { $k =~ /\Apublicinbox\.([^\.]+)\.watch\z/ or next; my $name = $1; my $watch = $config->{$k}; @@ -27,8 +40,9 @@ sub new { my $new = "$watch/new"; my $cur = "$watch/cur"; push @mdir, $new, $cur; - $mdmap{$new} = $inbox; - $mdmap{$cur} = $inbox; + die "$new already in use\n" if $mdmap{$new}; + die "$cur already in use\n" if $mdmap{$cur}; + $mdmap{$new} = $mdmap{$cur} = $inbox; } else { warn "watch unsupported: $k=$watch\n"; } @@ -55,6 +69,42 @@ sub _try_fsn_paths { _done_for_now($self); } +sub _check_spam { + my ($self, $path) = @_; + my $mime = _path_to_mime($path) or return; + _force_mid($mime); + foreach my $inbox (values %{$self->{mdmap}}) { + next unless ref $inbox; + my $im = _importer_for($self, $inbox); + $im->remove($mime); + if (my $scrub = _scrubber_for($inbox)) { + my $scrubbed = $scrub->scrub($mime) or next; + $im->remove($scrubbed); + } + } +} + +# used to hash the relevant portions of a message when there are conflicts +sub _hash_mime2 { + my ($mime) = @_; + require Digest::SHA; + my $dig = Digest::SHA->new('SHA-1'); + $dig->add($mime->header_obj->header_raw('Subject')); + $dig->add($mime->body_raw); + $dig->hexdigest; +} + +sub _force_mid { + my ($mime) = @_; + # probably a bad idea, but we inject a Message-Id if + # one is missing, here.. + my $mid = $mime->header_obj->header_raw('Message-Id'); + if (!defined $mid || $mid =~ /\A\s*\z/) { + $mid = '<' . _hash_mime2($mime) . '@generated>'; + $mime->header_set('Message-Id', $mid); + } +} + sub _try_path { my ($self, $path) = @_; if ($path !~ $self->{mdre}) { @@ -66,44 +116,22 @@ sub _try_path { warn "unmappable dir: $1\n"; return; } - my $im = $inbox->{-import} ||= eval { - my $git = $inbox->git; - my $name = $inbox->{name}; - my $addr = $inbox->{-primary_address}; - PublicInbox::Import->new($git, $name, $addr); - }; - $self->{importers}->{"$im"} = $im; - my $mime; - if (open my $fh, '<', $path) { - local $/; - my $str = <$fh>; - $str or return; - $mime = Email::MIME->new(\$str); - } elsif ($!{ENOENT}) { - return; - } else { - warn "failed to open $path: $!\n"; - return; + if (!ref($inbox) && $inbox eq 'watchspam') { + return _check_spam($self, $path); } - + my $im = _importer_for($self, $inbox); + my $mime = _path_to_mime($path) or return; $mime->header_set($_) foreach @PublicInbox::MDA::BAD_HEADERS; my $wm = $inbox->{-watchheader}; if ($wm) { my $v = $mime->header_obj->header_raw($wm->[0]); return unless ($v && $v =~ $wm->[1]); } - my $f = $inbox->{filter}; - if ($f && $f =~ /::/) { - eval "require $f"; - if ($@) { - warn $@; - } else { - $f = $f->new; - $mime = $f->scrub($mime); - } + if (my $scrub = _scrubber_for($inbox)) { + $mime = $scrub->scrub($mime) or return; } - $mime or return; - my $mid = $mime->header_obj->header_raw('Message-Id'); + + _force_mid($mime); $im->add($mime); } @@ -140,4 +168,44 @@ sub scan { _done_for_now($self); } +sub _path_to_mime { + my ($path) = @_; + if (open my $fh, '<', $path) { + local $/; + my $str = <$fh>; + $str or return; + return Email::MIME->new(\$str); + } elsif ($!{ENOENT}) { + return; + } else { + warn "failed to open $path: $!\n"; + return; + } +} + +sub _importer_for { + my ($self, $inbox) = @_; + my $im = $inbox->{-import} ||= eval { + my $git = $inbox->git; + my $name = $inbox->{name}; + my $addr = $inbox->{-primary_address}; + PublicInbox::Import->new($git, $name, $addr); + }; + $self->{importers}->{"$im"} = $im; +} + +sub _scrubber_for { + my ($inbox) = @_; + my $f = $inbox->{filter}; + if ($f && $f =~ /::/) { + eval "require $f"; + if ($@) { + warn $@; + } else { + return $f->new; + } + } + undef; +} + 1; diff --git a/t/watch_maildir.t b/t/watch_maildir.t index 6564a86..8a2c934 100644 --- a/t/watch_maildir.t +++ b/t/watch_maildir.t @@ -8,6 +8,7 @@ use PublicInbox::Config; my $tmpdir = tempdir('watch_maildir-XXXXXX', TMPDIR => 1, CLEANUP => 1); my $git_dir = "$tmpdir/test.git"; my $maildir = "$tmpdir/md"; +my $spamdir = "$tmpdir/spam"; use_ok 'PublicInbox::WatchMaildir'; use_ok 'PublicInbox::Emergency'; my $cfgpfx = "publicinbox.test"; @@ -21,14 +22,17 @@ Subject: spam Message-Id: Date: Sat, 18 Jun 2016 00:00:00 +0000 -msg +something EOF PublicInbox::Emergency->new($maildir)->prepare(\$msg); +my $sem = PublicInbox::Emergency->new($spamdir); # create dirs my $config = PublicInbox::Config->new({ "$cfgpfx.address" => $addr, "$cfgpfx.mainrepo" => $git_dir, "$cfgpfx.watch" => "maildir:$maildir", + "$cfgpfx.filter" => 'PublicInbox::Filter::Vger', + "publicinboxlearn.watchspam" => "maildir:$spamdir", }); PublicInbox::WatchMaildir->new($config)->scan; @@ -36,4 +40,44 @@ my $git = PublicInbox::Git->new($git_dir); my @list = $git->qx(qw(rev-list refs/heads/master)); is(scalar @list, 1, 'one revision in rev-list'); +my $write_spam = sub { + is(scalar glob("$spamdir/new/*"), undef, 'no spam existing'); + $sem->prepare(\$msg); + $sem->commit; + my @new = glob("$spamdir/new/*"); + is(scalar @new, 1); + my @p = split(m!/+!, $new[0]); + ok(link($new[0], "$spamdir/cur/".$p[-1])); + is(unlink($new[0]), 1); +}; +$write_spam->(); +is(unlink(glob("$maildir/new/*")), 1, 'unlinked old spam'); +PublicInbox::WatchMaildir->new($config)->scan; +@list = $git->qx(qw(rev-list refs/heads/master)); +is(scalar @list, 2, 'two revisions in rev-list'); +@list = $git->qx(qw(ls-tree -r --name-only refs/heads/master)); +is(scalar @list, 0, 'tree is empty'); + +# check with scrubbing +{ + $msg .= qq(-- +To unsubscribe from this list: send the line "unsubscribe git" in +the body of a message to majordomo\@vger.kernel.org +More majordomo info at http://vger.kernel.org/majordomo-info.html\n); + PublicInbox::Emergency->new($maildir)->prepare(\$msg); + PublicInbox::WatchMaildir->new($config)->scan; + @list = $git->qx(qw(ls-tree -r --name-only refs/heads/master)); + is(scalar @list, 1, 'tree has one file'); + my $mref = $git->cat_file('HEAD:'.$list[0]); + like($$mref, qr/something\n\z/s, 'message scrubbed on import'); + + is(unlink(glob("$maildir/new/*")), 1, 'unlinked spam'); + $write_spam->(); + PublicInbox::WatchMaildir->new($config)->scan; + @list = $git->qx(qw(ls-tree -r --name-only refs/heads/master)); + is(scalar @list, 0, 'tree is empty'); + @list = $git->qx(qw(rev-list refs/heads/master)); + is(scalar @list, 4, 'four revisions in rev-list'); +} + done_testing;