[PATCH 0/4] misc updates

[PATCH 0/4] misc updates

[Eric Wong]

Some internal cleanups.  This also adds the filter=scrub option
to make us more tolerant of crap email and ease integration with
external lists.

Eric Wong (4):
      nntpd: executable permission
      rename mid_compress to id_compress
      drop Message-IDs longer than 244 bytes
      mda: support a 'filter=scrub' option for external lists

 lib/PublicInbox/Config.pm    |  2 +-
 lib/PublicInbox/MDA.pm       |  2 ++
 lib/PublicInbox/MID.pm       | 19 +++++++------------
 lib/PublicInbox/Search.pm    |  6 +++---
 lib/PublicInbox/SearchIdx.pm |  9 +++++++--
 lib/PublicInbox/View.pm      |  4 ++--
 public-inbox-mda             | 13 ++++++++++++-
 public-inbox-nntpd           |  0
 t/view.t                     | 11 +++++------
 9 files changed, 39 insertions(+), 27 deletions(-)

[PATCH 1/4] nntpd: executable permission

[Eric Wong]

Ensure we are executable for consistency and documentation.
MakeMaker already makes this executable, but we might as
well do the same...
---
 public-inbox-nntpd | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 mode change 100644 => 100755 public-inbox-nntpd

diff --git a/public-inbox-nntpd b/public-inbox-nntpd
old mode 100644
new mode 100755
-- 
EW

[PATCH 2/4] rename mid_compress to id_compress

[Eric Wong]

We use it as a general compressor for identifiers such as
subject paths, so using the "mid_" prefix probably is not
appropriate.
---
 lib/PublicInbox/MID.pm       | 19 +++++++------------
 lib/PublicInbox/Search.pm    |  6 +++---
 lib/PublicInbox/SearchIdx.pm |  4 ++--
 lib/PublicInbox/View.pm      |  4 ++--
 t/view.t                     | 11 +++++------
 5 files changed, 19 insertions(+), 25 deletions(-)

diff --git a/lib/PublicInbox/MID.pm b/lib/PublicInbox/MID.pm
index 677a9d5..3d404ef 100644
--- a/lib/PublicInbox/MID.pm
+++ b/lib/PublicInbox/MID.pm
@@ -4,7 +4,7 @@ package PublicInbox::MID;
 use strict;
 use warnings;
 use base qw/Exporter/;
-our @EXPORT_OK = qw/mid_clean mid_compress mid2path/;
+our @EXPORT_OK = qw/mid_clean id_compress mid2path/;
 use Digest::SHA qw/sha1_hex/;
 use constant MID_MAX => 40; # SHA-1 hex length
 
@@ -19,18 +19,13 @@ sub mid_clean {
 }
 
 # this is idempotent
-sub mid_compress {
-	my ($mid, $force) = @_;
+sub id_compress {
+	my ($id, $force) = @_;
 
-	# XXX dirty hack! FIXME!
-	# Some HTTP servers (apache2 2.2.22-13+deb7u5 on my system)
-	# apparently do not handle "%25" in the URL path component correctly.
-	# I'm not yet sure if it's something weird with my rewrite rules
-	# or what; will need to debug...
-	return sha1_hex($mid) if (index($mid, '%') >= 0);
-
-	return $mid if (!$force && length($mid) <= MID_MAX);
-	sha1_hex($mid);
+	if ($force || $id =~ /[^\w\-]/ || length($id) > MID_MAX) {
+		return sha1_hex($id);
+	}
+	$id;
 }
 
 sub mid2path {
diff --git a/lib/PublicInbox/Search.pm b/lib/PublicInbox/Search.pm
index 1d13f4b..fbc6882 100644
--- a/lib/PublicInbox/Search.pm
+++ b/lib/PublicInbox/Search.pm
@@ -14,7 +14,7 @@ use constant LINES => 3; # :lines as defined in RFC 3977
 use Search::Xapian qw/:standard/;
 use PublicInbox::SearchMsg;
 use Email::MIME;
-use PublicInbox::MID qw/mid_clean mid_compress/;
+use PublicInbox::MID qw/mid_clean id_compress/;
 
 # This is English-only, everything else is non-standard and may be confused as
 # a prefix common in patch emails
@@ -25,7 +25,7 @@ use constant {
 	# SCHEMA_VERSION history
 	# 0 - initial
 	# 1 - subject_path is lower-cased
-	# 2 - subject_path is mid_compress in the index, only
+	# 2 - subject_path is id_compress in the index, only
 	# 3 - message-ID is compressed if it includes '%' (hack!)
 	# 4 - change "Re: " normalization, avoid circular Reference ghosts
 	# 5 - subject_path drops trailing '.'
@@ -104,7 +104,7 @@ sub get_thread {
 
 	return { total => 0, msgs => [] } unless $smsg;
 	my $qtid = Search::Xapian::Query->new(xpfx('thread').$smsg->thread_id);
-	my $path = mid_compress($smsg->path);
+	my $path = id_compress($smsg->path);
 	my $qsub = Search::Xapian::Query->new(xpfx('path').$path);
 	my $query = Search::Xapian::Query->new(OP_OR, $qtid, $qsub);
 	$self->do_enquire($query, $opts);
diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm
index f98ba3e..8184dc7 100644
--- a/lib/PublicInbox/SearchIdx.pm
+++ b/lib/PublicInbox/SearchIdx.pm
@@ -5,7 +5,7 @@ package PublicInbox::SearchIdx;
 use strict;
 use warnings;
 use base qw(PublicInbox::Search);
-use PublicInbox::MID qw/mid_clean mid_compress/;
+use PublicInbox::MID qw/mid_clean id_compress/;
 *xpfx = *PublicInbox::Search::xpfx;
 
 use constant {
@@ -81,7 +81,7 @@ sub add_message {
 
 		if ($subj ne '') {
 			my $path = $self->subject_path($subj);
-			$doc->add_term(xpfx('path') . mid_compress($path));
+			$doc->add_term(xpfx('path') . id_compress($path));
 		}
 
 		add_val($doc, &PublicInbox::Search::TS, $smsg->ts);
diff --git a/lib/PublicInbox/View.pm b/lib/PublicInbox/View.pm
index ccdcde2..c9be770 100644
--- a/lib/PublicInbox/View.pm
+++ b/lib/PublicInbox/View.pm
@@ -9,7 +9,7 @@ use Encode qw/find_encoding/;
 use Encode::MIME::Header;
 use Email::MIME::ContentType qw/parse_content_type/;
 use PublicInbox::Hval;
-use PublicInbox::MID qw/mid_clean mid_compress mid2path/;
+use PublicInbox::MID qw/mid_clean id_compress mid2path/;
 use Digest::SHA qw/sha1_hex/;
 my $SALT = rand;
 require POSIX;
@@ -586,7 +586,7 @@ sub anchor_for {
 	my ($msgid) = @_;
 	my $id = $msgid;
 	if ($id !~ /\A[a-f0-9]{40}\z/) {
-		$id = mid_compress(mid_clean($id), 1);
+		$id = id_compress(mid_clean($id), 1);
 	}
 	'm' . $id;
 }
diff --git a/t/view.t b/t/view.t
index 325f509..568ab30 100644
--- a/t/view.t
+++ b/t/view.t
@@ -145,13 +145,12 @@ EOF
 	like($html, qr/\bhi = bye\b/, "HTML output decoded QP");
 }
 
-
-{	# XXX dirty hack
-	use PublicInbox::MID qw/mid_compress/;
-	like(mid_compress('foo%bar@wtf'), qr/\A[a-f0-9]{40}\z/,
+{
+	use PublicInbox::MID qw/id_compress/;
+	like(id_compress('foo%bar@wtf'), qr/\A[a-f0-9]{40}\z/,
 		"percent always converted to sha1 to workaround buggy httpds");
-	is(mid_compress('foobar@wtf'), 'foobar@wtf',
-		'regular MID not compressed');
+	is(id_compress('foobar-wtf'), 'foobar-wtf',
+		'regular ID not compressed');
 }
 
 done_testing();
-- 
EW

[PATCH 3/4] drop Message-IDs longer than 244 bytes

[Eric Wong]

Xapian has this limit for terms, and there are likely no
legitimate Message-IDs (or single header lines) this long; so
there's no need to workaround this limit.
---
 lib/PublicInbox/MDA.pm       | 2 ++
 lib/PublicInbox/SearchIdx.pm | 5 +++++
 2 files changed, 7 insertions(+)

diff --git a/lib/PublicInbox/MDA.pm b/lib/PublicInbox/MDA.pm
index 25d6ae5..7025fb3 100644
--- a/lib/PublicInbox/MDA.pm
+++ b/lib/PublicInbox/MDA.pm
@@ -6,6 +6,7 @@ use warnings;
 use Email::Address;
 use Date::Parse qw(strptime);
 use constant MAX_SIZE => 1024 * 500; # same as spamc default, should be tunable
+use constant MAX_MID_SIZE => 244; # max term size - 1 in Xapian
 use constant cmd => qw/ssoma-mda -1/;
 
 # drop plus addressing for matching
@@ -20,6 +21,7 @@ sub precheck {
 	my ($klass, $filter, $address) = @_;
 	my $simple = $filter->simple;
 	my $mid = $simple->header("Message-ID");
+	return 0 if (length($mid) > MAX_MID_SIZE);
 	return 0 unless usable_str(length('<m@h>'), $mid) && $mid =~ /\@/;
 	return 0 unless usable_str(length('u@h'), $filter->from);
 	return 0 unless usable_str(length(':o'), $simple->header("Subject"));
diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm
index 8184dc7..0646cfb 100644
--- a/lib/PublicInbox/SearchIdx.pm
+++ b/lib/PublicInbox/SearchIdx.pm
@@ -8,6 +8,7 @@ use base qw(PublicInbox::Search);
 use PublicInbox::MID qw/mid_clean id_compress/;
 *xpfx = *PublicInbox::Search::xpfx;
 
+use constant MAX_MID_SIZE => 244; # max term size - 1 in Xapian
 use constant {
 	PERM_UMASK => 0,
 	OLD_PERM_GROUP => 1,
@@ -52,6 +53,7 @@ sub add_message {
 	my $ct_msg = $mime->header('Content-Type') || 'text/plain';
 
 	eval {
+		die 'Message-ID too long' if length($mid) > MAX_MID_SIZE;
 		my $smsg = $self->lookup_message($mid);
 		my $doc;
 
@@ -230,6 +232,9 @@ sub link_message_to_parents {
 
 		# prevent circular references via References: here:
 		foreach my $ref (@orig_refs) {
+			if (length($ref) > MAX_MID_SIZE) {
+				warn "References: <$ref> too long, ignoring\n";
+			}
 			next if $uniq{$ref};
 			$uniq{$ref} = 1;
 			push @refs, $ref;
-- 
EW

[PATCH 4/4] mda: support a 'filter=scrub' option for external lists

[Eric Wong]

For list where we are not the primary archival entry point,
defaulting to filter=scrub makes sense since their list
conventions may be more tolerant of HTML and other crap
than we are.
---
 lib/PublicInbox/Config.pm |  2 +-
 public-inbox-mda          | 13 ++++++++++++-
 2 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/lib/PublicInbox/Config.pm b/lib/PublicInbox/Config.pm
index 315d788..0d73a86 100644
--- a/lib/PublicInbox/Config.pm
+++ b/lib/PublicInbox/Config.pm
@@ -61,7 +61,7 @@ sub lookup {
 	defined $pfx or return;
 
 	my %rv;
-	foreach my $k (qw(mainrepo address)) {
+	foreach my $k (qw(mainrepo address filter)) {
 		my $v = $self->{"$pfx.$k"};
 		$rv{$k} = $v if defined $v;
 	}
diff --git a/public-inbox-mda b/public-inbox-mda
index 1a9469b..df8ca38 100755
--- a/public-inbox-mda
+++ b/public-inbox-mda
@@ -38,7 +38,18 @@ if (PublicInbox::MDA->precheck($filter, $dst->{address}) &&
 	$filtered = undef;
 	$filter->simple($msg);
 
-	if (PublicInbox::Filter->run($msg, $filter)) {
+	my $filter_arg;
+	my $fcfg = $dst->{filter};
+	if (!defined $fcfg || $filter eq 'reject') {
+		$filter_arg = $filter;
+	} elsif ($fcfg eq 'scrub') {
+		$filter_arg = undef; # the default for legacy versions
+	} else {
+		warn "publicinbox.$dst->{listname}.filter=$fcfg invalid\n";
+		warn "must be either 'scrub' or 'reject' (the default)\n";
+	}
+
+	if (PublicInbox::Filter->run($msg, $filter_arg)) {
 		# run spamc again on the HTML-free message
 		if (do_spamc($msg, \$filtered)) {
 			$msg = Email::MIME->new(\$filtered);
-- 
EW