From 97c6b564fd79e47ae6fca8de273c2aeaf2f5bea5 Mon Sep 17 00:00:00 2001
From: Eric Wong <e@80x24.org>
Date: Sat, 5 Jan 2019 10:41:15 +0000
Subject: filter/rubylang: fix SQLite DB lifetime problems

Clearly the AltId stuff was never tested for v2.  Ensure
this tricky filter (which reuses Msgmap to avoid introducing
new serial numbers) doesn't trigger deadlocks SQLite due
to opening a DB for writing multiple times.

I went through several iterations of this change before
going with this one, which is the least intrusive I could
fine.
---
 lib/PublicInbox/InboxWritable.pm | 14 +++++++++++---
 lib/PublicInbox/V2Writable.pm    | 13 +++++++++++++
 lib/PublicInbox/WatchMaildir.pm  |  5 +++--
 3 files changed, 27 insertions(+), 5 deletions(-)

(limited to 'lib')

diff --git a/lib/PublicInbox/InboxWritable.pm b/lib/PublicInbox/InboxWritable.pm
index 87c9ada9..2f1ca6f0 100644
--- a/lib/PublicInbox/InboxWritable.pm
+++ b/lib/PublicInbox/InboxWritable.pm
@@ -46,9 +46,16 @@ sub importer {
 }
 
 sub filter {
-	my ($self) = @_;
+	my ($self, $im) = @_;
 	my $f = $self->{filter};
 	if ($f && $f =~ /::/) {
+		# v2 keeps msgmap open, which causes conflicts for filters
+		# such as PublicInbox::Filter::RubyLang which overload msgmap
+		# for a predictable serial number.
+		if ($im && ($self->{version} || 1) >= 2 && $self->{altid}) {
+			$im->done;
+		}
+
 		my @args = (-inbox => $self);
 		# basic line splitting, only
 		# Perhaps we can have proper quote splitting one day...
@@ -100,7 +107,7 @@ sub maildir_path_load ($) {
 sub import_maildir {
 	my ($self, $dir) = @_;
 	my $im = $self->importer(1);
-	my $filter = $self->filter;
+
 	foreach my $sub (qw(cur new tmp)) {
 		-d "$dir/$sub" or die "$dir is not a Maildir (missing $sub)\n";
 	}
@@ -109,7 +116,8 @@ sub import_maildir {
 		while (defined(my $fn = readdir($dh))) {
 			next unless is_maildir_basename($fn);
 			my $mime = maildir_file_load("$dir/$fn") or next;
-			if ($filter) {
+
+			if (my $filter = $self->filter($im)) {
 				my $ret = $filter->scrub($mime) or return;
 				return if $ret == REJECT();
 				$mime = $ret;
diff --git a/lib/PublicInbox/V2Writable.pm b/lib/PublicInbox/V2Writable.pm
index 07319646..93babed5 100644
--- a/lib/PublicInbox/V2Writable.pm
+++ b/lib/PublicInbox/V2Writable.pm
@@ -163,6 +163,19 @@ sub num_for {
 			return if $existing;
 		}
 
+		# AltId may pre-populate article numbers (e.g. X-Mail-Count
+		# or NNTP article number), use that article number if it's
+		# not in Over.
+		my $altid = $self->{-inbox}->{altid};
+		if ($altid && grep(/:file=msgmap\.sqlite3\z/, @$altid)) {
+			my $num = $self->{mm}->num_for($mid);
+
+			if (defined $num && !$self->{over}->get_art($num)) {
+				$$mid0 = $mid;
+				return $num;
+			}
+		}
+
 		# very unlikely:
 		warn "<$mid> reused for mismatched content\n";
 
diff --git a/lib/PublicInbox/WatchMaildir.pm b/lib/PublicInbox/WatchMaildir.pm
index 84080ba9..2d4c6f43 100644
--- a/lib/PublicInbox/WatchMaildir.pm
+++ b/lib/PublicInbox/WatchMaildir.pm
@@ -121,7 +121,7 @@ sub _remove_spam {
 		eval {
 			my $im = _importer_for($self, $ibx);
 			$im->remove($mime, 'spam');
-			if (my $scrub = $ibx->filter) {
+			if (my $scrub = $ibx->filter($im)) {
 				my $scrubbed = $scrub->scrub($mime, 1);
 				$scrubbed or return;
 				$scrubbed == REJECT() and return;
@@ -159,7 +159,8 @@ sub _try_path {
 			my $v = $mime->header_obj->header_raw($wm->[0]);
 			next unless ($v && $v =~ $wm->[1]);
 		}
-		if (my $scrub = $ibx->filter) {
+
+		if (my $scrub = $ibx->filter($im)) {
 			my $ret = $scrub->scrub($mime) or next;
 			$ret == REJECT() and next;
 			$mime = $ret;
-- 
cgit v1.2.3-24-ge0c7