about summary refs log tree commit homepage
diff options
context:
space:
mode:
authorEric Wong <e@80x24.org>2019-06-24 23:29:37 +0000
committerEric Wong <e@80x24.org>2019-06-24 23:45:01 +0000
commit2752bc2b24d1e2c96c18b5f18372fc25e0ffc1fa (patch)
treec02e1d2e1ffc203d7a92ad94c3da9eb20e9d0e89
parentf4f0a3be0864721d90f9557ffe1c513b0289a74b (diff)
downloadpublic-inbox-2752bc2b24d1e2c96c18b5f18372fc25e0ffc1fa.tar.gz
"INSERT OR IGNORE" still bumps the auto-increment counter in
SQLite, which causes gaps to appear in NNTP article numbering.

This bug appeared in v2 repos where V2Writable may call ->add
repeatedly on the same message.  This bug is apparent with
public-inbox-watch and work-in-progress IMAP watchers which may
rescan and (attempt to) reinsert the same message on mailbox
changes.

Most uses of public-inbox-mda were not affected, unless the
same message is actually delivered multiple times to the mda.
v1 is not affected, either, since deduplication is only based
on Message-ID and msgmap never sees the duplicate.

Reported-by: "Eric W. Biederman" <ebiederm@xmission.com>
-rw-r--r--lib/PublicInbox/Msgmap.pm4
-rw-r--r--t/msgmap.t3
-rw-r--r--t/v2writable.t2
3 files changed, 7 insertions, 2 deletions
diff --git a/lib/PublicInbox/Msgmap.pm b/lib/PublicInbox/Msgmap.pm
index 0035c9e3..5a89b85a 100644
--- a/lib/PublicInbox/Msgmap.pm
+++ b/lib/PublicInbox/Msgmap.pm
@@ -126,9 +126,9 @@ sub mid_insert {
         my ($self, $mid) = @_;
         my $dbh = $self->{dbh};
         my $sth = $dbh->prepare_cached(<<'');
-INSERT OR IGNORE INTO msgmap (mid) VALUES (?)
+INSERT INTO msgmap (mid) VALUES (?)
 
-        return if $sth->execute($mid) == 0;
+        return unless eval { $sth->execute($mid) };
         my $num = $dbh->last_insert_id(undef, undef, 'msgmap', 'num');
         $self->num_highwater($num) if defined($num);
         $num;
diff --git a/t/msgmap.t b/t/msgmap.t
index 4dddd0a8..20985ce8 100644
--- a/t/msgmap.t
+++ b/t/msgmap.t
@@ -30,6 +30,9 @@ $@ = undef;
 my $ret = $d->mid_insert('a@b');
 is($ret, undef, 'duplicate mid_insert in undef result');
 is($d->num_for('a@b'), $mid2num{'a@b'}, 'existing number not clobbered');
+my $next = (sort(keys %num2mid))[-1];
+is($d->mid_insert('ok@unique'), $next + 1,
+        'got expected num after failing mid_insert');
 
 foreach my $n (keys %num2mid) {
         is($d->mid_for($n), $num2mid{$n}, "num:$n maps correctly");
diff --git a/t/v2writable.t b/t/v2writable.t
index 88df2d64..8f32fbe5 100644
--- a/t/v2writable.t
+++ b/t/v2writable.t
@@ -118,6 +118,8 @@ if ('ensure git configs are correct') {
         $mime->header_set('References', '<zz-mid@b>');
         ok($im->add($mime), 'message with multiple Message-ID');
         $im->done;
+        my ($total, undef) = $ibx->over->recent;
+        is($ibx->mm->num_highwater, $total, 'got expected highwater value');
         my $srch = $ibx->search;
         my $mset1 = $srch->reopen->query('m:abcde@1', { mset => 1 });
         is($mset1->size, 1, 'message found by first MID');