Merge branch 'eb/index-incremental'

Incremental indexing fixes from Eric W. Biederman.

These prevents the highest message number in msgmap from
being reassigned after deletes in rare cases and ensures
messages are deleted from msgmap in v2.

* eb/index-incremental:
  V2Writeable.pm: In unindex_oid delete the message from msgmap
  V2Writeable.pm: Ensure that a found message number is in the msgmap
  SearchIdx,V2Writeable: Update num_highwater on optimized deletes
  t/v[12]reindex.t: Verify the num highwater is as expected
  t/v[12]reindex.t Verify num_highwater
  Msgmap.pm: Track the largest value of num ever assigned
  SearchIdx.pm: Always assign numbers backwards during incremental indexing
  t/v[12]reindex.t: Test incremental indexing works
  t/v[12]reindex.t: Test that the resulting msgmap is as expected
  t/v[12]reindex.t: Place expected second in Xapian tests
  t/v2reindex.t: Isolate the test cases more
  t/v1reindex.t: Isolate the test cases
  Import.pm: Don't assume {in} and {out} always exist

2 files changed, 613 insertions, 81 deletions

diff --git a/t/v1reindex.t b/t/v1reindex.t
index 75380f0f..8be95149 100644
--- a/t/v1reindex.t
+++ b/t/v1reindex.t
@@ -22,7 +22,6 @@ my $ibx_config = {
         -primary_address => 'test@example.com',
         indexlevel => 'full',
 };
-my $ibx = PublicInbox::Inbox->new($ibx_config);
 my $mime = PublicInbox::MIME->create(
         header => [
                 From => 'a@example.com',
@@ -32,55 +31,101 @@ my $mime = PublicInbox::MIME->create(
         ],
         body => "hello world\n",
 );
-my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
-foreach my $i (1..10) {
-        $mime->header_set('Message-Id', "<$i\@example.com>");
-        ok($im->add($mime), "message $i added");
-        if ($i == 4) {
+my $minmax;
+my $msgmap;
+my ($mark1, $mark2, $mark3, $mark4);
+{
+        my %config = %$ibx_config;
+        my $ibx = PublicInbox::Inbox->new(\%config);
+        my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
+        foreach my $i (1..10) {
+                $mime->header_set('Message-Id', "<$i\@example.com>");
+                ok($im->add($mime), "message $i added");
+                if ($i == 4) {
+                        $mark1 = $im->get_mark($im->{tip});
+                        $im->remove($mime);
+                        $mark2 = $im->get_mark($im->{tip});
+                }
+        }
+
+        if ('test remove later') {
+                $mark3 = $im->get_mark($im->{tip});
+                $mime->header_set('Message-Id', "<5\@example.com>");
                 $im->remove($mime);
+                $mark4 = $im->get_mark($im->{tip});
         }
-}
 
-if ('test remove later') {
-        $mime->header_set('Message-Id', "<5\@example.com>");
-        $im->remove($mime);
+        $im->done;
+        my $rw = PublicInbox::SearchIdx->new($ibx, 1);
+        eval { $rw->index_sync() };
+        is($@, '', 'no error from indexing');
+
+        $minmax = [ $ibx->mm->minmax ];
+        ok(defined $minmax->[0] && defined $minmax->[1], 'minmax defined');
+        is_deeply($minmax, [ 1, 10 ], 'minmax as expected');
+        is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
+
+        my ($min, $max) = @$minmax;
+        $msgmap = $ibx->mm->msg_range(\$min, $max);
+        is_deeply($msgmap, [
+                          [1, '1@example.com' ],
+                          [2, '2@example.com' ],
+                          [3, '3@example.com' ],
+                          [6, '6@example.com' ],
+                          [7, '7@example.com' ],
+                          [8, '8@example.com' ],
+                          [9, '9@example.com' ],
+                          [10, '10@example.com' ],
+                  ], 'msgmap as expected');
 }
 
-$im->done;
-my $rw = PublicInbox::SearchIdx->new($ibx, 1);
-eval { $rw->index_sync() };
-is($@, '', 'no error from indexing');
+{
+        my %config = %$ibx_config;
+        my $ibx = PublicInbox::Inbox->new(\%config);
+        my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
+        my $rw = PublicInbox::SearchIdx->new($ibx, 1);
+        eval { $rw->index_sync({reindex => 1}) };
+        is($@, '', 'no error from reindexing');
+        $im->done;
 
-my $minmax = [ $ibx->mm->minmax ];
-ok(defined $minmax->[0] && defined $minmax->[1], 'minmax defined');
-is_deeply($minmax, [ 1, 10 ], 'minmax as expected');
+        is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
 
-$rw = PublicInbox::SearchIdx->new($ibx, 1);
-eval { $rw->index_sync({reindex => 1}) };
-is($@, '', 'no error from reindexing');
-$im->done;
+        my ($min, $max) = $ibx->mm->minmax;
+        is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged');
+}
 
 my $xap = "$mainrepo/public-inbox/xapian".PublicInbox::Search::SCHEMA_VERSION();
 remove_tree($xap);
 ok(!-d $xap, 'Xapian directories removed');
-$rw = PublicInbox::SearchIdx->new($ibx, 1);
+{
+        my %config = %$ibx_config;
+        my $ibx = PublicInbox::Inbox->new(\%config);
+        my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
+        my $rw = PublicInbox::SearchIdx->new($ibx, 1);
 
-eval { $rw->index_sync({reindex => 1}) };
-is($@, '', 'no error from reindexing');
-$im->done;
-ok(-d $xap, 'Xapian directories recreated');
+        eval { $rw->index_sync({reindex => 1}) };
+        is($@, '', 'no error from reindexing');
+        $im->done;
+        ok(-d $xap, 'Xapian directories recreated');
 
-delete $ibx->{mm};
-is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
+        delete $ibx->{mm};
+        is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
+        is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
+
+        my ($min, $max) = $ibx->mm->minmax;
+        is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged');
+}
 
 ok(unlink "$mainrepo/public-inbox/msgmap.sqlite3", 'remove msgmap');
 remove_tree($xap);
 ok(!-d $xap, 'Xapian directories removed again');
-
-$rw = PublicInbox::SearchIdx->new($ibx, 1);
 {
         my @warn;
         local $SIG{__WARN__} = sub { push @warn, @_ };
+        my %config = %$ibx_config;
+        my $ibx = PublicInbox::Inbox->new(\%config);
+        my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
+        my $rw = PublicInbox::SearchIdx->new($ibx, 1);
         eval { $rw->index_sync({reindex => 1}) };
         is($@, '', 'no error from reindexing without msgmap');
         is(scalar(@warn), 0, 'no warnings from reindexing');
@@ -88,16 +133,22 @@ $rw = PublicInbox::SearchIdx->new($ibx, 1);
         ok(-d $xap, 'Xapian directories recreated');
         delete $ibx->{mm};
         is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
+        is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
+
+        my ($min, $max) = $ibx->mm->minmax;
+        is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged');
 }
 
 ok(unlink "$mainrepo/public-inbox/msgmap.sqlite3", 'remove msgmap');
 remove_tree($xap);
 ok(!-d $xap, 'Xapian directories removed again');
-
-$rw = PublicInbox::SearchIdx->new($ibx, 1);
 {
         my @warn;
         local $SIG{__WARN__} = sub { push @warn, @_ };
+        my %config = %$ibx_config;
+        my $ibx = PublicInbox::Inbox->new(\%config);
+        my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
+        my $rw = PublicInbox::SearchIdx->new($ibx, 1);
         eval { $rw->index_sync({reindex => 1}) };
         is($@, '', 'no error from reindexing without msgmap');
         is_deeply(\@warn, [], 'no warnings');
@@ -105,18 +156,23 @@ $rw = PublicInbox::SearchIdx->new($ibx, 1);
         ok(-d $xap, 'Xapian directories recreated');
         delete $ibx->{mm};
         is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
+        is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
+
+        my ($min, $max) = @$minmax;
+        is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged');
 }
 
 ok(unlink "$mainrepo/public-inbox/msgmap.sqlite3", 'remove msgmap');
 remove_tree($xap);
 ok(!-d $xap, 'Xapian directories removed again');
-
-$ibx_config->{indexlevel} = 'medium';
-$ibx = PublicInbox::Inbox->new($ibx_config);
-$rw = PublicInbox::SearchIdx->new($ibx, 1);
 {
         my @warn;
         local $SIG{__WARN__} = sub { push @warn, @_ };
+        my %config = %$ibx_config;
+        $config{indexlevel} = 'medium';
+        my $ibx = PublicInbox::Inbox->new(\%config);
+        my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
+        my $rw = PublicInbox::SearchIdx->new($ibx, 1);
         eval { $rw->index_sync({reindex => 1}) };
         is($@, '', 'no error from reindexing without msgmap');
         is_deeply(\@warn, [], 'no warnings');
@@ -124,20 +180,25 @@ $rw = PublicInbox::SearchIdx->new($ibx, 1);
         ok(-d $xap, 'Xapian directories recreated');
         delete $ibx->{mm};
         is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
+        is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
         my $mset = $ibx->search->query('hello world', {mset=>1});
-        isnt(0, $mset->size, 'got Xapian search results');
+        isnt($mset->size, 0, 'got Xapian search results');
+
+        my ($min, $max) = $ibx->mm->minmax;
+        is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged');
 }
 
 ok(unlink "$mainrepo/public-inbox/msgmap.sqlite3", 'remove msgmap');
 remove_tree($xap);
 ok(!-d $xap, 'Xapian directories removed again');
-
-$ibx_config->{indexlevel} = 'basic';
-$ibx = PublicInbox::Inbox->new($ibx_config);
-$rw = PublicInbox::SearchIdx->new($ibx, 1);
 {
         my @warn;
         local $SIG{__WARN__} = sub { push @warn, @_ };
+        my %config = %$ibx_config;
+        $config{indexlevel} = 'basic';
+        my $ibx = PublicInbox::Inbox->new(\%config);
+        my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
+        my $rw = PublicInbox::SearchIdx->new($ibx, 1);
         eval { $rw->index_sync({reindex => 1}) };
         is($@, '', 'no error from reindexing without msgmap');
         is_deeply(\@warn, [], 'no warnings');
@@ -145,25 +206,231 @@ $rw = PublicInbox::SearchIdx->new($ibx, 1);
         ok(-d $xap, 'Xapian directories recreated');
         delete $ibx->{mm};
         is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
+        is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
         my $mset = $ibx->search->reopen->query('hello world', {mset=>1});
-        is(0, $mset->size, "no Xapian search results");
+        is($mset->size, 0, "no Xapian search results");
+
+        my ($min, $max) = $ibx->mm->minmax;
+        is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged');
 }
 
 # upgrade existing basic to medium
 # note: changing indexlevels is not yet supported in v2,
 # and may not be without more effort
-$ibx_config->{indexlevel} = 'medium';
-$ibx = PublicInbox::Inbox->new($ibx_config);
-$rw = PublicInbox::SearchIdx->new($ibx, 1);
 # no removals
 {
         my @warn;
         local $SIG{__WARN__} = sub { push @warn, @_ };
+        my %config = %$ibx_config;
+        $config{indexleve} = 'medium';
+        my $ibx = PublicInbox::Inbox->new(\%config);
+        my $rw = PublicInbox::SearchIdx->new($ibx, 1);
         eval { $rw->index_sync };
         is($@, '', 'no error from indexing');
         is_deeply(\@warn, [], 'no warnings');
         my $mset = $ibx->search->reopen->query('hello world', {mset=>1});
-        isnt(0, $mset->size, 'search OK after basic -> medium');
+        isnt($mset->size, 0, 'search OK after basic -> medium');
+
+        is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
+
+        my ($min, $max) = $ibx->mm->minmax;
+        is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged');
+}
+
+# An incremental indexing test
+ok(unlink "$mainrepo/public-inbox/msgmap.sqlite3", 'remove msgmap');
+remove_tree($xap);
+ok(!-d $xap, 'Xapian directories removed again');
+{
+        my @warn;
+        local $SIG{__WARN__} = sub { push @warn, @_ };
+        my %config = %$ibx_config;
+        my $ibx = PublicInbox::Inbox->new(\%config);
+        my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
+        my $rw = PublicInbox::SearchIdx->new($ibx, 1);
+        # mark1 4 simple additions in the same index_sync
+        eval { $rw->index_sync({ref => $mark1}) };
+        is($@, '', 'no error from reindexing without msgmap');
+        is_deeply(\@warn, [], 'no warnings');
+        $im->done;
+        my ($min, $max) = $ibx->mm->minmax;
+        is($min, 1, 'min as expected');
+        is($max, 4, 'max as expected');
+        is($ibx->mm->num_highwater, 4, 'num_highwater as expected');
+        is_deeply($ibx->mm->msg_range(\$min, $max),
+                  [
+                   [1, '1@example.com' ],
+                   [2, '2@example.com' ],
+                   [3, '3@example.com' ],
+                   [4, '4@example.com' ],
+                  ], 'msgmap as expected' );
 }
+{
+        my @warn;
+        local $SIG{__WARN__} = sub { push @warn, @_ };
+        my %config = %$ibx_config;
+        my $ibx = PublicInbox::Inbox->new(\%config);
+        my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
+        my $rw = PublicInbox::SearchIdx->new($ibx, 1);
+        # mark2 A delete separated form and add in the same index_sync
+        eval { $rw->index_sync({ref => $mark2}) };
+        is($@, '', 'no error from reindexing without msgmap');
+        is_deeply(\@warn, [], 'no warnings');
+        $im->done;
+        my ($min, $max) = $ibx->mm->minmax;
+        is($min, 1, 'min as expected');
+        is($max, 3, 'max as expected');
+        is($ibx->mm->num_highwater, 4, 'num_highwater as expected');
+        is_deeply($ibx->mm->msg_range(\$min, $max),
+                  [
+                   [1, '1@example.com' ],
+                   [2, '2@example.com' ],
+                   [3, '3@example.com' ],
+                  ], 'msgmap as expected' );
+}
+{
+        my @warn;
+        local $SIG{__WARN__} = sub { push @warn, @_ };
+        my %config = %$ibx_config;
+        my $ibx = PublicInbox::Inbox->new(\%config);
+        my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
+        my $rw = PublicInbox::SearchIdx->new($ibx, 1);
+        # mark3 adds following the delete at mark2
+        eval { $rw->index_sync({ref => $mark3}) };
+        is($@, '', 'no error from reindexing without msgmap');
+        is_deeply(\@warn, [], 'no warnings');
+        $im->done;
+        my ($min, $max) = $ibx->mm->minmax;
+        is($min, 1, 'min as expected');
+        is($max, 10, 'max as expected');
+        is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
+        is_deeply($ibx->mm->msg_range(\$min, $max),
+                  [
+                   [1, '1@example.com' ],
+                   [2, '2@example.com' ],
+                   [3, '3@example.com' ],
+                   [5, '5@example.com' ],
+                   [6, '6@example.com' ],
+                   [7, '7@example.com' ],
+                   [8, '8@example.com' ],
+                   [9, '9@example.com' ],
+                   [10, '10@example.com' ],
+                  ], 'msgmap as expected' );
+}
+{
+        my @warn;
+        local $SIG{__WARN__} = sub { push @warn, @_ };
+        my %config = %$ibx_config;
+        my $ibx = PublicInbox::Inbox->new(\%config);
+        my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
+        my $rw = PublicInbox::SearchIdx->new($ibx, 1);
+        # mark4 A delete of an older message
+        eval { $rw->index_sync({ref => $mark4}) };
+        is($@, '', 'no error from reindexing without msgmap');
+        is_deeply(\@warn, [], 'no warnings');
+        $im->done;
+        my ($min, $max) = $ibx->mm->minmax;
+        is($min, 1, 'min as expected');
+        is($max, 10, 'max as expected');
+        is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
+        is_deeply($ibx->mm->msg_range(\$min, $max),
+                  [
+                   [1, '1@example.com' ],
+                   [2, '2@example.com' ],
+                   [3, '3@example.com' ],
+                   [6, '6@example.com' ],
+                   [7, '7@example.com' ],
+                   [8, '8@example.com' ],
+                   [9, '9@example.com' ],
+                   [10, '10@example.com' ],
+                  ], 'msgmap as expected' );
+}
+
+
+# Another incremental indexing test
+ok(unlink "$mainrepo/public-inbox/msgmap.sqlite3", 'remove msgmap');
+remove_tree($xap);
+ok(!-d $xap, 'Xapian directories removed again');
+{
+        my @warn;
+        local $SIG{__WARN__} = sub { push @warn, @_ };
+        my %config = %$ibx_config;
+        my $ibx = PublicInbox::Inbox->new(\%config);
+        my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
+        my $rw = PublicInbox::SearchIdx->new($ibx, 1);
+        # mark2 an add and it's delete in the same index_sync
+        eval { $rw->index_sync({ref => $mark2}) };
+        is($@, '', 'no error from reindexing without msgmap');
+        is_deeply(\@warn, [], 'no warnings');
+        $im->done;
+        my ($min, $max) = $ibx->mm->minmax;
+        is($min, 1, 'min as expected');
+        is($max, 3, 'max as expected');
+        is($ibx->mm->num_highwater, 4, 'num_highwater as expected');
+        is_deeply($ibx->mm->msg_range(\$min, $max),
+                  [
+                   [1, '1@example.com' ],
+                   [2, '2@example.com' ],
+                   [3, '3@example.com' ],
+                  ], 'msgmap as expected' );
+}
+{
+        my @warn;
+        local $SIG{__WARN__} = sub { push @warn, @_ };
+        my %config = %$ibx_config;
+        my $ibx = PublicInbox::Inbox->new(\%config);
+        my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
+        my $rw = PublicInbox::SearchIdx->new($ibx, 1);
+        # mark3 adds following the delete at mark2
+        eval { $rw->index_sync({ref => $mark3}) };
+        is($@, '', 'no error from reindexing without msgmap');
+        is_deeply(\@warn, [], 'no warnings');
+        $im->done;
+        my ($min, $max) = $ibx->mm->minmax;
+        is($min, 1, 'min as expected');
+        is($max, 10, 'max as expected');
+        is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
+        is_deeply($ibx->mm->msg_range(\$min, $max),
+                  [
+                   [1, '1@example.com' ],
+                   [2, '2@example.com' ],
+                   [3, '3@example.com' ],
+                   [5, '5@example.com' ],
+                   [6, '6@example.com' ],
+                   [7, '7@example.com' ],
+                   [8, '8@example.com' ],
+                   [9, '9@example.com' ],
+                   [10, '10@example.com' ],
+                  ], 'msgmap as expected' );
+}
+{
+        my @warn;
+        local $SIG{__WARN__} = sub { push @warn, @_ };
+        my %config = %$ibx_config;
+        my $ibx = PublicInbox::Inbox->new(\%config);
+        my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
+        my $rw = PublicInbox::SearchIdx->new($ibx, 1);
+        # mark4 A delete of an older message
+        eval { $rw->index_sync({ref => $mark4}) };
+        is($@, '', 'no error from reindexing without msgmap');
+        is_deeply(\@warn, [], 'no warnings');
+        $im->done;
+        my ($min, $max) = $ibx->mm->minmax;
+        is($min, 1, 'min as expected');
+        is($max, 10, 'max as expected');
+        is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
+        is_deeply($ibx->mm->msg_range(\$min, $max),
+                  [
+                   [1, '1@example.com' ],
+                   [2, '2@example.com' ],
+                   [3, '3@example.com' ],
+                   [6, '6@example.com' ],
+                   [7, '7@example.com' ],
+                   [8, '8@example.com' ],
+                   [9, '9@example.com' ],
+                   [10, '10@example.com' ],
+                  ], 'msgmap as expected' );
+}
+
 
 done_testing();
diff --git a/t/v2reindex.t b/t/v2reindex.t
index 1543309c..a5454a22 100644
--- a/t/v2reindex.t
+++ b/t/v2reindex.t
@@ -21,7 +21,6 @@ my $ibx_config = {
         -primary_address => 'test@example.com',
         indexlevel => 'full',
 };
-my $ibx = PublicInbox::Inbox->new($ibx_config);
 my $mime = PublicInbox::MIME->create(
         header => [
                 From => 'a@example.com',
@@ -32,39 +31,86 @@ my $mime = PublicInbox::MIME->create(
         body => "hello world\n",
 );
 local $ENV{NPROC} = 2;
-my $im = PublicInbox::V2Writable->new($ibx, 1);
-foreach my $i (1..10) {
-        $mime->header_set('Message-Id', "<$i\@example.com>");
-        ok($im->add($mime), "message $i added");
-        if ($i == 4) {
+my $minmax;
+my $msgmap;
+my ($mark1, $mark2, $mark3, $mark4);
+{
+        my %config = %$ibx_config;
+        my $ibx = PublicInbox::Inbox->new(\%config);
+        my $im = PublicInbox::V2Writable->new($ibx, 1);
+        my $im0 = $im->importer();
+        foreach my $i (1..10) {
+                $mime->header_set('Message-Id', "<$i\@example.com>");
+                ok($im->add($mime), "message $i added");
+                if ($i == 4) {
+                        $mark1 = $im0->get_mark($im0->{tip});
+                        $im->remove($mime);
+                        $mark2 = $im0->get_mark($im0->{tip});
+                }
+        }
+
+        if ('test remove later') {
+                $mark3 = $im0->get_mark($im0->{tip});
+                $mime->header_set('Message-Id', "<5\@example.com>");
                 $im->remove($mime);
+                $mark4 = $im0->get_mark($im0->{tip});
         }
-}
 
-if ('test remove later') {
-        $mime->header_set('Message-Id', "<5\@example.com>");
-        $im->remove($mime);
+        $im->done;
+        $minmax = [ $ibx->mm->minmax ];
+        ok(defined $minmax->[0] && defined $minmax->[1], 'minmax defined');
+        is_deeply($minmax, [ 1, 10 ], 'minmax as expected');
+        is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
+
+        my ($min, $max) = @$minmax;
+        $msgmap = $ibx->mm->msg_range(\$min, $max);
+        is_deeply($msgmap, [
+                          [1, '1@example.com' ],
+                          [2, '2@example.com' ],
+                          [3, '3@example.com' ],
+                          [6, '6@example.com' ],
+                          [7, '7@example.com' ],
+                          [8, '8@example.com' ],
+                          [9, '9@example.com' ],
+                          [10, '10@example.com' ],
+                  ], 'msgmap as expected');
 }
 
-$im->done;
-my $minmax = [ $ibx->mm->minmax ];
-ok(defined $minmax->[0] && defined $minmax->[1], 'minmax defined');
-is_deeply($minmax, [ 1, 10 ], 'minmax as expected');
+{
+        my %config = %$ibx_config;
+        my $ibx = PublicInbox::Inbox->new(\%config);
+        my $im = PublicInbox::V2Writable->new($ibx, 1);
+        eval { $im->index_sync({reindex => 1}) };
+        is($@, '', 'no error from reindexing');
+        $im->done;
+
+        delete $ibx->{mm};
+        is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
+        is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
 
-eval { $im->index_sync({reindex => 1}) };
-is($@, '', 'no error from reindexing');
-$im->done;
+        my ($min, $max) = $ibx->mm->minmax;
+        is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged');
+}
 
 my $xap = "$mainrepo/xap".PublicInbox::Search::SCHEMA_VERSION();
 remove_tree($xap);
 ok(!-d $xap, 'Xapian directories removed');
-eval { $im->index_sync({reindex => 1}) };
-is($@, '', 'no error from reindexing');
-$im->done;
-ok(-d $xap, 'Xapian directories recreated');
+{
+        my %config = %$ibx_config;
+        my $ibx = PublicInbox::Inbox->new(\%config);
+        my $im = PublicInbox::V2Writable->new($ibx, 1);
+        eval { $im->index_sync({reindex => 1}) };
+        is($@, '', 'no error from reindexing');
+        $im->done;
+        ok(-d $xap, 'Xapian directories recreated');
 
-delete $ibx->{mm};
-is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
+        delete $ibx->{mm};
+        is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
+        is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
+
+        my ($min, $max) = $ibx->mm->minmax;
+        is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged');
+}
 
 ok(unlink "$mainrepo/msgmap.sqlite3", 'remove msgmap');
 remove_tree($xap);
@@ -72,6 +118,9 @@ ok(!-d $xap, 'Xapian directories removed again');
 {
         my @warn;
         local $SIG{__WARN__} = sub { push @warn, @_ };
+        my %config = %$ibx_config;
+        my $ibx = PublicInbox::Inbox->new(\%config);
+        my $im = PublicInbox::V2Writable->new($ibx, 1);
         eval { $im->index_sync({reindex => 1}) };
         is($@, '', 'no error from reindexing without msgmap');
         is(scalar(@warn), 0, 'no warnings from reindexing');
@@ -79,6 +128,10 @@ ok(!-d $xap, 'Xapian directories removed again');
         ok(-d $xap, 'Xapian directories recreated');
         delete $ibx->{mm};
         is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
+        is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
+
+        my ($min, $max) = $ibx->mm->minmax;
+        is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged');
 }
 
 my %sizes;
@@ -88,6 +141,9 @@ ok(!-d $xap, 'Xapian directories removed again');
 {
         my @warn;
         local $SIG{__WARN__} = sub { push @warn, @_ };
+        my %config = %$ibx_config;
+        my $ibx = PublicInbox::Inbox->new(\%config);
+        my $im = PublicInbox::V2Writable->new($ibx, 1);
         eval { $im->index_sync({reindex => 1}) };
         is($@, '', 'no error from reindexing without msgmap');
         is_deeply(\@warn, [], 'no warnings');
@@ -95,21 +151,25 @@ ok(!-d $xap, 'Xapian directories removed again');
         ok(-d $xap, 'Xapian directories recreated');
         delete $ibx->{mm};
         is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
+        is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
         my $mset = $ibx->search->query('"hello world"', {mset=>1});
-        isnt(0, $mset->size, "phrase search succeeds on indexlevel=full");
+        isnt($mset->size, 0, "phrase search succeeds on indexlevel=full");
         for (<"$xap/*/*">) { $sizes{$ibx->{indexlevel}} += -s _ if -f $_ }
+
+        my ($min, $max) = $ibx->mm->minmax;
+        is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged');
 }
 
 ok(unlink "$mainrepo/msgmap.sqlite3", 'remove msgmap');
 remove_tree($xap);
 ok(!-d $xap, 'Xapian directories removed again');
-
-$ibx_config->{indexlevel} = 'medium';
-$ibx = PublicInbox::Inbox->new($ibx_config);
-$im = PublicInbox::V2Writable->new($ibx);
 {
         my @warn;
         local $SIG{__WARN__} = sub { push @warn, @_ };
+        my %config = %$ibx_config;
+        $config{indexlevel} = 'medium';
+        my $ibx = PublicInbox::Inbox->new(\%config);
+        my $im = PublicInbox::V2Writable->new($ibx);
         eval { $im->index_sync({reindex => 1}) };
         is($@, '', 'no error from reindexing without msgmap');
         is_deeply(\@warn, [], 'no warnings');
@@ -117,31 +177,36 @@ $im = PublicInbox::V2Writable->new($ibx);
         ok(-d $xap, 'Xapian directories recreated');
         delete $ibx->{mm};
         is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
+        is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
 
         if (0) {
                 # not sure why, but Xapian seems to fallback to terms and
                 # phrase searches still work
                 delete $ibx->{search};
                 my $mset = $ibx->search->query('"hello world"', {mset=>1});
-                is(0, $mset->size, 'phrase search does not work on medium');
+                is($mset->size, 0, 'phrase search does not work on medium');
         }
 
         my $mset = $ibx->search->query('hello world', {mset=>1});
-        isnt(0, $mset->size, "normal search works on indexlevel=medium");
+        isnt($mset->size, 0, "normal search works on indexlevel=medium");
         for (<"$xap/*/*">) { $sizes{$ibx->{indexlevel}} += -s _ if -f $_ }
         ok($sizes{full} > $sizes{medium}, 'medium is smaller than full');
+
+
+        my ($min, $max) = $ibx->mm->minmax;
+        is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged');
 }
 
 ok(unlink "$mainrepo/msgmap.sqlite3", 'remove msgmap');
 remove_tree($xap);
 ok(!-d $xap, 'Xapian directories removed again');
-
-$ibx_config->{indexlevel} = 'basic';
-$ibx = PublicInbox::Inbox->new($ibx_config);
-$im = PublicInbox::V2Writable->new($ibx);
 {
         my @warn;
         local $SIG{__WARN__} = sub { push @warn, @_ };
+        my %config = %$ibx_config;
+        $config{indexlevel} = 'basic';
+        my $ibx = PublicInbox::Inbox->new(\%config);
+        my $im = PublicInbox::V2Writable->new($ibx);
         eval { $im->index_sync({reindex => 1}) };
         is($@, '', 'no error from reindexing without msgmap');
         is_deeply(\@warn, [], 'no warnings');
@@ -149,10 +214,210 @@ $im = PublicInbox::V2Writable->new($ibx);
         ok(-d $xap, 'Xapian directories recreated');
         delete $ibx->{mm};
         is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
+        is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
         my $mset = $ibx->search->query('hello', {mset=>1});
-        is(0, $mset->size, "search fails on indexlevel='basic'");
+        is($mset->size, 0, "search fails on indexlevel='basic'");
         for (<"$xap/*/*">) { $sizes{$ibx->{indexlevel}} += -s _ if -f $_ }
         ok($sizes{medium} > $sizes{basic}, 'basic is smaller than medium');
+
+        my ($min, $max) = $ibx->mm->minmax;
+        is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged');
+}
+
+
+# An incremental indexing test
+ok(unlink "$mainrepo/msgmap.sqlite3", 'remove msgmap');
+remove_tree($xap);
+ok(!-d $xap, 'Xapian directories removed again');
+{
+        my @warn;
+        local $SIG{__WARN__} = sub { push @warn, @_ };
+        my %config = %$ibx_config;
+        my $ibx = PublicInbox::Inbox->new(\%config);
+        # mark1 4 simple additions in the same index_sync
+        $ibx->{ref_head} = $mark1;
+        my $im = PublicInbox::V2Writable->new($ibx);
+        eval { $im->index_sync() };
+        is($@, '', 'no error from reindexing without msgmap');
+        is_deeply(\@warn, [], 'no warnings');
+        $im->done;
+        my ($min, $max) = $ibx->mm->minmax;
+        is($min, 1, 'min as expected');
+        is($max, 4, 'max as expected');
+        is($ibx->mm->num_highwater, 4, 'num_highwater as expected');
+        is_deeply($ibx->mm->msg_range(\$min, $max),
+                  [
+                   [1, '1@example.com' ],
+                   [2, '2@example.com' ],
+                   [3, '3@example.com' ],
+                   [4, '4@example.com' ],
+                  ], 'msgmap as expected' );
+}
+{
+        my @warn;
+        local $SIG{__WARN__} = sub { push @warn, @_ };
+        my %config = %$ibx_config;
+        my $ibx = PublicInbox::Inbox->new(\%config);
+        # mark2 A delete separated from an add in the same index_sync
+        $ibx->{ref_head} = $mark2;
+        my $im = PublicInbox::V2Writable->new($ibx);
+        eval { $im->index_sync() };
+        is($@, '', 'no error from reindexing without msgmap');
+        is_deeply(\@warn, [], 'no warnings');
+        $im->done;
+        my ($min, $max) = $ibx->mm->minmax;
+        is($min, 1, 'min as expected');
+        is($max, 3, 'max as expected');
+        is($ibx->mm->num_highwater, 4, 'num_highwater as expected');
+        is_deeply($ibx->mm->msg_range(\$min, $max),
+                  [
+                   [1, '1@example.com' ],
+                   [2, '2@example.com' ],
+                   [3, '3@example.com' ],
+                  ], 'msgmap as expected' );
+}
+{
+        my @warn;
+        local $SIG{__WARN__} = sub { push @warn, @_ };
+        my %config = %$ibx_config;
+        my $ibx = PublicInbox::Inbox->new(\%config);
+        # mark3 adds following the delete at mark2
+        $ibx->{ref_head} = $mark3;
+        my $im = PublicInbox::V2Writable->new($ibx);
+        eval { $im->index_sync() };
+        is($@, '', 'no error from reindexing without msgmap');
+        is_deeply(\@warn, [], 'no warnings');
+        $im->done;
+        my ($min, $max) = $ibx->mm->minmax;
+        is($min, 1, 'min as expected');
+        is($max, 10, 'max as expected');
+        is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
+        is_deeply($ibx->mm->msg_range(\$min, $max),
+                  [
+                   [1, '1@example.com' ],
+                   [2, '2@example.com' ],
+                   [3, '3@example.com' ],
+                   [5, '5@example.com' ],
+                   [6, '6@example.com' ],
+                   [7, '7@example.com' ],
+                   [8, '8@example.com' ],
+                   [9, '9@example.com' ],
+                   [10, '10@example.com' ],
+                  ], 'msgmap as expected' );
+}
+{
+        my @warn;
+        local $SIG{__WARN__} = sub { push @warn, @_ };
+        my %config = %$ibx_config;
+        my $ibx = PublicInbox::Inbox->new(\%config);
+        # mark4 A delete of an older message
+        $ibx->{ref_head} = $mark4;
+        my $im = PublicInbox::V2Writable->new($ibx);
+        eval { $im->index_sync() };
+        is($@, '', 'no error from reindexing without msgmap');
+        is_deeply(\@warn, [], 'no warnings');
+        $im->done;
+        my ($min, $max) = $ibx->mm->minmax;
+        is($min, 1, 'min as expected');
+        is($max, 10, 'max as expected');
+        is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
+        is_deeply($ibx->mm->msg_range(\$min, $max),
+                  [
+                   [1, '1@example.com' ],
+                   [2, '2@example.com' ],
+                   [3, '3@example.com' ],
+                   [6, '6@example.com' ],
+                   [7, '7@example.com' ],
+                   [8, '8@example.com' ],
+                   [9, '9@example.com' ],
+                   [10, '10@example.com' ],
+                  ], 'msgmap as expected' );
+}
+
+
+# Another incremental indexing test
+ok(unlink "$mainrepo/msgmap.sqlite3", 'remove msgmap');
+remove_tree($xap);
+ok(!-d $xap, 'Xapian directories removed again');
+{
+        my @warn;
+        local $SIG{__WARN__} = sub { push @warn, @_ };
+        my %config = %$ibx_config;
+        my $ibx = PublicInbox::Inbox->new(\%config);
+        # mark2 an add and it's delete in the same index_sync
+        $ibx->{ref_head} = $mark2;
+        my $im = PublicInbox::V2Writable->new($ibx);
+        eval { $im->index_sync() };
+        is($@, '', 'no error from reindexing without msgmap');
+        is_deeply(\@warn, [], 'no warnings');
+        $im->done;
+        my ($min, $max) = $ibx->mm->minmax;
+        is($min, 1, 'min as expected');
+        is($max, 3, 'max as expected');
+        is($ibx->mm->num_highwater, 4, 'num_highwater as expected');
+        is_deeply($ibx->mm->msg_range(\$min, $max),
+                  [
+                   [1, '1@example.com' ],
+                   [2, '2@example.com' ],
+                   [3, '3@example.com' ],
+                  ], 'msgmap as expected' );
+}
+{
+        my @warn;
+        local $SIG{__WARN__} = sub { push @warn, @_ };
+        my %config = %$ibx_config;
+        my $ibx = PublicInbox::Inbox->new(\%config);
+        # mark3 adds following the delete at mark2
+        $ibx->{ref_head} = $mark3;
+        my $im = PublicInbox::V2Writable->new($ibx);
+        eval { $im->index_sync() };
+        is($@, '', 'no error from reindexing without msgmap');
+        is_deeply(\@warn, [], 'no warnings');
+        $im->done;
+        my ($min, $max) = $ibx->mm->minmax;
+        is($min, 1, 'min as expected');
+        is($max, 10, 'max as expected');
+        is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
+        is_deeply($ibx->mm->msg_range(\$min, $max),
+                  [
+                   [1, '1@example.com' ],
+                   [2, '2@example.com' ],
+                   [3, '3@example.com' ],
+                   [5, '5@example.com' ],
+                   [6, '6@example.com' ],
+                   [7, '7@example.com' ],
+                   [8, '8@example.com' ],
+                   [9, '9@example.com' ],
+                   [10, '10@example.com' ],
+                  ], 'msgmap as expected' );
+}
+{
+        my @warn;
+        local $SIG{__WARN__} = sub { push @warn, @_ };
+        my %config = %$ibx_config;
+        my $ibx = PublicInbox::Inbox->new(\%config);
+        # mark4 A delete of an older message
+        $ibx->{ref_head} = $mark4;
+        my $im = PublicInbox::V2Writable->new($ibx);
+        eval { $im->index_sync() };
+        is($@, '', 'no error from reindexing without msgmap');
+        is_deeply(\@warn, [], 'no warnings');
+        $im->done;
+        my ($min, $max) = $ibx->mm->minmax;
+        is($min, 1, 'min as expected');
+        is($max, 10, 'max as expected');
+        is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
+        is_deeply($ibx->mm->msg_range(\$min, $max),
+                  [
+                   [1, '1@example.com' ],
+                   [2, '2@example.com' ],
+                   [3, '3@example.com' ],
+                   [6, '6@example.com' ],
+                   [7, '7@example.com' ],
+                   [8, '8@example.com' ],
+                   [9, '9@example.com' ],
+                   [10, '10@example.com' ],
+                  ], 'msgmap as expected' );
 }
 
 done_testing();