about summary refs log tree commit homepage
diff options
context:
space:
mode:
-rw-r--r--lib/PublicInbox/Import.pm1
-rw-r--r--lib/PublicInbox/Msgmap.pm23
-rw-r--r--lib/PublicInbox/SearchIdx.pm30
-rw-r--r--lib/PublicInbox/V2Writable.pm20
-rw-r--r--t/v1reindex.t359
-rw-r--r--t/v2reindex.t335
6 files changed, 666 insertions, 102 deletions
diff --git a/lib/PublicInbox/Import.pm b/lib/PublicInbox/Import.pm
index 4e3b4c55..bfa7a805 100644
--- a/lib/PublicInbox/Import.pm
+++ b/lib/PublicInbox/Import.pm
@@ -451,6 +451,7 @@ sub done {
 sub atfork_child {
         my ($self) = @_;
         foreach my $f (qw(in out)) {
+                next unless defined($self->{$f});
                 close $self->{$f} or die "failed to close import[$f]: $!\n";
         }
 }
diff --git a/lib/PublicInbox/Msgmap.pm b/lib/PublicInbox/Msgmap.pm
index fdc71e46..d474bade 100644
--- a/lib/PublicInbox/Msgmap.pm
+++ b/lib/PublicInbox/Msgmap.pm
@@ -50,6 +50,10 @@ sub new_file {
                 create_tables($dbh);
                 $dbh->begin_work;
                 $self->created_at(time) unless $self->created_at;
+
+                my (undef, $max) = $self->minmax();
+                $max ||= 0;
+                $self->num_highwater($max);
                 $dbh->commit;
         }
         $self;
@@ -107,6 +111,17 @@ sub created_at {
         $self->meta_accessor('created_at', $second);
 }
 
+sub num_highwater {
+        my ($self, $num) = @_;
+        my $high = $self->{num_highwater} ||=
+            $self->meta_accessor('num_highwater');
+        if (defined($num) && (!defined($high) || ($num > $high))) {
+                $self->{num_highwater} = $num;
+                $self->meta_accessor('num_highwater', $num);
+        }
+        $self->{num_highwater};
+}
+
 sub mid_insert {
         my ($self, $mid) = @_;
         my $dbh = $self->{dbh};
@@ -114,7 +129,9 @@ sub mid_insert {
 INSERT OR IGNORE INTO msgmap (mid) VALUES (?)
 
         return if $sth->execute($mid) == 0;
-        $dbh->last_insert_id(undef, undef, 'msgmap', 'num');
+        my $num = $dbh->last_insert_id(undef, undef, 'msgmap', 'num');
+        $self->num_highwater($num) unless !defined($num);
+        $num;
 }
 
 sub mid_for {
@@ -213,7 +230,9 @@ sub mid_set {
                 $self->{dbh}->prepare(
                         'INSERT OR IGNORE INTO msgmap (num,mid) VALUES (?,?)');
         };
-        $sth->execute($num, $mid);
+        my $result = $sth->execute($num, $mid);
+        $self->num_highwater($num) if (defined($result) && $result == 1);
+        $result;
 }
 
 sub DESTROY {
diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm
index 29868d99..ca832ad3 100644
--- a/lib/PublicInbox/SearchIdx.pm
+++ b/lib/PublicInbox/SearchIdx.pm
@@ -582,7 +582,8 @@ sub read_log {
                         my $blob = $1;
                         if (delete $D{$blob}) {
                                 if (defined $self->{regen_down}) {
-                                        $self->{regen_down}--;
+                                        my $num = $self->{regen_down}--;
+                                        $self->{mm}->num_highwater($num);
                                 }
                                 next;
                         }
@@ -618,23 +619,28 @@ sub _git_log {
         my ($self, $range) = @_;
         my $git = $self->{git};
 
+        # Count the new files so they can be added newest to oldest
+        # and still have numbers increasing from oldest to newest
+        my $fcount = 0;
+        # can't use 'rev-list --count' if we use --diff-filter
+        my $fh = $git->popen(qw(log --pretty=tformat:%h
+                             --no-notes --no-color --no-renames
+                             --diff-filter=AM), $range);
+        ++$fcount while <$fh>;
+        my $high = $self->{mm}->num_highwater;
+
         if (index($range, '..') < 0) {
-                my $regen_max = 0;
-                # can't use 'rev-list --count' if we use --diff-filter
-                my $fh = $git->popen(qw(log --pretty=tformat:%h
-                                --no-notes --no-color --no-renames
-                                --diff-filter=AM), $range);
-                ++$regen_max while <$fh>;
-                my (undef, $max) = $self->{mm}->minmax;
-
-                if ($max && $max == $regen_max) {
+                if ($high && $high == $fcount) {
                         # fix up old bugs in full indexes which caused messages to
                         # not appear in Msgmap
-                        $self->{regen_up} = $max;
+                        $self->{regen_up} = $high;
                 } else {
                         # normal regen is for for fresh data
-                        $self->{regen_down} = $regen_max;
+                        $self->{regen_down} = $fcount;
                 }
+        } else {
+                # Give oldest messages the smallest numbers
+                $self->{regen_down} = $high + $fcount;
         }
 
         $git->popen(qw/log --no-notes --no-color --no-renames
diff --git a/lib/PublicInbox/V2Writable.pm b/lib/PublicInbox/V2Writable.pm
index 934640eb..0396d9f5 100644
--- a/lib/PublicInbox/V2Writable.pm
+++ b/lib/PublicInbox/V2Writable.pm
@@ -658,7 +658,7 @@ sub mark_deleted {
 }
 
 sub reindex_oid {
-        my ($self, $mm_tmp, $D, $git, $oid, $regen) = @_;
+        my ($self, $mm_tmp, $D, $git, $oid, $regen, $reindex) = @_;
         my $len;
         my $msgref = $git->cat_file($oid, \$len);
         my $mime = PublicInbox::MIME->new($$msgref);
@@ -676,6 +676,7 @@ sub reindex_oid {
                 if (defined $n && $n > $num) {
                         $mid0 = $mid;
                         $num = $n;
+                        $self->{mm}->mid_set($num, $mid0);
                 }
         }
         if (!defined($mid0) && $regen && !$del) {
@@ -700,7 +701,8 @@ sub reindex_oid {
 
         if (!defined($mid0) || $del) {
                 if (!defined($mid0) && $del) { # expected for deletes
-                        $$regen--;
+                        $num = $$regen--;
+                        $self->{mm}->num_highwater($num) unless $reindex;
                         return
                 }
 
@@ -840,7 +842,10 @@ sub unindex_oid {
                         warn "BUG: multiple articles linked to $oid\n",
                                 join(',',sort keys %gone), "\n";
                 }
-                $self->{unindexed}->{$_}++ foreach keys %gone;
+                foreach my $num (keys %gone) {
+                        $self->{unindexed}->{$_}++;
+                        $self->{mm}->num_delete($num);
+                }
                 $self->unindex_oid_remote($oid, $mid);
         }
 }
@@ -877,11 +882,12 @@ sub index_sync {
         return unless defined $latest;
         $self->idx_init; # acquire lock
         my $mm_tmp = $self->{mm}->tmp_clone;
-        my $ranges = $opts->{reindex} ? [] : $self->last_commits($epoch_max);
+        my $reindex = $opts->{reindex};
+        my $ranges = $reindex ? [] : $self->last_commits($epoch_max);
 
-        my ($min, $max) = $mm_tmp->minmax;
+        my $high = $self->{mm}->num_highwater();
         my $regen = $self->index_prepare($opts, $epoch_max, $ranges);
-        $$regen += $max if $max;
+        $$regen += $high if $high;
         my $D = {}; # "$mid\0$cid" => $oid
         my @cmd = qw(log --raw -r --pretty=tformat:%H
                         --no-notes --no-color --no-abbrev --no-renames);
@@ -903,7 +909,7 @@ sub index_sync {
                                 chomp($cmt = $_);
                         } elsif (/\A:\d{6} 100644 $x40 ($x40) [AM]\tm$/o) {
                                 $self->reindex_oid($mm_tmp, $D, $git, $1,
-                                                $regen);
+                                                $regen, $reindex);
                         } elsif (/\A:\d{6} 100644 $x40 ($x40) [AM]\td$/o) {
                                 $self->mark_deleted($D, $git, $1);
                         }
diff --git a/t/v1reindex.t b/t/v1reindex.t
index 75380f0f..8be95149 100644
--- a/t/v1reindex.t
+++ b/t/v1reindex.t
@@ -22,7 +22,6 @@ my $ibx_config = {
         -primary_address => 'test@example.com',
         indexlevel => 'full',
 };
-my $ibx = PublicInbox::Inbox->new($ibx_config);
 my $mime = PublicInbox::MIME->create(
         header => [
                 From => 'a@example.com',
@@ -32,55 +31,101 @@ my $mime = PublicInbox::MIME->create(
         ],
         body => "hello world\n",
 );
-my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
-foreach my $i (1..10) {
-        $mime->header_set('Message-Id', "<$i\@example.com>");
-        ok($im->add($mime), "message $i added");
-        if ($i == 4) {
+my $minmax;
+my $msgmap;
+my ($mark1, $mark2, $mark3, $mark4);
+{
+        my %config = %$ibx_config;
+        my $ibx = PublicInbox::Inbox->new(\%config);
+        my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
+        foreach my $i (1..10) {
+                $mime->header_set('Message-Id', "<$i\@example.com>");
+                ok($im->add($mime), "message $i added");
+                if ($i == 4) {
+                        $mark1 = $im->get_mark($im->{tip});
+                        $im->remove($mime);
+                        $mark2 = $im->get_mark($im->{tip});
+                }
+        }
+
+        if ('test remove later') {
+                $mark3 = $im->get_mark($im->{tip});
+                $mime->header_set('Message-Id', "<5\@example.com>");
                 $im->remove($mime);
+                $mark4 = $im->get_mark($im->{tip});
         }
-}
 
-if ('test remove later') {
-        $mime->header_set('Message-Id', "<5\@example.com>");
-        $im->remove($mime);
+        $im->done;
+        my $rw = PublicInbox::SearchIdx->new($ibx, 1);
+        eval { $rw->index_sync() };
+        is($@, '', 'no error from indexing');
+
+        $minmax = [ $ibx->mm->minmax ];
+        ok(defined $minmax->[0] && defined $minmax->[1], 'minmax defined');
+        is_deeply($minmax, [ 1, 10 ], 'minmax as expected');
+        is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
+
+        my ($min, $max) = @$minmax;
+        $msgmap = $ibx->mm->msg_range(\$min, $max);
+        is_deeply($msgmap, [
+                          [1, '1@example.com' ],
+                          [2, '2@example.com' ],
+                          [3, '3@example.com' ],
+                          [6, '6@example.com' ],
+                          [7, '7@example.com' ],
+                          [8, '8@example.com' ],
+                          [9, '9@example.com' ],
+                          [10, '10@example.com' ],
+                  ], 'msgmap as expected');
 }
 
-$im->done;
-my $rw = PublicInbox::SearchIdx->new($ibx, 1);
-eval { $rw->index_sync() };
-is($@, '', 'no error from indexing');
+{
+        my %config = %$ibx_config;
+        my $ibx = PublicInbox::Inbox->new(\%config);
+        my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
+        my $rw = PublicInbox::SearchIdx->new($ibx, 1);
+        eval { $rw->index_sync({reindex => 1}) };
+        is($@, '', 'no error from reindexing');
+        $im->done;
 
-my $minmax = [ $ibx->mm->minmax ];
-ok(defined $minmax->[0] && defined $minmax->[1], 'minmax defined');
-is_deeply($minmax, [ 1, 10 ], 'minmax as expected');
+        is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
 
-$rw = PublicInbox::SearchIdx->new($ibx, 1);
-eval { $rw->index_sync({reindex => 1}) };
-is($@, '', 'no error from reindexing');
-$im->done;
+        my ($min, $max) = $ibx->mm->minmax;
+        is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged');
+}
 
 my $xap = "$mainrepo/public-inbox/xapian".PublicInbox::Search::SCHEMA_VERSION();
 remove_tree($xap);
 ok(!-d $xap, 'Xapian directories removed');
-$rw = PublicInbox::SearchIdx->new($ibx, 1);
+{
+        my %config = %$ibx_config;
+        my $ibx = PublicInbox::Inbox->new(\%config);
+        my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
+        my $rw = PublicInbox::SearchIdx->new($ibx, 1);
 
-eval { $rw->index_sync({reindex => 1}) };
-is($@, '', 'no error from reindexing');
-$im->done;
-ok(-d $xap, 'Xapian directories recreated');
+        eval { $rw->index_sync({reindex => 1}) };
+        is($@, '', 'no error from reindexing');
+        $im->done;
+        ok(-d $xap, 'Xapian directories recreated');
 
-delete $ibx->{mm};
-is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
+        delete $ibx->{mm};
+        is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
+        is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
+
+        my ($min, $max) = $ibx->mm->minmax;
+        is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged');
+}
 
 ok(unlink "$mainrepo/public-inbox/msgmap.sqlite3", 'remove msgmap');
 remove_tree($xap);
 ok(!-d $xap, 'Xapian directories removed again');
-
-$rw = PublicInbox::SearchIdx->new($ibx, 1);
 {
         my @warn;
         local $SIG{__WARN__} = sub { push @warn, @_ };
+        my %config = %$ibx_config;
+        my $ibx = PublicInbox::Inbox->new(\%config);
+        my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
+        my $rw = PublicInbox::SearchIdx->new($ibx, 1);
         eval { $rw->index_sync({reindex => 1}) };
         is($@, '', 'no error from reindexing without msgmap');
         is(scalar(@warn), 0, 'no warnings from reindexing');
@@ -88,16 +133,22 @@ $rw = PublicInbox::SearchIdx->new($ibx, 1);
         ok(-d $xap, 'Xapian directories recreated');
         delete $ibx->{mm};
         is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
+        is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
+
+        my ($min, $max) = $ibx->mm->minmax;
+        is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged');
 }
 
 ok(unlink "$mainrepo/public-inbox/msgmap.sqlite3", 'remove msgmap');
 remove_tree($xap);
 ok(!-d $xap, 'Xapian directories removed again');
-
-$rw = PublicInbox::SearchIdx->new($ibx, 1);
 {
         my @warn;
         local $SIG{__WARN__} = sub { push @warn, @_ };
+        my %config = %$ibx_config;
+        my $ibx = PublicInbox::Inbox->new(\%config);
+        my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
+        my $rw = PublicInbox::SearchIdx->new($ibx, 1);
         eval { $rw->index_sync({reindex => 1}) };
         is($@, '', 'no error from reindexing without msgmap');
         is_deeply(\@warn, [], 'no warnings');
@@ -105,18 +156,23 @@ $rw = PublicInbox::SearchIdx->new($ibx, 1);
         ok(-d $xap, 'Xapian directories recreated');
         delete $ibx->{mm};
         is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
+        is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
+
+        my ($min, $max) = @$minmax;
+        is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged');
 }
 
 ok(unlink "$mainrepo/public-inbox/msgmap.sqlite3", 'remove msgmap');
 remove_tree($xap);
 ok(!-d $xap, 'Xapian directories removed again');
-
-$ibx_config->{indexlevel} = 'medium';
-$ibx = PublicInbox::Inbox->new($ibx_config);
-$rw = PublicInbox::SearchIdx->new($ibx, 1);
 {
         my @warn;
         local $SIG{__WARN__} = sub { push @warn, @_ };
+        my %config = %$ibx_config;
+        $config{indexlevel} = 'medium';
+        my $ibx = PublicInbox::Inbox->new(\%config);
+        my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
+        my $rw = PublicInbox::SearchIdx->new($ibx, 1);
         eval { $rw->index_sync({reindex => 1}) };
         is($@, '', 'no error from reindexing without msgmap');
         is_deeply(\@warn, [], 'no warnings');
@@ -124,20 +180,25 @@ $rw = PublicInbox::SearchIdx->new($ibx, 1);
         ok(-d $xap, 'Xapian directories recreated');
         delete $ibx->{mm};
         is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
+        is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
         my $mset = $ibx->search->query('hello world', {mset=>1});
-        isnt(0, $mset->size, 'got Xapian search results');
+        isnt($mset->size, 0, 'got Xapian search results');
+
+        my ($min, $max) = $ibx->mm->minmax;
+        is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged');
 }
 
 ok(unlink "$mainrepo/public-inbox/msgmap.sqlite3", 'remove msgmap');
 remove_tree($xap);
 ok(!-d $xap, 'Xapian directories removed again');
-
-$ibx_config->{indexlevel} = 'basic';
-$ibx = PublicInbox::Inbox->new($ibx_config);
-$rw = PublicInbox::SearchIdx->new($ibx, 1);
 {
         my @warn;
         local $SIG{__WARN__} = sub { push @warn, @_ };
+        my %config = %$ibx_config;
+        $config{indexlevel} = 'basic';
+        my $ibx = PublicInbox::Inbox->new(\%config);
+        my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
+        my $rw = PublicInbox::SearchIdx->new($ibx, 1);
         eval { $rw->index_sync({reindex => 1}) };
         is($@, '', 'no error from reindexing without msgmap');
         is_deeply(\@warn, [], 'no warnings');
@@ -145,25 +206,231 @@ $rw = PublicInbox::SearchIdx->new($ibx, 1);
         ok(-d $xap, 'Xapian directories recreated');
         delete $ibx->{mm};
         is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
+        is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
         my $mset = $ibx->search->reopen->query('hello world', {mset=>1});
-        is(0, $mset->size, "no Xapian search results");
+        is($mset->size, 0, "no Xapian search results");
+
+        my ($min, $max) = $ibx->mm->minmax;
+        is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged');
 }
 
 # upgrade existing basic to medium
 # note: changing indexlevels is not yet supported in v2,
 # and may not be without more effort
-$ibx_config->{indexlevel} = 'medium';
-$ibx = PublicInbox::Inbox->new($ibx_config);
-$rw = PublicInbox::SearchIdx->new($ibx, 1);
 # no removals
 {
         my @warn;
         local $SIG{__WARN__} = sub { push @warn, @_ };
+        my %config = %$ibx_config;
+        $config{indexleve} = 'medium';
+        my $ibx = PublicInbox::Inbox->new(\%config);
+        my $rw = PublicInbox::SearchIdx->new($ibx, 1);
         eval { $rw->index_sync };
         is($@, '', 'no error from indexing');
         is_deeply(\@warn, [], 'no warnings');
         my $mset = $ibx->search->reopen->query('hello world', {mset=>1});
-        isnt(0, $mset->size, 'search OK after basic -> medium');
+        isnt($mset->size, 0, 'search OK after basic -> medium');
+
+        is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
+
+        my ($min, $max) = $ibx->mm->minmax;
+        is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged');
+}
+
+# An incremental indexing test
+ok(unlink "$mainrepo/public-inbox/msgmap.sqlite3", 'remove msgmap');
+remove_tree($xap);
+ok(!-d $xap, 'Xapian directories removed again');
+{
+        my @warn;
+        local $SIG{__WARN__} = sub { push @warn, @_ };
+        my %config = %$ibx_config;
+        my $ibx = PublicInbox::Inbox->new(\%config);
+        my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
+        my $rw = PublicInbox::SearchIdx->new($ibx, 1);
+        # mark1 4 simple additions in the same index_sync
+        eval { $rw->index_sync({ref => $mark1}) };
+        is($@, '', 'no error from reindexing without msgmap');
+        is_deeply(\@warn, [], 'no warnings');
+        $im->done;
+        my ($min, $max) = $ibx->mm->minmax;
+        is($min, 1, 'min as expected');
+        is($max, 4, 'max as expected');
+        is($ibx->mm->num_highwater, 4, 'num_highwater as expected');
+        is_deeply($ibx->mm->msg_range(\$min, $max),
+                  [
+                   [1, '1@example.com' ],
+                   [2, '2@example.com' ],
+                   [3, '3@example.com' ],
+                   [4, '4@example.com' ],
+                  ], 'msgmap as expected' );
 }
+{
+        my @warn;
+        local $SIG{__WARN__} = sub { push @warn, @_ };
+        my %config = %$ibx_config;
+        my $ibx = PublicInbox::Inbox->new(\%config);
+        my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
+        my $rw = PublicInbox::SearchIdx->new($ibx, 1);
+        # mark2 A delete separated form and add in the same index_sync
+        eval { $rw->index_sync({ref => $mark2}) };
+        is($@, '', 'no error from reindexing without msgmap');
+        is_deeply(\@warn, [], 'no warnings');
+        $im->done;
+        my ($min, $max) = $ibx->mm->minmax;
+        is($min, 1, 'min as expected');
+        is($max, 3, 'max as expected');
+        is($ibx->mm->num_highwater, 4, 'num_highwater as expected');
+        is_deeply($ibx->mm->msg_range(\$min, $max),
+                  [
+                   [1, '1@example.com' ],
+                   [2, '2@example.com' ],
+                   [3, '3@example.com' ],
+                  ], 'msgmap as expected' );
+}
+{
+        my @warn;
+        local $SIG{__WARN__} = sub { push @warn, @_ };
+        my %config = %$ibx_config;
+        my $ibx = PublicInbox::Inbox->new(\%config);
+        my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
+        my $rw = PublicInbox::SearchIdx->new($ibx, 1);
+        # mark3 adds following the delete at mark2
+        eval { $rw->index_sync({ref => $mark3}) };
+        is($@, '', 'no error from reindexing without msgmap');
+        is_deeply(\@warn, [], 'no warnings');
+        $im->done;
+        my ($min, $max) = $ibx->mm->minmax;
+        is($min, 1, 'min as expected');
+        is($max, 10, 'max as expected');
+        is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
+        is_deeply($ibx->mm->msg_range(\$min, $max),
+                  [
+                   [1, '1@example.com' ],
+                   [2, '2@example.com' ],
+                   [3, '3@example.com' ],
+                   [5, '5@example.com' ],
+                   [6, '6@example.com' ],
+                   [7, '7@example.com' ],
+                   [8, '8@example.com' ],
+                   [9, '9@example.com' ],
+                   [10, '10@example.com' ],
+                  ], 'msgmap as expected' );
+}
+{
+        my @warn;
+        local $SIG{__WARN__} = sub { push @warn, @_ };
+        my %config = %$ibx_config;
+        my $ibx = PublicInbox::Inbox->new(\%config);
+        my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
+        my $rw = PublicInbox::SearchIdx->new($ibx, 1);
+        # mark4 A delete of an older message
+        eval { $rw->index_sync({ref => $mark4}) };
+        is($@, '', 'no error from reindexing without msgmap');
+        is_deeply(\@warn, [], 'no warnings');
+        $im->done;
+        my ($min, $max) = $ibx->mm->minmax;
+        is($min, 1, 'min as expected');
+        is($max, 10, 'max as expected');
+        is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
+        is_deeply($ibx->mm->msg_range(\$min, $max),
+                  [
+                   [1, '1@example.com' ],
+                   [2, '2@example.com' ],
+                   [3, '3@example.com' ],
+                   [6, '6@example.com' ],
+                   [7, '7@example.com' ],
+                   [8, '8@example.com' ],
+                   [9, '9@example.com' ],
+                   [10, '10@example.com' ],
+                  ], 'msgmap as expected' );
+}
+
+
+# Another incremental indexing test
+ok(unlink "$mainrepo/public-inbox/msgmap.sqlite3", 'remove msgmap');
+remove_tree($xap);
+ok(!-d $xap, 'Xapian directories removed again');
+{
+        my @warn;
+        local $SIG{__WARN__} = sub { push @warn, @_ };
+        my %config = %$ibx_config;
+        my $ibx = PublicInbox::Inbox->new(\%config);
+        my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
+        my $rw = PublicInbox::SearchIdx->new($ibx, 1);
+        # mark2 an add and it's delete in the same index_sync
+        eval { $rw->index_sync({ref => $mark2}) };
+        is($@, '', 'no error from reindexing without msgmap');
+        is_deeply(\@warn, [], 'no warnings');
+        $im->done;
+        my ($min, $max) = $ibx->mm->minmax;
+        is($min, 1, 'min as expected');
+        is($max, 3, 'max as expected');
+        is($ibx->mm->num_highwater, 4, 'num_highwater as expected');
+        is_deeply($ibx->mm->msg_range(\$min, $max),
+                  [
+                   [1, '1@example.com' ],
+                   [2, '2@example.com' ],
+                   [3, '3@example.com' ],
+                  ], 'msgmap as expected' );
+}
+{
+        my @warn;
+        local $SIG{__WARN__} = sub { push @warn, @_ };
+        my %config = %$ibx_config;
+        my $ibx = PublicInbox::Inbox->new(\%config);
+        my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
+        my $rw = PublicInbox::SearchIdx->new($ibx, 1);
+        # mark3 adds following the delete at mark2
+        eval { $rw->index_sync({ref => $mark3}) };
+        is($@, '', 'no error from reindexing without msgmap');
+        is_deeply(\@warn, [], 'no warnings');
+        $im->done;
+        my ($min, $max) = $ibx->mm->minmax;
+        is($min, 1, 'min as expected');
+        is($max, 10, 'max as expected');
+        is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
+        is_deeply($ibx->mm->msg_range(\$min, $max),
+                  [
+                   [1, '1@example.com' ],
+                   [2, '2@example.com' ],
+                   [3, '3@example.com' ],
+                   [5, '5@example.com' ],
+                   [6, '6@example.com' ],
+                   [7, '7@example.com' ],
+                   [8, '8@example.com' ],
+                   [9, '9@example.com' ],
+                   [10, '10@example.com' ],
+                  ], 'msgmap as expected' );
+}
+{
+        my @warn;
+        local $SIG{__WARN__} = sub { push @warn, @_ };
+        my %config = %$ibx_config;
+        my $ibx = PublicInbox::Inbox->new(\%config);
+        my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
+        my $rw = PublicInbox::SearchIdx->new($ibx, 1);
+        # mark4 A delete of an older message
+        eval { $rw->index_sync({ref => $mark4}) };
+        is($@, '', 'no error from reindexing without msgmap');
+        is_deeply(\@warn, [], 'no warnings');
+        $im->done;
+        my ($min, $max) = $ibx->mm->minmax;
+        is($min, 1, 'min as expected');
+        is($max, 10, 'max as expected');
+        is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
+        is_deeply($ibx->mm->msg_range(\$min, $max),
+                  [
+                   [1, '1@example.com' ],
+                   [2, '2@example.com' ],
+                   [3, '3@example.com' ],
+                   [6, '6@example.com' ],
+                   [7, '7@example.com' ],
+                   [8, '8@example.com' ],
+                   [9, '9@example.com' ],
+                   [10, '10@example.com' ],
+                  ], 'msgmap as expected' );
+}
+
 
 done_testing();
diff --git a/t/v2reindex.t b/t/v2reindex.t
index 1543309c..a5454a22 100644
--- a/t/v2reindex.t
+++ b/t/v2reindex.t
@@ -21,7 +21,6 @@ my $ibx_config = {
         -primary_address => 'test@example.com',
         indexlevel => 'full',
 };
-my $ibx = PublicInbox::Inbox->new($ibx_config);
 my $mime = PublicInbox::MIME->create(
         header => [
                 From => 'a@example.com',
@@ -32,39 +31,86 @@ my $mime = PublicInbox::MIME->create(
         body => "hello world\n",
 );
 local $ENV{NPROC} = 2;
-my $im = PublicInbox::V2Writable->new($ibx, 1);
-foreach my $i (1..10) {
-        $mime->header_set('Message-Id', "<$i\@example.com>");
-        ok($im->add($mime), "message $i added");
-        if ($i == 4) {
+my $minmax;
+my $msgmap;
+my ($mark1, $mark2, $mark3, $mark4);
+{
+        my %config = %$ibx_config;
+        my $ibx = PublicInbox::Inbox->new(\%config);
+        my $im = PublicInbox::V2Writable->new($ibx, 1);
+        my $im0 = $im->importer();
+        foreach my $i (1..10) {
+                $mime->header_set('Message-Id', "<$i\@example.com>");
+                ok($im->add($mime), "message $i added");
+                if ($i == 4) {
+                        $mark1 = $im0->get_mark($im0->{tip});
+                        $im->remove($mime);
+                        $mark2 = $im0->get_mark($im0->{tip});
+                }
+        }
+
+        if ('test remove later') {
+                $mark3 = $im0->get_mark($im0->{tip});
+                $mime->header_set('Message-Id', "<5\@example.com>");
                 $im->remove($mime);
+                $mark4 = $im0->get_mark($im0->{tip});
         }
-}
 
-if ('test remove later') {
-        $mime->header_set('Message-Id', "<5\@example.com>");
-        $im->remove($mime);
+        $im->done;
+        $minmax = [ $ibx->mm->minmax ];
+        ok(defined $minmax->[0] && defined $minmax->[1], 'minmax defined');
+        is_deeply($minmax, [ 1, 10 ], 'minmax as expected');
+        is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
+
+        my ($min, $max) = @$minmax;
+        $msgmap = $ibx->mm->msg_range(\$min, $max);
+        is_deeply($msgmap, [
+                          [1, '1@example.com' ],
+                          [2, '2@example.com' ],
+                          [3, '3@example.com' ],
+                          [6, '6@example.com' ],
+                          [7, '7@example.com' ],
+                          [8, '8@example.com' ],
+                          [9, '9@example.com' ],
+                          [10, '10@example.com' ],
+                  ], 'msgmap as expected');
 }
 
-$im->done;
-my $minmax = [ $ibx->mm->minmax ];
-ok(defined $minmax->[0] && defined $minmax->[1], 'minmax defined');
-is_deeply($minmax, [ 1, 10 ], 'minmax as expected');
+{
+        my %config = %$ibx_config;
+        my $ibx = PublicInbox::Inbox->new(\%config);
+        my $im = PublicInbox::V2Writable->new($ibx, 1);
+        eval { $im->index_sync({reindex => 1}) };
+        is($@, '', 'no error from reindexing');
+        $im->done;
+
+        delete $ibx->{mm};
+        is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
+        is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
 
-eval { $im->index_sync({reindex => 1}) };
-is($@, '', 'no error from reindexing');
-$im->done;
+        my ($min, $max) = $ibx->mm->minmax;
+        is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged');
+}
 
 my $xap = "$mainrepo/xap".PublicInbox::Search::SCHEMA_VERSION();
 remove_tree($xap);
 ok(!-d $xap, 'Xapian directories removed');
-eval { $im->index_sync({reindex => 1}) };
-is($@, '', 'no error from reindexing');
-$im->done;
-ok(-d $xap, 'Xapian directories recreated');
+{
+        my %config = %$ibx_config;
+        my $ibx = PublicInbox::Inbox->new(\%config);
+        my $im = PublicInbox::V2Writable->new($ibx, 1);
+        eval { $im->index_sync({reindex => 1}) };
+        is($@, '', 'no error from reindexing');
+        $im->done;
+        ok(-d $xap, 'Xapian directories recreated');
 
-delete $ibx->{mm};
-is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
+        delete $ibx->{mm};
+        is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
+        is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
+
+        my ($min, $max) = $ibx->mm->minmax;
+        is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged');
+}
 
 ok(unlink "$mainrepo/msgmap.sqlite3", 'remove msgmap');
 remove_tree($xap);
@@ -72,6 +118,9 @@ ok(!-d $xap, 'Xapian directories removed again');
 {
         my @warn;
         local $SIG{__WARN__} = sub { push @warn, @_ };
+        my %config = %$ibx_config;
+        my $ibx = PublicInbox::Inbox->new(\%config);
+        my $im = PublicInbox::V2Writable->new($ibx, 1);
         eval { $im->index_sync({reindex => 1}) };
         is($@, '', 'no error from reindexing without msgmap');
         is(scalar(@warn), 0, 'no warnings from reindexing');
@@ -79,6 +128,10 @@ ok(!-d $xap, 'Xapian directories removed again');
         ok(-d $xap, 'Xapian directories recreated');
         delete $ibx->{mm};
         is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
+        is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
+
+        my ($min, $max) = $ibx->mm->minmax;
+        is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged');
 }
 
 my %sizes;
@@ -88,6 +141,9 @@ ok(!-d $xap, 'Xapian directories removed again');
 {
         my @warn;
         local $SIG{__WARN__} = sub { push @warn, @_ };
+        my %config = %$ibx_config;
+        my $ibx = PublicInbox::Inbox->new(\%config);
+        my $im = PublicInbox::V2Writable->new($ibx, 1);
         eval { $im->index_sync({reindex => 1}) };
         is($@, '', 'no error from reindexing without msgmap');
         is_deeply(\@warn, [], 'no warnings');
@@ -95,21 +151,25 @@ ok(!-d $xap, 'Xapian directories removed again');
         ok(-d $xap, 'Xapian directories recreated');
         delete $ibx->{mm};
         is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
+        is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
         my $mset = $ibx->search->query('"hello world"', {mset=>1});
-        isnt(0, $mset->size, "phrase search succeeds on indexlevel=full");
+        isnt($mset->size, 0, "phrase search succeeds on indexlevel=full");
         for (<"$xap/*/*">) { $sizes{$ibx->{indexlevel}} += -s _ if -f $_ }
+
+        my ($min, $max) = $ibx->mm->minmax;
+        is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged');
 }
 
 ok(unlink "$mainrepo/msgmap.sqlite3", 'remove msgmap');
 remove_tree($xap);
 ok(!-d $xap, 'Xapian directories removed again');
-
-$ibx_config->{indexlevel} = 'medium';
-$ibx = PublicInbox::Inbox->new($ibx_config);
-$im = PublicInbox::V2Writable->new($ibx);
 {
         my @warn;
         local $SIG{__WARN__} = sub { push @warn, @_ };
+        my %config = %$ibx_config;
+        $config{indexlevel} = 'medium';
+        my $ibx = PublicInbox::Inbox->new(\%config);
+        my $im = PublicInbox::V2Writable->new($ibx);
         eval { $im->index_sync({reindex => 1}) };
         is($@, '', 'no error from reindexing without msgmap');
         is_deeply(\@warn, [], 'no warnings');
@@ -117,31 +177,36 @@ $im = PublicInbox::V2Writable->new($ibx);
         ok(-d $xap, 'Xapian directories recreated');
         delete $ibx->{mm};
         is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
+        is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
 
         if (0) {
                 # not sure why, but Xapian seems to fallback to terms and
                 # phrase searches still work
                 delete $ibx->{search};
                 my $mset = $ibx->search->query('"hello world"', {mset=>1});
-                is(0, $mset->size, 'phrase search does not work on medium');
+                is($mset->size, 0, 'phrase search does not work on medium');
         }
 
         my $mset = $ibx->search->query('hello world', {mset=>1});
-        isnt(0, $mset->size, "normal search works on indexlevel=medium");
+        isnt($mset->size, 0, "normal search works on indexlevel=medium");
         for (<"$xap/*/*">) { $sizes{$ibx->{indexlevel}} += -s _ if -f $_ }
         ok($sizes{full} > $sizes{medium}, 'medium is smaller than full');
+
+
+        my ($min, $max) = $ibx->mm->minmax;
+        is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged');
 }
 
 ok(unlink "$mainrepo/msgmap.sqlite3", 'remove msgmap');
 remove_tree($xap);
 ok(!-d $xap, 'Xapian directories removed again');
-
-$ibx_config->{indexlevel} = 'basic';
-$ibx = PublicInbox::Inbox->new($ibx_config);
-$im = PublicInbox::V2Writable->new($ibx);
 {
         my @warn;
         local $SIG{__WARN__} = sub { push @warn, @_ };
+        my %config = %$ibx_config;
+        $config{indexlevel} = 'basic';
+        my $ibx = PublicInbox::Inbox->new(\%config);
+        my $im = PublicInbox::V2Writable->new($ibx);
         eval { $im->index_sync({reindex => 1}) };
         is($@, '', 'no error from reindexing without msgmap');
         is_deeply(\@warn, [], 'no warnings');
@@ -149,10 +214,210 @@ $im = PublicInbox::V2Writable->new($ibx);
         ok(-d $xap, 'Xapian directories recreated');
         delete $ibx->{mm};
         is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
+        is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
         my $mset = $ibx->search->query('hello', {mset=>1});
-        is(0, $mset->size, "search fails on indexlevel='basic'");
+        is($mset->size, 0, "search fails on indexlevel='basic'");
         for (<"$xap/*/*">) { $sizes{$ibx->{indexlevel}} += -s _ if -f $_ }
         ok($sizes{medium} > $sizes{basic}, 'basic is smaller than medium');
+
+        my ($min, $max) = $ibx->mm->minmax;
+        is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged');
+}
+
+
+# An incremental indexing test
+ok(unlink "$mainrepo/msgmap.sqlite3", 'remove msgmap');
+remove_tree($xap);
+ok(!-d $xap, 'Xapian directories removed again');
+{
+        my @warn;
+        local $SIG{__WARN__} = sub { push @warn, @_ };
+        my %config = %$ibx_config;
+        my $ibx = PublicInbox::Inbox->new(\%config);
+        # mark1 4 simple additions in the same index_sync
+        $ibx->{ref_head} = $mark1;
+        my $im = PublicInbox::V2Writable->new($ibx);
+        eval { $im->index_sync() };
+        is($@, '', 'no error from reindexing without msgmap');
+        is_deeply(\@warn, [], 'no warnings');
+        $im->done;
+        my ($min, $max) = $ibx->mm->minmax;
+        is($min, 1, 'min as expected');
+        is($max, 4, 'max as expected');
+        is($ibx->mm->num_highwater, 4, 'num_highwater as expected');
+        is_deeply($ibx->mm->msg_range(\$min, $max),
+                  [
+                   [1, '1@example.com' ],
+                   [2, '2@example.com' ],
+                   [3, '3@example.com' ],
+                   [4, '4@example.com' ],
+                  ], 'msgmap as expected' );
+}
+{
+        my @warn;
+        local $SIG{__WARN__} = sub { push @warn, @_ };
+        my %config = %$ibx_config;
+        my $ibx = PublicInbox::Inbox->new(\%config);
+        # mark2 A delete separated from an add in the same index_sync
+        $ibx->{ref_head} = $mark2;
+        my $im = PublicInbox::V2Writable->new($ibx);
+        eval { $im->index_sync() };
+        is($@, '', 'no error from reindexing without msgmap');
+        is_deeply(\@warn, [], 'no warnings');
+        $im->done;
+        my ($min, $max) = $ibx->mm->minmax;
+        is($min, 1, 'min as expected');
+        is($max, 3, 'max as expected');
+        is($ibx->mm->num_highwater, 4, 'num_highwater as expected');
+        is_deeply($ibx->mm->msg_range(\$min, $max),
+                  [
+                   [1, '1@example.com' ],
+                   [2, '2@example.com' ],
+                   [3, '3@example.com' ],
+                  ], 'msgmap as expected' );
+}
+{
+        my @warn;
+        local $SIG{__WARN__} = sub { push @warn, @_ };
+        my %config = %$ibx_config;
+        my $ibx = PublicInbox::Inbox->new(\%config);
+        # mark3 adds following the delete at mark2
+        $ibx->{ref_head} = $mark3;
+        my $im = PublicInbox::V2Writable->new($ibx);
+        eval { $im->index_sync() };
+        is($@, '', 'no error from reindexing without msgmap');
+        is_deeply(\@warn, [], 'no warnings');
+        $im->done;
+        my ($min, $max) = $ibx->mm->minmax;
+        is($min, 1, 'min as expected');
+        is($max, 10, 'max as expected');
+        is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
+        is_deeply($ibx->mm->msg_range(\$min, $max),
+                  [
+                   [1, '1@example.com' ],
+                   [2, '2@example.com' ],
+                   [3, '3@example.com' ],
+                   [5, '5@example.com' ],
+                   [6, '6@example.com' ],
+                   [7, '7@example.com' ],
+                   [8, '8@example.com' ],
+                   [9, '9@example.com' ],
+                   [10, '10@example.com' ],
+                  ], 'msgmap as expected' );
+}
+{
+        my @warn;
+        local $SIG{__WARN__} = sub { push @warn, @_ };
+        my %config = %$ibx_config;
+        my $ibx = PublicInbox::Inbox->new(\%config);
+        # mark4 A delete of an older message
+        $ibx->{ref_head} = $mark4;
+        my $im = PublicInbox::V2Writable->new($ibx);
+        eval { $im->index_sync() };
+        is($@, '', 'no error from reindexing without msgmap');
+        is_deeply(\@warn, [], 'no warnings');
+        $im->done;
+        my ($min, $max) = $ibx->mm->minmax;
+        is($min, 1, 'min as expected');
+        is($max, 10, 'max as expected');
+        is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
+        is_deeply($ibx->mm->msg_range(\$min, $max),
+                  [
+                   [1, '1@example.com' ],
+                   [2, '2@example.com' ],
+                   [3, '3@example.com' ],
+                   [6, '6@example.com' ],
+                   [7, '7@example.com' ],
+                   [8, '8@example.com' ],
+                   [9, '9@example.com' ],
+                   [10, '10@example.com' ],
+                  ], 'msgmap as expected' );
+}
+
+
+# Another incremental indexing test
+ok(unlink "$mainrepo/msgmap.sqlite3", 'remove msgmap');
+remove_tree($xap);
+ok(!-d $xap, 'Xapian directories removed again');
+{
+        my @warn;
+        local $SIG{__WARN__} = sub { push @warn, @_ };
+        my %config = %$ibx_config;
+        my $ibx = PublicInbox::Inbox->new(\%config);
+        # mark2 an add and it's delete in the same index_sync
+        $ibx->{ref_head} = $mark2;
+        my $im = PublicInbox::V2Writable->new($ibx);
+        eval { $im->index_sync() };
+        is($@, '', 'no error from reindexing without msgmap');
+        is_deeply(\@warn, [], 'no warnings');
+        $im->done;
+        my ($min, $max) = $ibx->mm->minmax;
+        is($min, 1, 'min as expected');
+        is($max, 3, 'max as expected');
+        is($ibx->mm->num_highwater, 4, 'num_highwater as expected');
+        is_deeply($ibx->mm->msg_range(\$min, $max),
+                  [
+                   [1, '1@example.com' ],
+                   [2, '2@example.com' ],
+                   [3, '3@example.com' ],
+                  ], 'msgmap as expected' );
+}
+{
+        my @warn;
+        local $SIG{__WARN__} = sub { push @warn, @_ };
+        my %config = %$ibx_config;
+        my $ibx = PublicInbox::Inbox->new(\%config);
+        # mark3 adds following the delete at mark2
+        $ibx->{ref_head} = $mark3;
+        my $im = PublicInbox::V2Writable->new($ibx);
+        eval { $im->index_sync() };
+        is($@, '', 'no error from reindexing without msgmap');
+        is_deeply(\@warn, [], 'no warnings');
+        $im->done;
+        my ($min, $max) = $ibx->mm->minmax;
+        is($min, 1, 'min as expected');
+        is($max, 10, 'max as expected');
+        is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
+        is_deeply($ibx->mm->msg_range(\$min, $max),
+                  [
+                   [1, '1@example.com' ],
+                   [2, '2@example.com' ],
+                   [3, '3@example.com' ],
+                   [5, '5@example.com' ],
+                   [6, '6@example.com' ],
+                   [7, '7@example.com' ],
+                   [8, '8@example.com' ],
+                   [9, '9@example.com' ],
+                   [10, '10@example.com' ],
+                  ], 'msgmap as expected' );
+}
+{
+        my @warn;
+        local $SIG{__WARN__} = sub { push @warn, @_ };
+        my %config = %$ibx_config;
+        my $ibx = PublicInbox::Inbox->new(\%config);
+        # mark4 A delete of an older message
+        $ibx->{ref_head} = $mark4;
+        my $im = PublicInbox::V2Writable->new($ibx);
+        eval { $im->index_sync() };
+        is($@, '', 'no error from reindexing without msgmap');
+        is_deeply(\@warn, [], 'no warnings');
+        $im->done;
+        my ($min, $max) = $ibx->mm->minmax;
+        is($min, 1, 'min as expected');
+        is($max, 10, 'max as expected');
+        is($ibx->mm->num_highwater, 10, 'num_highwater as expected');
+        is_deeply($ibx->mm->msg_range(\$min, $max),
+                  [
+                   [1, '1@example.com' ],
+                   [2, '2@example.com' ],
+                   [3, '3@example.com' ],
+                   [6, '6@example.com' ],
+                   [7, '7@example.com' ],
+                   [8, '8@example.com' ],
+                   [9, '9@example.com' ],
+                   [10, '10@example.com' ],
+                  ], 'msgmap as expected' );
 }
 
 done_testing();