diff options
-rw-r--r-- | lib/PublicInbox/Import.pm | 1 | ||||
-rw-r--r-- | lib/PublicInbox/Msgmap.pm | 23 | ||||
-rw-r--r-- | lib/PublicInbox/SearchIdx.pm | 30 | ||||
-rw-r--r-- | lib/PublicInbox/V2Writable.pm | 20 | ||||
-rw-r--r-- | t/v1reindex.t | 359 | ||||
-rw-r--r-- | t/v2reindex.t | 335 |
6 files changed, 666 insertions, 102 deletions
diff --git a/lib/PublicInbox/Import.pm b/lib/PublicInbox/Import.pm index 4e3b4c55..bfa7a805 100644 --- a/lib/PublicInbox/Import.pm +++ b/lib/PublicInbox/Import.pm @@ -451,6 +451,7 @@ sub done { sub atfork_child { my ($self) = @_; foreach my $f (qw(in out)) { + next unless defined($self->{$f}); close $self->{$f} or die "failed to close import[$f]: $!\n"; } } diff --git a/lib/PublicInbox/Msgmap.pm b/lib/PublicInbox/Msgmap.pm index fdc71e46..d474bade 100644 --- a/lib/PublicInbox/Msgmap.pm +++ b/lib/PublicInbox/Msgmap.pm @@ -50,6 +50,10 @@ sub new_file { create_tables($dbh); $dbh->begin_work; $self->created_at(time) unless $self->created_at; + + my (undef, $max) = $self->minmax(); + $max ||= 0; + $self->num_highwater($max); $dbh->commit; } $self; @@ -107,6 +111,17 @@ sub created_at { $self->meta_accessor('created_at', $second); } +sub num_highwater { + my ($self, $num) = @_; + my $high = $self->{num_highwater} ||= + $self->meta_accessor('num_highwater'); + if (defined($num) && (!defined($high) || ($num > $high))) { + $self->{num_highwater} = $num; + $self->meta_accessor('num_highwater', $num); + } + $self->{num_highwater}; +} + sub mid_insert { my ($self, $mid) = @_; my $dbh = $self->{dbh}; @@ -114,7 +129,9 @@ sub mid_insert { INSERT OR IGNORE INTO msgmap (mid) VALUES (?) return if $sth->execute($mid) == 0; - $dbh->last_insert_id(undef, undef, 'msgmap', 'num'); + my $num = $dbh->last_insert_id(undef, undef, 'msgmap', 'num'); + $self->num_highwater($num) unless !defined($num); + $num; } sub mid_for { @@ -213,7 +230,9 @@ sub mid_set { $self->{dbh}->prepare( 'INSERT OR IGNORE INTO msgmap (num,mid) VALUES (?,?)'); }; - $sth->execute($num, $mid); + my $result = $sth->execute($num, $mid); + $self->num_highwater($num) if (defined($result) && $result == 1); + $result; } sub DESTROY { diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm index 29868d99..ca832ad3 100644 --- a/lib/PublicInbox/SearchIdx.pm +++ b/lib/PublicInbox/SearchIdx.pm @@ -582,7 +582,8 @@ sub read_log { my $blob = $1; if (delete $D{$blob}) { if (defined $self->{regen_down}) { - $self->{regen_down}--; + my $num = $self->{regen_down}--; + $self->{mm}->num_highwater($num); } next; } @@ -618,23 +619,28 @@ sub _git_log { my ($self, $range) = @_; my $git = $self->{git}; + # Count the new files so they can be added newest to oldest + # and still have numbers increasing from oldest to newest + my $fcount = 0; + # can't use 'rev-list --count' if we use --diff-filter + my $fh = $git->popen(qw(log --pretty=tformat:%h + --no-notes --no-color --no-renames + --diff-filter=AM), $range); + ++$fcount while <$fh>; + my $high = $self->{mm}->num_highwater; + if (index($range, '..') < 0) { - my $regen_max = 0; - # can't use 'rev-list --count' if we use --diff-filter - my $fh = $git->popen(qw(log --pretty=tformat:%h - --no-notes --no-color --no-renames - --diff-filter=AM), $range); - ++$regen_max while <$fh>; - my (undef, $max) = $self->{mm}->minmax; - - if ($max && $max == $regen_max) { + if ($high && $high == $fcount) { # fix up old bugs in full indexes which caused messages to # not appear in Msgmap - $self->{regen_up} = $max; + $self->{regen_up} = $high; } else { # normal regen is for for fresh data - $self->{regen_down} = $regen_max; + $self->{regen_down} = $fcount; } + } else { + # Give oldest messages the smallest numbers + $self->{regen_down} = $high + $fcount; } $git->popen(qw/log --no-notes --no-color --no-renames diff --git a/lib/PublicInbox/V2Writable.pm b/lib/PublicInbox/V2Writable.pm index 934640eb..0396d9f5 100644 --- a/lib/PublicInbox/V2Writable.pm +++ b/lib/PublicInbox/V2Writable.pm @@ -658,7 +658,7 @@ sub mark_deleted { } sub reindex_oid { - my ($self, $mm_tmp, $D, $git, $oid, $regen) = @_; + my ($self, $mm_tmp, $D, $git, $oid, $regen, $reindex) = @_; my $len; my $msgref = $git->cat_file($oid, \$len); my $mime = PublicInbox::MIME->new($$msgref); @@ -676,6 +676,7 @@ sub reindex_oid { if (defined $n && $n > $num) { $mid0 = $mid; $num = $n; + $self->{mm}->mid_set($num, $mid0); } } if (!defined($mid0) && $regen && !$del) { @@ -700,7 +701,8 @@ sub reindex_oid { if (!defined($mid0) || $del) { if (!defined($mid0) && $del) { # expected for deletes - $$regen--; + $num = $$regen--; + $self->{mm}->num_highwater($num) unless $reindex; return } @@ -840,7 +842,10 @@ sub unindex_oid { warn "BUG: multiple articles linked to $oid\n", join(',',sort keys %gone), "\n"; } - $self->{unindexed}->{$_}++ foreach keys %gone; + foreach my $num (keys %gone) { + $self->{unindexed}->{$_}++; + $self->{mm}->num_delete($num); + } $self->unindex_oid_remote($oid, $mid); } } @@ -877,11 +882,12 @@ sub index_sync { return unless defined $latest; $self->idx_init; # acquire lock my $mm_tmp = $self->{mm}->tmp_clone; - my $ranges = $opts->{reindex} ? [] : $self->last_commits($epoch_max); + my $reindex = $opts->{reindex}; + my $ranges = $reindex ? [] : $self->last_commits($epoch_max); - my ($min, $max) = $mm_tmp->minmax; + my $high = $self->{mm}->num_highwater(); my $regen = $self->index_prepare($opts, $epoch_max, $ranges); - $$regen += $max if $max; + $$regen += $high if $high; my $D = {}; # "$mid\0$cid" => $oid my @cmd = qw(log --raw -r --pretty=tformat:%H --no-notes --no-color --no-abbrev --no-renames); @@ -903,7 +909,7 @@ sub index_sync { chomp($cmt = $_); } elsif (/\A:\d{6} 100644 $x40 ($x40) [AM]\tm$/o) { $self->reindex_oid($mm_tmp, $D, $git, $1, - $regen); + $regen, $reindex); } elsif (/\A:\d{6} 100644 $x40 ($x40) [AM]\td$/o) { $self->mark_deleted($D, $git, $1); } diff --git a/t/v1reindex.t b/t/v1reindex.t index 75380f0f..8be95149 100644 --- a/t/v1reindex.t +++ b/t/v1reindex.t @@ -22,7 +22,6 @@ my $ibx_config = { -primary_address => 'test@example.com', indexlevel => 'full', }; -my $ibx = PublicInbox::Inbox->new($ibx_config); my $mime = PublicInbox::MIME->create( header => [ From => 'a@example.com', @@ -32,55 +31,101 @@ my $mime = PublicInbox::MIME->create( ], body => "hello world\n", ); -my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx); -foreach my $i (1..10) { - $mime->header_set('Message-Id', "<$i\@example.com>"); - ok($im->add($mime), "message $i added"); - if ($i == 4) { +my $minmax; +my $msgmap; +my ($mark1, $mark2, $mark3, $mark4); +{ + my %config = %$ibx_config; + my $ibx = PublicInbox::Inbox->new(\%config); + my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx); + foreach my $i (1..10) { + $mime->header_set('Message-Id', "<$i\@example.com>"); + ok($im->add($mime), "message $i added"); + if ($i == 4) { + $mark1 = $im->get_mark($im->{tip}); + $im->remove($mime); + $mark2 = $im->get_mark($im->{tip}); + } + } + + if ('test remove later') { + $mark3 = $im->get_mark($im->{tip}); + $mime->header_set('Message-Id', "<5\@example.com>"); $im->remove($mime); + $mark4 = $im->get_mark($im->{tip}); } -} -if ('test remove later') { - $mime->header_set('Message-Id', "<5\@example.com>"); - $im->remove($mime); + $im->done; + my $rw = PublicInbox::SearchIdx->new($ibx, 1); + eval { $rw->index_sync() }; + is($@, '', 'no error from indexing'); + + $minmax = [ $ibx->mm->minmax ]; + ok(defined $minmax->[0] && defined $minmax->[1], 'minmax defined'); + is_deeply($minmax, [ 1, 10 ], 'minmax as expected'); + is($ibx->mm->num_highwater, 10, 'num_highwater as expected'); + + my ($min, $max) = @$minmax; + $msgmap = $ibx->mm->msg_range(\$min, $max); + is_deeply($msgmap, [ + [1, '1@example.com' ], + [2, '2@example.com' ], + [3, '3@example.com' ], + [6, '6@example.com' ], + [7, '7@example.com' ], + [8, '8@example.com' ], + [9, '9@example.com' ], + [10, '10@example.com' ], + ], 'msgmap as expected'); } -$im->done; -my $rw = PublicInbox::SearchIdx->new($ibx, 1); -eval { $rw->index_sync() }; -is($@, '', 'no error from indexing'); +{ + my %config = %$ibx_config; + my $ibx = PublicInbox::Inbox->new(\%config); + my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx); + my $rw = PublicInbox::SearchIdx->new($ibx, 1); + eval { $rw->index_sync({reindex => 1}) }; + is($@, '', 'no error from reindexing'); + $im->done; -my $minmax = [ $ibx->mm->minmax ]; -ok(defined $minmax->[0] && defined $minmax->[1], 'minmax defined'); -is_deeply($minmax, [ 1, 10 ], 'minmax as expected'); + is($ibx->mm->num_highwater, 10, 'num_highwater as expected'); -$rw = PublicInbox::SearchIdx->new($ibx, 1); -eval { $rw->index_sync({reindex => 1}) }; -is($@, '', 'no error from reindexing'); -$im->done; + my ($min, $max) = $ibx->mm->minmax; + is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged'); +} my $xap = "$mainrepo/public-inbox/xapian".PublicInbox::Search::SCHEMA_VERSION(); remove_tree($xap); ok(!-d $xap, 'Xapian directories removed'); -$rw = PublicInbox::SearchIdx->new($ibx, 1); +{ + my %config = %$ibx_config; + my $ibx = PublicInbox::Inbox->new(\%config); + my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx); + my $rw = PublicInbox::SearchIdx->new($ibx, 1); -eval { $rw->index_sync({reindex => 1}) }; -is($@, '', 'no error from reindexing'); -$im->done; -ok(-d $xap, 'Xapian directories recreated'); + eval { $rw->index_sync({reindex => 1}) }; + is($@, '', 'no error from reindexing'); + $im->done; + ok(-d $xap, 'Xapian directories recreated'); -delete $ibx->{mm}; -is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged'); + delete $ibx->{mm}; + is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged'); + is($ibx->mm->num_highwater, 10, 'num_highwater as expected'); + + my ($min, $max) = $ibx->mm->minmax; + is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged'); +} ok(unlink "$mainrepo/public-inbox/msgmap.sqlite3", 'remove msgmap'); remove_tree($xap); ok(!-d $xap, 'Xapian directories removed again'); - -$rw = PublicInbox::SearchIdx->new($ibx, 1); { my @warn; local $SIG{__WARN__} = sub { push @warn, @_ }; + my %config = %$ibx_config; + my $ibx = PublicInbox::Inbox->new(\%config); + my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx); + my $rw = PublicInbox::SearchIdx->new($ibx, 1); eval { $rw->index_sync({reindex => 1}) }; is($@, '', 'no error from reindexing without msgmap'); is(scalar(@warn), 0, 'no warnings from reindexing'); @@ -88,16 +133,22 @@ $rw = PublicInbox::SearchIdx->new($ibx, 1); ok(-d $xap, 'Xapian directories recreated'); delete $ibx->{mm}; is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged'); + is($ibx->mm->num_highwater, 10, 'num_highwater as expected'); + + my ($min, $max) = $ibx->mm->minmax; + is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged'); } ok(unlink "$mainrepo/public-inbox/msgmap.sqlite3", 'remove msgmap'); remove_tree($xap); ok(!-d $xap, 'Xapian directories removed again'); - -$rw = PublicInbox::SearchIdx->new($ibx, 1); { my @warn; local $SIG{__WARN__} = sub { push @warn, @_ }; + my %config = %$ibx_config; + my $ibx = PublicInbox::Inbox->new(\%config); + my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx); + my $rw = PublicInbox::SearchIdx->new($ibx, 1); eval { $rw->index_sync({reindex => 1}) }; is($@, '', 'no error from reindexing without msgmap'); is_deeply(\@warn, [], 'no warnings'); @@ -105,18 +156,23 @@ $rw = PublicInbox::SearchIdx->new($ibx, 1); ok(-d $xap, 'Xapian directories recreated'); delete $ibx->{mm}; is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged'); + is($ibx->mm->num_highwater, 10, 'num_highwater as expected'); + + my ($min, $max) = @$minmax; + is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged'); } ok(unlink "$mainrepo/public-inbox/msgmap.sqlite3", 'remove msgmap'); remove_tree($xap); ok(!-d $xap, 'Xapian directories removed again'); - -$ibx_config->{indexlevel} = 'medium'; -$ibx = PublicInbox::Inbox->new($ibx_config); -$rw = PublicInbox::SearchIdx->new($ibx, 1); { my @warn; local $SIG{__WARN__} = sub { push @warn, @_ }; + my %config = %$ibx_config; + $config{indexlevel} = 'medium'; + my $ibx = PublicInbox::Inbox->new(\%config); + my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx); + my $rw = PublicInbox::SearchIdx->new($ibx, 1); eval { $rw->index_sync({reindex => 1}) }; is($@, '', 'no error from reindexing without msgmap'); is_deeply(\@warn, [], 'no warnings'); @@ -124,20 +180,25 @@ $rw = PublicInbox::SearchIdx->new($ibx, 1); ok(-d $xap, 'Xapian directories recreated'); delete $ibx->{mm}; is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged'); + is($ibx->mm->num_highwater, 10, 'num_highwater as expected'); my $mset = $ibx->search->query('hello world', {mset=>1}); - isnt(0, $mset->size, 'got Xapian search results'); + isnt($mset->size, 0, 'got Xapian search results'); + + my ($min, $max) = $ibx->mm->minmax; + is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged'); } ok(unlink "$mainrepo/public-inbox/msgmap.sqlite3", 'remove msgmap'); remove_tree($xap); ok(!-d $xap, 'Xapian directories removed again'); - -$ibx_config->{indexlevel} = 'basic'; -$ibx = PublicInbox::Inbox->new($ibx_config); -$rw = PublicInbox::SearchIdx->new($ibx, 1); { my @warn; local $SIG{__WARN__} = sub { push @warn, @_ }; + my %config = %$ibx_config; + $config{indexlevel} = 'basic'; + my $ibx = PublicInbox::Inbox->new(\%config); + my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx); + my $rw = PublicInbox::SearchIdx->new($ibx, 1); eval { $rw->index_sync({reindex => 1}) }; is($@, '', 'no error from reindexing without msgmap'); is_deeply(\@warn, [], 'no warnings'); @@ -145,25 +206,231 @@ $rw = PublicInbox::SearchIdx->new($ibx, 1); ok(-d $xap, 'Xapian directories recreated'); delete $ibx->{mm}; is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged'); + is($ibx->mm->num_highwater, 10, 'num_highwater as expected'); my $mset = $ibx->search->reopen->query('hello world', {mset=>1}); - is(0, $mset->size, "no Xapian search results"); + is($mset->size, 0, "no Xapian search results"); + + my ($min, $max) = $ibx->mm->minmax; + is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged'); } # upgrade existing basic to medium # note: changing indexlevels is not yet supported in v2, # and may not be without more effort -$ibx_config->{indexlevel} = 'medium'; -$ibx = PublicInbox::Inbox->new($ibx_config); -$rw = PublicInbox::SearchIdx->new($ibx, 1); # no removals { my @warn; local $SIG{__WARN__} = sub { push @warn, @_ }; + my %config = %$ibx_config; + $config{indexleve} = 'medium'; + my $ibx = PublicInbox::Inbox->new(\%config); + my $rw = PublicInbox::SearchIdx->new($ibx, 1); eval { $rw->index_sync }; is($@, '', 'no error from indexing'); is_deeply(\@warn, [], 'no warnings'); my $mset = $ibx->search->reopen->query('hello world', {mset=>1}); - isnt(0, $mset->size, 'search OK after basic -> medium'); + isnt($mset->size, 0, 'search OK after basic -> medium'); + + is($ibx->mm->num_highwater, 10, 'num_highwater as expected'); + + my ($min, $max) = $ibx->mm->minmax; + is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged'); +} + +# An incremental indexing test +ok(unlink "$mainrepo/public-inbox/msgmap.sqlite3", 'remove msgmap'); +remove_tree($xap); +ok(!-d $xap, 'Xapian directories removed again'); +{ + my @warn; + local $SIG{__WARN__} = sub { push @warn, @_ }; + my %config = %$ibx_config; + my $ibx = PublicInbox::Inbox->new(\%config); + my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx); + my $rw = PublicInbox::SearchIdx->new($ibx, 1); + # mark1 4 simple additions in the same index_sync + eval { $rw->index_sync({ref => $mark1}) }; + is($@, '', 'no error from reindexing without msgmap'); + is_deeply(\@warn, [], 'no warnings'); + $im->done; + my ($min, $max) = $ibx->mm->minmax; + is($min, 1, 'min as expected'); + is($max, 4, 'max as expected'); + is($ibx->mm->num_highwater, 4, 'num_highwater as expected'); + is_deeply($ibx->mm->msg_range(\$min, $max), + [ + [1, '1@example.com' ], + [2, '2@example.com' ], + [3, '3@example.com' ], + [4, '4@example.com' ], + ], 'msgmap as expected' ); } +{ + my @warn; + local $SIG{__WARN__} = sub { push @warn, @_ }; + my %config = %$ibx_config; + my $ibx = PublicInbox::Inbox->new(\%config); + my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx); + my $rw = PublicInbox::SearchIdx->new($ibx, 1); + # mark2 A delete separated form and add in the same index_sync + eval { $rw->index_sync({ref => $mark2}) }; + is($@, '', 'no error from reindexing without msgmap'); + is_deeply(\@warn, [], 'no warnings'); + $im->done; + my ($min, $max) = $ibx->mm->minmax; + is($min, 1, 'min as expected'); + is($max, 3, 'max as expected'); + is($ibx->mm->num_highwater, 4, 'num_highwater as expected'); + is_deeply($ibx->mm->msg_range(\$min, $max), + [ + [1, '1@example.com' ], + [2, '2@example.com' ], + [3, '3@example.com' ], + ], 'msgmap as expected' ); +} +{ + my @warn; + local $SIG{__WARN__} = sub { push @warn, @_ }; + my %config = %$ibx_config; + my $ibx = PublicInbox::Inbox->new(\%config); + my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx); + my $rw = PublicInbox::SearchIdx->new($ibx, 1); + # mark3 adds following the delete at mark2 + eval { $rw->index_sync({ref => $mark3}) }; + is($@, '', 'no error from reindexing without msgmap'); + is_deeply(\@warn, [], 'no warnings'); + $im->done; + my ($min, $max) = $ibx->mm->minmax; + is($min, 1, 'min as expected'); + is($max, 10, 'max as expected'); + is($ibx->mm->num_highwater, 10, 'num_highwater as expected'); + is_deeply($ibx->mm->msg_range(\$min, $max), + [ + [1, '1@example.com' ], + [2, '2@example.com' ], + [3, '3@example.com' ], + [5, '5@example.com' ], + [6, '6@example.com' ], + [7, '7@example.com' ], + [8, '8@example.com' ], + [9, '9@example.com' ], + [10, '10@example.com' ], + ], 'msgmap as expected' ); +} +{ + my @warn; + local $SIG{__WARN__} = sub { push @warn, @_ }; + my %config = %$ibx_config; + my $ibx = PublicInbox::Inbox->new(\%config); + my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx); + my $rw = PublicInbox::SearchIdx->new($ibx, 1); + # mark4 A delete of an older message + eval { $rw->index_sync({ref => $mark4}) }; + is($@, '', 'no error from reindexing without msgmap'); + is_deeply(\@warn, [], 'no warnings'); + $im->done; + my ($min, $max) = $ibx->mm->minmax; + is($min, 1, 'min as expected'); + is($max, 10, 'max as expected'); + is($ibx->mm->num_highwater, 10, 'num_highwater as expected'); + is_deeply($ibx->mm->msg_range(\$min, $max), + [ + [1, '1@example.com' ], + [2, '2@example.com' ], + [3, '3@example.com' ], + [6, '6@example.com' ], + [7, '7@example.com' ], + [8, '8@example.com' ], + [9, '9@example.com' ], + [10, '10@example.com' ], + ], 'msgmap as expected' ); +} + + +# Another incremental indexing test +ok(unlink "$mainrepo/public-inbox/msgmap.sqlite3", 'remove msgmap'); +remove_tree($xap); +ok(!-d $xap, 'Xapian directories removed again'); +{ + my @warn; + local $SIG{__WARN__} = sub { push @warn, @_ }; + my %config = %$ibx_config; + my $ibx = PublicInbox::Inbox->new(\%config); + my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx); + my $rw = PublicInbox::SearchIdx->new($ibx, 1); + # mark2 an add and it's delete in the same index_sync + eval { $rw->index_sync({ref => $mark2}) }; + is($@, '', 'no error from reindexing without msgmap'); + is_deeply(\@warn, [], 'no warnings'); + $im->done; + my ($min, $max) = $ibx->mm->minmax; + is($min, 1, 'min as expected'); + is($max, 3, 'max as expected'); + is($ibx->mm->num_highwater, 4, 'num_highwater as expected'); + is_deeply($ibx->mm->msg_range(\$min, $max), + [ + [1, '1@example.com' ], + [2, '2@example.com' ], + [3, '3@example.com' ], + ], 'msgmap as expected' ); +} +{ + my @warn; + local $SIG{__WARN__} = sub { push @warn, @_ }; + my %config = %$ibx_config; + my $ibx = PublicInbox::Inbox->new(\%config); + my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx); + my $rw = PublicInbox::SearchIdx->new($ibx, 1); + # mark3 adds following the delete at mark2 + eval { $rw->index_sync({ref => $mark3}) }; + is($@, '', 'no error from reindexing without msgmap'); + is_deeply(\@warn, [], 'no warnings'); + $im->done; + my ($min, $max) = $ibx->mm->minmax; + is($min, 1, 'min as expected'); + is($max, 10, 'max as expected'); + is($ibx->mm->num_highwater, 10, 'num_highwater as expected'); + is_deeply($ibx->mm->msg_range(\$min, $max), + [ + [1, '1@example.com' ], + [2, '2@example.com' ], + [3, '3@example.com' ], + [5, '5@example.com' ], + [6, '6@example.com' ], + [7, '7@example.com' ], + [8, '8@example.com' ], + [9, '9@example.com' ], + [10, '10@example.com' ], + ], 'msgmap as expected' ); +} +{ + my @warn; + local $SIG{__WARN__} = sub { push @warn, @_ }; + my %config = %$ibx_config; + my $ibx = PublicInbox::Inbox->new(\%config); + my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx); + my $rw = PublicInbox::SearchIdx->new($ibx, 1); + # mark4 A delete of an older message + eval { $rw->index_sync({ref => $mark4}) }; + is($@, '', 'no error from reindexing without msgmap'); + is_deeply(\@warn, [], 'no warnings'); + $im->done; + my ($min, $max) = $ibx->mm->minmax; + is($min, 1, 'min as expected'); + is($max, 10, 'max as expected'); + is($ibx->mm->num_highwater, 10, 'num_highwater as expected'); + is_deeply($ibx->mm->msg_range(\$min, $max), + [ + [1, '1@example.com' ], + [2, '2@example.com' ], + [3, '3@example.com' ], + [6, '6@example.com' ], + [7, '7@example.com' ], + [8, '8@example.com' ], + [9, '9@example.com' ], + [10, '10@example.com' ], + ], 'msgmap as expected' ); +} + done_testing(); diff --git a/t/v2reindex.t b/t/v2reindex.t index 1543309c..a5454a22 100644 --- a/t/v2reindex.t +++ b/t/v2reindex.t @@ -21,7 +21,6 @@ my $ibx_config = { -primary_address => 'test@example.com', indexlevel => 'full', }; -my $ibx = PublicInbox::Inbox->new($ibx_config); my $mime = PublicInbox::MIME->create( header => [ From => 'a@example.com', @@ -32,39 +31,86 @@ my $mime = PublicInbox::MIME->create( body => "hello world\n", ); local $ENV{NPROC} = 2; -my $im = PublicInbox::V2Writable->new($ibx, 1); -foreach my $i (1..10) { - $mime->header_set('Message-Id', "<$i\@example.com>"); - ok($im->add($mime), "message $i added"); - if ($i == 4) { +my $minmax; +my $msgmap; +my ($mark1, $mark2, $mark3, $mark4); +{ + my %config = %$ibx_config; + my $ibx = PublicInbox::Inbox->new(\%config); + my $im = PublicInbox::V2Writable->new($ibx, 1); + my $im0 = $im->importer(); + foreach my $i (1..10) { + $mime->header_set('Message-Id', "<$i\@example.com>"); + ok($im->add($mime), "message $i added"); + if ($i == 4) { + $mark1 = $im0->get_mark($im0->{tip}); + $im->remove($mime); + $mark2 = $im0->get_mark($im0->{tip}); + } + } + + if ('test remove later') { + $mark3 = $im0->get_mark($im0->{tip}); + $mime->header_set('Message-Id', "<5\@example.com>"); $im->remove($mime); + $mark4 = $im0->get_mark($im0->{tip}); } -} -if ('test remove later') { - $mime->header_set('Message-Id', "<5\@example.com>"); - $im->remove($mime); + $im->done; + $minmax = [ $ibx->mm->minmax ]; + ok(defined $minmax->[0] && defined $minmax->[1], 'minmax defined'); + is_deeply($minmax, [ 1, 10 ], 'minmax as expected'); + is($ibx->mm->num_highwater, 10, 'num_highwater as expected'); + + my ($min, $max) = @$minmax; + $msgmap = $ibx->mm->msg_range(\$min, $max); + is_deeply($msgmap, [ + [1, '1@example.com' ], + [2, '2@example.com' ], + [3, '3@example.com' ], + [6, '6@example.com' ], + [7, '7@example.com' ], + [8, '8@example.com' ], + [9, '9@example.com' ], + [10, '10@example.com' ], + ], 'msgmap as expected'); } -$im->done; -my $minmax = [ $ibx->mm->minmax ]; -ok(defined $minmax->[0] && defined $minmax->[1], 'minmax defined'); -is_deeply($minmax, [ 1, 10 ], 'minmax as expected'); +{ + my %config = %$ibx_config; + my $ibx = PublicInbox::Inbox->new(\%config); + my $im = PublicInbox::V2Writable->new($ibx, 1); + eval { $im->index_sync({reindex => 1}) }; + is($@, '', 'no error from reindexing'); + $im->done; + + delete $ibx->{mm}; + is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged'); + is($ibx->mm->num_highwater, 10, 'num_highwater as expected'); -eval { $im->index_sync({reindex => 1}) }; -is($@, '', 'no error from reindexing'); -$im->done; + my ($min, $max) = $ibx->mm->minmax; + is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged'); +} my $xap = "$mainrepo/xap".PublicInbox::Search::SCHEMA_VERSION(); remove_tree($xap); ok(!-d $xap, 'Xapian directories removed'); -eval { $im->index_sync({reindex => 1}) }; -is($@, '', 'no error from reindexing'); -$im->done; -ok(-d $xap, 'Xapian directories recreated'); +{ + my %config = %$ibx_config; + my $ibx = PublicInbox::Inbox->new(\%config); + my $im = PublicInbox::V2Writable->new($ibx, 1); + eval { $im->index_sync({reindex => 1}) }; + is($@, '', 'no error from reindexing'); + $im->done; + ok(-d $xap, 'Xapian directories recreated'); -delete $ibx->{mm}; -is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged'); + delete $ibx->{mm}; + is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged'); + is($ibx->mm->num_highwater, 10, 'num_highwater as expected'); + + my ($min, $max) = $ibx->mm->minmax; + is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged'); +} ok(unlink "$mainrepo/msgmap.sqlite3", 'remove msgmap'); remove_tree($xap); @@ -72,6 +118,9 @@ ok(!-d $xap, 'Xapian directories removed again'); { my @warn; local $SIG{__WARN__} = sub { push @warn, @_ }; + my %config = %$ibx_config; + my $ibx = PublicInbox::Inbox->new(\%config); + my $im = PublicInbox::V2Writable->new($ibx, 1); eval { $im->index_sync({reindex => 1}) }; is($@, '', 'no error from reindexing without msgmap'); is(scalar(@warn), 0, 'no warnings from reindexing'); @@ -79,6 +128,10 @@ ok(!-d $xap, 'Xapian directories removed again'); ok(-d $xap, 'Xapian directories recreated'); delete $ibx->{mm}; is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged'); + is($ibx->mm->num_highwater, 10, 'num_highwater as expected'); + + my ($min, $max) = $ibx->mm->minmax; + is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged'); } my %sizes; @@ -88,6 +141,9 @@ ok(!-d $xap, 'Xapian directories removed again'); { my @warn; local $SIG{__WARN__} = sub { push @warn, @_ }; + my %config = %$ibx_config; + my $ibx = PublicInbox::Inbox->new(\%config); + my $im = PublicInbox::V2Writable->new($ibx, 1); eval { $im->index_sync({reindex => 1}) }; is($@, '', 'no error from reindexing without msgmap'); is_deeply(\@warn, [], 'no warnings'); @@ -95,21 +151,25 @@ ok(!-d $xap, 'Xapian directories removed again'); ok(-d $xap, 'Xapian directories recreated'); delete $ibx->{mm}; is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged'); + is($ibx->mm->num_highwater, 10, 'num_highwater as expected'); my $mset = $ibx->search->query('"hello world"', {mset=>1}); - isnt(0, $mset->size, "phrase search succeeds on indexlevel=full"); + isnt($mset->size, 0, "phrase search succeeds on indexlevel=full"); for (<"$xap/*/*">) { $sizes{$ibx->{indexlevel}} += -s _ if -f $_ } + + my ($min, $max) = $ibx->mm->minmax; + is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged'); } ok(unlink "$mainrepo/msgmap.sqlite3", 'remove msgmap'); remove_tree($xap); ok(!-d $xap, 'Xapian directories removed again'); - -$ibx_config->{indexlevel} = 'medium'; -$ibx = PublicInbox::Inbox->new($ibx_config); -$im = PublicInbox::V2Writable->new($ibx); { my @warn; local $SIG{__WARN__} = sub { push @warn, @_ }; + my %config = %$ibx_config; + $config{indexlevel} = 'medium'; + my $ibx = PublicInbox::Inbox->new(\%config); + my $im = PublicInbox::V2Writable->new($ibx); eval { $im->index_sync({reindex => 1}) }; is($@, '', 'no error from reindexing without msgmap'); is_deeply(\@warn, [], 'no warnings'); @@ -117,31 +177,36 @@ $im = PublicInbox::V2Writable->new($ibx); ok(-d $xap, 'Xapian directories recreated'); delete $ibx->{mm}; is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged'); + is($ibx->mm->num_highwater, 10, 'num_highwater as expected'); if (0) { # not sure why, but Xapian seems to fallback to terms and # phrase searches still work delete $ibx->{search}; my $mset = $ibx->search->query('"hello world"', {mset=>1}); - is(0, $mset->size, 'phrase search does not work on medium'); + is($mset->size, 0, 'phrase search does not work on medium'); } my $mset = $ibx->search->query('hello world', {mset=>1}); - isnt(0, $mset->size, "normal search works on indexlevel=medium"); + isnt($mset->size, 0, "normal search works on indexlevel=medium"); for (<"$xap/*/*">) { $sizes{$ibx->{indexlevel}} += -s _ if -f $_ } ok($sizes{full} > $sizes{medium}, 'medium is smaller than full'); + + + my ($min, $max) = $ibx->mm->minmax; + is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged'); } ok(unlink "$mainrepo/msgmap.sqlite3", 'remove msgmap'); remove_tree($xap); ok(!-d $xap, 'Xapian directories removed again'); - -$ibx_config->{indexlevel} = 'basic'; -$ibx = PublicInbox::Inbox->new($ibx_config); -$im = PublicInbox::V2Writable->new($ibx); { my @warn; local $SIG{__WARN__} = sub { push @warn, @_ }; + my %config = %$ibx_config; + $config{indexlevel} = 'basic'; + my $ibx = PublicInbox::Inbox->new(\%config); + my $im = PublicInbox::V2Writable->new($ibx); eval { $im->index_sync({reindex => 1}) }; is($@, '', 'no error from reindexing without msgmap'); is_deeply(\@warn, [], 'no warnings'); @@ -149,10 +214,210 @@ $im = PublicInbox::V2Writable->new($ibx); ok(-d $xap, 'Xapian directories recreated'); delete $ibx->{mm}; is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged'); + is($ibx->mm->num_highwater, 10, 'num_highwater as expected'); my $mset = $ibx->search->query('hello', {mset=>1}); - is(0, $mset->size, "search fails on indexlevel='basic'"); + is($mset->size, 0, "search fails on indexlevel='basic'"); for (<"$xap/*/*">) { $sizes{$ibx->{indexlevel}} += -s _ if -f $_ } ok($sizes{medium} > $sizes{basic}, 'basic is smaller than medium'); + + my ($min, $max) = $ibx->mm->minmax; + is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged'); +} + + +# An incremental indexing test +ok(unlink "$mainrepo/msgmap.sqlite3", 'remove msgmap'); +remove_tree($xap); +ok(!-d $xap, 'Xapian directories removed again'); +{ + my @warn; + local $SIG{__WARN__} = sub { push @warn, @_ }; + my %config = %$ibx_config; + my $ibx = PublicInbox::Inbox->new(\%config); + # mark1 4 simple additions in the same index_sync + $ibx->{ref_head} = $mark1; + my $im = PublicInbox::V2Writable->new($ibx); + eval { $im->index_sync() }; + is($@, '', 'no error from reindexing without msgmap'); + is_deeply(\@warn, [], 'no warnings'); + $im->done; + my ($min, $max) = $ibx->mm->minmax; + is($min, 1, 'min as expected'); + is($max, 4, 'max as expected'); + is($ibx->mm->num_highwater, 4, 'num_highwater as expected'); + is_deeply($ibx->mm->msg_range(\$min, $max), + [ + [1, '1@example.com' ], + [2, '2@example.com' ], + [3, '3@example.com' ], + [4, '4@example.com' ], + ], 'msgmap as expected' ); +} +{ + my @warn; + local $SIG{__WARN__} = sub { push @warn, @_ }; + my %config = %$ibx_config; + my $ibx = PublicInbox::Inbox->new(\%config); + # mark2 A delete separated from an add in the same index_sync + $ibx->{ref_head} = $mark2; + my $im = PublicInbox::V2Writable->new($ibx); + eval { $im->index_sync() }; + is($@, '', 'no error from reindexing without msgmap'); + is_deeply(\@warn, [], 'no warnings'); + $im->done; + my ($min, $max) = $ibx->mm->minmax; + is($min, 1, 'min as expected'); + is($max, 3, 'max as expected'); + is($ibx->mm->num_highwater, 4, 'num_highwater as expected'); + is_deeply($ibx->mm->msg_range(\$min, $max), + [ + [1, '1@example.com' ], + [2, '2@example.com' ], + [3, '3@example.com' ], + ], 'msgmap as expected' ); +} +{ + my @warn; + local $SIG{__WARN__} = sub { push @warn, @_ }; + my %config = %$ibx_config; + my $ibx = PublicInbox::Inbox->new(\%config); + # mark3 adds following the delete at mark2 + $ibx->{ref_head} = $mark3; + my $im = PublicInbox::V2Writable->new($ibx); + eval { $im->index_sync() }; + is($@, '', 'no error from reindexing without msgmap'); + is_deeply(\@warn, [], 'no warnings'); + $im->done; + my ($min, $max) = $ibx->mm->minmax; + is($min, 1, 'min as expected'); + is($max, 10, 'max as expected'); + is($ibx->mm->num_highwater, 10, 'num_highwater as expected'); + is_deeply($ibx->mm->msg_range(\$min, $max), + [ + [1, '1@example.com' ], + [2, '2@example.com' ], + [3, '3@example.com' ], + [5, '5@example.com' ], + [6, '6@example.com' ], + [7, '7@example.com' ], + [8, '8@example.com' ], + [9, '9@example.com' ], + [10, '10@example.com' ], + ], 'msgmap as expected' ); +} +{ + my @warn; + local $SIG{__WARN__} = sub { push @warn, @_ }; + my %config = %$ibx_config; + my $ibx = PublicInbox::Inbox->new(\%config); + # mark4 A delete of an older message + $ibx->{ref_head} = $mark4; + my $im = PublicInbox::V2Writable->new($ibx); + eval { $im->index_sync() }; + is($@, '', 'no error from reindexing without msgmap'); + is_deeply(\@warn, [], 'no warnings'); + $im->done; + my ($min, $max) = $ibx->mm->minmax; + is($min, 1, 'min as expected'); + is($max, 10, 'max as expected'); + is($ibx->mm->num_highwater, 10, 'num_highwater as expected'); + is_deeply($ibx->mm->msg_range(\$min, $max), + [ + [1, '1@example.com' ], + [2, '2@example.com' ], + [3, '3@example.com' ], + [6, '6@example.com' ], + [7, '7@example.com' ], + [8, '8@example.com' ], + [9, '9@example.com' ], + [10, '10@example.com' ], + ], 'msgmap as expected' ); +} + + +# Another incremental indexing test +ok(unlink "$mainrepo/msgmap.sqlite3", 'remove msgmap'); +remove_tree($xap); +ok(!-d $xap, 'Xapian directories removed again'); +{ + my @warn; + local $SIG{__WARN__} = sub { push @warn, @_ }; + my %config = %$ibx_config; + my $ibx = PublicInbox::Inbox->new(\%config); + # mark2 an add and it's delete in the same index_sync + $ibx->{ref_head} = $mark2; + my $im = PublicInbox::V2Writable->new($ibx); + eval { $im->index_sync() }; + is($@, '', 'no error from reindexing without msgmap'); + is_deeply(\@warn, [], 'no warnings'); + $im->done; + my ($min, $max) = $ibx->mm->minmax; + is($min, 1, 'min as expected'); + is($max, 3, 'max as expected'); + is($ibx->mm->num_highwater, 4, 'num_highwater as expected'); + is_deeply($ibx->mm->msg_range(\$min, $max), + [ + [1, '1@example.com' ], + [2, '2@example.com' ], + [3, '3@example.com' ], + ], 'msgmap as expected' ); +} +{ + my @warn; + local $SIG{__WARN__} = sub { push @warn, @_ }; + my %config = %$ibx_config; + my $ibx = PublicInbox::Inbox->new(\%config); + # mark3 adds following the delete at mark2 + $ibx->{ref_head} = $mark3; + my $im = PublicInbox::V2Writable->new($ibx); + eval { $im->index_sync() }; + is($@, '', 'no error from reindexing without msgmap'); + is_deeply(\@warn, [], 'no warnings'); + $im->done; + my ($min, $max) = $ibx->mm->minmax; + is($min, 1, 'min as expected'); + is($max, 10, 'max as expected'); + is($ibx->mm->num_highwater, 10, 'num_highwater as expected'); + is_deeply($ibx->mm->msg_range(\$min, $max), + [ + [1, '1@example.com' ], + [2, '2@example.com' ], + [3, '3@example.com' ], + [5, '5@example.com' ], + [6, '6@example.com' ], + [7, '7@example.com' ], + [8, '8@example.com' ], + [9, '9@example.com' ], + [10, '10@example.com' ], + ], 'msgmap as expected' ); +} +{ + my @warn; + local $SIG{__WARN__} = sub { push @warn, @_ }; + my %config = %$ibx_config; + my $ibx = PublicInbox::Inbox->new(\%config); + # mark4 A delete of an older message + $ibx->{ref_head} = $mark4; + my $im = PublicInbox::V2Writable->new($ibx); + eval { $im->index_sync() }; + is($@, '', 'no error from reindexing without msgmap'); + is_deeply(\@warn, [], 'no warnings'); + $im->done; + my ($min, $max) = $ibx->mm->minmax; + is($min, 1, 'min as expected'); + is($max, 10, 'max as expected'); + is($ibx->mm->num_highwater, 10, 'num_highwater as expected'); + is_deeply($ibx->mm->msg_range(\$min, $max), + [ + [1, '1@example.com' ], + [2, '2@example.com' ], + [3, '3@example.com' ], + [6, '6@example.com' ], + [7, '7@example.com' ], + [8, '8@example.com' ], + [9, '9@example.com' ], + [10, '10@example.com' ], + ], 'msgmap as expected' ); } done_testing(); |