about summary refs log tree commit homepage
diff options
context:
space:
mode:
-rw-r--r--MANIFEST1
-rw-r--r--lib/PublicInbox/SearchIdx.pm7
-rw-r--r--t/v1reindex.t109
3 files changed, 116 insertions, 1 deletions
diff --git a/MANIFEST b/MANIFEST
index a2fcae9f..fd74a435 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -199,6 +199,7 @@ t/thread-cycle.t
 t/time.t
 t/utf8.mbox
 t/v1-add-remove-add.t
+t/v1reindex.t
 t/v2-add-remove-add.t
 t/v2mda.t
 t/v2mirror.t
diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm
index 107cd345..0e0796c1 100644
--- a/lib/PublicInbox/SearchIdx.pm
+++ b/lib/PublicInbox/SearchIdx.pm
@@ -561,7 +561,12 @@ sub read_log {
         while (defined($line = <$log>)) {
                 if ($line =~ /$addmsg/o) {
                         my $blob = $1;
-                        delete $D{$blob} and next;
+                        if (delete $D{$blob}) {
+                                if (defined $self->{regen_down}) {
+                                        $self->{regen_down}--;
+                                }
+                                next;
+                        }
                         my $mime = do_cat_mail($git, $blob, \$bytes) or next;
                         batch_adjust(\$max, $bytes, $batch_cb, $latest);
                         $add_cb->($self, $mime, $bytes, $blob);
diff --git a/t/v1reindex.t b/t/v1reindex.t
new file mode 100644
index 00000000..0df36d3f
--- /dev/null
+++ b/t/v1reindex.t
@@ -0,0 +1,109 @@
+# Copyright (C) 2018 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+use strict;
+use warnings;
+use Test::More;
+use PublicInbox::MIME;
+use PublicInbox::ContentId qw(content_digest);
+use File::Temp qw/tempdir/;
+use File::Path qw(remove_tree);
+
+foreach my $mod (qw(DBD::SQLite Search::Xapian)) {
+        eval "require $mod";
+        plan skip_all => "$mod missing for v1reindex.t" if $@;
+}
+use_ok 'PublicInbox::SearchIdx';
+use_ok 'PublicInbox::Import';
+my $mainrepo = tempdir('pi-v1reindex-XXXXXX', TMPDIR => 1, CLEANUP => 1);
+is(system(qw(git init -q --bare), $mainrepo), 0);
+my $ibx_config = {
+        mainrepo => $mainrepo,
+        name => 'test-v1reindex',
+        -primary_address => 'test@example.com',
+};
+my $ibx = PublicInbox::Inbox->new($ibx_config);
+my $mime = PublicInbox::MIME->create(
+        header => [
+                From => 'a@example.com',
+                To => 'test@example.com',
+                Subject => 'this is a subject',
+                Date => 'Fri, 02 Oct 1993 00:00:00 +0000',
+        ],
+        body => "hello world\n",
+);
+my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
+foreach my $i (1..10) {
+        $mime->header_set('Message-Id', "<$i\@example.com>");
+        ok($im->add($mime), "message $i added");
+        if ($i == 4) {
+                $im->remove($mime);
+        }
+}
+
+if ('test remove later') {
+        $mime->header_set('Message-Id', "<5\@example.com>");
+        $im->remove($mime);
+}
+
+$im->done;
+my $rw = PublicInbox::SearchIdx->new($ibx, 1);
+eval { $rw->index_sync() };
+is($@, '', 'no error from indexing');
+
+my $minmax = [ $ibx->mm->minmax ];
+ok(defined $minmax->[0] && defined $minmax->[1], 'minmax defined');
+is_deeply($minmax, [ 1, 10 ], 'minmax as expected');
+
+$rw = PublicInbox::SearchIdx->new($ibx, 1);
+eval { $rw->index_sync({reindex => 1}) };
+is($@, '', 'no error from reindexing');
+$im->done;
+
+my $xap = "$mainrepo/public-inbox/xapian".PublicInbox::Search::SCHEMA_VERSION();
+remove_tree($xap);
+ok(!-d $xap, 'Xapian directories removed');
+$rw = PublicInbox::SearchIdx->new($ibx, 1);
+
+eval { $rw->index_sync({reindex => 1}) };
+is($@, '', 'no error from reindexing');
+$im->done;
+ok(-d $xap, 'Xapian directories recreated');
+
+delete $ibx->{mm};
+is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
+
+ok(unlink "$mainrepo/public-inbox/msgmap.sqlite3", 'remove msgmap');
+remove_tree($xap);
+$rw = PublicInbox::SearchIdx->new($ibx, 1);
+
+ok(!-d $xap, 'Xapian directories removed again');
+{
+        my @warn;
+        local $SIG{__WARN__} = sub { push @warn, @_ };
+        eval { $rw->index_sync({reindex => 1}) };
+        is($@, '', 'no error from reindexing without msgmap');
+        is(scalar(@warn), 0, 'no warnings from reindexing');
+        $im->done;
+        ok(-d $xap, 'Xapian directories recreated');
+        delete $ibx->{mm};
+        is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
+}
+
+ok(unlink "$mainrepo/public-inbox/msgmap.sqlite3", 'remove msgmap');
+remove_tree($xap);
+$rw = PublicInbox::SearchIdx->new($ibx, 1);
+
+ok(!-d $xap, 'Xapian directories removed again');
+{
+        my @warn;
+        local $SIG{__WARN__} = sub { push @warn, @_ };
+        eval { $rw->index_sync({reindex => 1}) };
+        is($@, '', 'no error from reindexing without msgmap');
+        is_deeply(\@warn, [], 'no warnings');
+        $im->done;
+        ok(-d $xap, 'Xapian directories recreated');
+        delete $ibx->{mm};
+        is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
+}
+
+done_testing();