SearchIdx: Allow the amount of indexing be configured

This adds a new inbox configuration option 'indexlevel' that can take
the values 'full', 'medium', and 'basic'.

When set to 'full' everything is indexed including the positions
of all terms.

When set to 'medium' everything except the positions of terms is
indexed.

When set to 'basic' terms and positions are not indexed.  Just the
Overview database for NNTP is created.  Which is still quite good and
allows searching for messages by Message-ID.  But there are no indexes to support
searching inside the email messages themselves.

Update the reindex tests to exercise the full medium and basic code paths

Signed-off-by: "Eric W. Biederman" <ebiederm@xmission.com>

1 files changed, 42 insertions, 1 deletions

diff --git a/t/v1reindex.t b/t/v1reindex.t
index 0df36d3f..ff32750f 100644
--- a/t/v1reindex.t
+++ b/t/v1reindex.t
@@ -20,6 +20,7 @@ my $ibx_config = {
         mainrepo => $mainrepo,
         name => 'test-v1reindex',
         -primary_address => 'test@example.com',
+        indexlevel => 'full',
 };
 my $ibx = PublicInbox::Inbox->new($ibx_config);
 my $mime = PublicInbox::MIME->create(
@@ -74,15 +75,32 @@ is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
 
 ok(unlink "$mainrepo/public-inbox/msgmap.sqlite3", 'remove msgmap');
 remove_tree($xap);
+ok(!-d $xap, 'Xapian directories removed again');
+
 $rw = PublicInbox::SearchIdx->new($ibx, 1);
+{
+        my @warn;
+        local $SIG{__WARN__} = sub { push @warn, @_ };
+        eval { $rw->index_sync({reindex => 1}) };
+        is($@, '', 'no error from reindexing without msgmap');
+        is(scalar(@warn), 0, 'no warnings from reindexing');
+        $im->done;
+        ok(-d $xap, 'Xapian directories recreated');
+        delete $ibx->{mm};
+        is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
+}
 
+ok(unlink "$mainrepo/public-inbox/msgmap.sqlite3", 'remove msgmap');
+remove_tree($xap);
 ok(!-d $xap, 'Xapian directories removed again');
+
+$rw = PublicInbox::SearchIdx->new($ibx, 1);
 {
         my @warn;
         local $SIG{__WARN__} = sub { push @warn, @_ };
         eval { $rw->index_sync({reindex => 1}) };
         is($@, '', 'no error from reindexing without msgmap');
-        is(scalar(@warn), 0, 'no warnings from reindexing');
+        is_deeply(\@warn, [], 'no warnings');
         $im->done;
         ok(-d $xap, 'Xapian directories recreated');
         delete $ibx->{mm};
@@ -91,9 +109,31 @@ ok(!-d $xap, 'Xapian directories removed again');
 
 ok(unlink "$mainrepo/public-inbox/msgmap.sqlite3", 'remove msgmap');
 remove_tree($xap);
+ok(!-d $xap, 'Xapian directories removed again');
+
+$ibx_config->{index_level} = 'medium';
+$ibx = PublicInbox::Inbox->new($ibx_config);
 $rw = PublicInbox::SearchIdx->new($ibx, 1);
+{
+        my @warn;
+        local $SIG{__WARN__} = sub { push @warn, @_ };
+        eval { $rw->index_sync({reindex => 1}) };
+        is($@, '', 'no error from reindexing without msgmap');
+        is_deeply(\@warn, [], 'no warnings');
+        $im->done;
+        ok(-d $xap, 'Xapian directories recreated');
+        delete $ibx->{mm};
+        is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
+}
 
+
+ok(unlink "$mainrepo/public-inbox/msgmap.sqlite3", 'remove msgmap');
+remove_tree($xap);
 ok(!-d $xap, 'Xapian directories removed again');
+
+$ibx_config->{index_level} = 'basic';
+$ibx = PublicInbox::Inbox->new($ibx_config);
+$rw = PublicInbox::SearchIdx->new($ibx, 1);
 {
         my @warn;
         local $SIG{__WARN__} = sub { push @warn, @_ };
@@ -106,4 +146,5 @@ ok(!-d $xap, 'Xapian directories removed again');
         is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged');
 }
 
+
 done_testing();