init+index: support --skip-docdata for Xapian

Since we no longer read document data from Xapian, allow users
to opt-out of storing it.

This breaks compatibility with previous releases of
public-inbox, but gives us a ~1.5% space savings on Xapian
storage (and associated I/O and page cache pressure reduction).

3 files changed, 24 insertions, 2 deletions

diff --git a/t/inbox_idle.t b/t/inbox_idle.t
index 61287200..e16ee11b 100644
--- a/t/inbox_idle.t
+++ b/t/inbox_idle.t
@@ -29,7 +29,7 @@ for my $V (1, 2) {
         if ($V == 1) {
                 my $sidx = PublicInbox::SearchIdx->new($ibx, 1);
                 $sidx->idx_acquire;
-                $sidx->set_indexlevel;
+                $sidx->set_metadata_once;
                 $sidx->idx_release; # allow watching on lockfile
         }
         my $pi_config = PublicInbox::Config->new(\<<EOF);
diff --git a/t/index-git-times.t b/t/index-git-times.t
index 2e9e88e8..8f80c866 100644
--- a/t/index-git-times.t
+++ b/t/index-git-times.t
@@ -4,6 +4,7 @@ use Test::More;
 use PublicInbox::TestCommon;
 use PublicInbox::Import;
 use PublicInbox::Config;
+use PublicInbox::Admin;
 use File::Path qw(remove_tree);
 
 require_mods(qw(DBD::SQLite Search::Xapian));
@@ -47,11 +48,15 @@ EOF
         PublicInbox::Import::run_die($cmd, undef, { 0 => $r });
 }
 
-run_script(['-index', $v1dir]) or die 'v1 index failed';
+run_script(['-index', '--skip-docdata', $v1dir]) or die 'v1 index failed';
+
 my $smsg;
 {
         my $cfg = PublicInbox::Config->new;
         my $ibx = $cfg->lookup($addr);
+        my $lvl = PublicInbox::Admin::detect_indexlevel($ibx);
+        is($lvl, 'medium', 'indexlevel detected');
+        is($ibx->{-skip_docdata}, 1, '--skip-docdata flag set on -index');
         $smsg = $ibx->over->get_art(1);
         is($smsg->{ds}, 749520000, 'datestamp from git author time');
         is($smsg->{ts}, 1285977600, 'timestamp from git committer time');
@@ -70,6 +75,10 @@ SKIP: {
         my $check_v2 = sub {
                 my $ibx = PublicInbox::Inbox->new({inboxdir => $v2dir,
                                 address => $addr});
+                my $lvl = PublicInbox::Admin::detect_indexlevel($ibx);
+                is($lvl, 'medium', 'indexlevel detected after convert');
+                is($ibx->{-skip_docdata}, 1,
+                        '--skip-docdata preserved after convert');
                 my $v2smsg = $ibx->over->get_art(1);
                 is($v2smsg->{ds}, $smsg->{ds},
                         'v2 datestamp from git author time');
diff --git a/t/init.t b/t/init.t
index 4d2c5049..dad09435 100644
--- a/t/init.t
+++ b/t/init.t
@@ -95,6 +95,19 @@ SKIP: {
                 my $ibx = PublicInbox::Inbox->new({ inboxdir => $dir });
                 is(PublicInbox::Admin::detect_indexlevel($ibx), $lvl,
                         'detected expected level w/o config');
+                ok(!$ibx->{-skip_docdata}, 'docdata written by default');
+        }
+        for my $v (1, 2) {
+                my $name = "v$v-skip-docdata";
+                my $dir = "$tmpdir/$name";
+                $cmd = [ '-init', $name, "-V$v", '--skip-docdata',
+                        $dir, "http://example.com/$name",
+                        "$name\@example.com" ];
+                ok(run_script($cmd), "-init -V$v --skip-docdata");
+                my $ibx = PublicInbox::Inbox->new({ inboxdir => $dir });
+                is(PublicInbox::Admin::detect_indexlevel($ibx), 'full',
+                        "detected default indexlevel -V$v");
+                ok($ibx->{-skip_docdata}, "docdata skip set -V$v");
         }
 
         # loop for idempotency