replace Xapian skeleton with SQLite overview DB

This ought to provide better performance and scalability
which is less dependent on inbox size.  Xapian does not
seem optimized for some queries used by the WWW homepage,
Atom feeds, XOVER and NEWNEWS NNTP commands.

This can actually make Xapian optional for NNTP usage,
and allow more functionality to work without Xapian
installed.

Indexing performance was extremely bad at first, but
DBI::Profile helped me optimize away problematic queries.

4 files changed, 54 insertions, 11 deletions

diff --git a/t/over.t b/t/over.t
new file mode 100644
index 00000000..1d3f9b37
--- /dev/null
+++ b/t/over.t
@@ -0,0 +1,38 @@
+# Copyright (C) 2018 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+use strict;
+use warnings;
+use Test::More;
+use File::Temp qw/tempdir/;
+foreach my $mod (qw(DBD::SQLite)) {
+        eval "require $mod";
+        plan skip_all => "$mod missing for over.t" if $@;
+}
+
+use_ok 'PublicInbox::OverIdx';
+my $tmpdir = tempdir('pi-over-XXXXXX', TMPDIR => 1, CLEANUP => 1);
+my $over = PublicInbox::OverIdx->new("$tmpdir/over.sqlite3");
+$over->connect;
+my $x = $over->next_tid;
+is(int($x), $x, 'integer tid');
+my $y = $over->next_tid;
+is($y, $x+1, 'tid increases');
+
+$x = $over->sid('hello-world');
+is(int($x), $x, 'integer sid');
+$y = $over->sid('hello-WORLD');
+is($y, $x+1, 'sid ncreases');
+is($over->sid('hello-world'), $x, 'idempotent');
+$over->disconnect;
+
+$over = PublicInbox::OverIdx->new("$tmpdir/over.sqlite3");
+$over->connect;
+is($over->sid('hello-world'), $x, 'idempotent across reopen');
+$over->each_by_mid('never', sub { fail('should not be called') });
+
+$x = $over->create_ghost('never');
+is(int($x), $x, 'integer tid for ghost');
+$y = $over->create_ghost('NEVAR');
+is($y, $x + 1, 'integer tid for ghost increases');
+
+done_testing();
diff --git a/t/psgi_search.t b/t/psgi_search.t
index 1df38691..60a44bde 100644
--- a/t/psgi_search.t
+++ b/t/psgi_search.t
@@ -30,8 +30,7 @@ EOF
 
 my $num = 0;
 # nb. using internal API, fragile!
-my $xdb = $rw->_xdb_acquire;
-$xdb->begin_transaction;
+$rw->begin_txn_lazy;
 
 foreach (reverse split(/\n\n/, $data)) {
         $_ .= "\n";
@@ -42,8 +41,7 @@ foreach (reverse split(/\n\n/, $data)) {
         ok($doc_id, 'message added: '. $mid);
 }
 
-$xdb->commit_transaction;
-$rw = undef;
+$rw->commit_txn_lazy;
 
 my $cfgpfx = "publicinbox.test";
 my $config = PublicInbox::Config->new({
diff --git a/t/search-thr-index.t b/t/search-thr-index.t
index 9549976d..3ddef809 100644
--- a/t/search-thr-index.t
+++ b/t/search-thr-index.t
@@ -32,8 +32,7 @@ EOF
 
 my $num = 0;
 # nb. using internal API, fragile!
-my $xdb = $rw->_xdb_acquire;
-$xdb->begin_transaction;
+my $xdb = $rw->begin_txn_lazy;
 my @mids;
 
 foreach (reverse split(/\n\n/, $data)) {
@@ -50,10 +49,12 @@ foreach (reverse split(/\n\n/, $data)) {
 
 my $prev;
 foreach my $mid (@mids) {
-        my $res = $rw->get_thread($mid);
+        my $res = $rw->{over}->get_thread($mid);
         is(3, $res->{total}, "got all messages from $mid");
 }
 
+$rw->commit_txn_lazy;
+
 done_testing();
 
 1;
diff --git a/t/search.t b/t/search.t
index 9ab15f77..51adb9fb 100644
--- a/t/search.t
+++ b/t/search.t
@@ -22,9 +22,9 @@ my $ibx = $rw->{-inbox};
 $rw = undef;
 my $ro = PublicInbox::Search->new($git_dir);
 my $rw_commit = sub {
-        $rw->{xdb}->commit_transaction if $rw && $rw->{xdb};
+        $rw->commit_txn_lazy if $rw;
         $rw = PublicInbox::SearchIdx->new($git_dir, 1);
-        $rw->_xdb_acquire->begin_transaction;
+        $rw->begin_txn_lazy;
 };
 
 {
@@ -93,7 +93,6 @@ sub filter_mids {
         ok($found, "message found");
         is($root_id, $found->{doc_id}, 'doc_id set correctly');
         is($found->mid, 'root@s', 'mid set correctly');
-        ok(int($found->thread_id) > 0, 'thread_id is an integer');
 
         my ($res, @res);
         my @exp = sort qw(root@s last@s);
@@ -148,7 +147,13 @@ sub filter_mids {
 
         my $ghost_id = $rw->add_message($was_ghost);
         is($ghost_id, int($ghost_id), "ghost_id is an integer: $ghost_id");
-        ok($ghost_id < $reply_id, "ghost vivified from earlier message");
+        my $msgs = $rw->{over}->get_thread('ghost-message@s')->{msgs};
+        is(scalar(@$msgs), 2, 'got both messages in ghost thread');
+        foreach (qw(sid tid)) {
+                is($msgs->[0]->{$_}, $msgs->[1]->{$_}, "{$_} match");
+        }
+        isnt($msgs->[0]->{num}, $msgs->[1]->{num}, "num do not match");
+        ok($_->{num} > 0, 'positive art num') foreach @$msgs
 }
 
 # search thread on ghost
@@ -400,6 +405,7 @@ sub filter_mids {
         is($txt->{msgs}->[0]->mid, $res->{msgs}->[0]->mid,
                 'search inside text attachments works');
 }
+$rw->commit_txn_lazy;
 
 done_testing();