nntp: xref: use ->ALL extindex if available

Getting Xref for cross-posted messages is an O(n) operation
where `n' is the number of newsgroups on the server.  This works
acceptably when there are dozens of groups, but would be
unnacceptable when there's tens of thousands of newsgroups.

With ~140 newsgroups, a lore.kernel.org mirror already handles
"XHDR Xref $MESSAGE_ID" requests around 30% faster after
creating the xref3.idx_nntp index.

The SQL additions to ExtSearch.pm may be a bit strange and
seem more appropriate for Over.pm; however it currently makes
sense to me since those bits of over.sqlite3 access are
exclusive to ExtSearch and can't be used by traditional
v1/v2 inboxes...

2 files changed, 40 insertions, 7 deletions

diff --git a/t/extsearch.t b/t/extsearch.t
index 778ba32d..f9f74e5c 100644
--- a/t/extsearch.t
+++ b/t/extsearch.t
@@ -11,6 +11,8 @@ require_git(2.6);
 require_mods(qw(DBD::SQLite Search::Xapian));
 use_ok 'PublicInbox::ExtSearch';
 use_ok 'PublicInbox::ExtSearchIdx';
+my $sock = tcp_server();
+my $host_port = $sock->sockhost . ':' . $sock->sockport;
 my ($home, $for_destroy) = tmpdir();
 local $ENV{HOME} = $home;
 mkdir "$home/.public-inbox" or BAIL_OUT $!;
@@ -35,7 +37,7 @@ seek($fh, 0, SEEK_SET) or BAIL_OUT $!;
 
 run_script(['-mda', '--no-precheck'], $env, { 0 => $fh }) or BAIL_OUT '-mda';
 
-ok(run_script([qw(-init -V1 v1test), "$home/v1test",
+ok(run_script([qw(-init -V1 v1test --newsgroup v1.example), "$home/v1test",
         'http://example.com/v1test', $v1addr ]), 'v1test init');
 
 $eml->header_set('List-Id', '<v1.example.com>');
@@ -51,6 +53,36 @@ run_script(['-index', "$home/v1test"]) or BAIL_OUT "index $?";
 
 ok(run_script([qw(-extindex --all), "$home/extindex"]), 'extindex init');
 
+
+{ # TODO: -extindex should write this to config
+        open $fh, '>>', "$home/.public-inbox/config" or BAIL_OUT $!;
+        print $fh <<EOF or BAIL_OUT $!;
+; for ->ALL
+[extindex "all"]
+        topdir = $home/extindex
+EOF
+        close $fh or BAIL_OUT $!;
+
+        my $pi_cfg = PublicInbox::Config->new;
+        $pi_cfg->fill_all;
+        ok($pi_cfg->ALL, '->ALL');
+        my $ibx = $pi_cfg->{-by_newsgroup}->{'v2.example'};
+        my $ret = $pi_cfg->ALL->nntp_xref_for($ibx, $ibx->over->get_art(1));
+        is_deeply($ret, ['v1.example:1'], '->nntp_xref_for');
+}
+
+SKIP: {
+        require_mods(qw(Net::NNTP), 1);
+        my ($out, $err) = ("$home/nntpd.out.log", "$home/nntpd.err.log");
+        my $cmd = [ '-nntpd', '-W0' ]; #, "--stdout=$out", "--stderr=$err" ];
+        my $td = start_script($cmd, undef, { 3 => $sock });
+        my $n = Net::NNTP->new($host_port);
+        $n->group('v1.example');
+        my $res = $n->head(1);
+        @$res = grep(/^Xref: /, @$res);
+        like($res->[0], qr/ v1\.example:1 v2\.example:1/, 'nntp_xref works');
+}
+
 my $es = PublicInbox::ExtSearch->new("$home/extindex");
 {
         my $smsg = $es->over->get_art(1);
@@ -58,7 +90,7 @@ my $es = PublicInbox::ExtSearch->new("$home/extindex");
         is($es->over->get_art(2), undef, 'only one added');
         my $xref3 = $es->over->get_xref3(1);
         like($xref3->[0], qr/\A\Qv2.example\E:1:/, 'order preserved 1');
-        like($xref3->[1], qr!\A\Q$home/v1test\E:1:!, 'order preserved 2');
+        like($xref3->[1], qr/\A\Qv1.example\E:1:/, 'order preserved 2');
         is(scalar(@$xref3), 2, 'only to entries');
 }
 
@@ -93,9 +125,5 @@ my @it = $misc->mset('')->items;
 is(scalar(@it), 2, 'two inboxes');
 like($it[0]->get_document->get_data, qr/v2test/, 'docdata matched v2');
 like($it[1]->get_document->get_data, qr/v1test/, 'docdata matched v1');
-my $pi_cfg = PublicInbox::Config->new;
-$pi_cfg->fill_all;
-my $ret = $misc->newsgroup_matches('', $pi_cfg);
-is_deeply($pi_cfg->{-by_newsgroup}, $ret, '->newsgroup_matches');
 
 done_testing;
diff --git a/t/nntp.t b/t/nntp.t
index 9a482acb..91a2aff7 100644
--- a/t/nntp.t
+++ b/t/nntp.t
@@ -8,6 +8,7 @@ use PublicInbox::Eml;
 require_mods(qw(DBD::SQLite Data::Dumper));
 use_ok 'PublicInbox::NNTP';
 use_ok 'PublicInbox::Inbox';
+use PublicInbox::Config;
 
 {
         sub quote_str {
@@ -110,7 +111,11 @@ use_ok 'PublicInbox::Inbox';
         my $mime = PublicInbox::Eml->new("Message-ID: <$mid>\r\n\r\n");
         my $hdr = $mime->header_obj;
         my $mock_self = {
-                nntpd => { grouplist => [], servername => 'example.com' },
+                nntpd => {
+                        grouplist => [],
+                        servername => 'example.com',
+                        pi_config => bless {}, 'PublicInbox::Config',
+                },
                 ng => $ng,
         };
         my $smsg = { num => 1, mid => $mid, nntp => $mock_self, -ibx => $ng };