nntp: xref: use ->ALL extindex if available

Getting Xref for cross-posted messages is an O(n) operation
where `n' is the number of newsgroups on the server.  This works
acceptably when there are dozens of groups, but would be
unnacceptable when there's tens of thousands of newsgroups.

With ~140 newsgroups, a lore.kernel.org mirror already handles
"XHDR Xref $MESSAGE_ID" requests around 30% faster after
creating the xref3.idx_nntp index.

The SQL additions to ExtSearch.pm may be a bit strange and
seem more appropriate for Over.pm; however it currently makes
sense to me since those bits of over.sqlite3 access are
exclusive to ExtSearch and can't be used by traditional
v1/v2 inboxes...

1 files changed, 15 insertions, 8 deletions

diff --git a/lib/PublicInbox/NNTP.pm b/lib/PublicInbox/NNTP.pm
index 39ff5257..8eec6b91 100644
--- a/lib/PublicInbox/NNTP.pm
+++ b/lib/PublicInbox/NNTP.pm
@@ -413,14 +413,21 @@ sub xref ($$$) {
         my $nntpd = $self->{nntpd};
         my $cur_ngname = $cur_ibx->{newsgroup};
         my $ret = "$nntpd->{servername} $cur_ngname:$smsg->{num}";
-
-        my $mid = $smsg->{mid};
-        my $groups = $nntpd->{pi_config}->{-by_newsgroup};
-        for my $xngname (@{$nntpd->{groupnames}}) {
-                next if $cur_ngname eq $xngname;
-                my $xibx = $groups->{$xngname} or next;
-                my $num = eval { $xibx->mm->num_for($mid) } or next;
-                $ret .= " $xngname:$num";
+        if (my $ALL = $nntpd->{pi_config}->ALL) {
+                if (my $ary = $ALL->nntp_xref_for($cur_ibx, $smsg)) {
+                        $ret .= join(' ', '', @$ary) if scalar(@$ary);
+                }
+                # better off wrong than slow if there's thousands of groups,
+                # so no fallback to the slow path below:
+        } else { # slow path
+                my $mid = $smsg->{mid};
+                my $groups = $nntpd->{pi_config}->{-by_newsgroup};
+                for my $xngname (@{$nntpd->{groupnames}}) {
+                        next if $cur_ngname eq $xngname;
+                        my $xibx = $groups->{$xngname} or next;
+                        my $num = eval { $xibx->mm->num_for($mid) } or next;
+                        $ret .= " $xngname:$num";
+                }
         }
         $ret;
 }