From 811b8d3cbaa790f59b7b107140b86248da16499b Mon Sep 17 00:00:00 2001 From: Eric Wong Date: Fri, 27 Nov 2020 09:52:54 +0000 Subject: nntp: xref: use ->ALL extindex if available Getting Xref for cross-posted messages is an O(n) operation where `n' is the number of newsgroups on the server. This works acceptably when there are dozens of groups, but would be unnacceptable when there's tens of thousands of newsgroups. With ~140 newsgroups, a lore.kernel.org mirror already handles "XHDR Xref $MESSAGE_ID" requests around 30% faster after creating the xref3.idx_nntp index. The SQL additions to ExtSearch.pm may be a bit strange and seem more appropriate for Over.pm; however it currently makes sense to me since those bits of over.sqlite3 access are exclusive to ExtSearch and can't be used by traditional v1/v2 inboxes... --- lib/PublicInbox/NNTP.pm | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) (limited to 'lib/PublicInbox/NNTP.pm') diff --git a/lib/PublicInbox/NNTP.pm b/lib/PublicInbox/NNTP.pm index 39ff5257..8eec6b91 100644 --- a/lib/PublicInbox/NNTP.pm +++ b/lib/PublicInbox/NNTP.pm @@ -413,14 +413,21 @@ sub xref ($$$) { my $nntpd = $self->{nntpd}; my $cur_ngname = $cur_ibx->{newsgroup}; my $ret = "$nntpd->{servername} $cur_ngname:$smsg->{num}"; - - my $mid = $smsg->{mid}; - my $groups = $nntpd->{pi_config}->{-by_newsgroup}; - for my $xngname (@{$nntpd->{groupnames}}) { - next if $cur_ngname eq $xngname; - my $xibx = $groups->{$xngname} or next; - my $num = eval { $xibx->mm->num_for($mid) } or next; - $ret .= " $xngname:$num"; + if (my $ALL = $nntpd->{pi_config}->ALL) { + if (my $ary = $ALL->nntp_xref_for($cur_ibx, $smsg)) { + $ret .= join(' ', '', @$ary) if scalar(@$ary); + } + # better off wrong than slow if there's thousands of groups, + # so no fallback to the slow path below: + } else { # slow path + my $mid = $smsg->{mid}; + my $groups = $nntpd->{pi_config}->{-by_newsgroup}; + for my $xngname (@{$nntpd->{groupnames}}) { + next if $cur_ngname eq $xngname; + my $xibx = $groups->{$xngname} or next; + my $num = eval { $xibx->mm->num_for($mid) } or next; + $ret .= " $xngname:$num"; + } } $ret; } -- cgit v1.2.3-24-ge0c7