From 96aa41ea2abe5ee03404b043d428d9bfee3def50 Mon Sep 17 00:00:00 2001
From: Eric Wong <e@80x24.org>
Date: Sat, 28 Nov 2020 05:09:13 +0000
Subject: nntp: speed up mid_lookup() using ->ALL extindex

We can reuse "xref3" information in extindex to quickly match
messages matching a given Message-ID across hundreds or
thousands of newsgroups with a few SQL statements.

"XHDR Xref $MESSAGE_ID" is around 40% faster, on top of
previous speedups.
---
 lib/PublicInbox/NNTP.pm | 34 ++++++++++++++++++++++++++++++----
 1 file changed, 30 insertions(+), 4 deletions(-)

(limited to 'lib/PublicInbox/NNTP.pm')

diff --git a/lib/PublicInbox/NNTP.pm b/lib/PublicInbox/NNTP.pm
index cc6534b9..7b3b1ffe 100644
--- a/lib/PublicInbox/NNTP.pm
+++ b/lib/PublicInbox/NNTP.pm
@@ -730,10 +730,36 @@ sub mid_lookup ($$) {
 		my $n = $self_ng->mm->num_for($mid);
 		return ($self_ng, $n) if defined $n;
 	}
-	foreach my $ng (values %{$self->{nntpd}->{groups}}) {
-		next if defined $self_ng && $ng eq $self_ng;
-		my $n = $ng->mm->num_for($mid);
-		return ($ng, $n) if defined $n;
+	my $pi_cfg = $self->{nntpd}->{pi_config};
+	if (my $ALL = $pi_cfg->ALL) {
+		my ($id, $prev);
+		while (my $smsg = $ALL->over->next_by_mid($mid, \$id, \$prev)) {
+			my $xr3 = $ALL->over->get_xref3($smsg->{num});
+			if (my @x = grep(/:$smsg->{blob}\z/, @$xr3)) {
+				my ($ngname, $xnum) = split(/:/, $x[0]);
+				my $ibx = $pi_cfg->{-by_newsgroup}->{$ngname};
+				return ($ibx, $xnum) if $ibx;
+				# fall through to trying all xref3s
+			} else {
+				warn <<EOF;
+W: xref3 missing for <$mid> ($smsg->{blob}) in $ALL->{topdir}, -extindex bug?
+EOF
+			}
+			# try all xref3s
+			for my $x (@$xr3) {
+				my ($ngname, $xnum) = split(/:/, $x);
+				my $ibx = $pi_cfg->{-by_newsgroup}->{$ngname};
+				return ($ibx, $xnum) if $ibx;
+				warn "W: `$ngname' does not exist for #$xnum\n";
+			}
+		}
+		# no warning here, $mid is just invalid
+	} else { # slow path for non-ALL users
+		foreach my $ibx (values %{$self->{nntpd}->{groups}}) {
+			next if defined $self_ng && $ibx eq $self_ng;
+			my $n = $ibx->mm->num_for($mid);
+			return ($ibx, $n) if defined $n;
+		}
 	}
 	(undef, undef);
 }
-- 
cgit v1.2.3-24-ge0c7