about summary refs log tree commit homepage
path: root/lib
diff options
context:
space:
mode:
authorEric Wong <e@80x24.org>2020-11-28 05:09:13 +0000
committerEric Wong <e@80x24.org>2020-11-29 02:25:49 +0000
commit96aa41ea2abe5ee03404b043d428d9bfee3def50 (patch)
treefa84ad1a509252dc1ccda701ca19712994b9b66a /lib
parent24e103d37b423c1c718e7f0f6285419005a98be5 (diff)
downloadpublic-inbox-96aa41ea2abe5ee03404b043d428d9bfee3def50.tar.gz
nntp: speed up mid_lookup() using ->ALL extindex
We can reuse "xref3" information in extindex to quickly match
messages matching a given Message-ID across hundreds or
thousands of newsgroups with a few SQL statements.

"XHDR Xref $MESSAGE_ID" is around 40% faster, on top of
previous speedups.
Diffstat (limited to 'lib')
-rw-r--r--lib/PublicInbox/NNTP.pm34
1 files changed, 30 insertions, 4 deletions
diff --git a/lib/PublicInbox/NNTP.pm b/lib/PublicInbox/NNTP.pm
index cc6534b9..7b3b1ffe 100644
--- a/lib/PublicInbox/NNTP.pm
+++ b/lib/PublicInbox/NNTP.pm
@@ -730,10 +730,36 @@ sub mid_lookup ($$) {
                 my $n = $self_ng->mm->num_for($mid);
                 return ($self_ng, $n) if defined $n;
         }
-        foreach my $ng (values %{$self->{nntpd}->{groups}}) {
-                next if defined $self_ng && $ng eq $self_ng;
-                my $n = $ng->mm->num_for($mid);
-                return ($ng, $n) if defined $n;
+        my $pi_cfg = $self->{nntpd}->{pi_config};
+        if (my $ALL = $pi_cfg->ALL) {
+                my ($id, $prev);
+                while (my $smsg = $ALL->over->next_by_mid($mid, \$id, \$prev)) {
+                        my $xr3 = $ALL->over->get_xref3($smsg->{num});
+                        if (my @x = grep(/:$smsg->{blob}\z/, @$xr3)) {
+                                my ($ngname, $xnum) = split(/:/, $x[0]);
+                                my $ibx = $pi_cfg->{-by_newsgroup}->{$ngname};
+                                return ($ibx, $xnum) if $ibx;
+                                # fall through to trying all xref3s
+                        } else {
+                                warn <<EOF;
+W: xref3 missing for <$mid> ($smsg->{blob}) in $ALL->{topdir}, -extindex bug?
+EOF
+                        }
+                        # try all xref3s
+                        for my $x (@$xr3) {
+                                my ($ngname, $xnum) = split(/:/, $x);
+                                my $ibx = $pi_cfg->{-by_newsgroup}->{$ngname};
+                                return ($ibx, $xnum) if $ibx;
+                                warn "W: `$ngname' does not exist for #$xnum\n";
+                        }
+                }
+                # no warning here, $mid is just invalid
+        } else { # slow path for non-ALL users
+                foreach my $ibx (values %{$self->{nntpd}->{groups}}) {
+                        next if defined $self_ng && $ibx eq $self_ng;
+                        my $n = $ibx->mm->num_for($mid);
+                        return ($ibx, $n) if defined $n;
+                }
         }
         (undef, undef);
 }