diff options
author | Eric Wong <e@80x24.org> | 2020-11-28 05:09:13 +0000 |
---|---|---|
committer | Eric Wong <e@80x24.org> | 2020-11-29 02:25:49 +0000 |
commit | 96aa41ea2abe5ee03404b043d428d9bfee3def50 (patch) | |
tree | fa84ad1a509252dc1ccda701ca19712994b9b66a /lib | |
parent | 24e103d37b423c1c718e7f0f6285419005a98be5 (diff) | |
download | public-inbox-96aa41ea2abe5ee03404b043d428d9bfee3def50.tar.gz |
nntp: speed up mid_lookup() using ->ALL extindex
We can reuse "xref3" information in extindex to quickly match messages matching a given Message-ID across hundreds or thousands of newsgroups with a few SQL statements. "XHDR Xref $MESSAGE_ID" is around 40% faster, on top of previous speedups.
Diffstat (limited to 'lib')
-rw-r--r-- | lib/PublicInbox/NNTP.pm | 34 |
1 files changed, 30 insertions, 4 deletions
diff --git a/lib/PublicInbox/NNTP.pm b/lib/PublicInbox/NNTP.pm index cc6534b9..7b3b1ffe 100644 --- a/lib/PublicInbox/NNTP.pm +++ b/lib/PublicInbox/NNTP.pm @@ -730,10 +730,36 @@ sub mid_lookup ($$) { my $n = $self_ng->mm->num_for($mid); return ($self_ng, $n) if defined $n; } - foreach my $ng (values %{$self->{nntpd}->{groups}}) { - next if defined $self_ng && $ng eq $self_ng; - my $n = $ng->mm->num_for($mid); - return ($ng, $n) if defined $n; + my $pi_cfg = $self->{nntpd}->{pi_config}; + if (my $ALL = $pi_cfg->ALL) { + my ($id, $prev); + while (my $smsg = $ALL->over->next_by_mid($mid, \$id, \$prev)) { + my $xr3 = $ALL->over->get_xref3($smsg->{num}); + if (my @x = grep(/:$smsg->{blob}\z/, @$xr3)) { + my ($ngname, $xnum) = split(/:/, $x[0]); + my $ibx = $pi_cfg->{-by_newsgroup}->{$ngname}; + return ($ibx, $xnum) if $ibx; + # fall through to trying all xref3s + } else { + warn <<EOF; +W: xref3 missing for <$mid> ($smsg->{blob}) in $ALL->{topdir}, -extindex bug? +EOF + } + # try all xref3s + for my $x (@$xr3) { + my ($ngname, $xnum) = split(/:/, $x); + my $ibx = $pi_cfg->{-by_newsgroup}->{$ngname}; + return ($ibx, $xnum) if $ibx; + warn "W: `$ngname' does not exist for #$xnum\n"; + } + } + # no warning here, $mid is just invalid + } else { # slow path for non-ALL users + foreach my $ibx (values %{$self->{nntpd}->{groups}}) { + next if defined $self_ng && $ibx eq $self_ng; + my $n = $ibx->mm->num_for($mid); + return ($ibx, $n) if defined $n; + } } (undef, undef); } |