diff options
Diffstat (limited to 'lib')
| -rw-r--r-- | lib/PublicInbox/ExtSearch.pm | 56 | ||||
| -rw-r--r-- | lib/PublicInbox/NNTP.pm | 23 | ||||
| -rw-r--r-- | lib/PublicInbox/OverIdx.pm | 5 |
3 files changed, 76 insertions, 8 deletions
diff --git a/lib/PublicInbox/ExtSearch.pm b/lib/PublicInbox/ExtSearch.pm index dd93cd32..20ec3224 100644 --- a/lib/PublicInbox/ExtSearch.pm +++ b/lib/PublicInbox/ExtSearch.pm @@ -11,6 +11,7 @@ use PublicInbox::Over; use PublicInbox::Inbox; use File::Spec (); use PublicInbox::MiscSearch; +use DBI qw(:sql_types); # SQL_BLOB # for ->reopen, ->mset, ->mset_to_artnums use parent qw(PublicInbox::Search); @@ -49,6 +50,61 @@ sub git { $self->{git} //= PublicInbox::Git->new("$self->{topdir}/ALL.git"); } +# returns an arrayref of [ $NEWSGROUP_NAME:$ART_NO ] using +# the `xref3' table +sub nntp_xref_for { # NNTP only + my ($self, $xibx, $xsmsg) = @_; + my $dbh = over($self)->dbh; + + my $sth = $dbh->prepare_cached(<<'', undef, 1); +SELECT ibx_id FROM inboxes WHERE eidx_key = ? LIMIT 1 + + $sth->execute($xibx->{newsgroup}); + my $xibx_id = $sth->fetchrow_array // do { + warn "W: `$xibx->{newsgroup}' not found in $self->{topdir}\n"; + return; + }; + + $sth = $dbh->prepare_cached(<<'', undef, 1); +SELECT docid FROM xref3 WHERE oidbin = ? AND xnum = ? AND ibx_id = ? LIMIT 1 + + $sth->bind_param(1, pack('H*', $xsmsg->{blob}), SQL_BLOB); + $sth->bind_param(2, $xsmsg->{num}); + $sth->bind_param(3, $xibx_id); + $sth->execute; + my $docid = $sth->fetchrow_array // do { + warn <<EOF; +W: `$xibx->{newsgroup}:$xsmsg->{num}' not found in $self->{topdir}" +EOF + return; + }; + + # LIMIT is number of newsgroups on server: + $sth = $dbh->prepare_cached(<<'', undef, 1); +SELECT ibx_id,xnum FROM xref3 WHERE docid = ? + + $sth->execute($docid); + my $rows = $sth->fetchall_arrayref; + + my $eidx_key_sth = $dbh->prepare_cached(<<'', undef, 1); +SELECT eidx_key FROM inboxes WHERE ibx_id = ? LIMIT 1 + + my %xref = map { + my ($ibx_id, $xnum) = @$_; + if ($ibx_id == $xibx_id) { + (); + } else { + $eidx_key_sth->execute($ibx_id); + my $eidx_key = $eidx_key_sth->fetchrow_array; + + # only include if there's a newsgroup name + $eidx_key && index($eidx_key, '/') >= 0 ? + () : ($eidx_key => $xnum) + } + } @$rows; + [ map { "$_:$xref{$_}" } sort keys %xref ]; # match NNTP LIST order +} + sub mm { undef } sub altid_map { {} } diff --git a/lib/PublicInbox/NNTP.pm b/lib/PublicInbox/NNTP.pm index 39ff5257..8eec6b91 100644 --- a/lib/PublicInbox/NNTP.pm +++ b/lib/PublicInbox/NNTP.pm @@ -413,14 +413,21 @@ sub xref ($$$) { my $nntpd = $self->{nntpd}; my $cur_ngname = $cur_ibx->{newsgroup}; my $ret = "$nntpd->{servername} $cur_ngname:$smsg->{num}"; - - my $mid = $smsg->{mid}; - my $groups = $nntpd->{pi_config}->{-by_newsgroup}; - for my $xngname (@{$nntpd->{groupnames}}) { - next if $cur_ngname eq $xngname; - my $xibx = $groups->{$xngname} or next; - my $num = eval { $xibx->mm->num_for($mid) } or next; - $ret .= " $xngname:$num"; + if (my $ALL = $nntpd->{pi_config}->ALL) { + if (my $ary = $ALL->nntp_xref_for($cur_ibx, $smsg)) { + $ret .= join(' ', '', @$ary) if scalar(@$ary); + } + # better off wrong than slow if there's thousands of groups, + # so no fallback to the slow path below: + } else { # slow path + my $mid = $smsg->{mid}; + my $groups = $nntpd->{pi_config}->{-by_newsgroup}; + for my $xngname (@{$nntpd->{groupnames}}) { + next if $cur_ngname eq $xngname; + my $xibx = $groups->{$xngname} or next; + my $num = eval { $xibx->mm->num_for($mid) } or next; + $ret .= " $xngname:$num"; + } } $ret; } diff --git a/lib/PublicInbox/OverIdx.pm b/lib/PublicInbox/OverIdx.pm index 173e3220..8bec08da 100644 --- a/lib/PublicInbox/OverIdx.pm +++ b/lib/PublicInbox/OverIdx.pm @@ -542,6 +542,11 @@ CREATE TABLE IF NOT EXISTS xref3 ( $dbh->do('CREATE INDEX IF NOT EXISTS idx_docid ON xref3 (docid)'); + # performance critical, this is not UNIQUE since we may need to + # tolerate some old bugs from indexing mirrors + $dbh->do('CREATE INDEX IF NOT EXISTS idx_nntp ON '. + 'xref3 (oidbin,xnum,ibx_id)'); + $dbh->do(<<''); CREATE TABLE IF NOT EXISTS eidx_meta ( key VARCHAR(255) PRIMARY KEY, |