diff options
Diffstat (limited to 'lib/PublicInbox/LeiSearch.pm')
| -rw-r--r-- | lib/PublicInbox/LeiSearch.pm | 82 |
1 files changed, 39 insertions, 43 deletions
diff --git a/lib/PublicInbox/LeiSearch.pm b/lib/PublicInbox/LeiSearch.pm index a10e6e17..684668c5 100644 --- a/lib/PublicInbox/LeiSearch.pm +++ b/lib/PublicInbox/LeiSearch.pm @@ -1,4 +1,4 @@ -# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org> +# Copyright (C) all contributors <meta@public-inbox.org> # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt> # read-only counterpart for PublicInbox::LeiStore @@ -7,21 +7,15 @@ use strict; use v5.10.1; use parent qw(PublicInbox::ExtSearch); # PublicInbox::Search->reopen use PublicInbox::Search qw(xap_terms); -use PublicInbox::ContentHash qw(content_digest content_hash); +use PublicInbox::ContentHash qw(content_digest content_hash git_sha); use PublicInbox::MID qw(mids mids_for_index); - -# get combined docid from over.num: -# (not generic Xapian, only works with our sharding scheme) -sub num2docid ($$) { - my ($self, $num) = @_; - my $nshard = $self->{nshard}; - ($num - 1) * $nshard + $num % $nshard + 1; -} +use PublicInbox::Compat qw(uniqstr); +use Carp qw(croak); sub _msg_kw { # retry_reopen callback my ($self, $num) = @_; my $xdb = $self->xdb; # set {nshard} for num2docid; - xap_terms('K', $xdb, num2docid($self, $num)); + xap_terms('K', $xdb, $self->num2docid($num)); } sub msg_keywords { # array or hashref @@ -34,7 +28,7 @@ sub _oid_kw { # retry_reopen callback my $xdb = $self->xdb; # set {nshard}; my %kw; for my $num (@$nums) { # there should only be one... - my $doc = $xdb->get_document(num2docid($self, $num)); + my $doc = $xdb->get_document($self->num2docid($num)); my $x = xap_terms('K', $doc); %kw = (%kw, %$x); } @@ -51,20 +45,16 @@ sub oidbin_keywords { sub _xsmsg_vmd { # retry_reopen my ($self, $smsg, $want_label) = @_; my $xdb = $self->xdb; # set {nshard}; - my (%kw, %L, $doc, $x); - $kw{flagged} = 1 if delete($smsg->{lei_q_tt_flagged}); + my (@kw, @L, $doc, $x); + @kw = qw(flagged) if delete($smsg->{lei_q_tt_flagged}); my @num = $self->over->blob_exists($smsg->{blob}); for my $num (@num) { # there should only be one... - $doc = $xdb->get_document(num2docid($self, $num)); - $x = xap_terms('K', $doc); - %kw = (%kw, %$x); - if ($want_label) { # JSON/JMAP only - $x = xap_terms('L', $doc); - %L = (%L, %$x); - } + $doc = $xdb->get_document($self->num2docid($num)); + push @kw, xap_terms('K', $doc); + push @L, xap_terms('L', $doc) if $want_label # JSON/JMAP only } - $smsg->{kw} = [ sort keys %kw ] if scalar(keys(%kw)); - $smsg->{L} = [ sort keys %L ] if scalar(keys(%L)); + @{$smsg->{kw}} = sort(uniqstr(@kw)) if @kw; + @{$smsg->{L}} = uniqstr(@L) if @L; } # lookup keywords+labels for external messages @@ -108,37 +98,55 @@ sub xoids_for { my $git = $self->git; my $xoids = {}; # no lms when used via {ale}: - my $lms = $self->{-lms_ro} //= lms($self) if defined($self->{topdir}); + my $lms = $self->{-lms_rw} //= lms($self) if defined($self->{topdir}); for my $mid (@$mids) { for my $o (@overs) { my ($id, $prev); while (my $cur = $o->next_by_mid($mid, \$id, \$prev)) { + # {bytes} may be '' from old bug + $cur->{bytes} = 1 if $cur->{bytes} eq ''; next if $cur->{bytes} == 0 || $xoids->{$cur->{blob}}; $git->cat_async($cur->{blob}, \&_cmp_1st, [$chash, $xoids, $cur, $lms]); if ($min && scalar(keys %$xoids) >= $min) { - $git->cat_async_wait; + $git->async_wait_all; return $xoids; } } } } - $git->cat_async_wait; + $git->async_wait_all; + + # it could be an 'lei index'-ed file that just got renamed + if (scalar(keys %$xoids) < ($min // 1) && defined($self->{topdir})) { + my $hex = git_sha(1, $eml)->hexdigest; + my @n = $overs[0]->blob_exists($hex); + for (@n) { $xoids->{$hex} //= $_ } + } scalar(keys %$xoids) ? $xoids : undef; } # returns true if $eml is indexed by lei/store and keywords don't match sub kw_changed { my ($self, $eml, $new_kw_sorted, $docids) = @_; + my $cur_kw; if ($eml) { my $xoids = xoids_for($self, $eml) // return; $docids //= []; @$docids = sort { $a <=> $b } values %$xoids; + if (!@$docids && $self->over) { + my $bin = git_sha(1, $eml)->digest; + @$docids = $self->over->oidbin_exists($bin); + } + } + for my $id (@$docids) { + $cur_kw = eval { msg_keywords($self, $id) } and last; + } + if (!defined($cur_kw) && $@) { + $docids = join(', num:', @$docids); + croak "E: num:$docids keyword lookup failure: $@"; } - my $cur_kw = eval { msg_keywords($self, $docids->[0]) }; - die "E: #$docids->[0] keyword lookup failure: $@\n" if $@; - # RFC 5550 sec 5.9 on the $Forwarded keyword states: # "Once set, the flag SHOULD NOT be cleared" if (exists($cur_kw->{forwarded}) && @@ -149,23 +157,11 @@ sub kw_changed { join("\0", @$new_kw_sorted) eq $cur_kw ? 0 : 1; } -sub all_terms { - my ($self, $pfx) = @_; - my $xdb = $self->xdb; - my $cur = $xdb->allterms_begin($pfx); - my $end = $xdb->allterms_end($pfx); - my %ret; - for (; $cur != $end; $cur++) { - my $tn = $cur->get_termname; - index($tn, $pfx) == 0 and - $ret{substr($tn, length($pfx))} = undef; - } - wantarray ? (sort keys %ret) : \%ret; -} - sub qparse_new { my ($self) = @_; my $qp = $self->SUPER::qparse_new; # PublicInbox::Search + $self->{qp_flags} |= PublicInbox::Search::FLAG_PHRASE() | + PublicInbox::Search::FLAG_PURE_NOT(); $qp->add_boolean_prefix('kw', 'K'); $qp->add_boolean_prefix('L', 'L'); $qp |