about summary refs log tree commit homepage
path: root/lib
diff options
context:
space:
mode:
authorEric Wong <e@80x24.org>2020-10-27 07:54:44 +0000
committerEric Wong <e@80x24.org>2020-11-07 10:22:03 +0000
commit4c315ed49fe8a6224264d74c490e0ee552365b2f (patch)
treef36428594fa810ddc343a42a87d4adf0188c3b2b /lib
parent090238c3353438526dec707dad0718cb21efd07f (diff)
downloadpublic-inbox-4c315ed49fe8a6224264d74c490e0ee552365b2f.tar.gz
It doesn't seem worth storing xref3 data in Xapian now that
the same info is in over.sqlite3.
Diffstat (limited to 'lib')
-rw-r--r--lib/PublicInbox/ExtSearchIdx.pm10
-rw-r--r--lib/PublicInbox/SearchIdx.pm64
-rw-r--r--lib/PublicInbox/SearchIdxShard.pm28
-rw-r--r--lib/PublicInbox/Smsg.pm13
4 files changed, 44 insertions, 71 deletions
diff --git a/lib/PublicInbox/ExtSearchIdx.pm b/lib/PublicInbox/ExtSearchIdx.pm
index 790ee921..026e1377 100644
--- a/lib/PublicInbox/ExtSearchIdx.pm
+++ b/lib/PublicInbox/ExtSearchIdx.pm
@@ -116,9 +116,10 @@ sub do_xpost ($$) {
         my $eml = $req->{eml};
         if (my $new_smsg = $req->{new_smsg}) { # 'm' on cross-posted message
                 my $xnum = $req->{xnum};
-                $idx->shard_add_xref3($docid, $xnum, $oid, $xibx, $eml);
+                $self->{oidx}->add_xref3($docid, $xnum, $oid, $xibx->eidx_key);
+                $idx->shard_add_eidx_info($docid, $oid, $xibx, $eml);
         } else { # 'd'
-                $idx->shard_remove_xref3($docid, $oid, $xibx, $eml);
+                $idx->shard_remove_eidx_info($docid, $oid, $xibx, $eml);
         }
 }
 
@@ -135,7 +136,10 @@ sub index_unseen ($) {
         $new_smsg->{num} = $docid;
         my $idx = $self->idx_shard($docid);
         $self->{oidx}->add_overview($eml, $new_smsg);
-        $idx->index_raw(undef, $eml, $new_smsg, $req->{ibx});
+        my $oid = $new_smsg->{blob};
+        my $ibx = delete $req->{ibx} or die 'BUG: {ibx} unset';
+        $self->{oidx}->add_xref3($docid, $req->{xnum}, $oid, $ibx->eidx_key);
+        $idx->index_raw(undef, $eml, $new_smsg, $ibx);
 }
 
 sub do_finalize ($) {
diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm
index 32fa16f5..569efbb0 100644
--- a/lib/PublicInbox/SearchIdx.pm
+++ b/lib/PublicInbox/SearchIdx.pm
@@ -370,8 +370,6 @@ sub add_xapian ($$$$) {
 
         if (defined(my $eidx_key = $smsg->{eidx_key})) {
                 $doc->add_boolean_term('O'.$eidx_key);
-                $doc->add_boolean_term('P'.
-                                "$eidx_key:$smsg->{num}:$smsg->{blob}");
         }
         msg_iter($eml, \&index_xapian, [ $self, $doc ]);
         index_ids($self, $doc, $eml, $mids);
@@ -456,57 +454,41 @@ sub _get_doc ($$$) {
         }
 }
 
-sub add_xref3 {
-        my ($self, $docid, $xnum, $oid, $eidx_key, $eml) = @_;
+sub add_eidx_info {
+        my ($self, $docid, $oid, $eidx_key, $eml) = @_;
         begin_txn_lazy($self);
         my $doc = _get_doc($self, $docid, $oid) or return;
         term_generator($self)->set_document($doc);
         $doc->add_boolean_term('O'.$eidx_key);
-        $doc->add_boolean_term('P'."$eidx_key:$xnum:$oid");
         index_list_id($self, $doc, $eml);
         $self->{xdb}->replace_document($docid, $doc);
 }
 
-sub remove_xref3 {
+sub remove_eidx_info {
         my ($self, $docid, $oid, $eidx_key, $eml) = @_;
         begin_txn_lazy($self);
         my $doc = _get_doc($self, $docid, $oid) or return;
-        my $xref3 = PublicInbox::Smsg::xref3(undef, $doc);
-        my %x3 = map { $_ => undef } @$xref3;
-        for (grep(/\A\Q$eidx_key\E:[0-9]+:\Q$oid\E\z/, @$xref3)) {
-                delete $x3{$_};
-                $doc->remove_term('P' . $_);
-        }
-        if (scalar(keys(%x3)) == 0) {
-                $self->{xdb}->delete_document($docid);
-                if (my $del_fh = $self->{del_fh}) { # TODO
-                        print $del_fh $docid, "\n" or die "E: print $!";
-                }
-        } else {
-                if (!grep(/\A\Q$eidx_key\E:/, keys %x3)) {
-                        $doc->remove_term('O'.$eidx_key);
-                }
-                for my $l ($eml->header_raw('List-Id')) {
-                        $l =~ /<([^>]+)>/ or next;
-                        my $lid = lc $1;
-                        $doc->remove_term('G' . $lid);
-
-                        # nb: we don't remove the XL probabilistic terms
-                        # since terms may overlap if cross-posted.
-                        #
-                        # IOW, a message which has both <foo.example.com>
-                        # and <bar.example.com> would have overlapping
-                        # "XLexample" and "XLcom" as terms and which we
-                        # wouldn't know if they're safe to remove if we just
-                        # unindex <foo.example.com> while preserving
-                        # <bar.example.com>.
-                        #
-                        # In any case, this entire sub is will likely never
-                        # be needed and users using the "l:" prefix are probably
-                        # rarer.
-                }
-                $self->{xdb}->replace_document($docid, $doc);
+        $doc->remove_term('O'.$eidx_key);
+        for my $l ($eml->header_raw('List-Id')) {
+                $l =~ /<([^>]+)>/ or next;
+                my $lid = lc $1;
+                $doc->remove_term('G' . $lid);
+
+                # nb: we don't remove the XL probabilistic terms
+                # since terms may overlap if cross-posted.
+                #
+                # IOW, a message which has both <foo.example.com>
+                # and <bar.example.com> would have overlapping
+                # "XLexample" and "XLcom" as terms and which we
+                # wouldn't know if they're safe to remove if we just
+                # unindex <foo.example.com> while preserving
+                # <bar.example.com>.
+                #
+                # In any case, this entire sub is will likely never
+                # be needed and users using the "l:" prefix are probably
+                # rarer.
         }
+        $self->{xdb}->replace_document($docid, $doc);
 }
 
 sub get_val ($$) {
diff --git a/lib/PublicInbox/SearchIdxShard.pm b/lib/PublicInbox/SearchIdxShard.pm
index ac01340c..644d8b58 100644
--- a/lib/PublicInbox/SearchIdxShard.pm
+++ b/lib/PublicInbox/SearchIdxShard.pm
@@ -75,15 +75,15 @@ sub shard_worker_loop ($$$$$) {
                 } elsif ($line =~ /\AD ([a-f0-9]{40,}) ([0-9]+)\n\z/s) {
                         $self->remove_by_oid($1, $2 + 0);
                 } elsif ($line =~ s/\A\+X //) {
-                        my ($len, $docid, $xnum, $oid, $eidx_key) =
-                                                        split(/ /, $line, 5);
-                        $self->add_xref3($docid, $xnum, $oid, $eidx_key,
-                                                eml($r, $len));
+                        my ($len, $docid, $oid, $eidx_key) =
+                                                        split(/ /, $line, 4);
+                        $self->add_eidx_info($docid, $oid, $eidx_key,
+                                                        eml($r, $len));
                 } elsif ($line =~ s/\A-X //) {
-                        my ($len, $docid, $xnum, $oid, $eidx_key) =
-                                                        split(/ /, $line, 5);
-                        $self->remove_xref3($docid, $xnum, $oid,
-                                                $eidx_key, eml($r, $len));
+                        my ($len, $docid, $oid, $eidx_key) =
+                                                        split(/ /, $line, 4);
+                        $self->remove_eidx_info($docid, $oid, $eidx_key,
+                                                        eml($r, $len));
                 } else {
                         chomp $line;
                         my $eidx_key;
@@ -135,20 +135,20 @@ sub index_raw {
         }
 }
 
-sub shard_add_xref3 {
-        my ($self, $docid, $xnum, $oid, $xibx, $eml) = @_;
+sub shard_add_eidx_info {
+        my ($self, $docid, $oid, $xibx, $eml) = @_;
         my $eidx_key = $xibx->eidx_key;
         if (my $w = $self->{w}) {
                 my $hdr = $eml->header_obj->as_string;
                 my $len = length($hdr);
-                print $w "+X $len $docid $xnum $oid $eidx_key\n", $hdr or
+                print $w "+X $len $docid $oid $eidx_key\n", $hdr or
                         die "failed to write shard: $!";
         } else {
-                $self->add_xref3($docid, $xnum, $oid, $eidx_key, $eml);
+                $self->add_eidx_info($docid, $oid, $eidx_key, $eml);
         }
 }
 
-sub shard_remove_xref3 {
+sub shard_remove_eidx_info {
         my ($self, $docid, $oid, $xibx, $eml) = @_;
         my $eidx_key = $xibx->eidx_key;
         if (my $w = $self->{w}) {
@@ -157,7 +157,7 @@ sub shard_remove_xref3 {
                 print $w "-X $len $docid $oid $eidx_key\n", $hdr or
                         die "failed to write shard: $!";
         } else {
-                $self->remove_xref3($docid, $oid, $eidx_key, $eml);
+                $self->remove_eidx_info($docid, $oid, $eidx_key, $eml);
         }
 }
 
diff --git a/lib/PublicInbox/Smsg.pm b/lib/PublicInbox/Smsg.pm
index c0fd85fd..14086538 100644
--- a/lib/PublicInbox/Smsg.pm
+++ b/lib/PublicInbox/Smsg.pm
@@ -137,17 +137,4 @@ sub subject_normalized ($) {
         $subj;
 }
 
-sub xref3 {
-        my ($self, $doc) = @_;
-        my $end = $doc->termlist_end;
-        my $it = $doc->termlist_begin;
-        $it->skip_to('P');
-        my @ret;
-        for (; $it != $end; $it++) {
-                my $val = $it->get_termname;
-                $val =~ s/\AP// and push @ret, $val;
-        }
-        \@ret;
-}
-
 1;