user/dev discussion of public-inbox itself
 help / color / mirror / code / Atom feed
From: Eric Wong <e@80x24.org>
To: meta@public-inbox.org
Subject: [PATCH 08/52] searchidx: introduce "xref3" concept
Date: Tue, 27 Oct 2020 07:54:09 +0000	[thread overview]
Message-ID: <20201027075453.19163-9-e@80x24.org> (raw)
In-Reply-To: <20201027075453.19163-1-e@80x24.org>

This will be used to track cross-posted messages in the
external/detached index.
---
 lib/PublicInbox/SearchIdx.pm      | 78 ++++++++++++++++++++++++++-----
 lib/PublicInbox/SearchIdxShard.pm | 53 ++++++++++++++++++---
 lib/PublicInbox/Smsg.pm           | 13 ++++++
 t/search.t                        | 28 +++++++++--
 4 files changed, 150 insertions(+), 22 deletions(-)

diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm
index af707ced..283bdd6c 100644
--- a/lib/PublicInbox/SearchIdx.pm
+++ b/lib/PublicInbox/SearchIdx.pm
@@ -325,6 +325,16 @@ sub index_xapian { # msg_iter callback
 	}
 }
 
+sub index_list_id ($$$) {
+	my ($self, $doc, $hdr) = @_;
+	for my $l ($hdr->header_raw('List-Id')) {
+		$l =~ /<([^>]+)>/ or next;
+		my $lid = lc $1;
+		$doc->add_boolean_term('G' . $lid);
+		index_text($self, $lid, 1, 'XL'); # probabilistic
+	}
+}
+
 sub index_ids ($$$$) {
 	my ($self, $doc, $hdr, $mids) = @_;
 	for my $mid (@$mids) {
@@ -338,12 +348,7 @@ sub index_ids ($$$$) {
 		}
 	}
 	$doc->add_boolean_term('Q' . $_) for @$mids;
-	for my $l ($hdr->header_raw('List-Id')) {
-		$l =~ /<([^>]+)>/ or next;
-		my $lid = lc $1;
-		$doc->add_boolean_term('G' . $lid);
-		index_text($self, $lid, 1, 'XL'); # probabilistic
-	}
+	index_list_id($self, $doc, $hdr);
 }
 
 sub add_xapian ($$$$) {
@@ -363,6 +368,10 @@ sub add_xapian ($$$$) {
 	$tg->set_document($doc);
 	index_headers($self, $smsg);
 
+	if (my $ng_or_dir = $self->{ng_or_dir}) { # external index
+		$doc->add_boolean_term('P'.
+				"$ng_or_dir:$smsg->{num}:$smsg->{blob}");
+	}
 	msg_iter($eml, \&index_xapian, [ $self, $doc ]);
 	index_ids($self, $doc, $eml, $mids);
 
@@ -436,6 +445,56 @@ sub add_message {
 	$smsg->{num};
 }
 
+sub _get_doc ($$$) {
+	my ($self, $docid, $oid) = @_;
+	my $doc = eval { $self->{xdb}->get_document($docid) };
+	$doc // do {
+		warn "E: $@\n" if $@;
+		warn "E: #$docid $oid missing in Xapian\n";
+		undef;
+	}
+}
+
+sub add_xref3 {
+	my ($self, $docid, $xnum, $oid, $ng_or_dir, $eml) = @_;
+	begin_txn_lazy($self);
+	my $doc = _get_doc($self, $docid, $oid) or return;
+	term_generator($self)->set_document($doc);
+	$doc->add_boolean_term('P'."$ng_or_dir:$xnum:$oid");
+	index_list_id($self, $doc, $eml);
+	$self->{xdb}->replace_document($docid, $doc);
+}
+
+sub remove_xref3 {
+	my ($self, $docid, $oid, $ng_or_dir, $eml) = @_;
+	begin_txn_lazy($self);
+	my $doc = _get_doc($self, $docid, $oid) or return;
+	my $xref3 = PublicInbox::Smsg::xref3(undef, $doc);
+	for (grep(/\A\Q$ng_or_dir\E:[0-9]+:\Q$oid\E\z/, @$xref3)) {
+		$doc->remove_term('P' . $_);
+	}
+	for my $l ($eml->header_raw('List-Id')) {
+		$l =~ /<([^>]+)>/ or next;
+		my $lid = lc $1;
+		$doc->remove_term('G' . $lid);
+
+		# nb: we don't remove the XL probabilistic terms
+		# since terms may overlap if cross-posted.
+		#
+		# IOW, a message which has both <foo.example.com>
+		# and <bar.example.com> would have overlapping
+		# "XLexample" and "XLcom" as terms and which we
+		# wouldn't know if they're safe to remove if we just
+		# unindex <foo.example.com> while preserving
+		# <bar.example.com>.
+		#
+		# In any case, this entire sub is will likely never
+		# be needed and users using the "l:" prefix are probably
+		# rarer.
+	}
+	$self->{xdb}->replace_document($docid, $doc);
+}
+
 sub get_val ($$) {
 	my ($doc, $col) = @_;
 	sortable_unserialise($doc->get_value($col));
@@ -457,12 +516,7 @@ sub xdb_remove {
 	my ($self, $oid, @removed) = @_;
 	my $xdb = $self->{xdb} or return;
 	for my $num (@removed) {
-		my $doc = eval { $xdb->get_document($num) };
-		unless ($doc) {
-			warn "E: $@\n" if $@;
-			warn "E: #$num $oid missing in Xapian\n";
-			next;
-		}
+		my $doc = _get_doc($self, $num, $oid) or next;
 		my $smsg = smsg_from_doc($doc);
 		my $blob = $smsg->{blob}; # may be undef if --skip-docdata
 		if (!defined($blob) || $blob eq $oid) {
diff --git a/lib/PublicInbox/SearchIdxShard.pm b/lib/PublicInbox/SearchIdxShard.pm
index f23d23d0..8e24aa1b 100644
--- a/lib/PublicInbox/SearchIdxShard.pm
+++ b/lib/PublicInbox/SearchIdxShard.pm
@@ -7,6 +7,7 @@ package PublicInbox::SearchIdxShard;
 use strict;
 use v5.10.1;
 use parent qw(PublicInbox::SearchIdx);
+use bytes qw(length);
 use IO::Handle (); # autoflush
 use PublicInbox::Eml;
 
@@ -47,6 +48,13 @@ sub spawn_worker {
 	close $r or die "failed to close: $!";
 }
 
+sub eml ($$) {
+	my ($r, $len) = @_;
+	my $n = read($r, my $bref, $len) or die "read: $!\n";
+	$n == $len or die "short read: $n != $len\n";
+	PublicInbox::Eml->new(\$bref);
+}
+
 # this reads all the writes to $self->{w} from the parent process
 sub shard_worker_loop ($$$$$) {
 	my ($self, $v2w, $r, $shard, $bnote) = @_;
@@ -65,25 +73,32 @@ sub shard_worker_loop ($$$$$) {
 					die "write failed for barrier $!\n";
 		} elsif ($line =~ /\AD ([a-f0-9]{40,}) ([0-9]+)\n\z/s) {
 			$self->remove_by_oid($1, $2 + 0);
+		} elsif ($line =~ s/\A\+X //) {
+			my ($len, $docid, $xnum, $oid, $ng_or_dir) =
+							split(/ /, $line, 5);
+			$self->add_xref3($docid, $xnum, $oid, $ng_or_dir,
+						eml($r, $len));
+		} elsif ($line =~ s/\A-X //) {
+			my ($len, $docid, $xnum, $oid, $ng_or_dir) =
+							split(/ /, $line, 5);
+			$self->remove_xref3($docid, $xnum, $oid,
+						$ng_or_dir, eml($r, $len));
 		} else {
 			chomp $line;
 			# n.b. $mid may contain spaces(!)
-			my ($to_read, $bytes, $num, $blob, $ds, $ts, $tid, $mid)
+			my ($len, $bytes, $num, $oid, $ds, $ts, $tid, $mid)
 				= split(/ /, $line, 8);
 			$self->begin_txn_lazy;
-			my $n = read($r, my $msg, $to_read) or die "read: $!\n";
-			$n == $to_read or die "short read: $n != $to_read\n";
-			my $mime = PublicInbox::Eml->new(\$msg);
 			my $smsg = bless {
 				bytes => $bytes,
 				num => $num + 0,
-				blob => $blob,
+				blob => $oid,
 				mid => $mid,
 				tid => $tid,
 				ds => $ds,
 				ts => $ts,
 			}, 'PublicInbox::Smsg';
-			$self->add_message($mime, $smsg);
+			$self->add_message(eml($r, $len), $smsg);
 		}
 	}
 	$self->worker_done;
@@ -107,6 +122,32 @@ sub index_raw {
 	}
 }
 
+sub shard_add_xref3 {
+	my ($self, $docid, $xnum, $oid, $xibx, $eml) = @_;
+	my $ng_or_dir = $xibx->{newsgroup} // $xibx->{inboxdir};
+	if (my $w = $self->{w}) {
+		my $hdr = $eml->header_obj->as_string;
+		my $len = length($hdr);
+		print $w "+X $len $docid $xnum $oid $ng_or_dir\n", $hdr or
+			die "failed to write shard: $!";
+	} else {
+		$self->add_xref3($docid, $xnum, $oid, $ng_or_dir, $eml);
+	}
+}
+
+sub shard_remove_xref3 {
+	my ($self, $docid, $oid, $xibx, $eml) = @_;
+	my $ng_or_dir = $xibx->{newsgroup} // $xibx->{inboxdir};
+	if (my $w = $self->{w}) {
+		my $hdr = $eml->header_obj->as_string;
+		my $len = length($hdr);
+		print $w "-X $len $docid $oid $ng_or_dir\n", $hdr or
+			die "failed to write shard: $!";
+	} else {
+		$self->remove_xref3($docid, $oid, $ng_or_dir, $eml);
+	}
+}
+
 sub atfork_child {
 	close $_[0]->{w} or die "failed to close write pipe: $!\n";
 }
diff --git a/lib/PublicInbox/Smsg.pm b/lib/PublicInbox/Smsg.pm
index 14086538..c0fd85fd 100644
--- a/lib/PublicInbox/Smsg.pm
+++ b/lib/PublicInbox/Smsg.pm
@@ -137,4 +137,17 @@ sub subject_normalized ($) {
 	$subj;
 }
 
+sub xref3 {
+	my ($self, $doc) = @_;
+	my $end = $doc->termlist_end;
+	my $it = $doc->termlist_begin;
+	$it->skip_to('P');
+	my @ret;
+	for (; $it != $end; $it++) {
+		my $val = $it->get_termname;
+		$val =~ s/\AP// and push @ret, $val;
+	}
+	\@ret;
+}
+
 1;
diff --git a/t/search.t b/t/search.t
index a66aec36..e789b81e 100644
--- a/t/search.t
+++ b/t/search.t
@@ -341,6 +341,14 @@ $ibx->with_umask(sub {
 		my $uid = PublicInbox::SearchIdx::get_val($doc, $col);
 		is($uid, $smsg->{num}, 'UID column matches {num}');
 		is($uid, $m->get_docid, 'UID column matches docid');
+
+		# check ->xref3 for external index:
+		is_deeply($smsg->xref3($doc), [], 'xref3 empty by default');
+		my $exp = "inbox.com.example:$uid:deadbeef";
+		$doc->add_boolean_term('P'.$exp);
+		is_deeply($smsg->xref3($doc), [ $exp ], 'xref3 can be set');
+		$doc->remove_term('P'.$exp);
+		is_deeply($smsg->xref3($doc), [], 'xref3 can be unset');
 	}
 
 	$mset = $ibx->search->mset('tc:list@example.com');
@@ -513,8 +521,13 @@ $ibx->with_umask(sub {
 	$rw_commit->();
 	my $doc_id = $rw->add_message(eml_load('t/data/message_embed.eml'));
 	ok($doc_id > 0, 'messages within messages');
-	$rw->commit_txn_lazy;
-	$ibx->search->reopen;
+
+	my $eml = PublicInbox::Eml->new(<<EOF);
+List-Id: <blahblah.example.com>
+
+EOF
+	$rw->add_xref3($doc_id, 1, 'deadbeef', 'newsgroup1.example', $eml);
+	$rw_commit->();
 	my $n_test_eml = $query->('n:test.eml');
 	is(scalar(@$n_test_eml), 1, 'got a result');
 	my $n_embed2x_eml = $query->('n:embed2x.eml');
@@ -532,8 +545,15 @@ $ibx->with_umask(sub {
 	is($query->('s:"mail header experiments"')->[0]->{mid},
 		'20200418222508.GA13918@dcvr',
 		'Subject search reaches inside message/rfc822');
+	is($query->('l:blahblah.example.com')->[0]->{num}, $doc_id,
+		'xref3 List-Id probabilistic works');
+	is($query->('lid:blahblah.example.com')->[0]->{num}, $doc_id,
+		'xref3 List-Id boolean term works');
+	$rw->remove_xref3($doc_id, 'deadbeef', 'newsgroup1.example', $eml);
+	$rw->commit_txn_lazy;
+	$ibx->search->reopen;
+	my $res = $query->('lid:blahblah.example.com');
+	is_deeply($res, [], '->remove_xref3 dropped boolean term');
 });
 
 done_testing();
-
-1;

  parent reply	other threads:[~2020-10-27  7:54 UTC|newest]

Thread overview: 57+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2020-10-27  7:54 [PATCH 00/52] detached external index: mostly Eric Wong
2020-10-27  7:54 ` [PATCH 01/52] doc/standards: add RFCs for URL schemes Eric Wong
2020-11-05  7:54   ` [PATCH v2] " Eric Wong
2020-10-27  7:54 ` [PATCH 02/52] search: hoist out _xdb_sharded for v2 inboxes Eric Wong
2020-10-27  7:54 ` [PATCH 03/52] extsearch: start mocking out Eric Wong
2020-10-27  7:54 ` [PATCH 04/52] searchidx: expose INDEXLEVELS as `our' Eric Wong
2020-10-27  7:54 ` [PATCH 05/52] v2writable: add git method Eric Wong
2020-10-27  7:54 ` [PATCH 06/52] v2writable: make OO calls to last_commit-related methods Eric Wong
2020-10-27  7:54 ` [PATCH 07/52] search: xdb_sharded: make this a public method for ExtSearch Eric Wong
2020-10-27  7:54 ` Eric Wong [this message]
2020-10-27  7:54 ` [PATCH 09/52] v2writable: prepare initialization for external indices Eric Wong
2020-10-27  7:54 ` [PATCH 10/52] v2writable: hoist out write_alternates Eric Wong
2020-10-27  7:54 ` [PATCH 11/52] searchidxshard: allow msgref to be undef Eric Wong
2020-10-27  7:54 ` [PATCH 12/52] v2writable: idx_shard: simplify callers Eric Wong
2020-10-27  7:54 ` [PATCH 13/52] v2writable: count_shards: allow working without {ibx} Eric Wong
2020-10-27  7:54 ` [PATCH 14/52] overidx: introduce changes for external index Eric Wong
2020-10-27  7:54 ` [PATCH 15/52] v2: some changes for ExtSearchIdx compatibility Eric Wong
2020-10-27  7:54 ` [PATCH 16/52] inboxwritable: eidx_key for external index Eric Wong
2020-10-27  7:54 ` [PATCH 17/52] v2writable: rename remaining "remote" terminology Eric Wong
2020-10-27  7:54 ` [PATCH 18/52] v2writable: checkpoint: account for lack of {mm} Eric Wong
2020-10-27  7:54 ` [PATCH 19/52] extsearchidx: initial implementation Eric Wong
2020-10-27  7:54 ` [PATCH 20/52] searchidx: index eidx_key as a boolean term Eric Wong
2020-10-27  7:54 ` [PATCH 21/52] searchidx: xref3 delete support Eric Wong
2020-10-27  7:54 ` [PATCH 22/52] searchidxshard: special init for eidx Eric Wong
2020-10-27  7:54 ` [PATCH 23/52] searchidx: put {ibx} into $sync state Eric Wong
2020-10-27  7:54 ` [PATCH 24/52] searchidx: log2stack: simplify callers Eric Wong
2020-10-27  7:54 ` [PATCH 25/52] v2writable: more generic sync setup code Eric Wong
2020-10-27  7:54 ` [PATCH 26/52] v2writable: allow OO method references Eric Wong
2020-10-27  7:54 ` [PATCH 27/52] v2writable: rename {v2w} field to {self} Eric Wong
2020-10-27  7:54 ` [PATCH 28/52] v2writable: make *last_commits and sync_prepare OO methods Eric Wong
2020-10-27  7:54 ` [PATCH 29/52] v2writable: move size check init to sync_prepare Eric Wong
2020-10-27  7:54 ` [PATCH 30/52] extsearchidx: more compatibility with V2Writable callers Eric Wong
2020-10-27  7:54 ` [PATCH 31/52] v2writable: reduce scope of epoch-aware code Eric Wong
2020-10-27  7:54 ` [PATCH 32/52] extsearchidx: remove {unindex_range} field Eric Wong
2020-10-27  7:54 ` [PATCH 33/52] v2writable: pass oid to uindex_oid Eric Wong
2020-10-27  7:54 ` [PATCH 34/52] extsearchidx: sync unit updates Eric Wong
2020-10-27  7:54 ` [PATCH 35/52] searchidx: export prepare_stack Eric Wong
2020-10-27  7:54 ` [PATCH 36/52] extsearchidx: sync updates Eric Wong
2020-10-27  7:54 ` [PATCH 37/52] searchidx: reduce inbox-dependency, wrap ->with_umask Eric Wong
2020-10-27  7:54 ` [PATCH 38/52] searchidx: favor $sync->{ibx} (over $self->{ibx}) Eric Wong
2020-10-27  7:54 ` [PATCH 39/52] Makefile.PL: do not build manpage if POD is missing Eric Wong
2020-10-27  7:54 ` [PATCH 40/52] script: add preliminary eindex implementation Eric Wong
2020-10-27  7:54 ` [PATCH 41/52] index: eindex wiring Eric Wong
2020-10-27  7:54 ` [PATCH 42/52] over: store xref3 data in over.sqlite3 Eric Wong
2020-10-27  7:54 ` [PATCH 43/52] searchidx: remove xref3 support for Xapian Eric Wong
2020-10-27  7:54 ` [PATCH 44/52] t/extsearch.t: verify results and xref3 ordering Eric Wong
2020-10-27  7:54 ` [PATCH 45/52] t/v2writable: remove pointless ->barrier call Eric Wong
2020-10-27  7:54 ` [PATCH 46/52] extsearch: wire up smsg_eml Eric Wong
2020-10-27  7:54 ` [PATCH 47/52] extsearchidx: handle edits Eric Wong
2020-10-27  7:54 ` [PATCH 48/52] extsearch: wire up remaining Inbox-like methods for WWW Eric Wong
2020-10-27  7:54 ` [PATCH 49/52] searchidx: ignore exceptions from ->remove_term Eric Wong
2020-10-27  7:54 ` [PATCH 50/52] extsearchidx: set current_info in warning callbacks Eric Wong
2020-10-27  7:54 ` [PATCH 51/52] extsearchidx: support --batch-size checkpoints Eric Wong
2020-11-03  9:08   ` Eric Wong
2020-10-27  7:54 ` [PATCH 52/52] searchidxshard: make warnings with eidx_key less confusing Eric Wong
2020-10-27 12:08 ` [PATCH 00/52] detached external index: mostly Konstantin Ryabitsev
2020-11-10 18:53 ` detached external index: performance note Eric Wong

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

  List information: https://public-inbox.org/README

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20201027075453.19163-9-e@80x24.org \
    --to=e@80x24.org \
    --cc=meta@public-inbox.org \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
Code repositories for project(s) associated with this public inbox

	https://80x24.org/public-inbox.git

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).