user/dev discussion of public-inbox itself
 help / color / mirror / code / Atom feed
Search results ordered by [date|relevance]  view[summary|nested|Atom feed]
thread overview below | download mbox.gz: |
* [PATCH 3/7] searchidxshard: IPC conversion, part 2
  2021-01-03  2:06  7% [PATCH 0/7] v2: swap in new IPC package Eric Wong
@ 2021-01-03  2:06  5% ` Eric Wong
  0 siblings, 0 replies; 4+ results
From: Eric Wong @ 2021-01-03  2:06 UTC (permalink / raw)
  To: meta

We can remove some now-pointless wrapper functions by using
->ipc_do in even more places.
---
 lib/PublicInbox/ExtSearchIdx.pm   | 23 ++++++++++++-----------
 lib/PublicInbox/LeiStore.pm       | 13 +++++++------
 lib/PublicInbox/SearchIdx.pm      |  8 +-------
 lib/PublicInbox/SearchIdxShard.pm | 30 ------------------------------
 lib/PublicInbox/V2Writable.pm     |  3 +--
 5 files changed, 21 insertions(+), 56 deletions(-)

diff --git a/lib/PublicInbox/ExtSearchIdx.pm b/lib/PublicInbox/ExtSearchIdx.pm
index c3581628..064d9939 100644
--- a/lib/PublicInbox/ExtSearchIdx.pm
+++ b/lib/PublicInbox/ExtSearchIdx.pm
@@ -102,7 +102,7 @@ sub do_xpost ($$) {
 	if (my $new_smsg = $req->{new_smsg}) { # 'm' on cross-posted message
 		my $xnum = $req->{xnum};
 		$self->{oidx}->add_xref3($docid, $xnum, $oid, $eidx_key);
-		$idx->shard_add_eidx_info($docid, $eidx_key, $eml);
+		$idx->ipc_do('add_eidx_info', $docid, $eidx_key, $eml);
 		check_batch_limit($req);
 	} else { # 'd'
 		my $rm_eidx_info;
@@ -110,9 +110,10 @@ sub do_xpost ($$) {
 							\$rm_eidx_info);
 		if ($nr == 0) {
 			$self->{oidx}->eidxq_del($docid);
-			$idx->shard_remove($docid);
+			$idx->ipc_do('xdb_remove', $docid);
 		} elsif ($rm_eidx_info) {
-			$idx->shard_remove_eidx_info($docid, $eidx_key, $eml);
+			$idx->ipc_do('remove_eidx_info',
+					$docid, $eidx_key, $eml);
 			$self->{oidx}->eidxq_add($docid); # yes, add
 		}
 	}
@@ -327,7 +328,7 @@ DELETE FROM xref3 WHERE docid = ? AND ibx_id = ?
 		}
 	} else {
 		warn "I: remove #$docid $eidx_key @oid\n";
-		$self->idx_shard($docid)->shard_remove($docid);
+		$self->idx_shard($docid)->ipc_do('xdb_remove', $docid);
 	}
 }
 
@@ -440,7 +441,7 @@ sub _reindex_finalize ($$$) {
 	for my $x (reverse @$stable) {
 		$ibx = _ibx_for($self, $sync, $x);
 		my $hdr = delete $x->{hdr} // die 'BUG: no {hdr}';
-		$idx->shard_add_eidx_info($docid, $ibx->eidx_key, $hdr);
+		$idx->ipc_do('add_eidx_info', $docid, $ibx->eidx_key, $hdr);
 	}
 	return if $nr == 1; # likely, all good
 
@@ -483,12 +484,12 @@ sub _reindex_oid { # git->cat_async callback
 		my $remain = $self->{oidx}->remove_xref3($docid, $expect_oid);
 		if ($remain == 0) {
 			warn "W: #$docid gone or corrupted\n";
-			$self->idx_shard($docid)->shard_remove($docid);
+			$self->idx_shard($docid)->ipc_do('xdb_remove', $docid);
 		} elsif (my $next_oid = $req->{xr3r}->[++$req->{ix}]->[2]) {
 			$self->git->cat_async($next_oid, \&_reindex_oid, $req);
 		} else {
 			warn "BUG: #$docid gone (UNEXPECTED)\n";
-			$self->idx_shard($docid)->shard_remove($docid);
+			$self->idx_shard($docid)->ipc_do('xdb_remove', $docid);
 		}
 		return;
 	}
@@ -522,7 +523,7 @@ sub _reindex_smsg ($$$) {
 BUG? #$docid $smsg->{blob} is not referenced by inboxes during reindex
 
 		$self->{oidx}->delete_by_num($docid);
-		$self->idx_shard($docid)->shard_remove($docid);
+		$self->idx_shard($docid)->ipc_do('xdb_remove', $docid);
 		return;
 	}
 
@@ -799,10 +800,10 @@ DELETE FROM xref3 WHERE ibx_id = ? AND xnum = ? AND oidbin = ?
 			if (scalar(@$xr3) == 0) { # all gone
 				$self->{oidx}->delete_by_num($docid);
 				$self->{oidx}->eidxq_del($docid);
-				$idx->shard_remove($docid);
+				$idx->ipc_do('xdb_remove', $docid);
 			} else { # enqueue for reindex of remaining messages
-				$idx->shard_remove_eidx_info($docid,
-							$ibx->eidx_key);
+				$idx->ipc_do('remove_eidx_info',
+						$docid, $ibx->eidx_key);
 				$self->{oidx}->eidxq_add($docid); # yes, add
 			}
 		}
diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm
index 07a3198a..d686e95a 100644
--- a/lib/PublicInbox/LeiStore.pm
+++ b/lib/PublicInbox/LeiStore.pm
@@ -146,7 +146,7 @@ sub set_eml_keywords {
 	my $eidx = eidx_init($self);
 	my @docids = _docids_for($self, $eml);
 	for my $docid (@docids) {
-		$eidx->idx_shard($docid)->shard_set_keywords($docid, @kw);
+		$eidx->idx_shard($docid)->ipc_do('set_keywords', $docid, @kw);
 	}
 	\@docids;
 }
@@ -156,7 +156,7 @@ sub add_eml_keywords {
 	my $eidx = eidx_init($self);
 	my @docids = _docids_for($self, $eml);
 	for my $docid (@docids) {
-		$eidx->idx_shard($docid)->shard_add_keywords($docid, @kw);
+		$eidx->idx_shard($docid)->ipc_do('add_keywords', $docid, @kw);
 	}
 	\@docids;
 }
@@ -166,7 +166,7 @@ sub remove_eml_keywords {
 	my $eidx = eidx_init($self);
 	my @docids = _docids_for($self, $eml);
 	for my $docid (@docids) {
-		$eidx->idx_shard($docid)->shard_remove_keywords($docid, @kw);
+		$eidx->idx_shard($docid)->ipc_do('remove_keywords', $docid, @kw)
 	}
 	\@docids;
 }
@@ -205,8 +205,9 @@ sub add_eml {
 		for my $docid (@docids) {
 			my $idx = $eidx->idx_shard($docid);
 			$oidx->add_xref3($docid, -1, $smsg->{blob}, '.');
-			$idx->shard_add_eidx_info($docid, '.', $eml); # List-Id
-			$idx->shard_add_keywords($docid, @kw) if @kw;
+			# add_eidx_info for List-Id
+			$idx->ipc_do('add_eidx_info', $docid, '.', $eml);
+			$idx->ipc_do('add_keywords', $docid, @kw) if @kw;
 		}
 		\@docids;
 	} else {
@@ -215,7 +216,7 @@ sub add_eml {
 		$oidx->add_xref3($smsg->{num}, -1, $smsg->{blob}, '.');
 		my $idx = $eidx->idx_shard($smsg->{num});
 		$idx->index_raw($msgref, $eml, $smsg);
-		$idx->shard_add_keywords($smsg->{num}, @kw) if @kw;
+		$idx->ipc_do('add_keywords', $smsg->{num}, @kw) if @kw;
 		$smsg;
 	}
 }
diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm
index d83fd4ca..da3ac2e3 100644
--- a/lib/PublicInbox/SearchIdx.pm
+++ b/lib/PublicInbox/SearchIdx.pm
@@ -552,6 +552,7 @@ sub smsg_from_doc ($) {
 
 sub xdb_remove {
 	my ($self, @docids) = @_;
+	$self->begin_txn_lazy;
 	my $xdb = $self->{xdb} or return;
 	for my $docid (@docids) {
 		eval { $xdb->delete_document($docid) };
@@ -559,13 +560,6 @@ sub xdb_remove {
 	}
 }
 
-sub remove_by_docid {
-	my ($self, $num) = @_;
-	die "BUG: remove_by_docid is v2-only\n" if $self->{oidx};
-	$self->begin_txn_lazy;
-	xdb_remove($self, $num) if need_xapian($self);
-}
-
 sub index_git_blob_id {
 	my ($doc, $pfx, $objid) = @_;
 
diff --git a/lib/PublicInbox/SearchIdxShard.pm b/lib/PublicInbox/SearchIdxShard.pm
index 68644bc0..43dad959 100644
--- a/lib/PublicInbox/SearchIdxShard.pm
+++ b/lib/PublicInbox/SearchIdxShard.pm
@@ -54,16 +54,6 @@ sub index_raw {
 	$self->ipc_do('add_message', $eml, $smsg);
 }
 
-sub shard_add_eidx_info {
-	my ($self, $docid, $eidx_key, $eml) = @_;
-	$self->ipc_do('add_eidx_info', $docid, $eidx_key, $eml);
-}
-
-sub shard_remove_eidx_info {
-	my ($self, $docid, $eidx_key, $eml) = @_;
-	$self->ipc_do('remove_eidx_info', $docid, $eidx_key, $eml);
-}
-
 # needed when there's multiple IPC workers and the parent forking
 # causes newer siblings to inherit older siblings sockets
 sub shard_atfork_child {
@@ -93,26 +83,6 @@ sub shard_close {
 	$self->ipc_worker_stop;
 }
 
-sub shard_remove {
-	my ($self, $num) = @_;
-	$self->ipc_do('remove_by_docid', $num);
-}
-
-sub shard_set_keywords {
-	my ($self, $docid, @kw) = @_;
-	$self->ipc_do('set_keywords', $docid, @kw);
-}
-
-sub shard_remove_keywords {
-	my ($self, $docid, @kw) = @_;
-	$self->ipc_do('remove_keywords', $docid, @kw);
-}
-
-sub shard_add_keywords {
-	my ($self, $docid, @kw) = @_;
-	$self->ipc_do('add_keywords', $docid, @kw);
-}
-
 sub shard_over_check {
 	my ($self, $over) = @_;
 	if ($self->{-ipc_sock} && $over->{dbh}) {
diff --git a/lib/PublicInbox/V2Writable.pm b/lib/PublicInbox/V2Writable.pm
index cad559c5..885edbe9 100644
--- a/lib/PublicInbox/V2Writable.pm
+++ b/lib/PublicInbox/V2Writable.pm
@@ -1133,8 +1133,7 @@ sub unindex_oid_aux ($$$) {
 	my ($self, $oid, $mid) = @_;
 	my @removed = $self->{oidx}->remove_oid($oid, $mid);
 	for my $num (@removed) {
-		my $idx = idx_shard($self, $num);
-		$idx->shard_remove($num);
+		idx_shard($self, $num)->ipc_do('xdb_remove', $num);
 	}
 }
 

^ permalink raw reply related	[relevance 5%]

* [PATCH] overidx: update comment for new sub name
@ 2021-10-04 11:11  5% Eric Wong
  0 siblings, 0 replies; 4+ results
From: Eric Wong @ 2021-10-04 11:11 UTC (permalink / raw)
  To: meta

`shard_remove_eidx_info' was made unnecessary with commit
82b805db3ad9 (searchidxshard: IPC conversion, part 2, 2021-01-03)
and we now call `remove_eidx_info' directly.
---
 lib/PublicInbox/OverIdx.pm | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/PublicInbox/OverIdx.pm b/lib/PublicInbox/OverIdx.pm
index e0893337e784..2e3d4534f125 100644
--- a/lib/PublicInbox/OverIdx.pm
+++ b/lib/PublicInbox/OverIdx.pm
@@ -633,7 +633,7 @@ SELECT COUNT(*) FROM xref3 WHERE docid = ?
 		# if deduplication rules in ContentHash change, it's
 		# possible a docid can have multiple rows with the
 		# same ibx_id.  This governs whether or not we call
-		# ->shard_remove_eidx_info in ExtSearchIdx.
+		# ->remove_eidx_info in ExtSearchIdx.
 		$sth = $self->{dbh}->prepare_cached(<<'', undef, 1);
 SELECT COUNT(*) FROM xref3 WHERE docid = ? AND ibx_id = ?
 

^ permalink raw reply related	[relevance 5%]

* [PATCH 2/3] overidx: update comment for new sub name
  @ 2021-10-05  9:40  5% ` Eric Wong
  0 siblings, 0 replies; 4+ results
From: Eric Wong @ 2021-10-05  9:40 UTC (permalink / raw)
  To: meta

`shard_remove_eidx_info' was made unnecessary with commit
82b805db3ad9 (searchidxshard: IPC conversion, part 2, 2021-01-03)
and we now call `remove_eidx_info' directly.
---
 lib/PublicInbox/OverIdx.pm | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/PublicInbox/OverIdx.pm b/lib/PublicInbox/OverIdx.pm
index e0893337e784..2e3d4534f125 100644
--- a/lib/PublicInbox/OverIdx.pm
+++ b/lib/PublicInbox/OverIdx.pm
@@ -633,7 +633,7 @@ SELECT COUNT(*) FROM xref3 WHERE docid = ?
 		# if deduplication rules in ContentHash change, it's
 		# possible a docid can have multiple rows with the
 		# same ibx_id.  This governs whether or not we call
-		# ->shard_remove_eidx_info in ExtSearchIdx.
+		# ->remove_eidx_info in ExtSearchIdx.
 		$sth = $self->{dbh}->prepare_cached(<<'', undef, 1);
 SELECT COUNT(*) FROM xref3 WHERE docid = ? AND ibx_id = ?
 

^ permalink raw reply related	[relevance 5%]

* [PATCH 0/7] v2: swap in new IPC package
@ 2021-01-03  2:06  7% Eric Wong
  2021-01-03  2:06  5% ` [PATCH 3/7] searchidxshard: IPC conversion, part 2 Eric Wong
  0 siblings, 1 reply; 4+ results
From: Eric Wong @ 2021-01-03  2:06 UTC (permalink / raw)
  To: meta

SearchIdxShard was too big and adding the new extindex
stuff made things worse.  Since I intend to use IPC
in more places, I figured it'd be good to prove it with
works well by dropping it into the old v2 mix.

The below diffstat is nice

Eric Wong (7):
  ipc: some documentation comments
  searchidxshard: use PublicInbox::IPC to kill lots of code
  searchidxshard: IPC conversion, part 2
  searchidxshard: replace index_raw with index_eml
  use Eml (or MIME) objects for all indexing paths
  ipc: switch to one-way pipes
  searchidxshard: use add_xapian directly for v2

 lib/PublicInbox/ExtSearchIdx.pm   |  38 +++--
 lib/PublicInbox/IPC.pm            | 127 +++++++++------
 lib/PublicInbox/Import.pm         |   4 +-
 lib/PublicInbox/LeiStore.pm       |  18 +--
 lib/PublicInbox/SearchIdx.pm      |  35 +---
 lib/PublicInbox/SearchIdxShard.pm | 257 +++++-------------------------
 lib/PublicInbox/Smsg.pm           |  13 ++
 lib/PublicInbox/V2Writable.pm     | 102 +++++-------
 t/import.t                        |  12 +-
 t/search.t                        |   2 +-
 10 files changed, 206 insertions(+), 402 deletions(-)

^ permalink raw reply	[relevance 7%]

Results 1-4 of 4 | reverse | options above
-- pct% links below jump to the message on this page, permalinks otherwise --
2021-01-03  2:06  7% [PATCH 0/7] v2: swap in new IPC package Eric Wong
2021-01-03  2:06  5% ` [PATCH 3/7] searchidxshard: IPC conversion, part 2 Eric Wong
2021-10-04 11:11  5% [PATCH] overidx: update comment for new sub name Eric Wong
2021-10-05  9:40     [PATCH 0/3] some indexing stuff Eric Wong
2021-10-05  9:40  5% ` [PATCH 2/3] overidx: update comment for new sub name Eric Wong

Code repositories for project(s) associated with this public inbox

	https://80x24.org/public-inbox.git

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).