From ea885bf0f76dc1795dc771667be721ec0fed5482 Mon Sep 17 00:00:00 2001
From: Eric Wong <e@80x24.org>
Date: Sun, 29 Nov 2020 05:09:35 +0000
Subject: extindex: support `--gc' to remove dead inboxes

Inboxes may be removed or newsgroups renamed over time.
Introduce a switch to do garbage collection and eliminate stale
search and xref3 results based on inboxes which remain in the
config file.

This may also fixup stale results leftover from any bugs which
may leave stale data around.

This is also useful in case a clumsy BOFH (me :P) is swapping
between several PI_CONFIGs and accidentally indexed a bunch of
inboxes they didn't intend to.
---
 lib/PublicInbox/SearchIdxShard.pm | 21 ++++++++++++++++++---
 1 file changed, 18 insertions(+), 3 deletions(-)

(limited to 'lib/PublicInbox/SearchIdxShard.pm')

diff --git a/lib/PublicInbox/SearchIdxShard.pm b/lib/PublicInbox/SearchIdxShard.pm
index dcfeb0be..53fac9b6 100644
--- a/lib/PublicInbox/SearchIdxShard.pm
+++ b/lib/PublicInbox/SearchIdxShard.pm
@@ -57,6 +57,7 @@ sub spawn_worker {
 
 sub eml ($$) {
 	my ($r, $len) = @_;
+	return if $len == 0;
 	my $n = read($r, my $bref, $len) or die "read: $!\n";
 	$n == $len or die "short read: $n != $len\n";
 	PublicInbox::Eml->new(\$bref);
@@ -92,6 +93,10 @@ sub shard_worker_loop ($$$$$) {
 			chomp $eidx_key;
 			$self->remove_eidx_info($docid, $oid, $eidx_key,
 							eml($r, $len));
+		} elsif ($line =~ s/\AO ([^\n]+)\n//) {
+			my $over_fn = $1;
+			$over_fn =~ tr/\0/\n/;
+			$self->over_check(PublicInbox::Over->new($over_fn));
 		} else {
 			chomp $line;
 			my $eidx_key;
@@ -155,10 +160,9 @@ sub shard_add_eidx_info {
 }
 
 sub shard_remove_eidx_info {
-	my ($self, $docid, $oid, $xibx, $eml) = @_;
-	my $eidx_key = $xibx->eidx_key;
+	my ($self, $docid, $oid, $eidx_key, $eml) = @_;
 	if (my $w = $self->{w}) {
-		my $hdr = $eml->header_obj->as_string;
+		my $hdr = $eml ? $eml->header_obj->as_string : '';
 		my $len = length($hdr);
 		print $w "-X $len $docid $oid $eidx_key\n", $hdr or
 			die "failed to write shard: $!";
@@ -212,4 +216,15 @@ sub shard_remove {
 	}
 }
 
+sub shard_over_check {
+	my ($self, $over) = @_;
+	if (my $w = $self->{w}) { # triggers remove_by_oid in a shard child
+		my ($over_fn) = $over->{dbh}->sqlite_db_filename;
+		$over_fn =~ tr/\n/\0/;
+		print $w "O $over_fn\n" or die "failed to write over $!";
+	} else {
+		$self->over_check($over);
+	}
+}
+
 1;
-- 
cgit v1.2.3-24-ge0c7