From 3f27381a79782be136ae8d27c8f585edbbc6f6bf Mon Sep 17 00:00:00 2001
From: Eric Wong <e@80x24.org>
Date: Wed, 1 Sep 2021 00:17:32 +0000
Subject: extindex: --gc removes messages from over, too

While messages from removed inboxes were removed from Xapian
search, --gc failed to remove messages from over.sqlite3
entirely.  They no longer show up in the topic summary view.

Reported-by: Konstantin Ryabitsev <konstantin@linuxfoundation.org>
Link: https://public-inbox.org/20210830201723.dehoul4y6gpqf2cp@nitro.local/
---
 t/extsearch.t | 61 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 61 insertions(+)

(limited to 't')

diff --git a/t/extsearch.t b/t/extsearch.t
index b03adc17..ad4f2c6d 100644
--- a/t/extsearch.t
+++ b/t/extsearch.t
@@ -466,4 +466,65 @@ SKIP: {
 		'--gc works after compact');
 }
 
+{ # ensure --gc removes non-xposted messages
+	my $old_size = -s $cfg_path // xbail "stat $cfg_path $!";
+	my $tmp_addr = 'v2tmp@example.com';
+	run_script([qw(-init v2tmp --indexlevel basic
+		--newsgroup v2tmp.example),
+		"$home/v2tmp", 'http://example.com/v2tmp', $tmp_addr ])
+		or xbail '-init';
+	$env = { ORIGINAL_RECIPIENT => $tmp_addr };
+	open $fh, '+>', undef or xbail "open $!";
+	$fh->autoflush(1);
+	my $mid = 'tmpmsg@example.com';
+	print $fh <<EOM or xbail "print $!";
+From: b\@z
+To: b\@r
+Message-Id: <$mid>
+Subject: tmpmsg
+Date: Tue, 19 Jan 2038 03:14:07 +0000
+
+EOM
+	seek $fh, 0, SEEK_SET or xbail "seek $!";
+	run_script([qw(-mda --no-precheck)], $env, {0 => $fh}) or xbail '-mda';
+	ok(run_script([qw(-extindex --all), "$home/extindex"]), 'update');
+	my $nr;
+	{
+		my $es = PublicInbox::ExtSearch->new("$home/extindex");
+		my ($id, $prv);
+		my $smsg = $es->over->next_by_mid($mid, \$id, \$prv);
+		ok($smsg, 'tmpmsg indexed');
+		my $mset = $es->search->mset("mid:$mid");
+		is($mset->size, 1, 'new message found');
+		$mset = $es->search->mset('z:0..');
+		$nr = $mset->size;
+	}
+	truncate($cfg_path, $old_size) or xbail "truncate $!";
+	my $rdr = { 2 => \(my $err) };
+	ok(run_script([qw(-extindex --gc), "$home/extindex"], undef, $rdr),
+		'gc to get rid of removed inbox');
+	is_deeply([ grep(!/^(?:I:|#)/, split(/^/m, $err)) ], [],
+		'no non-informational errors in stderr');
+
+	my $es = PublicInbox::ExtSearch->new("$home/extindex");
+	my $mset = $es->search->mset("mid:$mid");
+	is($mset->size, 0, 'tmpmsg gone from search');
+	my ($id, $prv);
+	is($es->over->next_by_mid($mid, \$id, \$prv), undef,
+		'tmpmsg gone from over');
+	$id = $prv = undef;
+	is($es->over->next_by_mid('testmessage@example.com', \$id, \$prv),
+		undef, 'remaining message not indavderover');
+	$mset = $es->search->mset('z:0..');
+	is($mset->size, $nr - 1, 'existing messages not clobbered from search');
+	my $o = $es->over->{dbh}->selectall_arrayref(<<EOM);
+SELECT num FROM over ORDER BY num
+EOM
+	is(scalar(@$o), $mset->size, 'over row count matches Xapian');
+	my $x = $es->over->{dbh}->selectall_arrayref(<<EOM);
+SELECT DISTINCT(docid) FROM xref3 ORDER BY docid
+EOM
+	is_deeply($x, $o, 'xref3 and over docids match');
+}
+
 done_testing;
-- 
cgit v1.2.3-24-ge0c7