From mboxrd@z Thu Jan  1 00:00:00 1970
Return-Path: <e@80x24.org>
X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on dcvr.yhbt.net
X-Spam-Level: 
X-Spam-Status: No, score=-4.0 required=3.0 tests=ALL_TRUSTED,BAYES_00
	shortcircuit=no autolearn=ham autolearn_force=no version=3.4.2
Received: from localhost (dcvr.yhbt.net [127.0.0.1])
	by dcvr.yhbt.net (Postfix) with ESMTP id B258A1FA18
	for <meta@public-inbox.org>; Sun, 17 Jan 2021 07:10:00 +0000 (UTC)
From: Eric Wong <e@80x24.org>
To: meta@public-inbox.org
Subject: [PATCH 5/5] extindex: fix w/ Xapian 1.2.21..1.2.24
Date: Sun, 17 Jan 2021 07:09:59 +0000
Message-Id: <20210117070959.16826-6-e@80x24.org>
In-Reply-To: <20210117070959.16826-1-e@80x24.org>
References: <20210117070959.16826-1-e@80x24.org>
MIME-Version: 1.0
Content-Transfer-Encoding: 8bit
List-Id: <meta.public-inbox.org>

From: Eric Wong <e@yhbt.net>

Xapian v1.2.21..v1.2.24 failed to set the close-on-exec flag
on the flintlock FD, causing "git cat-file" processes to
hold onto the lock and prevent subsequent Xapian::WritableDatabase
from locking the DB.  So cleanup git processes after committing
the miscidx transaction.
---
 lib/PublicInbox/ExtSearchIdx.pm |  1 +
 lib/PublicInbox/SearchIdx.pm    |  5 ++++-
 lib/PublicInbox/V2Writable.pm   | 12 ++++++++++--
 3 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/lib/PublicInbox/ExtSearchIdx.pm b/lib/PublicInbox/ExtSearchIdx.pm
index 85959a95..c782a62a 100644
--- a/lib/PublicInbox/ExtSearchIdx.pm
+++ b/lib/PublicInbox/ExtSearchIdx.pm
@@ -1003,6 +1003,7 @@ sub idx_init { # similar to V2Writable
 	$self->with_umask(\&_idx_init, $self, $opt);
 	$self->{oidx}->begin_lazy;
 	$self->{oidx}->eidx_prep;
+	$self->git->batch_prepare;
 	$self->{midx}->begin_txn;
 }
 
diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm
index adced076..7f7b980d 100644
--- a/lib/PublicInbox/SearchIdx.pm
+++ b/lib/PublicInbox/SearchIdx.pm
@@ -105,8 +105,11 @@ sub load_xapian_writable () {
 	$DB_CREATE_OR_OPEN = eval($xap.'::DB_CREATE_OR_OPEN()');
 	$DB_OPEN = eval($xap.'::DB_OPEN()');
 	my $ver = (eval($xap.'::major_version()') << 16) |
-		(eval($xap.'::minor_version()') << 8);
+		(eval($xap.'::minor_version()') << 8) |
+		eval($xap.'::revision()');
 	$DB_NO_SYNC = 0x4 if $ver >= 0x10400;
+	# Xapian v1.2.21..v1.2.24 were missing close-on-exec on OFD locks
+	$X->{CLOEXEC_UNSET} = 1 if $ver >= 0x010215 && $ver <= 0x010218;
 	1;
 }
 
diff --git a/lib/PublicInbox/V2Writable.pm b/lib/PublicInbox/V2Writable.pm
index 54004fd7..0104f87a 100644
--- a/lib/PublicInbox/V2Writable.pm
+++ b/lib/PublicInbox/V2Writable.pm
@@ -17,6 +17,7 @@ use PublicInbox::InboxWritable;
 use PublicInbox::OverIdx;
 use PublicInbox::Msgmap;
 use PublicInbox::Spawn qw(spawn popen_rd run_die);
+use PublicInbox::Search;
 use PublicInbox::SearchIdx qw(log2stack is_ancestor check_size is_bad_blob);
 use IO::Handle; # ->autoflush
 use File::Temp ();
@@ -608,7 +609,11 @@ shard[$i] bad echo:$echo != $i waiting for txn commit
 		}
 
 		my $midx = $self->{midx}; # misc index
-		$midx->commit_txn if $midx;
+		if ($midx) {
+			$midx->commit_txn;
+			$PublicInbox::Search::X{CLOEXEC_UNSET} and
+				$self->git->cleanup;
+		}
 
 		# last_commit is special, don't commit these until
 		# Xapian shards are done:
@@ -618,7 +623,10 @@ shard[$i] bad echo:$echo != $i waiting for txn commit
 			$dbh->commit;
 			$dbh->begin_work;
 		}
-		$midx->begin_txn if $midx;
+		if ($midx) {
+			$self->git->batch_prepare;
+			$midx->begin_txn;
+		}
 	}
 	$self->{total_bytes} += $self->{transact_bytes};
 	$self->{transact_bytes} = 0;