From 35e31227139e401cf7845d824a1b4e6a1de56fe3 Mon Sep 17 00:00:00 2001
From: Eric Wong <e@yhbt.net>
Date: Sat, 27 Jun 2020 10:03:59 +0000
Subject: imaptracker: use flock(2) around writes

SQLite only issues non-blocking F_SETLK ops (not F_SETLKW) and
retries failures using a configurable busy_timeout.  SQLite's
busy loop sleeps for a millisecond and retries the lock until
the configured busy_timeout is hit.

Trying to set ->sqlite_busy_timeout to larger values (e.g. 30000
milliseconds) still leads to failure when running the new stress
test with 8 processes with TMPDIR on a 7200 RPM HDD.

Inspection of SQLite source reveals there's no built-in way to
use F_SETLKW, so tack on the existing flock(2) support we use to
synchronize git + SQLite + Xapian for inbox writing.  We use
flock(2) instead of POSIX fcntl(2) locks since Perl doesn't
provide a way to manipulate "struct flock" portably.
---
 t/imap_tracker.t | 30 +++++++++++++++++++++++++++++-
 1 file changed, 29 insertions(+), 1 deletion(-)

(limited to 't')

diff --git a/t/imap_tracker.t b/t/imap_tracker.t
index 8dc04ed7..01e1d0b1 100644
--- a/t/imap_tracker.t
+++ b/t/imap_tracker.t
@@ -9,8 +9,8 @@ my ($tmpdir, $for_destroy) = tmpdir();
 mkdir "$tmpdir/old" or die "mkdir $tmpdir/old: $!";
 my $old = "$tmpdir/old/imap.sqlite3";
 my $cur = "$tmpdir/data/public-inbox/imap.sqlite3";
+local $ENV{XDG_DATA_HOME} = "$tmpdir/data";
 {
-	local $ENV{XDG_DATA_HOME} = "$tmpdir/data";
 	local $ENV{PI_DIR} = "$tmpdir/old";
 
 	my $tracker = PublicInbox::IMAPTracker->new;
@@ -22,5 +22,33 @@ my $cur = "$tmpdir/data/public-inbox/imap.sqlite3";
 	$tracker = PublicInbox::IMAPTracker->new;
 	ok(!-f $cur, '->new does not create new file if old is present');
 }
+SKIP: {
+	my $nproc = $ENV{TEST_STRESS_NPROC};
+	skip 'TEST_STRESS_NPROC= not set', 1 unless $nproc;
+	my $nr = $ENV{TEST_STRESS_NR} // 10000;
+	diag "TEST_STRESS_NPROC=$nproc TEST_STRESS_NR=$nr";
+	require POSIX;
+	for my $n (1..$nproc) {
+		defined(my $pid = fork) or BAIL_OUT "fork: $!";
+		if ($pid == 0) {
+			my $url = "imap://example.com/INBOX.$$";
+			my $uidval = time;
+			eval {
+				my $itrk = PublicInbox::IMAPTracker->new($url);
+				for my $uid (1..$nr) {
+					$itrk->update_last($uidval, $uid);
+					my ($uv, $u) = $itrk->get_last;
+				}
+			};
+			warn "E: $n $$ - $@\n" if $@;
+			POSIX::_exit($@ ? 1 : 0);
+		}
+	}
+	while (1) {
+		my $pid = waitpid(-1, 0);
+		last if $pid < 0;
+		is($?, 0, "$pid exited");
+	}
+}
 
 done_testing;
-- 
cgit v1.2.3-24-ge0c7