user/dev discussion of public-inbox itself
 help / color / mirror / code / Atom feed
* [PATCH 0/5] fixes for older Perls and Xapian
@ 2021-01-17  7:09 Eric Wong
  2021-01-17  7:09 ` [PATCH 1/5] address: pairs: enable pure Perl version Eric Wong
                   ` (4 more replies)
  0 siblings, 5 replies; 6+ messages in thread
From: Eric Wong @ 2021-01-17  7:09 UTC (permalink / raw)
  To: meta

Eric Wong (5):
  address: pairs: enable pure Perl version
  initialize scalar for `vec' perlop modification
  t/thread-index-gap: support older DBI
  t/shared_kv: workaround old File::Spec
  extindex: fix w/ Xapian 1.2.21..1.2.24

 lib/PublicInbox/Address.pm      |  1 +
 lib/PublicInbox/ExtSearchIdx.pm |  1 +
 lib/PublicInbox/LEI.pm          |  2 +-
 lib/PublicInbox/SearchIdx.pm    |  5 ++++-
 lib/PublicInbox/Syscall.pm      |  2 +-
 lib/PublicInbox/V2Writable.pm   | 12 ++++++++++--
 t/shared_kv.t                   | 10 +++-------
 t/thread-index-gap.t            |  7 +++++--
 8 files changed, 26 insertions(+), 14 deletions(-)

^ permalink raw reply	[flat|nested] 6+ messages in thread

* [PATCH 1/5] address: pairs: enable pure Perl version
  2021-01-17  7:09 [PATCH 0/5] fixes for older Perls and Xapian Eric Wong
@ 2021-01-17  7:09 ` Eric Wong
  2021-01-17  7:09 ` [PATCH 2/5] initialize scalar for `vec' perlop modification Eric Wong
                   ` (3 subsequent siblings)
  4 siblings, 0 replies; 6+ messages in thread
From: Eric Wong @ 2021-01-17  7:09 UTC (permalink / raw)
  To: meta

Oops, this is needed for systems lacking Email::Address::XS
---
 lib/PublicInbox/Address.pm | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/PublicInbox/Address.pm b/lib/PublicInbox/Address.pm
index a090fa43..2c9c4395 100644
--- a/lib/PublicInbox/Address.pm
+++ b/lib/PublicInbox/Address.pm
@@ -37,6 +37,7 @@ if ($@) {
 	require PublicInbox::AddressPP;
 	*emails = \&PublicInbox::AddressPP::emails;
 	*names = \&PublicInbox::AddressPP::names;
+	*pairs = \&PublicInbox::AddressPP::pairs;
 }
 
 1;

^ permalink raw reply	[flat|nested] 6+ messages in thread

* [PATCH 2/5] initialize scalar for `vec' perlop modification
  2021-01-17  7:09 [PATCH 0/5] fixes for older Perls and Xapian Eric Wong
  2021-01-17  7:09 ` [PATCH 1/5] address: pairs: enable pure Perl version Eric Wong
@ 2021-01-17  7:09 ` Eric Wong
  2021-01-17  7:09 ` [PATCH 3/5] t/thread-index-gap: support older DBI Eric Wong
                   ` (2 subsequent siblings)
  4 siblings, 0 replies; 6+ messages in thread
From: Eric Wong @ 2021-01-17  7:09 UTC (permalink / raw)
  To: meta

From: Eric Wong <e@yhbt.net>

Older Perls (tested 5.16.3) would warn on uninitialized scalars while
newer (tested 5.28.1) do not.  Just initialize it to an empty string
since it'll be filled in by `vec'.
---
 lib/PublicInbox/LEI.pm     | 2 +-
 lib/PublicInbox/Syscall.pm | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 1f4a3082..2784ca6b 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -670,7 +670,7 @@ sub accept_dispatch { # Listener {post_accept} callback
 	my ($sock) = @_; # ignore other
 	$sock->autoflush(1);
 	my $self = bless { sock => $sock }, __PACKAGE__;
-	vec(my $rvec, fileno($sock), 1) = 1;
+	vec(my $rvec = '', fileno($sock), 1) = 1;
 	select($rvec, undef, undef, 1) or
 		return send($sock, 'timed out waiting to recv FDs', MSG_EOR);
 	my @fds = $recv_cmd->($sock, my $buf, 4096 * 33); # >MAX_ARG_STRLEN
diff --git a/lib/PublicInbox/Syscall.pm b/lib/PublicInbox/Syscall.pm
index a1f53235..5ff1d65f 100644
--- a/lib/PublicInbox/Syscall.pm
+++ b/lib/PublicInbox/Syscall.pm
@@ -224,7 +224,7 @@ sub epoll_ctl_mod8 {
 # epoll_wait wrapper
 # ARGS: (epfd, maxevents, timeout (milliseconds), arrayref)
 #  arrayref: values modified to be [$fd, $event]
-our $epoll_wait_events;
+our $epoll_wait_events = '';
 our $epoll_wait_size = 0;
 sub epoll_wait_mod4 {
 	my ($epfd, $maxevents, $timeout_msec, $events) = @_;

^ permalink raw reply	[flat|nested] 6+ messages in thread

* [PATCH 3/5] t/thread-index-gap: support older DBI
  2021-01-17  7:09 [PATCH 0/5] fixes for older Perls and Xapian Eric Wong
  2021-01-17  7:09 ` [PATCH 1/5] address: pairs: enable pure Perl version Eric Wong
  2021-01-17  7:09 ` [PATCH 2/5] initialize scalar for `vec' perlop modification Eric Wong
@ 2021-01-17  7:09 ` Eric Wong
  2021-01-17  7:09 ` [PATCH 4/5] t/shared_kv: workaround old File::Spec Eric Wong
  2021-01-17  7:09 ` [PATCH 5/5] extindex: fix w/ Xapian 1.2.21..1.2.24 Eric Wong
  4 siblings, 0 replies; 6+ messages in thread
From: Eric Wong @ 2021-01-17  7:09 UTC (permalink / raw)
  To: meta

From: Eric Wong <e@yhbt.net>

...by avoiding selectall_array in favor of selectall_arrayref.
Tested with DBI 1.627.
---
 t/thread-index-gap.t | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/t/thread-index-gap.t b/t/thread-index-gap.t
index f4e4bd09..83c3707d 100644
--- a/t/thread-index-gap.t
+++ b/t/thread-index-gap.t
@@ -45,12 +45,15 @@ for my $msgs (['orig', reverse @msgs], ['shuffle', shuffle(@msgs)]) {
 	}
 	$im->done;
 	my $over = $ibx->over;
-	my @tid = $over->dbh->selectall_array('SELECT DISTINCT(tid) FROM over');
+	my $dbh = $over->dbh;
+	my $tid = $dbh->selectall_arrayref('SELECT DISTINCT(tid) FROM over');
+	my @tid = map { $_->[0] } @$tid;
 	is(scalar(@tid), 1, "only one thread initially ($desc)");
 	$over->dbh_close;
 	run_script([qw(-index --reindex --rethread), $ibx->{inboxdir}]) or
 		BAIL_OUT 'rethread';
-	@tid = $over->dbh->selectall_array('SELECT DISTINCT(tid) FROM over');
+	$tid = $dbh->selectall_arrayref('SELECT DISTINCT(tid) FROM over');
+	@tid = map { $_->[0] } @$tid;
 	is(scalar(@tid), 1, "only one thread after rethread ($desc)");
 }
 

^ permalink raw reply	[flat|nested] 6+ messages in thread

* [PATCH 4/5] t/shared_kv: workaround old File::Spec
  2021-01-17  7:09 [PATCH 0/5] fixes for older Perls and Xapian Eric Wong
                   ` (2 preceding siblings ...)
  2021-01-17  7:09 ` [PATCH 3/5] t/thread-index-gap: support older DBI Eric Wong
@ 2021-01-17  7:09 ` Eric Wong
  2021-01-17  7:09 ` [PATCH 5/5] extindex: fix w/ Xapian 1.2.21..1.2.24 Eric Wong
  4 siblings, 0 replies; 6+ messages in thread
From: Eric Wong @ 2021-01-17  7:09 UTC (permalink / raw)
  To: meta

From: Eric Wong <e@yhbt.net>

The version of File::Spec shipped with Perl 5.16.3 memoizes the
value of File::Spec->tmpdir, causing changes to $ENV{TMPDIR}
after-the-fact to be ignored.

We'll only work around this in the test since it's innocuous and
unlikely to matter in real-world usage (and there's many places
where we'd have to workaround this in non-test code).
---
 t/shared_kv.t | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/t/shared_kv.t b/t/shared_kv.t
index e7a8e05e..6f6374f2 100644
--- a/t/shared_kv.t
+++ b/t/shared_kv.t
@@ -9,9 +9,8 @@ use_ok 'PublicInbox::SharedKV';
 my ($tmpdir, $for_destroy) = tmpdir();
 local $ENV{TMPDIR} = $tmpdir;
 my $skv = PublicInbox::SharedKV->new;
-opendir(my $dh, $tmpdir) or BAIL_OUT $!;
-my @ent = grep(!/\A\.\.?\z/, readdir($dh));
-is(scalar(@ent), 1, 'created a temporary dir');
+my $skv_tmpdir = $skv->{tmpdir};
+ok(-d $skv_tmpdir, 'created a temporary dir');
 $skv->dbh;
 my $dead = "\xde\xad";
 my $beef = "\xbe\xef";
@@ -48,10 +47,7 @@ is($skv->delete_by_val($dead), 2, 'delete_by_val hits');
 is($skv->delete_by_val($dead), 0, 'delete_by_val misses again');
 
 undef $skv;
-rewinddir($dh);
-@ent = grep(!/\A\.\.?\z/, readdir($dh));
-is(scalar(@ent), 0, 'temporary dir gone');
-undef $dh;
+ok(!-d $skv_tmpdir, 'temporary dir gone');
 $skv = PublicInbox::SharedKV->new("$tmpdir/dir", 'base');
 ok(-e "$tmpdir/dir/base.sqlite3", 'file created');
 

^ permalink raw reply	[flat|nested] 6+ messages in thread

* [PATCH 5/5] extindex: fix w/ Xapian 1.2.21..1.2.24
  2021-01-17  7:09 [PATCH 0/5] fixes for older Perls and Xapian Eric Wong
                   ` (3 preceding siblings ...)
  2021-01-17  7:09 ` [PATCH 4/5] t/shared_kv: workaround old File::Spec Eric Wong
@ 2021-01-17  7:09 ` Eric Wong
  4 siblings, 0 replies; 6+ messages in thread
From: Eric Wong @ 2021-01-17  7:09 UTC (permalink / raw)
  To: meta

From: Eric Wong <e@yhbt.net>

Xapian v1.2.21..v1.2.24 failed to set the close-on-exec flag
on the flintlock FD, causing "git cat-file" processes to
hold onto the lock and prevent subsequent Xapian::WritableDatabase
from locking the DB.  So cleanup git processes after committing
the miscidx transaction.
---
 lib/PublicInbox/ExtSearchIdx.pm |  1 +
 lib/PublicInbox/SearchIdx.pm    |  5 ++++-
 lib/PublicInbox/V2Writable.pm   | 12 ++++++++++--
 3 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/lib/PublicInbox/ExtSearchIdx.pm b/lib/PublicInbox/ExtSearchIdx.pm
index 85959a95..c782a62a 100644
--- a/lib/PublicInbox/ExtSearchIdx.pm
+++ b/lib/PublicInbox/ExtSearchIdx.pm
@@ -1003,6 +1003,7 @@ sub idx_init { # similar to V2Writable
 	$self->with_umask(\&_idx_init, $self, $opt);
 	$self->{oidx}->begin_lazy;
 	$self->{oidx}->eidx_prep;
+	$self->git->batch_prepare;
 	$self->{midx}->begin_txn;
 }
 
diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm
index adced076..7f7b980d 100644
--- a/lib/PublicInbox/SearchIdx.pm
+++ b/lib/PublicInbox/SearchIdx.pm
@@ -105,8 +105,11 @@ sub load_xapian_writable () {
 	$DB_CREATE_OR_OPEN = eval($xap.'::DB_CREATE_OR_OPEN()');
 	$DB_OPEN = eval($xap.'::DB_OPEN()');
 	my $ver = (eval($xap.'::major_version()') << 16) |
-		(eval($xap.'::minor_version()') << 8);
+		(eval($xap.'::minor_version()') << 8) |
+		eval($xap.'::revision()');
 	$DB_NO_SYNC = 0x4 if $ver >= 0x10400;
+	# Xapian v1.2.21..v1.2.24 were missing close-on-exec on OFD locks
+	$X->{CLOEXEC_UNSET} = 1 if $ver >= 0x010215 && $ver <= 0x010218;
 	1;
 }
 
diff --git a/lib/PublicInbox/V2Writable.pm b/lib/PublicInbox/V2Writable.pm
index 54004fd7..0104f87a 100644
--- a/lib/PublicInbox/V2Writable.pm
+++ b/lib/PublicInbox/V2Writable.pm
@@ -17,6 +17,7 @@ use PublicInbox::InboxWritable;
 use PublicInbox::OverIdx;
 use PublicInbox::Msgmap;
 use PublicInbox::Spawn qw(spawn popen_rd run_die);
+use PublicInbox::Search;
 use PublicInbox::SearchIdx qw(log2stack is_ancestor check_size is_bad_blob);
 use IO::Handle; # ->autoflush
 use File::Temp ();
@@ -608,7 +609,11 @@ shard[$i] bad echo:$echo != $i waiting for txn commit
 		}
 
 		my $midx = $self->{midx}; # misc index
-		$midx->commit_txn if $midx;
+		if ($midx) {
+			$midx->commit_txn;
+			$PublicInbox::Search::X{CLOEXEC_UNSET} and
+				$self->git->cleanup;
+		}
 
 		# last_commit is special, don't commit these until
 		# Xapian shards are done:
@@ -618,7 +623,10 @@ shard[$i] bad echo:$echo != $i waiting for txn commit
 			$dbh->commit;
 			$dbh->begin_work;
 		}
-		$midx->begin_txn if $midx;
+		if ($midx) {
+			$self->git->batch_prepare;
+			$midx->begin_txn;
+		}
 	}
 	$self->{total_bytes} += $self->{transact_bytes};
 	$self->{transact_bytes} = 0;

^ permalink raw reply	[flat|nested] 6+ messages in thread

end of thread, other threads:[~2021-01-17  7:10 UTC | newest]

Thread overview: 6+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2021-01-17  7:09 [PATCH 0/5] fixes for older Perls and Xapian Eric Wong
2021-01-17  7:09 ` [PATCH 1/5] address: pairs: enable pure Perl version Eric Wong
2021-01-17  7:09 ` [PATCH 2/5] initialize scalar for `vec' perlop modification Eric Wong
2021-01-17  7:09 ` [PATCH 3/5] t/thread-index-gap: support older DBI Eric Wong
2021-01-17  7:09 ` [PATCH 4/5] t/shared_kv: workaround old File::Spec Eric Wong
2021-01-17  7:09 ` [PATCH 5/5] extindex: fix w/ Xapian 1.2.21..1.2.24 Eric Wong

Code repositories for project(s) associated with this inbox:

	https://80x24.org/public-inbox.git

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).