user/dev discussion of public-inbox itself
 help / color / mirror / Atom feed
* [PATCH 0/2] Perl <5.22 fixes
@ 2020-08-08  4:59 Eric Wong
  2020-08-08  4:59 ` [PATCH 1/2] support setting No_COW on Perl <5.22 Eric Wong
  2020-08-08  4:59 ` [PATCH 2/2] dir_idle: require Perl 5.22+ for kqueue Eric Wong
  0 siblings, 2 replies; 3+ messages in thread
From: Eric Wong @ 2020-08-08  4:59 UTC (permalink / raw)
  To: meta

fileno(DIRHANDLE) didn't work until Perl 5.22, so we'll
fall back to using some Inline::C for setting No_COW on
btrfs and polling for -watch on *BSD with older Perl.

Eric Wong (2):
  support setting No_COW on Perl <5.22
  dir_idle: require Perl 5.22+ for kqueue

 lib/PublicInbox/DirIdle.pm   |  3 ++-
 lib/PublicInbox/Msgmap.pm    |  2 +-
 lib/PublicInbox/NDC_PP.pm    | 13 +++++++++----
 lib/PublicInbox/Over.pm      |  4 ++--
 lib/PublicInbox/SearchIdx.pm | 10 ++--------
 lib/PublicInbox/Spawn.pm     | 21 ++++++++++++++++++---
 lib/PublicInbox/Xapcmd.pm    |  4 ++--
 t/nodatacow.t                | 34 +++++++++++++++++++++++++---------
 8 files changed, 61 insertions(+), 30 deletions(-)

^ permalink raw reply	[flat|nested] 3+ messages in thread

* [PATCH 1/2] support setting No_COW on Perl <5.22
  2020-08-08  4:59 [PATCH 0/2] Perl <5.22 fixes Eric Wong
@ 2020-08-08  4:59 ` Eric Wong
  2020-08-08  4:59 ` [PATCH 2/2] dir_idle: require Perl 5.22+ for kqueue Eric Wong
  1 sibling, 0 replies; 3+ messages in thread
From: Eric Wong @ 2020-08-08  4:59 UTC (permalink / raw)
  To: meta

fileno(DIRHANDLE) only works on Perl 5.22+, so we need to use
dirfd(3) ourselves from Inline::C (or rely on chattr(1) being
installed).

While we're at it, rename `set_nodatacow' to `nodatacow_fd'
for consistency with `nodatacow_dir'.
---
 lib/PublicInbox/Msgmap.pm    |  2 +-
 lib/PublicInbox/NDC_PP.pm    | 13 +++++++++----
 lib/PublicInbox/Over.pm      |  4 ++--
 lib/PublicInbox/SearchIdx.pm | 10 ++--------
 lib/PublicInbox/Spawn.pm     | 21 ++++++++++++++++++---
 lib/PublicInbox/Xapcmd.pm    |  4 ++--
 t/nodatacow.t                | 34 +++++++++++++++++++++++++---------
 7 files changed, 59 insertions(+), 29 deletions(-)

diff --git a/lib/PublicInbox/Msgmap.pm b/lib/PublicInbox/Msgmap.pm
index eee8d6ca..e7f7e2c9 100644
--- a/lib/PublicInbox/Msgmap.pm
+++ b/lib/PublicInbox/Msgmap.pm
@@ -51,7 +51,7 @@ sub new_file {
 sub tmp_clone {
 	my ($self, $dir) = @_;
 	my ($fh, $fn) = tempfile('msgmap-XXXXXXXX', EXLOCK => 0, DIR => $dir);
-	PublicInbox::Spawn::set_nodatacow(fileno($fh));
+	PublicInbox::Spawn::nodatacow_fd(fileno($fh));
 	my $tmp;
 	if ($self->{dbh}->can('sqlite_backup_to_dbh')) {
 		$tmp = ref($self)->new_file($fn, 2);
diff --git a/lib/PublicInbox/NDC_PP.pm b/lib/PublicInbox/NDC_PP.pm
index 0d20030d..10a7ee2a 100644
--- a/lib/PublicInbox/NDC_PP.pm
+++ b/lib/PublicInbox/NDC_PP.pm
@@ -6,10 +6,8 @@ package PublicInbox::NDC_PP;
 use strict;
 use v5.10.1;
 
-sub set_nodatacow ($) {
-	my ($fd) = @_;
-	return if $^O ne 'linux';
-	defined(my $path = readlink("/proc/self/fd/$fd")) or return;
+sub nodatacow_dir ($) {
+	my ($path) = @_;
 	open my $mh, '<', '/proc/self/mounts' or return;
 	for (grep(/ btrfs /, <$mh>)) {
 		my (undef, $mnt_path, $type) = split(/ /);
@@ -26,4 +24,11 @@ sub set_nodatacow ($) {
 	}
 }
 
+sub nodatacow_fd ($) {
+	my ($fd) = @_;
+	return if $^O ne 'linux';
+	defined(my $path = readlink("/proc/self/fd/$fd")) or return;
+	nodatacow_dir($path);
+}
+
 1;
diff --git a/lib/PublicInbox/Over.pm b/lib/PublicInbox/Over.pm
index 0146414c..2b314882 100644
--- a/lib/PublicInbox/Over.pm
+++ b/lib/PublicInbox/Over.pm
@@ -20,10 +20,10 @@ sub dbh_new {
 		if ($rw) {
 			require PublicInbox::Spawn;
 			open my $fh, '+>>', $f or die "failed to open $f: $!";
-			PublicInbox::Spawn::set_nodatacow(fileno($fh));
+			PublicInbox::Spawn::nodatacow_fd(fileno($fh));
 			my $j = "$f-journal";
 			open $fh, '+>>', $j or die "failed to open $j: $!";
-			PublicInbox::Spawn::set_nodatacow(fileno($fh));
+			PublicInbox::Spawn::nodatacow_fd(fileno($fh));
 		} else {
 			$self->{filename} = $f; # die on stat() below:
 		}
diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm
index 01b9f52d..1cf3e66c 100644
--- a/lib/PublicInbox/SearchIdx.pm
+++ b/lib/PublicInbox/SearchIdx.pm
@@ -18,10 +18,10 @@ use PublicInbox::IdxStack;
 use Carp qw(croak);
 use POSIX qw(strftime);
 use PublicInbox::OverIdx;
-use PublicInbox::Spawn qw(spawn);
+use PublicInbox::Spawn qw(spawn nodatacow_dir);
 use PublicInbox::Git qw(git_unquote);
 use PublicInbox::MsgTime qw(msg_timestamp msg_datestamp);
-our @EXPORT_OK = qw(crlf_adjust log2stack is_ancestor check_size nodatacow_dir);
+our @EXPORT_OK = qw(crlf_adjust log2stack is_ancestor check_size);
 my $X = \%PublicInbox::Search::X;
 my ($DB_CREATE_OR_OPEN, $DB_OPEN);
 our $DB_NO_SYNC = 0;
@@ -109,12 +109,6 @@ sub load_xapian_writable () {
 	1;
 }
 
-sub nodatacow_dir ($) {
-	my ($dir) = @_;
-	opendir my $dh, $dir or die "opendir($dir): $!\n";
-	PublicInbox::Spawn::set_nodatacow(fileno($dh));
-}
-
 sub idx_acquire {
 	my ($self) = @_;
 	my $flag;
diff --git a/lib/PublicInbox/Spawn.pm b/lib/PublicInbox/Spawn.pm
index 508d43fd..cb16fcf6 100644
--- a/lib/PublicInbox/Spawn.pm
+++ b/lib/PublicInbox/Spawn.pm
@@ -19,7 +19,7 @@ use strict;
 use parent qw(Exporter);
 use Symbol qw(gensym);
 use PublicInbox::ProcessPipe;
-our @EXPORT_OK = qw/which spawn popen_rd/;
+our @EXPORT_OK = qw/which spawn popen_rd nodatacow_dir/;
 our @RLIMITS = qw(RLIMIT_CPU RLIMIT_CORE RLIMIT_DATA);
 
 my $vfork_spawn = <<'VFORK_SPAWN';
@@ -159,11 +159,12 @@ my $set_nodatacow = $^O eq 'linux' ? <<'SET_NODATACOW' : '';
 #include <sys/vfs.h>
 #include <linux/magic.h>
 #include <linux/fs.h>
+#include <dirent.h>
 #include <errno.h>
 #include <stdio.h>
 #include <string.h>
 
-void set_nodatacow(int fd)
+void nodatacow_fd(int fd)
 {
 	struct statfs buf;
 	int val = 0;
@@ -185,6 +186,19 @@ void set_nodatacow(int fd)
 	if (ioctl(fd, FS_IOC_SETFLAGS, &val) < 0)
 		fprintf(stderr, "FS_IOC_SET_FLAGS: %s\\n", strerror(errno));
 }
+
+void nodatacow_dir(const char *dir)
+{
+	DIR *dh = opendir(dir);
+	int fd;
+
+	if (!dh) croak("opendir(%s): %s", dir, strerror(errno));
+	fd = dirfd(dh);
+	if (fd >= 0)
+		nodatacow_fd(fd);
+	/* ENOTSUP probably won't happen under Linux... */
+	closedir(dh);
+}
 SET_NODATACOW
 
 my $inline_dir = $ENV{PERL_INLINE_DIRECTORY} //= (
@@ -226,7 +240,8 @@ unless (defined $vfork_spawn) {
 unless ($set_nodatacow) {
 	require PublicInbox::NDC_PP;
 	no warnings 'once';
-	*set_nodatacow = \&PublicInbox::NDC_PP::set_nodatacow;
+	*nodatacow_fd = \&PublicInbox::NDC_PP::nodatacow_fd;
+	*nodatacow_dir = \&PublicInbox::NDC_PP::nodatacow_dir;
 }
 undef $set_nodatacow;
 undef $vfork_spawn;
diff --git a/lib/PublicInbox/Xapcmd.pm b/lib/PublicInbox/Xapcmd.pm
index 8423194f..714f6859 100644
--- a/lib/PublicInbox/Xapcmd.pm
+++ b/lib/PublicInbox/Xapcmd.pm
@@ -3,9 +3,9 @@
 package PublicInbox::Xapcmd;
 use strict;
 use warnings;
-use PublicInbox::Spawn qw(which popen_rd);
+use PublicInbox::Spawn qw(which popen_rd nodatacow_dir);
 use PublicInbox::Over;
-use PublicInbox::SearchIdx qw(nodatacow_dir);
+use PublicInbox::SearchIdx;
 use File::Temp 0.19 (); # ->newdir
 use File::Path qw(remove_tree);
 use File::Basename qw(dirname);
diff --git a/t/nodatacow.t b/t/nodatacow.t
index 87b6bdf7..e5b742a2 100644
--- a/t/nodatacow.t
+++ b/t/nodatacow.t
@@ -3,7 +3,7 @@
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 use strict;
 use Test::More;
-use File::Temp qw(tempfile);
+use File::Temp 0.19;
 use PublicInbox::TestCommon;
 use PublicInbox::Spawn qw(which);
 use_ok 'PublicInbox::NDC_PP';
@@ -15,20 +15,36 @@ SKIP: {
 	skip 'BTRFS_TESTDIR not defined', $nr unless defined $dir;
 	skip 'chattr(1) not installed', $nr unless which('chattr');
 	my $lsattr = which('lsattr') or skip 'lsattr(1) not installed', $nr;
-	my ($fh, $name) = tempfile(DIR => $dir, UNLINK => 1);
-	BAIL_OUT "tempfile: $!" unless $fh && defined($name);
-	my $pp_sub = \&PublicInbox::NDC_PP::set_nodatacow;
+	my $tmp = File::Temp->newdir('nodatacow-XXXXX', DIR => $dir);
+	my $dn = $tmp->dirname;
+
+	my $name = "$dn/pp.f";
+	open my $fh, '>', $name or BAIL_OUT "open($name): $!";
+	my $pp_sub = \&PublicInbox::NDC_PP::nodatacow_fd;
 	$pp_sub->(fileno($fh));
 	my $res = xqx([$lsattr, $name]);
-	like($res, qr/C/, "`C' attribute set with pure Perl");
+	like($res, qr/C.*\Q$name\E/, "`C' attribute set on fd with pure Perl");
+
+	$name = "$dn/pp.d";
+	mkdir($name) or BAIL_OUT "mkdir($name) $!";
+	PublicInbox::NDC_PP::nodatacow_dir($name);
+	$res = xqx([$lsattr, '-d', $name]);
+	like($res, qr/C.*\Q$name\E/, "`C' attribute set on dir with pure Perl");
 
-	my $ic_sub = \&PublicInbox::Spawn::set_nodatacow;
+	$name = "$dn/ic.f";
+	my $ic_sub = \&PublicInbox::Spawn::nodatacow_fd;
 	$pp_sub == $ic_sub and
-		skip 'Inline::C or Linux kernel headers missing', 1;
-	($fh, $name) = tempfile(DIR => $dir, UNLINK => 1);
+		skip 'Inline::C or Linux kernel headers missing', 2;
+	open $fh, '>', $name or BAIL_OUT "open($name): $!";
 	$ic_sub->(fileno($fh));
 	$res = xqx([$lsattr, $name]);
-	like($res, qr/C/, "`C' attribute set with Inline::C");
+	like($res, qr/C.*\Q$name\E/, "`C' attribute set on fd with Inline::C");
+
+	$name = "$dn/ic.d";
+	mkdir($name) or BAIL_OUT "mkdir($name) $!";
+	PublicInbox::Spawn::nodatacow_dir($name);
+	$res = xqx([$lsattr, '-d', $name]);
+	like($res, qr/C.*\Q$name\E/, "`C' attribute set on dir with Inline::C");
 };
 
 done_testing;

^ permalink raw reply	[flat|nested] 3+ messages in thread

* [PATCH 2/2] dir_idle: require Perl 5.22+ for kqueue
  2020-08-08  4:59 [PATCH 0/2] Perl <5.22 fixes Eric Wong
  2020-08-08  4:59 ` [PATCH 1/2] support setting No_COW on Perl <5.22 Eric Wong
@ 2020-08-08  4:59 ` Eric Wong
  1 sibling, 0 replies; 3+ messages in thread
From: Eric Wong @ 2020-08-08  4:59 UTC (permalink / raw)
  To: meta

IO::KQueue requires us to use fileno(DIRHANDLE) for setting up
kqueue watches.  This use of fileno() is only supported since
Perl 5.22, so BSD users on older Perl will have to fall back to
old polling.

This affects users of -watch, currently; but will affect other
read-only Xapian users soon.
---
 lib/PublicInbox/DirIdle.pm | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/PublicInbox/DirIdle.pm b/lib/PublicInbox/DirIdle.pm
index daa2212b..458285e2 100644
--- a/lib/PublicInbox/DirIdle.pm
+++ b/lib/PublicInbox/DirIdle.pm
@@ -13,7 +13,8 @@ if ($^O eq 'linux' && eval { require Linux::Inotify2; 1 }) {
 	$MAIL_IN = Linux::Inotify2::IN_MOVED_TO() |
 		Linux::Inotify2::IN_CREATE();
 	$ino_cls = 'Linux::Inotify2';
-} elsif (eval { require PublicInbox::KQNotify }) {
+# Perl 5.22+ is needed for fileno(DIRHANDLE) support:
+} elsif ($^V ge v5.22 && eval { require PublicInbox::KQNotify }) {
 	$MAIL_IN = PublicInbox::KQNotify::MOVED_TO_OR_CREATE();
 	$ino_cls = 'PublicInbox::KQNotify';
 } else {

^ permalink raw reply	[flat|nested] 3+ messages in thread

end of thread, other threads:[~2020-08-08  4:59 UTC | newest]

Thread overview: 3+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2020-08-08  4:59 [PATCH 0/2] Perl <5.22 fixes Eric Wong
2020-08-08  4:59 ` [PATCH 1/2] support setting No_COW on Perl <5.22 Eric Wong
2020-08-08  4:59 ` [PATCH 2/2] dir_idle: require Perl 5.22+ for kqueue Eric Wong

user/dev discussion of public-inbox itself

This inbox may be cloned and mirrored by anyone:

	git clone --mirror https://public-inbox.org/meta
	git clone --mirror http://czquwvybam4bgbro.onion/meta
	git clone --mirror http://hjrcffqmbrq6wope.onion/meta
	git clone --mirror http://ou63pmih66umazou.onion/meta

	# If you have public-inbox 1.1+ installed, you may
	# initialize and index your mirror using the following commands:
	public-inbox-init -V1 meta meta/ https://public-inbox.org/meta \
		meta@public-inbox.org
	public-inbox-index meta

Example config snippet for mirrors.
Newsgroups are available over NNTP:
	nntp://news.public-inbox.org/inbox.comp.mail.public-inbox.meta
	nntp://ou63pmih66umazou.onion/inbox.comp.mail.public-inbox.meta
	nntp://czquwvybam4bgbro.onion/inbox.comp.mail.public-inbox.meta
	nntp://hjrcffqmbrq6wope.onion/inbox.comp.mail.public-inbox.meta
	nntp://news.gmane.io/gmane.mail.public-inbox.general
 note: .onion URLs require Tor: https://www.torproject.org/

code repositories for the project(s) associated with this inbox:

	https://80x24.org/public-inbox.git

AGPL code for this site: git clone https://public-inbox.org/public-inbox.git