* [PATCH 17/30] syscall: common $F_SETPIPE_SZ definition
2023-10-17 23:37 7% [PATCH 00/30] autodie-ification and code simplifications Eric Wong
@ 2023-10-17 23:38 6% ` Eric Wong
0 siblings, 0 replies; 2+ results
From: Eric Wong @ 2023-10-17 23:38 UTC (permalink / raw)
To: meta
We use this in various places to minimize or maximize pipe
size on Linux. So keep it all in one place.
---
lib/PublicInbox/CidxLogP.pm | 4 ++--
lib/PublicInbox/EOFpipe.pm | 6 +++---
lib/PublicInbox/LeiXSearch.pm | 2 +-
lib/PublicInbox/SearchIdxShard.pm | 14 +++++++-------
lib/PublicInbox/Syscall.pm | 16 ++++++++--------
t/gcf2.t | 5 +++--
t/lei-sigpipe.t | 7 +++----
7 files changed, 27 insertions(+), 27 deletions(-)
diff --git a/lib/PublicInbox/CidxLogP.pm b/lib/PublicInbox/CidxLogP.pm
index 7877d5ac..34f7201d 100644
--- a/lib/PublicInbox/CidxLogP.pm
+++ b/lib/PublicInbox/CidxLogP.pm
@@ -10,12 +10,12 @@
package PublicInbox::CidxLogP;
use v5.12;
use parent qw(PublicInbox::DS);
-use PublicInbox::Syscall qw(EPOLLIN EPOLLONESHOT);
+use PublicInbox::Syscall qw(EPOLLIN EPOLLONESHOT $F_SETPIPE_SZ);
sub new {
my ($cls, $rd, $cidx, $git, $roots) = @_;
my $self = bless { cidx => $cidx, git => $git, roots => $roots }, $cls;
- fcntl($rd, 1031, 1048576) if $^O eq 'linux'; # fatter pipes
+ fcntl($rd, $F_SETPIPE_SZ, 1048576) if $F_SETPIPE_SZ;
$self->SUPER::new($rd, EPOLLIN|EPOLLONESHOT);
}
diff --git a/lib/PublicInbox/EOFpipe.pm b/lib/PublicInbox/EOFpipe.pm
index 628e9366..3474874f 100644
--- a/lib/PublicInbox/EOFpipe.pm
+++ b/lib/PublicInbox/EOFpipe.pm
@@ -4,13 +4,13 @@
package PublicInbox::EOFpipe;
use v5.12;
use parent qw(PublicInbox::DS);
-use PublicInbox::Syscall qw(EPOLLIN EPOLLONESHOT);
+use PublicInbox::Syscall qw(EPOLLIN EPOLLONESHOT $F_SETPIPE_SZ);
sub new {
my (undef, $rd, $cb) = @_;
my $self = bless { cb => $cb }, __PACKAGE__;
- # 1031: F_SETPIPE_SZ, 4096: page size
- fcntl($rd, 1031, 4096) if $^O eq 'linux';
+ # 4096: page size
+ fcntl($rd, $F_SETPIPE_SZ, 4096) if $F_SETPIPE_SZ;
$self->SUPER::new($rd, EPOLLIN|EPOLLONESHOT);
}
diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index d83a403c..25b66b3b 100644
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -21,6 +21,7 @@ use Fcntl qw(SEEK_SET F_SETFL O_APPEND O_RDWR);
use PublicInbox::ContentHash qw(git_sha);
use POSIX qw(strftime);
use autodie qw(open read seek truncate);
+use PublicInbox::Syscall qw($F_SETPIPE_SZ);
sub new {
my ($class) = @_;
@@ -536,7 +537,6 @@ sub do_query {
if ($lei->{opt}->{augment} && delete $lei->{early_mua}) {
$lei->start_mua;
}
- my $F_SETPIPE_SZ = $^O eq 'linux' ? 1031 : undef;
if ($l2m->{-wq_nr_workers} > 1 &&
$l2m->{base_type} =~ /\A(?:maildir|mbox)\z/) {
# setup two barriers to coordinate ->has_entries
diff --git a/lib/PublicInbox/SearchIdxShard.pm b/lib/PublicInbox/SearchIdxShard.pm
index 21bd56c2..1630eb4a 100644
--- a/lib/PublicInbox/SearchIdxShard.pm
+++ b/lib/PublicInbox/SearchIdxShard.pm
@@ -7,6 +7,7 @@ package PublicInbox::SearchIdxShard;
use v5.12;
use parent qw(PublicInbox::SearchIdx PublicInbox::IPC);
use PublicInbox::OnDestroy;
+use PublicInbox::Syscall qw($F_SETPIPE_SZ);
sub new {
my ($class, $v2w, $shard) = @_; # v2w may be ExtSearchIdx
@@ -20,13 +21,12 @@ sub new {
if ($v2w->{parallel}) {
local $self->{-v2w_afc} = $v2w;
$self->ipc_worker_spawn("shard[$shard]");
- # F_SETPIPE_SZ = 1031 on Linux; increasing the pipe size for
- # inputs speeds V2Writable batch imports across 8 cores by
- # nearly 20%. Since any of our responses are small, make
- # the response pipe as small as possible
- if ($^O eq 'linux') {
- fcntl($self->{-ipc_req}, 1031, 1048576);
- fcntl($self->{-ipc_res}, 1031, 4096);
+ # Increasing the pipe size for requests speeds V2 batch imports
+ # across 8 cores by nearly 20%. Since many of our responses
+ # are small, make the response pipe as small as possible
+ if ($F_SETPIPE_SZ) {
+ fcntl($self->{-ipc_req}, $F_SETPIPE_SZ, 1048576);
+ fcntl($self->{-ipc_res}, $F_SETPIPE_SZ, 4096);
}
}
$self;
diff --git a/lib/PublicInbox/Syscall.pm b/lib/PublicInbox/Syscall.pm
index e83beb6a..78181bb6 100644
--- a/lib/PublicInbox/Syscall.pm
+++ b/lib/PublicInbox/Syscall.pm
@@ -28,7 +28,7 @@ our @EXPORT_OK = qw(epoll_ctl epoll_create epoll_wait
EPOLLIN EPOLLOUT EPOLLET
EPOLL_CTL_ADD EPOLL_CTL_DEL EPOLL_CTL_MOD
EPOLLONESHOT EPOLLEXCLUSIVE
- signalfd rename_noreplace %SIGNUM);
+ signalfd rename_noreplace %SIGNUM $F_SETPIPE_SZ);
use constant {
EPOLLIN => 1,
EPOLLOUT => 4,
@@ -55,13 +55,12 @@ use constant {
my @BYTES_4_hole = BYTES_4_hole ? (0) : ();
-our (
- $SYS_epoll_create,
- $SYS_epoll_ctl,
- $SYS_epoll_wait,
- $SYS_signalfd4,
- $SYS_renameat2,
- );
+our ($SYS_epoll_create,
+ $SYS_epoll_ctl,
+ $SYS_epoll_wait,
+ $SYS_signalfd4,
+ $SYS_renameat2,
+ $F_SETPIPE_SZ);
my ($SYS_sendmsg, $SYS_recvmsg);
my $SYS_fstatfs; # don't need fstatfs64, just statfs.f_type
@@ -70,6 +69,7 @@ my $SFD_CLOEXEC = 02000000; # Perl does not expose O_CLOEXEC
our $no_deprecated = 0;
if ($^O eq "linux") {
+ $F_SETPIPE_SZ = 1031;
my (undef, undef, $release, undef, $machine) = POSIX::uname();
my ($maj, $min) = ($release =~ /\A([0-9]+)\.([0-9]+)/);
$SYS_renameat2 = 0 if "$maj.$min" < 3.15;
diff --git a/t/gcf2.t b/t/gcf2.t
index d12a4420..33f3bbca 100644
--- a/t/gcf2.t
+++ b/t/gcf2.t
@@ -10,6 +10,7 @@ use POSIX qw(_exit);
use Cwd qw(abs_path);
require_mods('PublicInbox::Gcf2');
use_ok 'PublicInbox::Gcf2';
+use PublicInbox::Syscall qw($F_SETPIPE_SZ);
use PublicInbox::Import;
my ($tmpdir, $for_destroy) = tmpdir();
@@ -109,7 +110,7 @@ SKIP: {
for my $blk (1, 0) {
my ($r, $w);
pipe($r, $w) or BAIL_OUT $!;
- fcntl($w, 1031, 4096) or
+ fcntl($w, $F_SETPIPE_SZ, 4096) or
skip('Linux too old for F_SETPIPE_SZ', 14);
$w->blocking($blk);
seek($fh, 0, SEEK_SET) or BAIL_OUT "seek: $!";
@@ -129,7 +130,7 @@ SKIP: {
$ck_copying->("pipe blocking($blk)");
pipe($r, $w) or BAIL_OUT $!;
- fcntl($w, 1031, 4096) or BAIL_OUT $!;
+ fcntl($w, $F_SETPIPE_SZ, 4096) or BAIL_OUT $!;
$w->blocking($blk);
close $r;
local $SIG{PIPE} = 'IGNORE';
diff --git a/t/lei-sigpipe.t b/t/lei-sigpipe.t
index 55c208e2..622598a4 100644
--- a/t/lei-sigpipe.t
+++ b/t/lei-sigpipe.t
@@ -6,6 +6,7 @@ use v5.10.1;
use PublicInbox::TestCommon;
use POSIX qw(WTERMSIG WIFSIGNALED SIGPIPE);
use PublicInbox::OnDestroy;
+use PublicInbox::Syscall qw($F_SETPIPE_SZ);
# undo systemd (and similar) ignoring SIGPIPE, since lei expects to be run
# from an interactive terminal:
@@ -21,10 +22,8 @@ test_lei(sub {
my $imported;
for my $out ([], [qw(-f mboxcl2)], [qw(-f text)]) {
pipe(my ($r, $w)) or BAIL_OUT $!;
- my $size = 65536;
- if ($^O eq 'linux' && fcntl($w, 1031, 4096)) {
- $size = 4096;
- }
+ my $size = $F_SETPIPE_SZ && fcntl($w, $F_SETPIPE_SZ, 4096) ?
+ 4096 : 65536;
unless (-f $f) {
open my $fh, '>', $f or xbail "open $f: $!";
print $fh <<'EOM' or xbail;
^ permalink raw reply related [relevance 6%]
* [PATCH 00/30] autodie-ification and code simplifications
@ 2023-10-17 23:37 7% Eric Wong
2023-10-17 23:38 6% ` [PATCH 17/30] syscall: common $F_SETPIPE_SZ definition Eric Wong
0 siblings, 1 reply; 2+ results
From: Eric Wong @ 2023-10-17 23:37 UTC (permalink / raw)
To: meta
Noisy code is less pleasant to work on, so use autodie more and
a few more simplifications. There's a couple of small bugfixes
discovered along the way, too.
Eric Wong (30):
lei_mirror: start converting to autodie
lei_mirror: autodie most `close' calls
lei_mirror: use autodie for most `open' calls
git: introduce read_all function
import: use read_all to detect short reads
lei_mirror: use read_all
use read_all in more places to improve safety
xap_helper*: use autodie in more places
xap_helper: die more easily in both implementations
xap_helper: simplify SIGTERM exit checks
xap_helper: autodie for getsockopt
xap_client: autodie for pipe and socketpair
xt/git-http-backend: remove Net::HTTP usage
ds: introduce and use do_fork helper
ds: get rid of SetLoopTimeout
cindex: drop some unused functions
syscall: common $F_SETPIPE_SZ definition
t/lei-up: additional diagnostics for match failures
test_common: use autodie and read_all where possible
test_common: only hide TCP port in messages
test_common: use $cwdfh for every run_script command
init: drop extraneous `+'
init: use autodie to reduce distractions
xt/mem-imapd-tls: remove unused/broken epoll imports
xt/mem-imapd-tls: reduce FDs for lsof use
lei: use autodie where appropriate
lei_auth: update comments and use v5.12
lei_config: drop redundant open check
convert: use read_all to simplify error checks
idx_stack: use autodie + read_all
lib/PublicInbox/CidxLogP.pm | 4 +-
lib/PublicInbox/CodeSearchIdx.pm | 5 --
lib/PublicInbox/DS.pm | 36 ++++----
lib/PublicInbox/Daemon.pm | 16 ++--
lib/PublicInbox/EOFpipe.pm | 6 +-
lib/PublicInbox/Gcf2.pm | 7 +-
lib/PublicInbox/Git.pm | 19 +++--
lib/PublicInbox/IPC.pm | 12 +--
lib/PublicInbox/IdxStack.pm | 20 ++---
lib/PublicInbox/Import.pm | 8 +-
lib/PublicInbox/InboxWritable.pm | 6 +-
lib/PublicInbox/LEI.pm | 48 +++++------
lib/PublicInbox/LeiALE.pm | 11 +--
lib/PublicInbox/LeiAuth.pm | 7 +-
lib/PublicInbox/LeiBlob.pm | 6 +-
lib/PublicInbox/LeiConfig.pm | 4 +-
lib/PublicInbox/LeiMailSync.pm | 5 +-
lib/PublicInbox/LeiMirror.pm | 131 ++++++++++++++----------------
lib/PublicInbox/LeiSucks.pm | 5 +-
lib/PublicInbox/LeiXSearch.pm | 2 +-
lib/PublicInbox/MultiGit.pm | 3 +-
lib/PublicInbox/SearchIdxShard.pm | 14 ++--
lib/PublicInbox/Syscall.pm | 16 ++--
lib/PublicInbox/TestCommon.pm | 85 +++++++++----------
lib/PublicInbox/ViewVCS.pm | 12 ++-
lib/PublicInbox/WWW.pm | 4 +-
lib/PublicInbox/Watch.pm | 11 +--
lib/PublicInbox/XapClient.pm | 11 +--
lib/PublicInbox/XapHelper.pm | 24 ++----
lib/PublicInbox/XapHelperCxx.pm | 11 +--
lib/PublicInbox/Xapcmd.pm | 5 +-
lib/PublicInbox/xap_helper.h | 60 ++++++--------
script/public-inbox-convert | 8 +-
script/public-inbox-edit | 4 +-
script/public-inbox-init | 30 +++----
t/dir_idle.t | 2 +-
t/ds-leak.t | 4 +-
t/gcf2.t | 5 +-
t/init.t | 7 ++
t/lei-sigpipe.t | 7 +-
t/lei-up.t | 4 +-
xt/git-http-backend.t | 30 +++----
xt/mem-imapd-tls.t | 21 ++---
xt/mem-nntpd-tls.t | 8 +-
44 files changed, 335 insertions(+), 409 deletions(-)
^ permalink raw reply [relevance 7%]
Results 1-2 of 2 | reverse | options above
-- pct% links below jump to the message on this page, permalinks otherwise --
2023-10-17 23:37 7% [PATCH 00/30] autodie-ification and code simplifications Eric Wong
2023-10-17 23:38 6% ` [PATCH 17/30] syscall: common $F_SETPIPE_SZ definition Eric Wong
Code repositories for project(s) associated with this public inbox
https://80x24.org/public-inbox.git
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).