* [PATCH 02/21] ipc: switch wq to use the event loop
2021-02-01 8:28 7% [PATCH 00/21] lei2mail worker segfault finally fixed Eric Wong
@ 2021-02-01 8:28 6% ` Eric Wong
0 siblings, 0 replies; 2+ results
From: Eric Wong @ 2021-02-01 8:28 UTC (permalink / raw)
To: meta
This will let us to maximize the capability of our asynchronous
git API. This lets us avoid relying on EOF to notify lei2mail
workers; thus giving us the option of running fewer lei_xsearch
worker processes in parallel than local sources.
I tried using a synchronous git API; and even with libgit2 in
the same process to avoid the IPC cost failed to match the
throughput afforded by this change. This is because libgit2 is
built (at least on Debian) with the SHA-1 collision code enabled
and ubc_check stuff was dominating my profiles.
---
MANIFEST | 1 +
lib/PublicInbox/IPC.pm | 17 +++++++++++------
lib/PublicInbox/LeiToMail.pm | 26 +++++++++++---------------
lib/PublicInbox/WQWorker.pm | 34 ++++++++++++++++++++++++++++++++++
4 files changed, 57 insertions(+), 21 deletions(-)
create mode 100644 lib/PublicInbox/WQWorker.pm
diff --git a/MANIFEST b/MANIFEST
index 2077ab12..c10775e4 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -228,6 +228,7 @@ lib/PublicInbox/V2Writable.pm
lib/PublicInbox/View.pm
lib/PublicInbox/ViewDiff.pm
lib/PublicInbox/ViewVCS.pm
+lib/PublicInbox/WQWorker.pm
lib/PublicInbox/WWW.pm
lib/PublicInbox/WWW.pod
lib/PublicInbox/Watch.pm
diff --git a/lib/PublicInbox/IPC.pm b/lib/PublicInbox/IPC.pm
index d2ff038d..479c4377 100644
--- a/lib/PublicInbox/IPC.pm
+++ b/lib/PublicInbox/IPC.pm
@@ -14,6 +14,7 @@ use Carp qw(confess croak);
use PublicInbox::DS qw(dwaitpid);
use PublicInbox::Spawn;
use PublicInbox::OnDestroy;
+use PublicInbox::WQWorker;
use Socket qw(AF_UNIX MSG_EOR SOCK_STREAM);
use Errno qw(EMSGSIZE);
my $SEQPACKET = eval { Socket::SOCK_SEQPACKET() }; # portable enough?
@@ -151,6 +152,8 @@ sub wq_wait_old {
# for base class, override in sub classes
sub ipc_atfork_prepare {}
+sub wq_atexit_child {}
+
sub ipc_atfork_child {
my ($self) = @_;
my $io = delete($self->{-ipc_atfork_child_close}) or return;
@@ -251,10 +254,11 @@ sub ipc_sibling_atfork_child {
$pid == $$ and die "BUG: $$ ipc_atfork_child called on itself";
}
-sub _recv_and_run {
+sub recv_and_run {
my ($self, $s2, $len, $full_stream) = @_;
my @fds = $recv_cmd->($s2, my $buf, $len);
- my $n = length($buf // '') or return;
+ return if scalar(@fds) && !defined($fds[0]);
+ my $n = length($buf) or return 0;
my $nfd = 0;
for my $fd (@fds) {
if (open(my $cmdfh, '+<&=', $fd)) {
@@ -281,14 +285,15 @@ sub _recv_and_run {
sub wq_worker_loop ($) {
my ($self) = @_;
- my $len = $self->{wq_req_len} // (4096 * 33);
- my $s2 = $self->{-wq_s2} // die 'BUG: no -wq_s2';
- 1 while (_recv_and_run($self, $s2, $len));
+ my $wqw = PublicInbox::WQWorker->new($self);
+ PublicInbox::DS->SetPostLoopCallback(sub { $wqw->{sock} });
+ PublicInbox::DS->EventLoop;
+ PublicInbox::DS->Reset;
}
sub do_sock_stream { # via wq_do, for big requests
my ($self, $len) = @_;
- _recv_and_run($self, delete $self->{0}, $len, 1);
+ recv_and_run($self, delete $self->{0}, $len, 1);
}
sub wq_do { # always async
diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm
index 244bfb67..1f6c2a3b 100644
--- a/lib/PublicInbox/LeiToMail.pm
+++ b/lib/PublicInbox/LeiToMail.pm
@@ -12,11 +12,16 @@ use PublicInbox::ProcessPipe;
use PublicInbox::Spawn qw(which spawn popen_rd);
use PublicInbox::LeiDedupe;
use PublicInbox::OnDestroy;
+use PublicInbox::Git;
+use PublicInbox::GitAsyncCat;
use Symbol qw(gensym);
use IO::Handle; # ->autoflush
use Fcntl qw(SEEK_SET SEEK_END O_CREAT O_EXCL O_WRONLY);
use Errno qw(EEXIST ESPIPE ENOENT);
-use PublicInbox::Git;
+
+# struggles with short-lived repos, Gcf2Client makes little sense with lei;
+# but we may use in-process libgit2 in the future.
+$PublicInbox::GitAsyncCat::GCF2C = 0;
my %kw2char = ( # Maildir characters
draft => 'D',
@@ -467,27 +472,18 @@ sub write_mail { # via ->wq_do
$self->write_cb($lei);
};
my $git = $self->{"$$\0$git_dir"} //= PublicInbox::Git->new($git_dir);
- $git->cat_async($smsg->{blob}, \&git_to_mail, [$wcb, $smsg, $not_done]);
+ git_async_cat($git, $smsg->{blob}, \&git_to_mail,
+ [$wcb, $smsg, $not_done]);
}
-# We rely on OnDestroy to run this before ->DESTROY, since ->DESTROY
-# ordering is unstable at worker exit and may cause segfaults
-sub reap_gits {
+sub wq_atexit_child {
my ($self) = @_;
delete $self->{wcb};
for my $git (delete @$self{grep(/\A$$\0/, keys %$self)}) {
$git->async_wait_all;
}
-}
-
-sub DESTROY { delete $_[0]->{wcb} }
-
-sub ipc_atfork_child { # runs after IPC::wq_worker_loop
- my ($self) = @_;
- $self->SUPER::ipc_atfork_child;
- # reap_gits needs to run before $self->DESTROY,
- # IPC.pm will ensure that.
- PublicInbox::OnDestroy->new($$, \&reap_gits, $self);
+ $SIG{__WARN__} = 'DEFAULT';
+ $SIG{PIPE} = 'DEFAULT';
}
1;
diff --git a/lib/PublicInbox/WQWorker.pm b/lib/PublicInbox/WQWorker.pm
new file mode 100644
index 00000000..25a5e4fb
--- /dev/null
+++ b/lib/PublicInbox/WQWorker.pm
@@ -0,0 +1,34 @@
+# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# for PublicInbox::IPC wq_* (work queue) workers
+package PublicInbox::WQWorker;
+use strict;
+use v5.10.1;
+use parent qw(PublicInbox::DS);
+use PublicInbox::Syscall qw(EPOLLIN EPOLLEXCLUSIVE EPOLLET);
+use Errno qw(EAGAIN ECONNRESET);
+use IO::Handle (); # blocking
+
+sub new {
+ my (undef, $wq) = @_;
+ my $s2 = $wq->{-wq_s2} // die 'BUG: no -wq_s2';
+ $s2->blocking(0);
+ my $self = bless { sock => $s2, wq => $wq }, __PACKAGE__;
+ $self->SUPER::new($s2, EPOLLEXCLUSIVE|EPOLLIN|EPOLLET);
+ $self;
+}
+
+sub event_step {
+ my ($self) = @_;
+ my $n;
+ do {
+ $n = $self->{wq}->recv_and_run($self->{sock}, 4096 * 33);
+ } while ($n);
+ return if !defined($n) && $! == EAGAIN; # likely
+ warn "wq worker error: $!\n" if !defined($n) && $! != ECONNRESET;
+ $self->{wq}->wq_atexit_child;
+ $self->close; # PublicInbox::DS::close
+}
+
+1;
^ permalink raw reply related [relevance 6%]
* [PATCH 00/21] lei2mail worker segfault finally fixed
@ 2021-02-01 8:28 7% Eric Wong
2021-02-01 8:28 6% ` [PATCH 02/21] ipc: switch wq to use the event loop Eric Wong
0 siblings, 1 reply; 2+ results
From: Eric Wong @ 2021-02-01 8:28 UTC (permalink / raw)
To: meta
This lei2mail segfault turned out to be an old Perl 5 quirk
which plagued many before me. It was not consistently
reproducible, and random changes seemed to make it happen more
or less frequently. There were several times when I thought I
fixed it (and maybe this is still one of them!) only to have it
pop up again.
Still, I found many other little bugs and improvements worth
doing along the way. Hope things go more smoothly in the
future...
Anyways, [PATCH 18/21] is the fix (and I'll followup with more
on how I found the fix). 19/21 is purely defensive
future-proofing.
Eric Wong (21):
lei: more consistent dedupe and ovv_buf init
ipc: switch wq to use the event loop
lei: remove per-child SIG{__WARN__}
lei: remove SIGPIPE handler
ipc: more helpful ETOOMANYREFS error messages
lei: remove syslog dependency
sharedkv: release {dbh} before rmtree
lei: keep $lei around until workers are reaped
lei_dedupe: use Digest::SHA
lei_xsearch: load PublicInbox::Smsg
lei: deep clone {ovv} for l2m workers
sharedkv: lock and explicitly disconnect {dbh}
lei: increase initial timeout
sharedkv: use lock_for_scope_fast
lei_to_mail: reduce spew on Maildir removal
sharedkv: do not set cache_size by default
import: reap git-config(1) synchronously
ds: guard against stack-not-refcounted quirk of Perl 5
ds: next_tick: avoid $_ in top-level loop iterator
lei: avoid ETOOMANYREFS, cleanup imports
doc: note optional BSD::Resource use
Documentation/public-inbox-config.pod | 2 +-
INSTALL | 6 ++
MANIFEST | 2 +
lib/PublicInbox/DS.pm | 12 ++--
lib/PublicInbox/IPC.pm | 43 +++++++-----
lib/PublicInbox/Import.pm | 1 +
lib/PublicInbox/LEI.pm | 95 +++++++++++++++------------
lib/PublicInbox/LeiDedupe.pm | 6 +-
lib/PublicInbox/LeiExternal.pm | 3 +-
lib/PublicInbox/LeiOverview.pm | 51 +++++++-------
lib/PublicInbox/LeiToMail.pm | 84 +++++++++++------------
lib/PublicInbox/LeiXSearch.pm | 36 +++++-----
lib/PublicInbox/Lock.pm | 17 +++++
lib/PublicInbox/SharedKV.pm | 33 +++++++---
lib/PublicInbox/WQWorker.pm | 34 ++++++++++
script/lei | 28 +++++---
t/lei_to_mail.t | 31 +++++----
xt/stress-sharedkv.t | 50 ++++++++++++++
18 files changed, 342 insertions(+), 192 deletions(-)
create mode 100644 lib/PublicInbox/WQWorker.pm
create mode 100644 xt/stress-sharedkv.t
^ permalink raw reply [relevance 7%]
Results 1-2 of 2 | reverse | options above
-- pct% links below jump to the message on this page, permalinks otherwise --
2021-02-01 8:28 7% [PATCH 00/21] lei2mail worker segfault finally fixed Eric Wong
2021-02-01 8:28 6% ` [PATCH 02/21] ipc: switch wq to use the event loop Eric Wong
Code repositories for project(s) associated with this public inbox
https://80x24.org/public-inbox.git
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).