* [PATCH 12/19] lei: more consistent IPC exit and error handling
2021-02-07 8:51 7% [PATCH 00/19] lei import Maildir, remote mboxrd fixes Eric Wong
@ 2021-02-07 8:51 6% ` Eric Wong
0 siblings, 0 replies; 2+ results
From: Eric Wong @ 2021-02-07 8:51 UTC (permalink / raw)
To: meta
We're able to propagate $? from wq_workers in a consistent
manner, now.
---
lib/PublicInbox/IPC.pm | 22 +++++++++++-----------
lib/PublicInbox/LEI.pm | 6 +++---
lib/PublicInbox/LeiImport.pm | 14 ++++++++++----
lib/PublicInbox/LeiXSearch.pm | 12 +++++++++---
4 files changed, 33 insertions(+), 21 deletions(-)
diff --git a/lib/PublicInbox/IPC.pm b/lib/PublicInbox/IPC.pm
index 728f726c..c8673e26 100644
--- a/lib/PublicInbox/IPC.pm
+++ b/lib/PublicInbox/IPC.pm
@@ -140,10 +140,9 @@ sub ipc_worker_reap { # dwaitpid callback
}
sub wq_wait_old {
- my ($self, @args) = @_;
- my $cb = ref($args[0]) eq 'CODE' ? shift(@args) : \&ipc_worker_reap;
+ my ($self, $cb, @args) = @_;
my $pids = delete $self->{"-wq_old_pids.$$"} or return;
- dwaitpid($_, $cb, [$self, @args]) for @$pids;
+ dwaitpid($_, $cb // \&ipc_worker_reap, [$self, @args]) for @$pids;
}
# for base class, override in sub classes
@@ -348,13 +347,12 @@ sub wq_exit { # wakes up wq_worker_decr_wait
sub wq_worker_decr { # SIGTTOU handler, kills first idle worker
my ($self) = @_;
return unless wq_workers($self);
- my $s2 = $self->{-wq_s2} // die 'BUG: no wq_s2';
- $self->wq_io_do('wq_exit', [ $s2, $s2, $s2 ]);
+ $self->wq_io_do('wq_exit');
# caller must call wq_worker_decr_wait in main loop
}
sub wq_worker_decr_wait {
- my ($self, $timeout) = @_;
+ my ($self, $timeout, $cb, @args) = @_;
return if $self->{-wq_ppid} != $$; # can't reap siblings or parents
my $s1 = $self->{-wq_s1} // croak 'BUG: no wq_s1';
vec(my $rin = '', fileno($s1), 1) = 1;
@@ -363,17 +361,17 @@ sub wq_worker_decr_wait {
recv($s1, my $pid, 64, 0) // croak "recv: $!";
my $workers = $self->{-wq_workers} // croak 'BUG: no wq_workers';
delete $workers->{$pid} // croak "BUG: PID:$pid invalid";
- dwaitpid($pid, \&ipc_worker_reap, $self);
+ dwaitpid($pid, $cb // \&ipc_worker_reap, [ $self, @args ]);
}
# set or retrieve number of workers
sub wq_workers {
- my ($self, $nr) = @_;
+ my ($self, $nr, $cb, @args) = @_;
my $cur = $self->{-wq_workers} or return;
if (defined $nr) {
while (scalar(keys(%$cur)) > $nr) {
$self->wq_worker_decr;
- $self->wq_worker_decr_wait;
+ $self->wq_worker_decr_wait(undef, $cb, @args);
}
$self->wq_worker_incr while scalar(keys(%$cur)) < $nr;
}
@@ -381,7 +379,7 @@ sub wq_workers {
}
sub wq_close {
- my ($self, $nohang) = @_;
+ my ($self, $nohang, $cb, @args) = @_;
delete @$self{qw(-wq_s1 -wq_s2)} or return;
my $ppid = delete $self->{-wq_ppid} or return;
my $workers = delete $self->{-wq_workers} // die 'BUG: no wq_workers';
@@ -390,7 +388,9 @@ sub wq_close {
if ($nohang) {
push @{$self->{"-wq_old_pids.$$"}}, @pids;
} else {
- dwaitpid($_, \&ipc_worker_reap, $self) for @pids;
+ $cb //= \&ipc_worker_reap;
+ unshift @args, $self;
+ dwaitpid($_, $cb, \@args) for @pids;
}
}
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 515bc2a3..21862488 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -360,7 +360,7 @@ sub fail_handler ($;$$) {
my ($lei, $code, $io) = @_;
for my $f (@WQ_KEYS) {
my $wq = delete $lei->{$f} or next;
- $wq->wq_wait_old($lei) if $wq->wq_kill_old; # lei-daemon
+ $wq->wq_wait_old(undef, $lei) if $wq->wq_kill_old; # lei-daemon
}
close($io) if $io; # needed to avoid warnings on SIGPIPE
$lei->x_it($code // (1 >> 8));
@@ -827,9 +827,9 @@ sub dclose {
for my $f (@WQ_KEYS) {
my $wq = delete $self->{$f} or next;
if ($wq->wq_kill) {
- $wq->wq_close
+ $wq->wq_close(0, undef, $self);
} elsif ($wq->wq_kill_old) {
- $wq->wq_wait_old($self);
+ $wq->wq_wait_old(undef, $self);
}
}
close(delete $self->{1}) if $self->{1}; # may reap_compress
diff --git a/lib/PublicInbox/LeiImport.pm b/lib/PublicInbox/LeiImport.pm
index 3a99570e..2b2dc2f7 100644
--- a/lib/PublicInbox/LeiImport.pm
+++ b/lib/PublicInbox/LeiImport.pm
@@ -14,12 +14,18 @@ sub _import_eml { # MboxReader callback
$sto->ipc_do('set_eml', $eml, $set_kw ? $sto->mbox_keywords($eml) : ());
}
+sub import_done_wait { # dwaitpid callback
+ my ($arg, $pid) = @_;
+ my ($imp, $lei) = @$arg;
+ $lei->child_error($?, 'non-fatal errors during import') if $?;
+ my $ign = $lei->{sto}->ipc_do('done'); # PublicInbox::LeiStore::done
+ $lei->dclose;
+}
+
sub import_done { # EOF callback for main daemon
my ($lei) = @_;
- my $imp = delete $lei->{imp};
- $imp->wq_wait_old($lei) if $imp;
- my $wait = $lei->{sto}->ipc_do('done');
- $lei->dclose;
+ my $imp = delete $lei->{imp} or return;
+ $imp->wq_wait_old(\&import_done_wait, $lei);
}
sub call { # the main "lei import" method
diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index 1ba767c1..1024b020 100644
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -279,12 +279,18 @@ sub git_tmp ($) {
$git;
}
+sub xsearch_done_wait { # dwaitpid callback
+ my ($arg, $pid) = @_;
+ my ($wq, $lei) = @$arg;
+ $lei->child_error($?, 'non-fatal error from '.ref($wq)) if $?;
+}
+
sub query_done { # EOF callback for main daemon
my ($lei) = @_;
my $l2m = delete $lei->{l2m};
- $l2m->wq_wait_old($lei) if $l2m;
+ $l2m->wq_wait_old(\&xsearch_done_wait, $lei) if $l2m;
if (my $lxs = delete $lei->{lxs}) {
- $lxs->wq_wait_old($lei);
+ $lxs->wq_wait_old(\&xsearch_done_wait, $lei);
}
$lei->{ovv}->ovv_end($lei);
if ($l2m) { # close() calls LeiToMail reap_compress
@@ -309,7 +315,7 @@ sub do_post_augment {
if (my $err = $@) {
if (my $lxs = delete $lei->{lxs}) {
$lxs->wq_kill;
- $lxs->wq_close;
+ $lxs->wq_close(0, undef, $lei);
}
$lei->fail("$err");
}
^ permalink raw reply related [relevance 6%]
* [PATCH 00/19] lei import Maildir, remote mboxrd fixes
@ 2021-02-07 8:51 7% Eric Wong
2021-02-07 8:51 6% ` [PATCH 12/19] lei: more consistent IPC exit and error handling Eric Wong
0 siblings, 1 reply; 2+ results
From: Eric Wong @ 2021-02-07 8:51 UTC (permalink / raw)
To: meta
"lei q" with remote mboxrd + early MUA spawning is
nicer, too. Several risky constructs eliminated,
Interrupting "add-external --mirror" is less bad, now;
though it could probably support indexlevel=none in
case somebody wants to run index themselves.
Eric Wong (19):
spawn: pi_fork_exec: restore parent sigmask in child
spawn: pi_fork_exec: support "pgid"
lei add-external: handle interrupts with --mirror
spawn_pp: die more consistently in child
ipc: do not die inside wq_worker child process
ipc: trim down the Storable checks
Makefile.PL: depend on IO::Uncompress::Gunzip
xapcmd: avoid potential die surprise in children
tests: guard setup_public_inboxes for SQLite and Xapian
Revert "ipc: add support for asynchronous callbacks"
ipc: wq_do => wq_io_do
lei: more consistent IPC exit and error handling
lei: remove --mua-cmd alias for --mua
lei: replace --thread with --threads
lei q: improve remote mboxrd UX
lei q: SIGWINCH process group with the terminal
lei import: support Maildirs
imap: avoid unnecessary delete on stack
httpd/async: avoid unnecessary on-stack delete
Documentation/lei-q.pod | 4 +-
MANIFEST | 1 +
Makefile.PL | 1 +
lib/PublicInbox/HTTPD/Async.pm | 2 +-
lib/PublicInbox/IMAP.pm | 6 +-
lib/PublicInbox/IPC.pm | 105 +++++++-----------------
lib/PublicInbox/LEI.pm | 49 +++++++----
lib/PublicInbox/LeiCurl.pm | 11 ++-
lib/PublicInbox/LeiHelp.pm | 6 +-
lib/PublicInbox/LeiImport.pm | 38 ++++++---
lib/PublicInbox/LeiMirror.pm | 75 ++++++++++-------
lib/PublicInbox/LeiOverview.pm | 7 +-
lib/PublicInbox/LeiQuery.pm | 4 +-
lib/PublicInbox/LeiStore.pm | 8 +-
lib/PublicInbox/LeiToMail.pm | 37 ++++-----
lib/PublicInbox/LeiXSearch.pm | 143 ++++++++++++++++++++-------------
lib/PublicInbox/Mbox.pm | 2 +-
lib/PublicInbox/OnDestroy.pm | 2 +-
lib/PublicInbox/Search.pm | 2 +-
lib/PublicInbox/SearchView.pm | 2 +-
lib/PublicInbox/Spawn.pm | 63 +++++++++------
lib/PublicInbox/SpawnPP.pm | 44 +++++-----
lib/PublicInbox/Xapcmd.pm | 11 +--
script/lei | 8 +-
t/ipc.t | 39 ++-------
t/lei-externals.t | 2 +
t/lei-import-maildir.t | 33 ++++++++
t/lei-mirror.t | 14 ++++
t/lei.t | 2 +-
t/lei_to_mail.t | 6 +-
t/spawn.t | 18 +++++
xt/stress-sharedkv.t | 6 +-
32 files changed, 433 insertions(+), 318 deletions(-)
create mode 100644 t/lei-import-maildir.t
^ permalink raw reply [relevance 7%]
Results 1-2 of 2 | reverse | options above
-- pct% links below jump to the message on this page, permalinks otherwise --
2021-02-07 8:51 7% [PATCH 00/19] lei import Maildir, remote mboxrd fixes Eric Wong
2021-02-07 8:51 6% ` [PATCH 12/19] lei: more consistent IPC exit and error handling Eric Wong
Code repositories for project(s) associated with this public inbox
https://80x24.org/public-inbox.git
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).