* [PATCH 06/16] ipc: drop dynamic WQ process counts
@ 2021-09-19 12:50 6% ` Eric Wong
0 siblings, 0 replies; 3+ results
From: Eric Wong @ 2021-09-19 12:50 UTC (permalink / raw)
To: meta
In retrospect, I don't think it's needed; and trying to wire up
a user interface for lei to manage process counts doesn't seem
worthwhile. It could be resurrected for public-facing daemon
use in the future, but that's what version control systems are for.
This also lets us automatically avoid setting up broadcast
sockets
Followup-to: 7b7939d47b336fb7 ("lei: lock worker counts")
---
lib/PublicInbox/IPC.pm | 69 ++++++-------------------------------
lib/PublicInbox/LEI.pm | 1 -
lib/PublicInbox/LeiStore.pm | 1 -
t/ipc.t | 13 +------
4 files changed, 11 insertions(+), 73 deletions(-)
diff --git a/lib/PublicInbox/IPC.pm b/lib/PublicInbox/IPC.pm
index 92f35189..add5f3df 100644
--- a/lib/PublicInbox/IPC.pm
+++ b/lib/PublicInbox/IPC.pm
@@ -262,9 +262,10 @@ sub do_sock_stream { # via wq_io_do, for big requests
sub wq_broadcast {
my ($self, $sub, @args) = @_;
if (my $wkr = $self->{-wq_workers}) {
+ my $buf = ipc_freeze([$sub, @args]);
for my $bcast1 (values %$wkr) {
- my $buf = ipc_freeze([$sub, @args]);
- send($bcast1, $buf, MSG_EOR) // croak "send: $!";
+ my $sock = $bcast1 // $self->{-wq_s1} // next;
+ send($sock, $buf, MSG_EOR) // croak "send: $!";
# XXX shouldn't have to deal with EMSGSIZE here...
}
} else {
@@ -336,11 +337,10 @@ sub wq_do {
}
}
-sub _wq_worker_start ($$$) {
- my ($self, $oldset, $fields) = @_;
+sub _wq_worker_start ($$$$) {
+ my ($self, $oldset, $fields, $one) = @_;
my ($bcast1, $bcast2);
- $self->{-wq_no_bcast} or
- socketpair($bcast1, $bcast2, AF_UNIX, $SEQPACKET, 0) or
+ $one or socketpair($bcast1, $bcast2, AF_UNIX, $SEQPACKET, 0) or
die "socketpair: $!";
my $seed = rand(0xffffffff);
my $pid = fork // die "fork: $!";
@@ -380,66 +380,17 @@ sub wq_workers_start {
socketpair($self->{-wq_s1}, $self->{-wq_s2}, AF_UNIX, $SEQPACKET, 0) or
die "socketpair: $!";
$self->ipc_atfork_prepare;
- $nr_workers //= $self->{-wq_nr_workers};
+ $nr_workers //= $self->{-wq_nr_workers}; # was set earlier
my $sigset = $oldset // PublicInbox::DS::block_signals();
$self->{-wq_workers} = {};
$self->{-wq_ident} = $ident;
- _wq_worker_start($self, $sigset, $fields) for (1..$nr_workers);
+ my $one = $nr_workers == 1;
+ $self->{-wq_nr_workers} = $nr_workers;
+ _wq_worker_start($self, $sigset, $fields, $one) for (1..$nr_workers);
PublicInbox::DS::sig_setmask($sigset) unless $oldset;
$self->{-wq_ppid} = $$;
}
-sub wq_worker_incr { # SIGTTIN handler
- my ($self, $oldset, $fields) = @_;
- $self->{-wq_s2} or return;
- die "-wq_nr_workers locked" if defined $self->{-wq_nr_workers};
- $self->ipc_atfork_prepare;
- my $sigset = $oldset // PublicInbox::DS::block_signals();
- _wq_worker_start($self, $sigset, $fields);
- PublicInbox::DS::sig_setmask($sigset) unless $oldset;
-}
-
-sub wq_exit { # wakes up wq_worker_decr_wait
- send($_[0]->{-wq_s2}, $$, MSG_EOR) // die "$$ send: $!";
- exit;
-}
-
-sub wq_worker_decr { # SIGTTOU handler, kills first idle worker
- my ($self) = @_;
- return unless wq_workers($self);
- die "-wq_nr_workers locked" if defined $self->{-wq_nr_workers};
- $self->wq_io_do('wq_exit');
- # caller must call wq_worker_decr_wait in main loop
-}
-
-sub wq_worker_decr_wait {
- my ($self, $timeout, $cb, @args) = @_;
- return if $self->{-wq_ppid} != $$; # can't reap siblings or parents
- die "-wq_nr_workers locked" if defined $self->{-wq_nr_workers};
- my $s1 = $self->{-wq_s1} // croak 'BUG: no wq_s1';
- vec(my $rin = '', fileno($s1), 1) = 1;
- select(my $rout = $rin, undef, undef, $timeout) or
- croak 'timed out waiting for wq_exit';
- recv($s1, my $pid, 64, 0) // croak "recv: $!";
- my $workers = $self->{-wq_workers} // croak 'BUG: no wq_workers';
- delete $workers->{$pid} // croak "BUG: PID:$pid invalid";
- dwaitpid($pid, $cb // \&ipc_worker_reap, [ $self, @args ]);
-}
-
-# set or retrieve number of workers
-sub wq_workers {
- my ($self, $nr, $cb, @args) = @_;
- my $cur = $self->{-wq_workers} or return;
- if (defined $nr) {
- while (scalar(keys(%$cur)) > $nr) {
- $self->wq_worker_decr;
- $self->wq_worker_decr_wait(undef, $cb, @args);
- }
- $self->wq_worker_incr while scalar(keys(%$cur)) < $nr;
- }
- scalar(keys(%$cur));
-}
-
sub wq_close {
my ($self, $nohang, $cb, @args) = @_;
delete @$self{qw(-wq_s1 -wq_s2)} or return;
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index f62e82dc..def85ef1 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -627,7 +627,6 @@ sub workers_start {
my $end = $lei->pkt_op_pair;
my $ident = $wq->{-wq_ident} // "lei-$lei->{cmd} worker";
$flds->{lei} = $lei;
- $wq->{-wq_nr_workers} //= $jobs; # lock, no incrementing
$wq->wq_workers_start($ident, $jobs, $lei->oldset, $flds);
delete $lei->{pkt_op_p};
my $op_c = delete $lei->{pkt_op_c};
diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm
index 164a9f2d..b4f40912 100644
--- a/lib/PublicInbox/LeiStore.pm
+++ b/lib/PublicInbox/LeiStore.pm
@@ -565,7 +565,6 @@ sub write_prepare {
# Mail we import into lei are private, so headers filtered out
# by -mda for public mail are not appropriate
local @PublicInbox::MDA::BAD_HEADERS = ();
- $self->{-wq_no_bcast} = 1;
$self->wq_workers_start("lei/store $dir", 1, $lei->oldset, {
lei => $lei,
-err_wr => $w,
diff --git a/t/ipc.t b/t/ipc.t
index 202b1cc6..ce89f94b 100644
--- a/t/ipc.t
+++ b/t/ipc.t
@@ -180,24 +180,13 @@ SKIP: {
is($warn[2], $warn[1], 'worker did not die');
$SIG{__WARN__} = 'DEFAULT';
- is($ipc->wq_workers_start('wq', 1), $$, 'workers started again');
- is($ipc->wq_workers, 1, '1 worker started');
-
- $ipc->wq_worker_incr;
- is($ipc->wq_workers, 2, 'worker count bumped');
- $ipc->wq_worker_decr;
- $ipc->wq_worker_decr_wait(10);
- is($ipc->wq_workers, 1, 'worker count lowered');
- is($ipc->wq_workers(2), 2, 'worker count set');
- is($ipc->wq_workers, 2, 'worker count stayed set');
-
+ is($ipc->wq_workers_start('wq', 2), $$, 'workers started again');
$ipc->wq_broadcast('test_append_pid', "$tmpdir/append_pid");
$ipc->wq_close;
open my $fh, '<', "$tmpdir/append_pid" or BAIL_OUT "open: $!";
chomp(my @pids = <$fh>);
my %pids = map { $_ => 1 } grep(/\A[0-9]+\z/, @pids);
is(scalar keys %pids, 2, 'broadcast hit both PIDs');
- is($ipc->wq_workers, undef, 'workers undef after close');
}
done_testing;
^ permalink raw reply related [relevance 6%]
* [PATCH 1/9] lei: lock worker counts
2021-09-18 9:33 7% [PATCH 0/9] lei: a bunch of random stuff Eric Wong
@ 2021-09-18 9:33 6% ` Eric Wong
0 siblings, 0 replies; 3+ results
From: Eric Wong @ 2021-09-18 9:33 UTC (permalink / raw)
To: meta
It doesn't seem worthwhile to change worker counts dynamically
on a per-command-basis with lei, and I don't know how such an
interface would even work...
---
lib/PublicInbox/LEI.pm | 3 ++-
lib/PublicInbox/LeiExportKw.pm | 1 -
lib/PublicInbox/LeiImport.pm | 1 -
lib/PublicInbox/LeiLsMailSource.pm | 3 +--
lib/PublicInbox/LeiLsSearch.pm | 2 +-
lib/PublicInbox/LeiRefreshMailSync.pm | 4 +---
lib/PublicInbox/LeiRm.pm | 2 +-
lib/PublicInbox/LeiTag.pm | 3 +--
8 files changed, 7 insertions(+), 12 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 9794497b..41e761f8 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -258,7 +258,7 @@ our %CMD = ( # sorted in order of importance/use:
@c_opt ],
'import' => [ 'LOCATION...|--stdin',
'one-time import/update from URL or filesystem',
- qw(stdin| offset=i recursive|r exclude=s include|I=s jobs=s new-only
+ qw(stdin| offset=i recursive|r exclude=s include|I=s new-only
lock=s@ in-format|F=s kw! verbose|v+ incremental! mail-sync!),
@net_opt, @c_opt ],
'forget-mail-sync' => [ 'LOCATION...',
@@ -627,6 +627,7 @@ sub workers_start {
my $end = $lei->pkt_op_pair;
my $ident = $wq->{-wq_ident} // "lei-$lei->{cmd} worker";
$flds->{lei} = $lei;
+ $wq->{-wq_nr_workers} //= $jobs; # lock, no incrementing
$wq->wq_workers_start($ident, $jobs, $lei->oldset, $flds);
delete $lei->{pkt_op_p};
my $op_c = delete $lei->{pkt_op_c};
diff --git a/lib/PublicInbox/LeiExportKw.pm b/lib/PublicInbox/LeiExportKw.pm
index d37f3768..8b8aa373 100644
--- a/lib/PublicInbox/LeiExportKw.pm
+++ b/lib/PublicInbox/LeiExportKw.pm
@@ -124,7 +124,6 @@ EOM
my $ops = {};
$sto->write_prepare($lei);
$lei->{auth}->op_merge($ops, $self) if $lei->{auth};
- $self->{-wq_nr_workers} = $j // 1; # locked
(my $op_c, $ops) = $lei->workers_start($self, $j, $ops);
$lei->{wq1} = $self;
$lei->{-err_type} = 'non-fatal';
diff --git a/lib/PublicInbox/LeiImport.pm b/lib/PublicInbox/LeiImport.pm
index 7c563bd8..b1cb3940 100644
--- a/lib/PublicInbox/LeiImport.pm
+++ b/lib/PublicInbox/LeiImport.pm
@@ -101,7 +101,6 @@ sub do_import_index ($$@) {
}
my $ops = {};
$lei->{auth}->op_merge($ops, $self) if $lei->{auth};
- $self->{-wq_nr_workers} = $j // 1; # locked
$lei->{-eml_noisy} = 1;
(my $op_c, $ops) = $lei->workers_start($self, $j, $ops);
$lei->{wq1} = $self;
diff --git a/lib/PublicInbox/LeiLsMailSource.pm b/lib/PublicInbox/LeiLsMailSource.pm
index f012e10e..bcb1838e 100644
--- a/lib/PublicInbox/LeiLsMailSource.pm
+++ b/lib/PublicInbox/LeiLsMailSource.pm
@@ -96,8 +96,7 @@ sub lei_ls_mail_source {
$lei->start_pager if -t $lei->{1};
my $ops = {};
$lei->{auth}->op_merge($ops, $self);
- my $j = $self->{-wq_nr_workers} = 1; # locked
- (my $op_c, $ops) = $lei->workers_start($self, $j, $ops);
+ (my $op_c, $ops) = $lei->workers_start($self, 1, $ops);
$lei->{wq1} = $self;
$lei->{-err_type} = 'non-fatal';
net_merge_all_done($self) unless $lei->{auth};
diff --git a/lib/PublicInbox/LeiLsSearch.pm b/lib/PublicInbox/LeiLsSearch.pm
index 70136135..aebf0184 100644
--- a/lib/PublicInbox/LeiLsSearch.pm
+++ b/lib/PublicInbox/LeiLsSearch.pm
@@ -71,7 +71,7 @@ sub do_ls_search_long {
sub bg_worker ($$$) {
my ($lei, $pfx, $json) = @_;
- my $self = bless { -wq_nr_workers => 1, json => $json }, __PACKAGE__;
+ my $self = bless { json => $json }, __PACKAGE__;
my ($op_c, $ops) = $lei->workers_start($self, 1);
$lei->{wq1} = $self;
$self->wq_io_do('do_ls_search_long', [], $pfx);
diff --git a/lib/PublicInbox/LeiRefreshMailSync.pm b/lib/PublicInbox/LeiRefreshMailSync.pm
index 09a7ead0..cdd99725 100644
--- a/lib/PublicInbox/LeiRefreshMailSync.pm
+++ b/lib/PublicInbox/LeiRefreshMailSync.pm
@@ -84,11 +84,9 @@ EOM
my $self = bless { missing_ok => 1 }, __PACKAGE__;
$lei->{opt}->{'mail-sync'} = 1; # for prepare_inputs
$self->prepare_inputs($lei, \@folders) or return;
- my $j = $lei->{opt}->{jobs} || scalar(@{$self->{inputs}}) || 1;
my $ops = {};
$lei->{auth}->op_merge($ops, $self) if $lei->{auth};
- $self->{-wq_nr_workers} = $j // 1; # locked
- (my $op_c, $ops) = $lei->workers_start($self, $j, $ops);
+ (my $op_c, $ops) = $lei->workers_start($self, 1, $ops);
$lei->{wq1} = $self;
$lei->{-err_type} = 'non-fatal';
net_merge_all_done($self) unless $lei->{auth};
diff --git a/lib/PublicInbox/LeiRm.pm b/lib/PublicInbox/LeiRm.pm
index 778fa1de..3371f3ed 100644
--- a/lib/PublicInbox/LeiRm.pm
+++ b/lib/PublicInbox/LeiRm.pm
@@ -32,7 +32,7 @@ sub lei_rm {
my ($lei, @inputs) = @_;
$lei->_lei_store(1)->write_prepare($lei);
$lei->{opt}->{'in-format'} //= 'eml';
- my $self = bless { -wq_nr_workers => 1 }, __PACKAGE__;
+ my $self = bless {}, __PACKAGE__;
$self->prepare_inputs($lei, \@inputs) or return;
my ($op_c, $ops) = $lei->workers_start($self, 1);
$lei->{wq1} = $self;
diff --git a/lib/PublicInbox/LeiTag.pm b/lib/PublicInbox/LeiTag.pm
index 44d77b88..c4f5ecff 100644
--- a/lib/PublicInbox/LeiTag.pm
+++ b/lib/PublicInbox/LeiTag.pm
@@ -50,8 +50,7 @@ sub lei_tag { # the "lei tag" method
return $lei->fail('no keywords or labels specified');
my $ops = {};
$lei->{auth}->op_merge($ops, $self) if $lei->{auth};
- my $j = $self->{-wq_nr_workers} = 1; # locked for now
- (my $op_c, $ops) = $lei->workers_start($self, $j, $ops);
+ (my $op_c, $ops) = $lei->workers_start($self, 1, $ops);
$lei->{wq1} = $self;
$lei->{-err_type} = 'non-fatal';
net_merge_all_done($self) unless $lei->{auth};
^ permalink raw reply related [relevance 6%]
* [PATCH 0/9] lei: a bunch of random stuff
@ 2021-09-18 9:33 7% Eric Wong
2021-09-18 9:33 6% ` [PATCH 1/9] lei: lock worker counts Eric Wong
0 siblings, 1 reply; 3+ results
From: Eric Wong @ 2021-09-18 9:33 UTC (permalink / raw)
To: meta
The unique timers stuff will be used for "lei up" polling,
as will 9/9 to improve "lei up" usability.
The net_reader changes were noticed while getting imaps://
to work with socks5h:// (not just imap://).
There's still a lot of mail_sync stuff going on, but it's
getting closer...
Eric Wong (9):
lei: lock worker counts
lei_mail_sync: rely on flock(2), avoid IPC
lei_mail_sync: set nodatacow on btrfs
ds: support add unique timers
net_reader: tie SocksDebug to {imap,nntp}.Debug
net_reader: detect IMAP failures earlier
net_reader: support imaps:// w/ socks5h:// proxy
net_reader: set SO_KEEPALIVE on all Net::NNTP sockets
lei up: automatically use dt: for remote externals
Documentation/lei-up.pod | 15 ++++
lib/PublicInbox/DS.pm | 100 +++++++++++++-------------
lib/PublicInbox/LEI.pm | 40 +++++------
lib/PublicInbox/LeiExportKw.pm | 32 ++++-----
lib/PublicInbox/LeiForgetMailSync.pm | 6 +-
lib/PublicInbox/LeiImport.pm | 8 +--
lib/PublicInbox/LeiInput.pm | 2 +-
lib/PublicInbox/LeiInspect.pm | 5 +-
lib/PublicInbox/LeiLsMailSource.pm | 3 +-
lib/PublicInbox/LeiLsMailSync.pm | 3 +-
lib/PublicInbox/LeiLsSearch.pm | 2 +-
lib/PublicInbox/LeiMailSync.pm | 51 ++++++++++---
lib/PublicInbox/LeiNoteEvent.pm | 31 ++++----
lib/PublicInbox/LeiRefreshMailSync.pm | 35 ++++-----
lib/PublicInbox/LeiRm.pm | 2 +-
lib/PublicInbox/LeiSavedSearch.pm | 1 +
lib/PublicInbox/LeiStore.pm | 39 +---------
lib/PublicInbox/LeiTag.pm | 3 +-
lib/PublicInbox/LeiToMail.pm | 10 ++-
lib/PublicInbox/LeiUp.pm | 2 +-
lib/PublicInbox/LeiXSearch.pm | 50 ++++++++++---
lib/PublicInbox/NetReader.pm | 26 ++++---
t/lei-q-remote-import.t | 4 ++
23 files changed, 259 insertions(+), 211 deletions(-)
^ permalink raw reply [relevance 7%]
Results 1-3 of 3 | reverse | options above
-- pct% links below jump to the message on this page, permalinks otherwise --
2021-09-18 9:33 7% [PATCH 0/9] lei: a bunch of random stuff Eric Wong
2021-09-18 9:33 6% ` [PATCH 1/9] lei: lock worker counts Eric Wong
2021-09-19 12:50 [PATCH 00/16] lei IPC overhaul, NNTP fixes Eric Wong
2021-09-19 12:50 6% ` [PATCH 06/16] ipc: drop dynamic WQ process counts Eric Wong
Code repositories for project(s) associated with this public inbox
https://80x24.org/public-inbox.git
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).