ipc: add wq_broadcast

We'll give workqueues a broadcast mechanism to ensure all
workers see a certain message.  We'll also tag each worker
with {-wq_worker_nr} in preparation for work distribution.

This is intended to avoid extra connection and fork() costs
from LeiAuth in a future commit.

3 files changed, 53 insertions, 24 deletions

diff --git a/lib/PublicInbox/IPC.pm b/lib/PublicInbox/IPC.pm
index efac4c4d..2aeb6462 100644
--- a/lib/PublicInbox/IPC.pm
+++ b/lib/PublicInbox/IPC.pm
@@ -240,8 +240,9 @@ sub recv_and_run {
 }
 
 sub wq_worker_loop ($) {
-        my ($self) = @_;
+        my ($self, $bcast_a) = @_;
         my $wqw = PublicInbox::WQWorker->new($self);
+        PublicInbox::WQWorker->new($self, '-wq_bcast2');
         PublicInbox::DS->SetPostLoopCallback(sub { $wqw->{sock} });
         PublicInbox::DS->EventLoop;
         PublicInbox::DS->Reset;
@@ -252,6 +253,20 @@ sub do_sock_stream { # via wq_io_do, for big requests
         recv_and_run($self, delete $self->{0}, $len, 1);
 }
 
+sub wq_broadcast {
+        my ($self, $sub, @args) = @_;
+        if (my $wkr = $self->{-wq_workers}) {
+                for my $bcast1 (values %$wkr) {
+                        my $buf = ipc_freeze([$sub, @args]);
+                        send($bcast1, $buf, MSG_EOR) // croak "send: $!";
+                        # XXX shouldn't have to deal with EMSGSIZE here...
+                }
+        } else {
+                eval { $self->$sub(@args) };
+                warn "wq_broadcast: $@" if $@;
+        }
+}
+
 sub wq_io_do { # always async
         my ($self, $sub, $ios, @args) = @_;
         if (my $s1 = $self->{-wq_s1}) { # run in worker
@@ -284,15 +299,21 @@ sub wq_io_do { # always async
 
 sub _wq_worker_start ($$$) {
         my ($self, $oldset, $fields) = @_;
+        my ($bcast1, $bcast2);
+        socketpair($bcast1, $bcast2, AF_UNIX, $SEQPACKET, 0) or
+                                                die "socketpair: $!";
         my $seed = rand(0xffffffff);
         my $pid = fork // die "fork: $!";
         if ($pid == 0) {
                 srand($seed);
+                undef $bcast1;
                 eval { PublicInbox::DS->Reset };
-                delete @$self{qw(-wq_s1 -wq_workers -wq_ppid)};
+                delete @$self{qw(-wq_s1 -wq_ppid)};
+                $self->{-wq_worker_nr} =
+                                keys %{delete($self->{-wq_workers}) // {}};
                 $SIG{$_} = 'IGNORE' for (qw(PIPE));
                 $SIG{$_} = 'DEFAULT' for (qw(TTOU TTIN TERM QUIT INT CHLD));
-                local $0 = $self->{-wq_ident};
+                local $0 = "$self->{-wq_ident} $self->{-wq_worker_nr}";
                 # ensure we properly exit even if warn() dies:
                 my $end = PublicInbox::OnDestroy->new($$, sub { exit(!!$@) });
                 eval {
@@ -301,12 +322,13 @@ sub _wq_worker_start ($$$) {
                         my $on_destroy = $self->ipc_atfork_child;
                         local %SIG = %SIG;
                         PublicInbox::DS::sig_setmask($oldset);
+                        $self->{-wq_bcast2} = $bcast2;
                         wq_worker_loop($self);
                 };
                 warn "worker $self->{-wq_ident} PID:$$ died: $@" if $@;
                 undef $end; # trigger exit
         } else {
-                $self->{-wq_workers}->{$pid} = \undef;
+                $self->{-wq_workers}->{$pid} = $bcast1;
         }
 }
 
diff --git a/lib/PublicInbox/WQWorker.pm b/lib/PublicInbox/WQWorker.pm
index 25a5e4fb..3636321e 100644
--- a/lib/PublicInbox/WQWorker.pm
+++ b/lib/PublicInbox/WQWorker.pm
@@ -11,10 +11,10 @@ use Errno qw(EAGAIN ECONNRESET);
 use IO::Handle (); # blocking
 
 sub new {
-        my (undef, $wq) = @_;
-        my $s2 = $wq->{-wq_s2} // die 'BUG: no -wq_s2';
+        my ($cls, $wq, $field) = @_;
+        my $s2 = $wq->{$field // '-wq_s2'} // die "BUG: no {$field}";
         $s2->blocking(0);
-        my $self = bless { sock => $s2, wq => $wq }, __PACKAGE__;
+        my $self = bless { sock => $s2, wq => $wq }, $cls;
         $self->SUPER::new($s2, EPOLLEXCLUSIVE|EPOLLIN|EPOLLET);
         $self;
 }
@@ -27,7 +27,7 @@ sub event_step {
         } while ($n);
         return if !defined($n) && $! == EAGAIN; # likely
         warn "wq worker error: $!\n" if !defined($n) && $! != ECONNRESET;
-        $self->{wq}->wq_atexit_child;
+        $self->{wq}->wq_atexit_child if $self->{sock} == $self->{wq}->{-wq_s2};
         $self->close; # PublicInbox::DS::close
 }
 
diff --git a/t/ipc.t b/t/ipc.t
index 345024bd..ca88eb59 100644
--- a/t/ipc.t
+++ b/t/ipc.t
@@ -9,6 +9,7 @@ use Fcntl qw(SEEK_SET);
 use Digest::SHA qw(sha1_hex);
 require_mods(qw(Storable||Sereal));
 require_ok 'PublicInbox::IPC';
+my ($tmpdir, $for_destroy) = tmpdir();
 state $once = eval <<'';
 package PublicInbox::IPC;
 use strict;
@@ -31,6 +32,12 @@ sub test_sha {
         print { $self->{1} } sha1_hex($buf), "\n";
         $self->{1}->flush;
 }
+sub test_append_pid {
+        my ($self, $file) = @_;
+        open my $fh, '>>', $file or die "open: $!";
+        $fh->autoflush(1);
+        print $fh "$$\n" or die "print: $!";
+}
 1;
 
 my $ipc = bless {}, 'PublicInbox::IPC';
@@ -83,11 +90,8 @@ $test->('local');
         defined($pid) or BAIL_OUT 'no spawn, no test';
         is($ipc->ipc_do('test_pid'), $pid, 'worker pid returned');
         $test->('worker');
-        {
-                my ($tmp, $for_destroy) = tmpdir();
-                $ipc->ipc_lock_init("$tmp/lock");
-                is($ipc->ipc_do('test_pid'), $pid, 'worker pid returned');
-        }
+        $ipc->ipc_lock_init("$tmpdir/lock");
+        is($ipc->ipc_do('test_pid'), $pid, 'worker pid returned');
         $ipc->ipc_worker_stop;
         ok(!kill(0, $pid) && $!{ESRCH}, 'worker stopped');
 }
@@ -167,18 +171,21 @@ SKIP: {
         $SIG{__WARN__} = 'DEFAULT';
         is($ipc->wq_workers_start('wq', 1), $$, 'workers started again');
         is($ipc->wq_workers, 1, '1 worker started');
-        SKIP: {
-                $ipc->WQ_MAX_WORKERS > 1 or
-                        skip 'Inline::C or Socket::MsgHdr not available', 4;
-                $ipc->wq_worker_incr;
-                is($ipc->wq_workers, 2, 'worker count bumped');
-                $ipc->wq_worker_decr;
-                $ipc->wq_worker_decr_wait(10);
-                is($ipc->wq_workers, 1, 'worker count lowered');
-                is($ipc->wq_workers(2), 2, 'worker count set');
-                is($ipc->wq_workers, 2, 'worker count stayed set');
-        }
+
+        $ipc->wq_worker_incr;
+        is($ipc->wq_workers, 2, 'worker count bumped');
+        $ipc->wq_worker_decr;
+        $ipc->wq_worker_decr_wait(10);
+        is($ipc->wq_workers, 1, 'worker count lowered');
+        is($ipc->wq_workers(2), 2, 'worker count set');
+        is($ipc->wq_workers, 2, 'worker count stayed set');
+
+        $ipc->wq_broadcast('test_append_pid', "$tmpdir/append_pid");
         $ipc->wq_close;
+        open my $fh, '<', "$tmpdir/append_pid" or BAIL_OUT "open: $!";
+        chomp(my @pids = <$fh>);
+        my %pids = map { $_ => 1 } grep(/\A[0-9]+\z/, @pids);
+        is(scalar keys %pids, 2, 'broadcast hit both PIDs');
         is($ipc->wq_workers, undef, 'workers undef after close');
 }