diff options
author | Eric Wong <e@80x24.org> | 2021-01-31 22:28:14 -1000 |
---|---|---|
committer | Eric Wong <e@80x24.org> | 2021-02-01 11:38:08 +0000 |
commit | c2145231a10937edb7bc74a3b70fcb3d11938de3 (patch) | |
tree | fa6ac11102487794d7a6771371298dcc8bb022aa /lib/PublicInbox/WQWorker.pm | |
parent | 4e101abd56867b9f5286644d14ca05bcff49708c (diff) | |
download | public-inbox-c2145231a10937edb7bc74a3b70fcb3d11938de3.tar.gz |
This will let us to maximize the capability of our asynchronous git API. This lets us avoid relying on EOF to notify lei2mail workers; thus giving us the option of running fewer lei_xsearch worker processes in parallel than local sources. I tried using a synchronous git API; and even with libgit2 in the same process to avoid the IPC cost failed to match the throughput afforded by this change. This is because libgit2 is built (at least on Debian) with the SHA-1 collision code enabled and ubc_check stuff was dominating my profiles.
Diffstat (limited to 'lib/PublicInbox/WQWorker.pm')
-rw-r--r-- | lib/PublicInbox/WQWorker.pm | 34 |
1 files changed, 34 insertions, 0 deletions
diff --git a/lib/PublicInbox/WQWorker.pm b/lib/PublicInbox/WQWorker.pm new file mode 100644 index 00000000..25a5e4fb --- /dev/null +++ b/lib/PublicInbox/WQWorker.pm @@ -0,0 +1,34 @@ +# Copyright (C) 2021 all contributors <meta@public-inbox.org> +# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt> + +# for PublicInbox::IPC wq_* (work queue) workers +package PublicInbox::WQWorker; +use strict; +use v5.10.1; +use parent qw(PublicInbox::DS); +use PublicInbox::Syscall qw(EPOLLIN EPOLLEXCLUSIVE EPOLLET); +use Errno qw(EAGAIN ECONNRESET); +use IO::Handle (); # blocking + +sub new { + my (undef, $wq) = @_; + my $s2 = $wq->{-wq_s2} // die 'BUG: no -wq_s2'; + $s2->blocking(0); + my $self = bless { sock => $s2, wq => $wq }, __PACKAGE__; + $self->SUPER::new($s2, EPOLLEXCLUSIVE|EPOLLIN|EPOLLET); + $self; +} + +sub event_step { + my ($self) = @_; + my $n; + do { + $n = $self->{wq}->recv_and_run($self->{sock}, 4096 * 33); + } while ($n); + return if !defined($n) && $! == EAGAIN; # likely + warn "wq worker error: $!\n" if !defined($n) && $! != ECONNRESET; + $self->{wq}->wq_atexit_child; + $self->close; # PublicInbox::DS::close +} + +1; |