From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on dcvr.yhbt.net X-Spam-Level: X-Spam-ASN: X-Spam-Status: No, score=-4.0 required=3.0 tests=ALL_TRUSTED,BAYES_00 shortcircuit=no autolearn=ham autolearn_force=no version=3.4.2 Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id B8C311F488 for ; Sun, 5 May 2019 00:52:19 +0000 (UTC) From: Eric Wong To: meta@public-inbox.org Subject: [PATCH 2/4] listener: use EPOLLEXCLUSIVE for listen sockets Date: Sun, 5 May 2019 00:52:17 +0000 Message-Id: <20190505005219.31772-3-e@80x24.org> In-Reply-To: <20190505005219.31772-1-e@80x24.org> References: <20190505005219.31772-1-e@80x24.org> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit List-Id: Since our listen sockets are non-blocking and we may run multiple httpd|nntpd processes; we need a way to avoid thundering herds when there are multiple httpd|nntpd worker processes. EPOLLEXCLUSIVE was added just for that in Linux 4.5 --- TODO | 3 --- lib/PublicInbox/DS.pm | 22 ++++++++++++++++------ lib/PublicInbox/Listener.pm | 2 +- lib/PublicInbox/Syscall.pm | 7 +++++-- 4 files changed, 22 insertions(+), 12 deletions(-) diff --git a/TODO b/TODO index 372f733..ac255b8 100644 --- a/TODO +++ b/TODO @@ -56,9 +56,6 @@ all need to be considered for everything we introduce) ugh... https://rt.cpan.org/Ticket/Display.html?id=116615 (IO::KQueue is broken with Danga::Socket / PublicInbox::DS) -* EPOLLEXCLUSIVE for listen socket fairness across -httpd/nntpd - worker processes. - * improve documentation * linkify thread skeletons better diff --git a/lib/PublicInbox/DS.pm b/lib/PublicInbox/DS.pm index 543d3fd..3ccc275 100644 --- a/lib/PublicInbox/DS.pm +++ b/lib/PublicInbox/DS.pm @@ -78,6 +78,8 @@ our ( @Timers, # timers ); +# this may be set to zero with old kernels +our $EPOLLEXCLUSIVE = EPOLLEXCLUSIVE; Reset(); ##################################################################### @@ -666,11 +668,9 @@ This is normally (always?) called from your subclass via: =cut sub new { - my PublicInbox::DS $self = shift; + my ($self, $sock, $exclusive) = @_; $self = fields::new($self) unless ref $self; - my $sock = shift; - $self->{sock} = $sock; my $fd = fileno($sock); @@ -685,13 +685,23 @@ sub new { $self->{corked} = 0; $self->{read_push_back} = []; - $self->{event_watch} = POLLERR|POLLHUP|POLLNVAL; + my $ev = $self->{event_watch} = POLLERR|POLLHUP|POLLNVAL; _InitPoller(); if ($HaveEpoll) { - epoll_ctl($Epoll, EPOLL_CTL_ADD, $fd, $self->{event_watch}) - and die "couldn't add epoll watch for $fd\n"; + if ($exclusive) { + $ev = $self->{event_watch} = EPOLLIN|EPOLLERR|EPOLLHUP|$EPOLLEXCLUSIVE; + } +retry: + if (epoll_ctl($Epoll, EPOLL_CTL_ADD, $fd, $ev)) { + if ($!{EINVAL} && ($ev & $EPOLLEXCLUSIVE)) { + $EPOLLEXCLUSIVE = 0; # old kernel + $ev = $self->{event_watch} = EPOLLIN|EPOLLERR|EPOLLHUP; + goto retry; + } + die "couldn't add epoll watch for $fd: $!\n"; + } } elsif ($HaveKQueue) { # Add them to the queue but disabled for now diff --git a/lib/PublicInbox/Listener.pm b/lib/PublicInbox/Listener.pm index d1f0d2e..a75a6fd 100644 --- a/lib/PublicInbox/Listener.pm +++ b/lib/PublicInbox/Listener.pm @@ -17,7 +17,7 @@ sub new ($$$) { listen($s, 1024); IO::Handle::blocking($s, 0); my $self = fields::new($class); - $self->SUPER::new($s); # calls epoll_create for the first socket + $self->SUPER::new($s, 1); # calls epoll_create for the first socket $self->watch_read(1); $self->{post_accept} = $cb; $self diff --git a/lib/PublicInbox/Syscall.pm b/lib/PublicInbox/Syscall.pm index cf70045..9194364 100644 --- a/lib/PublicInbox/Syscall.pm +++ b/lib/PublicInbox/Syscall.pm @@ -23,10 +23,12 @@ $VERSION = "0.25"; @ISA = qw(Exporter); @EXPORT_OK = qw(sendfile epoll_ctl epoll_create epoll_wait EPOLLIN EPOLLOUT EPOLLERR EPOLLHUP EPOLLRDBAND - EPOLL_CTL_ADD EPOLL_CTL_DEL EPOLL_CTL_MOD); + EPOLL_CTL_ADD EPOLL_CTL_DEL EPOLL_CTL_MOD + EPOLLEXCLUSIVE); %EXPORT_TAGS = (epoll => [qw(epoll_ctl epoll_create epoll_wait EPOLLIN EPOLLOUT EPOLLERR EPOLLHUP EPOLLRDBAND - EPOLL_CTL_ADD EPOLL_CTL_DEL EPOLL_CTL_MOD)], + EPOLL_CTL_ADD EPOLL_CTL_DEL EPOLL_CTL_MOD + EPOLLEXCLUSIVE)], sendfile => [qw(sendfile)], ); @@ -35,6 +37,7 @@ use constant EPOLLOUT => 4; use constant EPOLLERR => 8; use constant EPOLLHUP => 16; use constant EPOLLRDBAND => 128; +use constant EPOLLEXCLUSIVE => (1 << 28); use constant EPOLL_CTL_ADD => 1; use constant EPOLL_CTL_DEL => 2; use constant EPOLL_CTL_MOD => 3; -- EW