From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.6 (2021-04-09) on dcvr.yhbt.net X-Spam-Level: X-Spam-ASN: X-Spam-Status: No, score=-4.2 required=3.0 tests=ALL_TRUSTED,AWL,BAYES_00, DKIM_SIGNED,DKIM_VALID,DKIM_VALID_AU,DKIM_VALID_EF shortcircuit=no autolearn=ham autolearn_force=no version=3.4.6 Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id 94A0A1F452 for ; Wed, 25 Oct 2023 00:29:49 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=80x24.org; s=selector1; t=1698193789; bh=KgxutE3mTym71hoxeewn+X7728jrQjiLGCCdAmX/qvQ=; h=From:To:Subject:Date:In-Reply-To:References:From; b=eTRT6Lzuk9MeqkRtC11P9mDle/lu89bEBX/fz8QTL79j+8ThPVO4HByxxsOHcirli BRsB4kvGBeRBK8BSPzm/J+WEDersYhnwCipbeDFfDT84KtgktpzjPJ4E/+3iSs0gDA jb3IWM9zj1LMiu9ClIr+0u59PVQ9+rlzCdLXmc7M= From: Eric Wong To: meta@public-inbox.org Subject: [PATCH 01/26] limiter: split out from qspawn Date: Wed, 25 Oct 2023 00:29:24 +0000 Message-ID: <20231025002949.3092193-2-e@80x24.org> In-Reply-To: <20231025002949.3092193-1-e@80x24.org> References: <20231025002949.3092193-1-e@80x24.org> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit List-Id: It's slightly better organized this way, especially since `publicinboxLimiter' has its own user-facing config section and knobs. I may use it in LeiMirror and CodeSearchIdx for process management. --- MANIFEST | 1 + lib/PublicInbox/Config.pm | 4 +-- lib/PublicInbox/GitHTTPBackend.pm | 3 +- lib/PublicInbox/Inbox.pm | 4 +-- lib/PublicInbox/Limiter.pm | 47 +++++++++++++++++++++++++++++++ lib/PublicInbox/MailDiff.pm | 1 + lib/PublicInbox/Qspawn.pm | 47 ++----------------------------- t/qspawn.t | 3 +- 8 files changed, 60 insertions(+), 50 deletions(-) create mode 100644 lib/PublicInbox/Limiter.pm diff --git a/MANIFEST b/MANIFEST index 791d91a7..dcce801c 100644 --- a/MANIFEST +++ b/MANIFEST @@ -287,6 +287,7 @@ lib/PublicInbox/LeiUp.pm lib/PublicInbox/LeiViewText.pm lib/PublicInbox/LeiWatch.pm lib/PublicInbox/LeiXSearch.pm +lib/PublicInbox/Limiter.pm lib/PublicInbox/Linkify.pm lib/PublicInbox/Listener.pm lib/PublicInbox/Lock.pm diff --git a/lib/PublicInbox/Config.pm b/lib/PublicInbox/Config.pm index 15e0872e..d156b2d3 100644 --- a/lib/PublicInbox/Config.pm +++ b/lib/PublicInbox/Config.pm @@ -124,9 +124,9 @@ sub lookup_newsgroup { sub limiter { my ($self, $name) = @_; $self->{-limiters}->{$name} //= do { - require PublicInbox::Qspawn; + require PublicInbox::Limiter; my $max = $self->{"publicinboxlimiter.$name.max"} || 1; - my $limiter = PublicInbox::Qspawn::Limiter->new($max); + my $limiter = PublicInbox::Limiter->new($max); $limiter->setup_rlimit($name, $self); $limiter; }; diff --git a/lib/PublicInbox/GitHTTPBackend.pm b/lib/PublicInbox/GitHTTPBackend.pm index 74432429..d69f5f8b 100644 --- a/lib/PublicInbox/GitHTTPBackend.pm +++ b/lib/PublicInbox/GitHTTPBackend.pm @@ -9,13 +9,14 @@ use v5.10.1; use Fcntl qw(:seek); use IO::Handle; # ->flush use HTTP::Date qw(time2str); +use PublicInbox::Limiter; use PublicInbox::Qspawn; use PublicInbox::Tmpfile; use PublicInbox::WwwStatic qw(r @NO_CACHE); use Carp (); # 32 is same as the git-daemon connection limit -my $default_limiter = PublicInbox::Qspawn::Limiter->new(32); +my $default_limiter = PublicInbox::Limiter->new(32); # n.b. serving "description" and "cloneurl" should be innocuous enough to # not cause problems. serving "config" might... diff --git a/lib/PublicInbox/Inbox.pm b/lib/PublicInbox/Inbox.pm index 9afbb478..3dad7004 100644 --- a/lib/PublicInbox/Inbox.pm +++ b/lib/PublicInbox/Inbox.pm @@ -55,8 +55,8 @@ sub _set_limiter ($$$) { my $val = $self->{$mkey} or return; my $lim; if ($val =~ /\A[0-9]+\z/) { - require PublicInbox::Qspawn; - $lim = PublicInbox::Qspawn::Limiter->new($val); + require PublicInbox::Limiter; + $lim = PublicInbox::Limiter->new($val); } elsif ($val =~ /\A[a-z][a-z0-9]*\z/) { $lim = $pi_cfg->limiter($val); warn "$mkey limiter=$val not found\n" if !$lim; diff --git a/lib/PublicInbox/Limiter.pm b/lib/PublicInbox/Limiter.pm new file mode 100644 index 00000000..48a2b6a3 --- /dev/null +++ b/lib/PublicInbox/Limiter.pm @@ -0,0 +1,47 @@ +# Copyright (C) all contributors +# License: AGPL-3.0+ + +package PublicInbox::Limiter; +use v5.12; +use PublicInbox::Spawn; + +sub new { + my ($class, $max) = @_; + bless { + # 32 is same as the git-daemon connection limit + max => $max || 32, + running => 0, + run_queue => [], + # RLIMIT_CPU => undef, + # RLIMIT_DATA => undef, + # RLIMIT_CORE => undef, + }, $class; +} + +sub setup_rlimit { + my ($self, $name, $cfg) = @_; + for my $rlim (@PublicInbox::Spawn::RLIMITS) { + my $k = lc($rlim); + $k =~ tr/_//d; + $k = "publicinboxlimiter.$name.$k"; + my $v = $cfg->{$k} // next; + my @rlimit = split(/\s*,\s*/, $v); + if (scalar(@rlimit) == 1) { + push @rlimit, $rlimit[0]; + } elsif (scalar(@rlimit) != 2) { + warn "could not parse $k: $v\n"; + } + eval { require BSD::Resource }; + if ($@) { + warn "BSD::Resource missing for $rlim"; + next; + } + for my $i (0..$#rlimit) { + next if $rlimit[$i] ne 'INFINITY'; + $rlimit[$i] = BSD::Resource::RLIM_INFINITY(); + } + $self->{$rlim} = \@rlimit; + } +} + +1; diff --git a/lib/PublicInbox/MailDiff.pm b/lib/PublicInbox/MailDiff.pm index 994c7851..c3ce9365 100644 --- a/lib/PublicInbox/MailDiff.pm +++ b/lib/PublicInbox/MailDiff.pm @@ -8,6 +8,7 @@ use PublicInbox::MsgIter qw(msg_part_text); use PublicInbox::ViewDiff qw(flush_diff); use PublicInbox::GitAsyncCat; use PublicInbox::ContentDigestDbg; +use PublicInbox::Qspawn; sub write_part { # Eml->each_part callback my ($ary, $self) = @_; diff --git a/lib/PublicInbox/Qspawn.pm b/lib/PublicInbox/Qspawn.pm index 0e52617c..a4d78e49 100644 --- a/lib/PublicInbox/Qspawn.pm +++ b/lib/PublicInbox/Qspawn.pm @@ -29,6 +29,7 @@ use v5.12; use PublicInbox::Spawn qw(popen_rd); use PublicInbox::GzipFilter; use Scalar::Util qw(blessed); +use PublicInbox::Limiter; # n.b.: we get EAGAIN with public-inbox-httpd, and EINTR on other PSGI servers use Errno qw(EAGAIN EINTR); @@ -183,7 +184,7 @@ sub psgi_qx { $self->{qx_arg} = $qx_arg; $self->{qx_fh} = $qx_fh; $self->{qx_buf} = \$qx_buf; - $limiter ||= $def_limiter ||= PublicInbox::Qspawn::Limiter->new(32); + $limiter ||= $def_limiter ||= PublicInbox::Limiter->new(32); start($self, $limiter, \&psgi_qx_start); } @@ -317,7 +318,7 @@ sub psgi_return { $self->{psgi_env} = $env; $self->{hdr_buf} = \(my $hdr_buf = ''); $self->{parse_hdr} = [ $parse_hdr, $hdr_arg ]; - $limiter ||= $def_limiter ||= PublicInbox::Qspawn::Limiter->new(32); + $limiter ||= $def_limiter ||= PublicInbox::Limiter->new(32); # the caller already captured the PSGI write callback from # the PSGI server, so we can call ->start, here: @@ -334,46 +335,4 @@ sub psgi_return { } } -package PublicInbox::Qspawn::Limiter; -use v5.12; - -sub new { - my ($class, $max) = @_; - bless { - # 32 is same as the git-daemon connection limit - max => $max || 32, - running => 0, - run_queue => [], - # RLIMIT_CPU => undef, - # RLIMIT_DATA => undef, - # RLIMIT_CORE => undef, - }, $class; -} - -sub setup_rlimit { - my ($self, $name, $cfg) = @_; - foreach my $rlim (@PublicInbox::Spawn::RLIMITS) { - my $k = lc($rlim); - $k =~ tr/_//d; - $k = "publicinboxlimiter.$name.$k"; - defined(my $v = $cfg->{$k}) or next; - my @rlimit = split(/\s*,\s*/, $v); - if (scalar(@rlimit) == 1) { - push @rlimit, $rlimit[0]; - } elsif (scalar(@rlimit) != 2) { - warn "could not parse $k: $v\n"; - } - eval { require BSD::Resource }; - if ($@) { - warn "BSD::Resource missing for $rlim"; - next; - } - foreach my $i (0..$#rlimit) { - next if $rlimit[$i] ne 'INFINITY'; - $rlimit[$i] = BSD::Resource::RLIM_INFINITY(); - } - $self->{$rlim} = \@rlimit; - } -} - 1; diff --git a/t/qspawn.t b/t/qspawn.t index 224e20db..507f86a5 100644 --- a/t/qspawn.t +++ b/t/qspawn.t @@ -3,6 +3,7 @@ use v5.12; use Test::More; use_ok 'PublicInbox::Qspawn'; +use_ok 'PublicInbox::Limiter'; { my $cmd = [qw(sh -c), 'echo >&2 err; echo out']; @@ -23,7 +24,7 @@ sub finish_err ($) { $qsp->{qsp_err} && ${$qsp->{qsp_err}}; } -my $limiter = PublicInbox::Qspawn::Limiter->new(1); +my $limiter = PublicInbox::Limiter->new(1); { my $x = PublicInbox::Qspawn->new([qw(true)]); $x->{qsp_err} = \(my $err = '');