From 2c972f3d70caf99488fff300341450e48be6ebf1 Mon Sep 17 00:00:00 2001 From: Eric Wong Date: Sat, 9 Jul 2016 03:18:35 +0000 Subject: www: add configurable limiters Currently only for git-http-backend use, this allows limiting the number of spawned processes per-inbox or by group, if there are multiple large inboxes amidst a sea of small ones. For example, a "big" repo limiter could be used for big inboxes: which would be shared between multiple repos: [limiter "big"] max = 4 [publicinbox "git"] address = git@vger.kernel.org mainrepo = /path/to/git.git ; shared limiter with giant: httpbackendmax = big [publicinbox "giant"] address = giant@project.org mainrepo = /path/to/giant.git ; shared limiter with git: httpbackendmax = big ; This is a tiny inbox, use the default limiter with 32 slots: [publicinbox "meta"] address = meta@public-inbox.org mainrepo = /path/to/meta.git --- MANIFEST | 1 + lib/PublicInbox/Config.pm | 13 +++++++++- lib/PublicInbox/GitHTTPBackend.pm | 10 ++++++-- lib/PublicInbox/Inbox.pm | 25 +++++++++++++++++++- lib/PublicInbox/Qspawn.pm | 7 +++--- t/config.t | 2 ++ t/config_limiter.t | 50 +++++++++++++++++++++++++++++++++++++++ 7 files changed, 101 insertions(+), 7 deletions(-) create mode 100644 t/config_limiter.t diff --git a/MANIFEST b/MANIFEST index ceb1a9da..75bb43e3 100644 --- a/MANIFEST +++ b/MANIFEST @@ -106,6 +106,7 @@ t/cgi.t t/check-www-inbox.perl t/common.perl t/config.t +t/config_limiter.t t/emergency.t t/fail-bin/spamc t/feed.t diff --git a/lib/PublicInbox/Config.pm b/lib/PublicInbox/Config.pm index d34d11ad..d7eaa3e8 100644 --- a/lib/PublicInbox/Config.pm +++ b/lib/PublicInbox/Config.pm @@ -20,6 +20,7 @@ sub new { $self->{-by_addr} ||= {}; $self->{-by_name} ||= {}; $self->{-by_newsgroup} ||= {}; + $self->{-limiters} ||= {}; $self; } @@ -85,6 +86,15 @@ sub lookup_newsgroup { undef; } +sub limiter { + my ($self, $name) = @_; + $self->{-limiters}->{$name} ||= do { + require PublicInbox::Qspawn; + my $key = "limiter.$name.max"; + PublicInbox::Qspawn::Limiter->new($self->{$key}); + }; +} + sub get { my ($self, $inbox, $key) = @_; @@ -131,7 +141,7 @@ sub _fill { my $rv = {}; foreach my $k (qw(mainrepo address filter url newsgroup - watch watchheader)) { + watch watchheader httpbackendmax)) { my $v = $self->{"$pfx.$k"}; $rv->{$k} = $v if defined $v; } @@ -139,6 +149,7 @@ sub _fill { my $name = $pfx; $name =~ s/\Apublicinbox\.//; $rv->{name} = $name; + $rv->{-pi_config} = $self; $rv = PublicInbox::Inbox->new($rv); my $v = $rv->{address}; if (ref($v) eq 'ARRAY') { diff --git a/lib/PublicInbox/GitHTTPBackend.pm b/lib/PublicInbox/GitHTTPBackend.pm index ed8fdf00..d4914795 100644 --- a/lib/PublicInbox/GitHTTPBackend.pm +++ b/lib/PublicInbox/GitHTTPBackend.pm @@ -179,7 +179,6 @@ sub prepare_range { # returns undef if 403 so it falls back to dumb HTTP sub serve_smart { my ($env, $git, $path) = @_; - my $limiter = $default_limiter; my $in = $env->{'psgi.input'}; my $fd = eval { fileno($in) }; unless (defined $fd && $fd >= 0) { @@ -197,7 +196,14 @@ sub serve_smart { my $val = $env->{$name}; $env{$name} = $val if defined $val; } - my $git_dir = ref $git ? $git->{git_dir} : $git; + my ($git_dir, $limiter); + if (ref $git) { + $limiter = $git->{-httpbackend_limiter} || $default_limiter; + $git_dir = $git->{git_dir}; + } else { + $limiter = $default_limiter; + $git_dir = $git; + } $env{GIT_HTTP_EXPORT_ALL} = '1'; $env{PATH_TRANSLATED} = "$git_dir/$path"; my %rdr = ( 0 => fileno($in) ); diff --git a/lib/PublicInbox/Inbox.pm b/lib/PublicInbox/Inbox.pm index dc9980b7..23b77213 100644 --- a/lib/PublicInbox/Inbox.pm +++ b/lib/PublicInbox/Inbox.pm @@ -34,6 +34,7 @@ sub new { my $v = $opts->{address} ||= 'public-inbox@example.com'; my $p = $opts->{-primary_address} = ref($v) eq 'ARRAY' ? $v->[0] : $v; $opts->{domain} = ($p =~ /\@(\S+)\z/) ? $1 : 'localhost'; + weaken($opts->{-pi_config}); bless $opts, $class; } @@ -44,11 +45,33 @@ sub _weaken_fields { } } +sub _set_limiter ($$$) { + my ($self, $git, $pfx) = @_; + my $lkey = "-${pfx}_limiter"; + $git->{$lkey} = $self->{$lkey} ||= eval { + my $mkey = $pfx.'max'; + my $val = $self->{$mkey} or return; + my $lim; + if ($val =~ /\A\d+\z/) { + require PublicInbox::Qspawn; + $lim = PublicInbox::Qspawn::Limiter->new($val); + } elsif ($val =~ /\A[a-z][a-z0-9]*\z/) { + $lim = $self->{-pi_config}->limiter($val); + warn "$mkey limiter=$val not found\n" if !$lim; + } else { + warn "$mkey limiter=$val not understood\n"; + } + $lim; + } +} + sub git { my ($self) = @_; $self->{git} ||= eval { _weaken_later($self); - PublicInbox::Git->new($self->{mainrepo}); + my $g = PublicInbox::Git->new($self->{mainrepo}); + _set_limiter($self, $g, 'httpbackend'); + $g; }; } diff --git a/lib/PublicInbox/Qspawn.pm b/lib/PublicInbox/Qspawn.pm index cc9c340d..697c55a1 100644 --- a/lib/PublicInbox/Qspawn.pm +++ b/lib/PublicInbox/Qspawn.pm @@ -47,7 +47,7 @@ sub start { my ($self, $limiter, $cb) = @_; $self->{limiter} = $limiter; - if ($limiter->{running} < $limiter->{limit}) { + if ($limiter->{running} < $limiter->{max}) { _do_spawn($self, $cb); } else { push @{$limiter->{run_queue}}, [ $self, $cb ]; @@ -59,9 +59,10 @@ use strict; use warnings; sub new { - my ($class, $limit) = @_; + my ($class, $max) = @_; bless { - limit => $limit || 1, + # 32 is same as the git-daemon connection limit + max => $max || 32, running => 0, run_queue => [], }, $class; diff --git a/t/config.t b/t/config.t index dc448cdf..77e8f4ac 100644 --- a/t/config.t +++ b/t/config.t @@ -30,6 +30,7 @@ my $tmpdir = tempdir('pi-config-XXXXXX', TMPDIR => 1, CLEANUP => 1); 'url' => 'http://example.com/meta', -primary_address => 'meta@public-inbox.org', 'name' => 'meta', + -pi_config => $cfg, }, "lookup matches expected output"); is($cfg->lookup('blah@example.com'), undef, @@ -45,6 +46,7 @@ my $tmpdir = tempdir('pi-config-XXXXXX', TMPDIR => 1, CLEANUP => 1); 'domain' => 'public-inbox.org', 'name' => 'test', 'url' => 'http://example.com/test', + -pi_config => $cfg, }, "lookup matches expected output for test"); } diff --git a/t/config_limiter.t b/t/config_limiter.t new file mode 100644 index 00000000..bfea1510 --- /dev/null +++ b/t/config_limiter.t @@ -0,0 +1,50 @@ +# Copyright (C) 2016 all contributors +# License: AGPL-3.0+ +use strict; +use warnings; +use Test::More; +use PublicInbox::Config; +my $cfgpfx = "publicinbox.test"; +{ + my $config = PublicInbox::Config->new({ + "$cfgpfx.address" => 'test@example.com', + "$cfgpfx.mainrepo" => '/path/to/non/existent', + "$cfgpfx.httpbackendmax" => 12, + }); + my $ibx = $config->lookup_name('test'); + my $git = $ibx->git; + my $old = "$git"; + my $lim = $git->{-httpbackend_limiter}; + ok($lim, 'Limiter exists'); + is($lim->{max}, 12, 'limiter has expected slots'); + $git = undef; + $ibx->{git} = undef; + $git = $ibx->git; + isnt($old, "$git", 'got new Git object'); + is("$git->{-httpbackend_limiter}", "$lim", 'same limiter'); +} + +{ + my $config = PublicInbox::Config->new({ + 'limiter.named.max' => 3, + "$cfgpfx.address" => 'test@example.com', + "$cfgpfx.mainrepo" => '/path/to/non/existent', + "$cfgpfx.httpbackendmax" => 'named', + }); + my $ibx = $config->lookup_name('test'); + my $git = $ibx->git; + ok($git, 'got git object'); + my $old = "$git"; + my $lim = $git->{-httpbackend_limiter}; + ok($lim, 'Limiter exists'); + is($lim->{max}, 3, 'limiter has expected slots'); + $git = undef; + $ibx->{git} = undef; + PublicInbox::Inbox::weaken_task; + $git = $ibx->git; + isnt($old, "$git", 'got new Git object'); + is("$git->{-httpbackend_limiter}", "$lim", 'same limiter'); + is($lim->{max}, 3, 'limiter has expected slots'); +} + +done_testing; -- cgit v1.2.3-24-ge0c7