about summary refs log tree commit homepage
diff options
context:
space:
mode:
-rw-r--r--MANIFEST1
-rw-r--r--lib/PublicInbox/Config.pm13
-rw-r--r--lib/PublicInbox/GitHTTPBackend.pm10
-rw-r--r--lib/PublicInbox/Inbox.pm25
-rw-r--r--lib/PublicInbox/Qspawn.pm7
-rw-r--r--t/config.t2
-rw-r--r--t/config_limiter.t50
7 files changed, 101 insertions, 7 deletions
diff --git a/MANIFEST b/MANIFEST
index ceb1a9da..75bb43e3 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -106,6 +106,7 @@ t/cgi.t
 t/check-www-inbox.perl
 t/common.perl
 t/config.t
+t/config_limiter.t
 t/emergency.t
 t/fail-bin/spamc
 t/feed.t
diff --git a/lib/PublicInbox/Config.pm b/lib/PublicInbox/Config.pm
index d34d11ad..d7eaa3e8 100644
--- a/lib/PublicInbox/Config.pm
+++ b/lib/PublicInbox/Config.pm
@@ -20,6 +20,7 @@ sub new {
         $self->{-by_addr} ||= {};
         $self->{-by_name} ||= {};
         $self->{-by_newsgroup} ||= {};
+        $self->{-limiters} ||= {};
         $self;
 }
 
@@ -85,6 +86,15 @@ sub lookup_newsgroup {
         undef;
 }
 
+sub limiter {
+        my ($self, $name) = @_;
+        $self->{-limiters}->{$name} ||= do {
+                require PublicInbox::Qspawn;
+                my $key = "limiter.$name.max";
+                PublicInbox::Qspawn::Limiter->new($self->{$key});
+        };
+}
+
 sub get {
         my ($self, $inbox, $key) = @_;
 
@@ -131,7 +141,7 @@ sub _fill {
         my $rv = {};
 
         foreach my $k (qw(mainrepo address filter url newsgroup
-                        watch watchheader)) {
+                        watch watchheader httpbackendmax)) {
                 my $v = $self->{"$pfx.$k"};
                 $rv->{$k} = $v if defined $v;
         }
@@ -139,6 +149,7 @@ sub _fill {
         my $name = $pfx;
         $name =~ s/\Apublicinbox\.//;
         $rv->{name} = $name;
+        $rv->{-pi_config} = $self;
         $rv = PublicInbox::Inbox->new($rv);
         my $v = $rv->{address};
         if (ref($v) eq 'ARRAY') {
diff --git a/lib/PublicInbox/GitHTTPBackend.pm b/lib/PublicInbox/GitHTTPBackend.pm
index ed8fdf00..d4914795 100644
--- a/lib/PublicInbox/GitHTTPBackend.pm
+++ b/lib/PublicInbox/GitHTTPBackend.pm
@@ -179,7 +179,6 @@ sub prepare_range {
 # returns undef if 403 so it falls back to dumb HTTP
 sub serve_smart {
         my ($env, $git, $path) = @_;
-        my $limiter = $default_limiter;
         my $in = $env->{'psgi.input'};
         my $fd = eval { fileno($in) };
         unless (defined $fd && $fd >= 0) {
@@ -197,7 +196,14 @@ sub serve_smart {
                 my $val = $env->{$name};
                 $env{$name} = $val if defined $val;
         }
-        my $git_dir = ref $git ? $git->{git_dir} : $git;
+        my ($git_dir, $limiter);
+        if (ref $git) {
+                $limiter = $git->{-httpbackend_limiter} || $default_limiter;
+                $git_dir = $git->{git_dir};
+        } else {
+                $limiter = $default_limiter;
+                $git_dir = $git;
+        }
         $env{GIT_HTTP_EXPORT_ALL} = '1';
         $env{PATH_TRANSLATED} = "$git_dir/$path";
         my %rdr = ( 0 => fileno($in) );
diff --git a/lib/PublicInbox/Inbox.pm b/lib/PublicInbox/Inbox.pm
index dc9980b7..23b77213 100644
--- a/lib/PublicInbox/Inbox.pm
+++ b/lib/PublicInbox/Inbox.pm
@@ -34,6 +34,7 @@ sub new {
         my $v = $opts->{address} ||= 'public-inbox@example.com';
         my $p = $opts->{-primary_address} = ref($v) eq 'ARRAY' ? $v->[0] : $v;
         $opts->{domain} = ($p =~ /\@(\S+)\z/) ? $1 : 'localhost';
+        weaken($opts->{-pi_config});
         bless $opts, $class;
 }
 
@@ -44,11 +45,33 @@ sub _weaken_fields {
         }
 }
 
+sub _set_limiter ($$$) {
+        my ($self, $git, $pfx) = @_;
+        my $lkey = "-${pfx}_limiter";
+        $git->{$lkey} = $self->{$lkey} ||= eval {
+                my $mkey = $pfx.'max';
+                my $val = $self->{$mkey} or return;
+                my $lim;
+                if ($val =~ /\A\d+\z/) {
+                        require PublicInbox::Qspawn;
+                        $lim = PublicInbox::Qspawn::Limiter->new($val);
+                } elsif ($val =~ /\A[a-z][a-z0-9]*\z/) {
+                        $lim = $self->{-pi_config}->limiter($val);
+                        warn "$mkey limiter=$val not found\n" if !$lim;
+                } else {
+                        warn "$mkey limiter=$val not understood\n";
+                }
+                $lim;
+        }
+}
+
 sub git {
         my ($self) = @_;
         $self->{git} ||= eval {
                 _weaken_later($self);
-                PublicInbox::Git->new($self->{mainrepo});
+                my $g = PublicInbox::Git->new($self->{mainrepo});
+                _set_limiter($self, $g, 'httpbackend');
+                $g;
         };
 }
 
diff --git a/lib/PublicInbox/Qspawn.pm b/lib/PublicInbox/Qspawn.pm
index cc9c340d..697c55a1 100644
--- a/lib/PublicInbox/Qspawn.pm
+++ b/lib/PublicInbox/Qspawn.pm
@@ -47,7 +47,7 @@ sub start {
         my ($self, $limiter, $cb) = @_;
         $self->{limiter} = $limiter;
 
-        if ($limiter->{running} < $limiter->{limit}) {
+        if ($limiter->{running} < $limiter->{max}) {
                 _do_spawn($self, $cb);
         } else {
                 push @{$limiter->{run_queue}}, [ $self, $cb ];
@@ -59,9 +59,10 @@ use strict;
 use warnings;
 
 sub new {
-        my ($class, $limit) = @_;
+        my ($class, $max) = @_;
         bless {
-                limit => $limit || 1,
+                # 32 is same as the git-daemon connection limit
+                max => $max || 32,
                 running => 0,
                 run_queue => [],
         }, $class;
diff --git a/t/config.t b/t/config.t
index dc448cdf..77e8f4ac 100644
--- a/t/config.t
+++ b/t/config.t
@@ -30,6 +30,7 @@ my $tmpdir = tempdir('pi-config-XXXXXX', TMPDIR => 1, CLEANUP => 1);
                 'url' => 'http://example.com/meta',
                 -primary_address => 'meta@public-inbox.org',
                 'name' => 'meta',
+                -pi_config => $cfg,
         }, "lookup matches expected output");
 
         is($cfg->lookup('blah@example.com'), undef,
@@ -45,6 +46,7 @@ my $tmpdir = tempdir('pi-config-XXXXXX', TMPDIR => 1, CLEANUP => 1);
                 'domain' => 'public-inbox.org',
                 'name' => 'test',
                 'url' => 'http://example.com/test',
+                -pi_config => $cfg,
         }, "lookup matches expected output for test");
 }
 
diff --git a/t/config_limiter.t b/t/config_limiter.t
new file mode 100644
index 00000000..bfea1510
--- /dev/null
+++ b/t/config_limiter.t
@@ -0,0 +1,50 @@
+# Copyright (C) 2016 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+use strict;
+use warnings;
+use Test::More;
+use PublicInbox::Config;
+my $cfgpfx = "publicinbox.test";
+{
+        my $config = PublicInbox::Config->new({
+                "$cfgpfx.address" => 'test@example.com',
+                "$cfgpfx.mainrepo" => '/path/to/non/existent',
+                "$cfgpfx.httpbackendmax" => 12,
+        });
+        my $ibx = $config->lookup_name('test');
+        my $git = $ibx->git;
+        my $old = "$git";
+        my $lim = $git->{-httpbackend_limiter};
+        ok($lim, 'Limiter exists');
+        is($lim->{max}, 12, 'limiter has expected slots');
+        $git = undef;
+        $ibx->{git} = undef;
+        $git = $ibx->git;
+        isnt($old, "$git", 'got new Git object');
+        is("$git->{-httpbackend_limiter}", "$lim", 'same limiter');
+}
+
+{
+        my $config = PublicInbox::Config->new({
+                'limiter.named.max' => 3,
+                "$cfgpfx.address" => 'test@example.com',
+                "$cfgpfx.mainrepo" => '/path/to/non/existent',
+                "$cfgpfx.httpbackendmax" => 'named',
+        });
+        my $ibx = $config->lookup_name('test');
+        my $git = $ibx->git;
+        ok($git, 'got git object');
+        my $old = "$git";
+        my $lim = $git->{-httpbackend_limiter};
+        ok($lim, 'Limiter exists');
+        is($lim->{max}, 3, 'limiter has expected slots');
+        $git = undef;
+        $ibx->{git} = undef;
+        PublicInbox::Inbox::weaken_task;
+        $git = $ibx->git;
+        isnt($old, "$git", 'got new Git object');
+        is("$git->{-httpbackend_limiter}", "$lim", 'same limiter');
+        is($lim->{max}, 3, 'limiter has expected slots');
+}
+
+done_testing;