user/dev discussion of public-inbox itself
 help / color / mirror / code / Atom feed
Search results ordered by [date|relevance]  view[summary|nested|Atom feed]
thread overview below | download mbox.gz: |
* [PATCH 6/6] www: add configurable limiters
  2016-07-09  3:18  7% [PATCH 0/6] bunch of cleanups and a new feature! Eric Wong
@ 2016-07-09  3:18  3% ` Eric Wong
  0 siblings, 0 replies; 2+ results
From: Eric Wong @ 2016-07-09  3:18 UTC (permalink / raw)
  To: meta

Currently only for git-http-backend use, this allows limiting
the number of spawned processes per-inbox or by group, if there
are multiple large inboxes amidst a sea of small ones.

For example, a "big" repo limiter could be used for big inboxes:
which would be shared between multiple repos:

[limiter "big"]
	max = 4
[publicinbox "git"]
	address = git@vger.kernel.org
	mainrepo = /path/to/git.git
	; shared limiter with giant:
	httpbackendmax = big
[publicinbox "giant"]
	address = giant@project.org
	mainrepo = /path/to/giant.git
	; shared limiter with git:
	httpbackendmax = big

; This is a tiny inbox, use the default limiter with 32 slots:
[publicinbox "meta"]
	address = meta@public-inbox.org
	mainrepo = /path/to/meta.git
---
 MANIFEST                          |  1 +
 lib/PublicInbox/Config.pm         | 13 +++++++++-
 lib/PublicInbox/GitHTTPBackend.pm | 10 ++++++--
 lib/PublicInbox/Inbox.pm          | 25 +++++++++++++++++++-
 lib/PublicInbox/Qspawn.pm         |  7 +++---
 t/config.t                        |  2 ++
 t/config_limiter.t                | 50 +++++++++++++++++++++++++++++++++++++++
 7 files changed, 101 insertions(+), 7 deletions(-)
 create mode 100644 t/config_limiter.t

diff --git a/MANIFEST b/MANIFEST
index ceb1a9d..75bb43e 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -106,6 +106,7 @@ t/cgi.t
 t/check-www-inbox.perl
 t/common.perl
 t/config.t
+t/config_limiter.t
 t/emergency.t
 t/fail-bin/spamc
 t/feed.t
diff --git a/lib/PublicInbox/Config.pm b/lib/PublicInbox/Config.pm
index d34d11a..d7eaa3e 100644
--- a/lib/PublicInbox/Config.pm
+++ b/lib/PublicInbox/Config.pm
@@ -20,6 +20,7 @@ sub new {
 	$self->{-by_addr} ||= {};
 	$self->{-by_name} ||= {};
 	$self->{-by_newsgroup} ||= {};
+	$self->{-limiters} ||= {};
 	$self;
 }
 
@@ -85,6 +86,15 @@ sub lookup_newsgroup {
 	undef;
 }
 
+sub limiter {
+	my ($self, $name) = @_;
+	$self->{-limiters}->{$name} ||= do {
+		require PublicInbox::Qspawn;
+		my $key = "limiter.$name.max";
+		PublicInbox::Qspawn::Limiter->new($self->{$key});
+	};
+}
+
 sub get {
 	my ($self, $inbox, $key) = @_;
 
@@ -131,7 +141,7 @@ sub _fill {
 	my $rv = {};
 
 	foreach my $k (qw(mainrepo address filter url newsgroup
-			watch watchheader)) {
+			watch watchheader httpbackendmax)) {
 		my $v = $self->{"$pfx.$k"};
 		$rv->{$k} = $v if defined $v;
 	}
@@ -139,6 +149,7 @@ sub _fill {
 	my $name = $pfx;
 	$name =~ s/\Apublicinbox\.//;
 	$rv->{name} = $name;
+	$rv->{-pi_config} = $self;
 	$rv = PublicInbox::Inbox->new($rv);
 	my $v = $rv->{address};
 	if (ref($v) eq 'ARRAY') {
diff --git a/lib/PublicInbox/GitHTTPBackend.pm b/lib/PublicInbox/GitHTTPBackend.pm
index ed8fdf0..d491479 100644
--- a/lib/PublicInbox/GitHTTPBackend.pm
+++ b/lib/PublicInbox/GitHTTPBackend.pm
@@ -179,7 +179,6 @@ sub prepare_range {
 # returns undef if 403 so it falls back to dumb HTTP
 sub serve_smart {
 	my ($env, $git, $path) = @_;
-	my $limiter = $default_limiter;
 	my $in = $env->{'psgi.input'};
 	my $fd = eval { fileno($in) };
 	unless (defined $fd && $fd >= 0) {
@@ -197,7 +196,14 @@ sub serve_smart {
 		my $val = $env->{$name};
 		$env{$name} = $val if defined $val;
 	}
-	my $git_dir = ref $git ? $git->{git_dir} : $git;
+	my ($git_dir, $limiter);
+	if (ref $git) {
+		$limiter = $git->{-httpbackend_limiter} || $default_limiter;
+		$git_dir = $git->{git_dir};
+	} else {
+		$limiter = $default_limiter;
+		$git_dir = $git;
+	}
 	$env{GIT_HTTP_EXPORT_ALL} = '1';
 	$env{PATH_TRANSLATED} = "$git_dir/$path";
 	my %rdr = ( 0 => fileno($in) );
diff --git a/lib/PublicInbox/Inbox.pm b/lib/PublicInbox/Inbox.pm
index dc9980b..23b7721 100644
--- a/lib/PublicInbox/Inbox.pm
+++ b/lib/PublicInbox/Inbox.pm
@@ -34,6 +34,7 @@ sub new {
 	my $v = $opts->{address} ||= 'public-inbox@example.com';
 	my $p = $opts->{-primary_address} = ref($v) eq 'ARRAY' ? $v->[0] : $v;
 	$opts->{domain} = ($p =~ /\@(\S+)\z/) ? $1 : 'localhost';
+	weaken($opts->{-pi_config});
 	bless $opts, $class;
 }
 
@@ -44,11 +45,33 @@ sub _weaken_fields {
 	}
 }
 
+sub _set_limiter ($$$) {
+	my ($self, $git, $pfx) = @_;
+	my $lkey = "-${pfx}_limiter";
+	$git->{$lkey} = $self->{$lkey} ||= eval {
+		my $mkey = $pfx.'max';
+		my $val = $self->{$mkey} or return;
+		my $lim;
+		if ($val =~ /\A\d+\z/) {
+			require PublicInbox::Qspawn;
+			$lim = PublicInbox::Qspawn::Limiter->new($val);
+		} elsif ($val =~ /\A[a-z][a-z0-9]*\z/) {
+			$lim = $self->{-pi_config}->limiter($val);
+			warn "$mkey limiter=$val not found\n" if !$lim;
+		} else {
+			warn "$mkey limiter=$val not understood\n";
+		}
+		$lim;
+	}
+}
+
 sub git {
 	my ($self) = @_;
 	$self->{git} ||= eval {
 		_weaken_later($self);
-		PublicInbox::Git->new($self->{mainrepo});
+		my $g = PublicInbox::Git->new($self->{mainrepo});
+		_set_limiter($self, $g, 'httpbackend');
+		$g;
 	};
 }
 
diff --git a/lib/PublicInbox/Qspawn.pm b/lib/PublicInbox/Qspawn.pm
index cc9c340..697c55a 100644
--- a/lib/PublicInbox/Qspawn.pm
+++ b/lib/PublicInbox/Qspawn.pm
@@ -47,7 +47,7 @@ sub start {
 	my ($self, $limiter, $cb) = @_;
 	$self->{limiter} = $limiter;
 
-	if ($limiter->{running} < $limiter->{limit}) {
+	if ($limiter->{running} < $limiter->{max}) {
 		_do_spawn($self, $cb);
 	} else {
 		push @{$limiter->{run_queue}}, [ $self, $cb ];
@@ -59,9 +59,10 @@ use strict;
 use warnings;
 
 sub new {
-	my ($class, $limit) = @_;
+	my ($class, $max) = @_;
 	bless {
-		limit => $limit || 1,
+		# 32 is same as the git-daemon connection limit
+		max => $max || 32,
 		running => 0,
 		run_queue => [],
 	}, $class;
diff --git a/t/config.t b/t/config.t
index dc448cd..77e8f4a 100644
--- a/t/config.t
+++ b/t/config.t
@@ -30,6 +30,7 @@ my $tmpdir = tempdir('pi-config-XXXXXX', TMPDIR => 1, CLEANUP => 1);
 		'url' => 'http://example.com/meta',
 		-primary_address => 'meta@public-inbox.org',
 		'name' => 'meta',
+		-pi_config => $cfg,
 	}, "lookup matches expected output");
 
 	is($cfg->lookup('blah@example.com'), undef,
@@ -45,6 +46,7 @@ my $tmpdir = tempdir('pi-config-XXXXXX', TMPDIR => 1, CLEANUP => 1);
 		'domain' => 'public-inbox.org',
 		'name' => 'test',
 		'url' => 'http://example.com/test',
+		-pi_config => $cfg,
 	}, "lookup matches expected output for test");
 }
 
diff --git a/t/config_limiter.t b/t/config_limiter.t
new file mode 100644
index 0000000..bfea151
--- /dev/null
+++ b/t/config_limiter.t
@@ -0,0 +1,50 @@
+# Copyright (C) 2016 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+use strict;
+use warnings;
+use Test::More;
+use PublicInbox::Config;
+my $cfgpfx = "publicinbox.test";
+{
+	my $config = PublicInbox::Config->new({
+		"$cfgpfx.address" => 'test@example.com',
+		"$cfgpfx.mainrepo" => '/path/to/non/existent',
+		"$cfgpfx.httpbackendmax" => 12,
+	});
+	my $ibx = $config->lookup_name('test');
+	my $git = $ibx->git;
+	my $old = "$git";
+	my $lim = $git->{-httpbackend_limiter};
+	ok($lim, 'Limiter exists');
+	is($lim->{max}, 12, 'limiter has expected slots');
+	$git = undef;
+	$ibx->{git} = undef;
+	$git = $ibx->git;
+	isnt($old, "$git", 'got new Git object');
+	is("$git->{-httpbackend_limiter}", "$lim", 'same limiter');
+}
+
+{
+	my $config = PublicInbox::Config->new({
+		'limiter.named.max' => 3,
+		"$cfgpfx.address" => 'test@example.com',
+		"$cfgpfx.mainrepo" => '/path/to/non/existent',
+		"$cfgpfx.httpbackendmax" => 'named',
+	});
+	my $ibx = $config->lookup_name('test');
+	my $git = $ibx->git;
+	ok($git, 'got git object');
+	my $old = "$git";
+	my $lim = $git->{-httpbackend_limiter};
+	ok($lim, 'Limiter exists');
+	is($lim->{max}, 3, 'limiter has expected slots');
+	$git = undef;
+	$ibx->{git} = undef;
+	PublicInbox::Inbox::weaken_task;
+	$git = $ibx->git;
+	isnt($old, "$git", 'got new Git object');
+	is("$git->{-httpbackend_limiter}", "$lim", 'same limiter');
+	is($lim->{max}, 3, 'limiter has expected slots');
+}
+
+done_testing;
-- 
EW


^ permalink raw reply related	[relevance 3%]

* [PATCH 0/6] bunch of cleanups and a new feature!
@ 2016-07-09  3:18  7% Eric Wong
  2016-07-09  3:18  3% ` [PATCH 6/6] www: add configurable limiters Eric Wong
  0 siblings, 1 reply; 2+ results
From: Eric Wong @ 2016-07-09  3:18 UTC (permalink / raw)
  To: meta

The final patch seems to be what's needed to maintain fairness
for small inboxes while serving large ones.

Eric Wong (6):
      www: drop unused constants
      www: cleanup parameter passing
      feed: remove dead code and unneeded use
      cleanup some unnecessary use/requires
      qspawn: allow configurable limiters
      www: add configurable limiters

 MANIFEST                          |  1 +
 lib/PublicInbox/Config.pm         | 13 +++++++-
 lib/PublicInbox/ExtMsg.pm         |  4 +--
 lib/PublicInbox/Feed.pm           | 23 -------------
 lib/PublicInbox/GitHTTPBackend.pm | 15 +++++++--
 lib/PublicInbox/Inbox.pm          | 25 +++++++++++++-
 lib/PublicInbox/Mbox.pm           |  1 -
 lib/PublicInbox/Qspawn.pm         | 39 ++++++++++++++++------
 lib/PublicInbox/View.pm           |  2 --
 lib/PublicInbox/WWW.pm            | 70 ++++++++++++++++++---------------------
 t/config.t                        |  2 ++
 t/config_limiter.t                | 50 ++++++++++++++++++++++++++++
 t/qspawn.t                        | 10 +++---
 13 files changed, 171 insertions(+), 84 deletions(-)

^ permalink raw reply	[relevance 7%]

Results 1-2 of 2 | reverse | sort options + mbox downloads above
-- links below jump to the message on this page --
2016-07-09  3:18  7% [PATCH 0/6] bunch of cleanups and a new feature! Eric Wong
2016-07-09  3:18  3% ` [PATCH 6/6] www: add configurable limiters Eric Wong

Code repositories for project(s) associated with this public inbox

	https://80x24.org/public-inbox.git

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).