diff options
author | Eric Wong <e@80x24.org> | 2016-05-24 03:41:53 +0000 |
---|---|---|
committer | Eric Wong <e@80x24.org> | 2016-05-24 04:12:51 +0000 |
commit | 64aea34d06f71828b0bdd6ae177b9bcf22d752b4 (patch) | |
tree | 4f7df75d112aa4fe0b9056cf24688a2e42032019 /lib/PublicInbox | |
parent | 8648f519a95872600689c3a5d6d87fd17770f9fc (diff) | |
download | public-inbox-64aea34d06f71828b0bdd6ae177b9bcf22d752b4.tar.gz |
Having an excessive amount of git-pack-objects processes is dangerous to the health of the server. Queue up process spawning for long-running responses and serve them sequentially, instead.
Diffstat (limited to 'lib/PublicInbox')
-rw-r--r-- | lib/PublicInbox/GitHTTPBackend.pm | 38 | ||||
-rw-r--r-- | lib/PublicInbox/Qspawn.pm | 52 |
2 files changed, 67 insertions, 23 deletions
diff --git a/lib/PublicInbox/GitHTTPBackend.pm b/lib/PublicInbox/GitHTTPBackend.pm index ded56b33..9464cb49 100644 --- a/lib/PublicInbox/GitHTTPBackend.pm +++ b/lib/PublicInbox/GitHTTPBackend.pm @@ -8,9 +8,9 @@ use strict; use warnings; use Fcntl qw(:seek); use IO::File; -use PublicInbox::Spawn qw(spawn); use HTTP::Date qw(time2str); use HTTP::Status qw(status_message); +use PublicInbox::Qspawn; # n.b. serving "description" and "cloneurl" should be innocuous enough to # not cause problems. serving "config" might... @@ -167,11 +167,6 @@ sub serve_smart { unless (defined $fd && $fd >= 0) { $in = input_to_file($env) or return r(500); } - my ($rpipe, $wpipe); - unless (pipe($rpipe, $wpipe)) { - err($env, "error creating pipe: $! - going static"); - return; - } my %env = %ENV; # GIT_COMMITTER_NAME, GIT_COMMITTER_EMAIL # may be set in the server-process and are passed as-is @@ -187,20 +182,13 @@ sub serve_smart { my $git_dir = $git->{git_dir}; $env{GIT_HTTP_EXPORT_ALL} = '1'; $env{PATH_TRANSLATED} = "$git_dir/$path"; - my %rdr = ( 0 => fileno($in), 1 => fileno($wpipe) ); - my $pid = spawn([qw(git http-backend)], \%env, \%rdr); - unless (defined $pid) { - err($env, "error spawning: $! - going static"); - return; - } - $wpipe = $in = undef; - my $fh; + my %rdr = ( 0 => fileno($in) ); + my $x = PublicInbox::Qspawn->new([qw(git http-backend)], \%env, \%rdr); + my ($fh, $rpipe); my $end = sub { $rpipe = undef; - my $e = $pid == waitpid($pid, 0) ? - $? : "PID:$pid still running?"; - if ($e) { - err($env, "git http-backend ($git_dir): $e"); + if (my $err = $x->finish) { + err($env, "git http-backend ($git_dir): $err"); drop_client($env); } $fh->close if $fh; # async-only @@ -248,11 +236,15 @@ sub serve_smart { # holding the input here is a waste of FDs and memory $env->{'psgi.input'} = undef; - if ($async) { - $async = $async->($rpipe, $cb, $end); - } else { # generic PSGI - $cb->() while $rd_hdr; - } + $x->start(sub { # may run later, much later... + ($rpipe) = @_; + $in = undef; + if ($async) { + $async = $async->($rpipe, $cb, $end); + } else { # generic PSGI + $cb->() while $rd_hdr; + } + }); }; } diff --git a/lib/PublicInbox/Qspawn.pm b/lib/PublicInbox/Qspawn.pm new file mode 100644 index 00000000..9e4c8e08 --- /dev/null +++ b/lib/PublicInbox/Qspawn.pm @@ -0,0 +1,52 @@ +# Copyright (C) 2016 all contributors <meta@public-inbox.org> +# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt> +package PublicInbox::Qspawn; +use strict; +use warnings; +use PublicInbox::Spawn qw(popen_rd); +our $LIMIT = 1; +my $running = 0; +my @run_queue; + +sub new ($$$;) { + my ($class, $cmd, $env, $opt) = @_; + bless { args => [ $cmd, $env, $opt ] }, $class; +} + +sub _do_spawn { + my ($self, $cb) = @_; + my $err; + ($self->{rpipe}, $self->{pid}) = popen_rd(@{$self->{args}}); + if ($self->{pid}) { + $running++; + } else { + $self->{err} = $!; + } + $cb->($self->{rpipe}); +} + +sub finish ($) { + my ($self) = @_; + if (delete $self->{rpipe}) { + my $pid = delete $self->{pid}; + $self->{err} = $pid == waitpid($pid, 0) ? $? : + "PID:$pid still running?"; + $running--; + } + if (my $next = shift @run_queue) { + _do_spawn(@$next); + } + $self->{err}; +} + +sub start ($$) { + my ($self, $cb) = @_; + + if ($running < $LIMIT) { + _do_spawn($self, $cb); + } else { + push @run_queue, [ $self, $cb ]; + } +} + +1; |