about summary refs log tree commit homepage
path: root/lib/PublicInbox
diff options
context:
space:
mode:
authorEric Wong <e@80x24.org>2016-05-24 03:41:53 +0000
committerEric Wong <e@80x24.org>2016-05-24 04:12:51 +0000
commit64aea34d06f71828b0bdd6ae177b9bcf22d752b4 (patch)
tree4f7df75d112aa4fe0b9056cf24688a2e42032019 /lib/PublicInbox
parent8648f519a95872600689c3a5d6d87fd17770f9fc (diff)
downloadpublic-inbox-64aea34d06f71828b0bdd6ae177b9bcf22d752b4.tar.gz
Having an excessive amount of git-pack-objects processes is
dangerous to the health of the server.  Queue up process spawning
for long-running responses and serve them sequentially, instead.
Diffstat (limited to 'lib/PublicInbox')
-rw-r--r--lib/PublicInbox/GitHTTPBackend.pm38
-rw-r--r--lib/PublicInbox/Qspawn.pm52
2 files changed, 67 insertions, 23 deletions
diff --git a/lib/PublicInbox/GitHTTPBackend.pm b/lib/PublicInbox/GitHTTPBackend.pm
index ded56b33..9464cb49 100644
--- a/lib/PublicInbox/GitHTTPBackend.pm
+++ b/lib/PublicInbox/GitHTTPBackend.pm
@@ -8,9 +8,9 @@ use strict;
 use warnings;
 use Fcntl qw(:seek);
 use IO::File;
-use PublicInbox::Spawn qw(spawn);
 use HTTP::Date qw(time2str);
 use HTTP::Status qw(status_message);
+use PublicInbox::Qspawn;
 
 # n.b. serving "description" and "cloneurl" should be innocuous enough to
 # not cause problems.  serving "config" might...
@@ -167,11 +167,6 @@ sub serve_smart {
         unless (defined $fd && $fd >= 0) {
                 $in = input_to_file($env) or return r(500);
         }
-        my ($rpipe, $wpipe);
-        unless (pipe($rpipe, $wpipe)) {
-                err($env, "error creating pipe: $! - going static");
-                return;
-        }
         my %env = %ENV;
         # GIT_COMMITTER_NAME, GIT_COMMITTER_EMAIL
         # may be set in the server-process and are passed as-is
@@ -187,20 +182,13 @@ sub serve_smart {
         my $git_dir = $git->{git_dir};
         $env{GIT_HTTP_EXPORT_ALL} = '1';
         $env{PATH_TRANSLATED} = "$git_dir/$path";
-        my %rdr = ( 0 => fileno($in), 1 => fileno($wpipe) );
-        my $pid = spawn([qw(git http-backend)], \%env, \%rdr);
-        unless (defined $pid) {
-                err($env, "error spawning: $! - going static");
-                return;
-        }
-        $wpipe = $in = undef;
-        my $fh;
+        my %rdr = ( 0 => fileno($in) );
+        my $x = PublicInbox::Qspawn->new([qw(git http-backend)], \%env, \%rdr);
+        my ($fh, $rpipe);
         my $end = sub {
                 $rpipe = undef;
-                my $e = $pid == waitpid($pid, 0) ?
-                        $? : "PID:$pid still running?";
-                if ($e) {
-                        err($env, "git http-backend ($git_dir): $e");
+                if (my $err = $x->finish) {
+                        err($env, "git http-backend ($git_dir): $err");
                         drop_client($env);
                 }
                 $fh->close if $fh; # async-only
@@ -248,11 +236,15 @@ sub serve_smart {
                 # holding the input here is a waste of FDs and memory
                 $env->{'psgi.input'} = undef;
 
-                if ($async) {
-                        $async = $async->($rpipe, $cb, $end);
-                } else { # generic PSGI
-                        $cb->() while $rd_hdr;
-                }
+                $x->start(sub { # may run later, much later...
+                        ($rpipe) = @_;
+                        $in = undef;
+                        if ($async) {
+                                $async = $async->($rpipe, $cb, $end);
+                        } else { # generic PSGI
+                                $cb->() while $rd_hdr;
+                        }
+                });
         };
 }
 
diff --git a/lib/PublicInbox/Qspawn.pm b/lib/PublicInbox/Qspawn.pm
new file mode 100644
index 00000000..9e4c8e08
--- /dev/null
+++ b/lib/PublicInbox/Qspawn.pm
@@ -0,0 +1,52 @@
+# Copyright (C) 2016 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+package PublicInbox::Qspawn;
+use strict;
+use warnings;
+use PublicInbox::Spawn qw(popen_rd);
+our $LIMIT = 1;
+my $running = 0;
+my @run_queue;
+
+sub new ($$$;) {
+        my ($class, $cmd, $env, $opt) = @_;
+        bless { args => [ $cmd, $env, $opt ] }, $class;
+}
+
+sub _do_spawn {
+        my ($self, $cb) = @_;
+        my $err;
+        ($self->{rpipe}, $self->{pid}) = popen_rd(@{$self->{args}});
+        if ($self->{pid}) {
+                $running++;
+        } else {
+                $self->{err} = $!;
+        }
+        $cb->($self->{rpipe});
+}
+
+sub finish ($) {
+        my ($self) = @_;
+        if (delete $self->{rpipe}) {
+                my $pid = delete $self->{pid};
+                $self->{err} = $pid == waitpid($pid, 0) ? $? :
+                                "PID:$pid still running?";
+                $running--;
+        }
+        if (my $next = shift @run_queue) {
+                _do_spawn(@$next);
+        }
+        $self->{err};
+}
+
+sub start ($$) {
+        my ($self, $cb) = @_;
+
+        if ($running < $LIMIT) {
+                _do_spawn($self, $cb);
+        } else {
+                push @run_queue, [ $self, $cb ];
+        }
+}
+
+1;