From: Eric Wong <e@80x24.org>
To: meta@public-inbox.org
Subject: [PATCH 2/6] qspawn: documentation updates
Date: Thu, 31 Jan 2019 10:51:49 +0000 [thread overview]
Message-ID: <20190131105153.25272-3-e@80x24.org> (raw)
In-Reply-To: <20190131105153.25272-1-e@80x24.org>
This will become critical for future changes to display
git commits, diffs, and trees.
Use "qspawn.wcb" instead of "qspawn.response" to enhance
readability.
---
lib/PublicInbox/Qspawn.pm | 77 +++++++++++++++++++++++++++++++++-----
lib/PublicInbox/ViewVCS.pm | 2 +-
2 files changed, 68 insertions(+), 11 deletions(-)
diff --git a/lib/PublicInbox/Qspawn.pm b/lib/PublicInbox/Qspawn.pm
index 913fac8..509a441 100644
--- a/lib/PublicInbox/Qspawn.pm
+++ b/lib/PublicInbox/Qspawn.pm
@@ -1,17 +1,40 @@
-# Copyright (C) 2016-2018 all contributors <meta@public-inbox.org>
+# Copyright (C) 2016-2019 all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
-# Limits the number of processes spawned
+# Like most Perl modules in public-inbox, this is internal and
+# NOT subject to any stability guarantees! It is only documented
+# for other hackers.
+#
+# This is used to limit the number of processes spawned by the
+# PSGI server, so it acts like a semaphore and queues up extra
+# commands to be run if currently at the limit. Multiple "limiters"
+# may be configured which give inboxes different channels to
+# operate in. This can be useful to ensure smaller inboxes can
+# be cloned while cloning of large inboxes is maxed out.
+#
# This does not depend on Danga::Socket or any other external
-# scheduling mechanism, you just need to call start and finish
-# appropriately
+# scheduling mechanism, you just need to call start() and finish()
+# appropriately. However, public-inbox-httpd (which uses Danga::Socket)
+# will be able to schedule this based on readability of stdout from
+# the spawned process. See GitHTTPBackend.pm and SolverGit.pm for
+# usage examples. It does not depend on any form of threading.
+#
+# This is useful for scheduling CGI execution of both long-lived
+# git-http-backend(1) process (for "git clone") as well as short-lived
+# processes such as git-apply(1).
+
package PublicInbox::Qspawn;
use strict;
use warnings;
use PublicInbox::Spawn qw(popen_rd);
require Plack::Util;
+
my $def_limiter;
+# declares a command to spawn (but does not spawn it).
+# $cmd is the command to spawn
+# $env is the environ for the child process
+# $opt can include redirects and perhaps other process spawning options
sub new ($$$;) {
my ($class, $cmd, $env, $opt) = @_;
bless { args => [ $cmd, $env, $opt ] }, $class;
@@ -79,6 +102,10 @@ sub _psgi_finish ($$) {
}
}
+# Similar to `backtick` or "qx" ("perldoc -f qx"), it calls $qx_cb with
+# the stdout of the given command when done; but respects the given limiter
+# $env is the PSGI env. As with ``/qx; only use this when output is small
+# and safe to slurp.
sub psgi_qx {
my ($self, $env, $limiter, $qx_cb) = @_;
my $qx = PublicInbox::Qspawn::Qx->new;
@@ -125,6 +152,28 @@ sub filter_fh ($$) {
});
}
+# Used for streaming the stdout of one process as a PSGI response.
+#
+# $env is the PSGI env.
+# optional keys in $env:
+# $env->{'qspawn.wcb'} - the write callback from the PSGI server
+# optional, use this if you've already
+# captured it elsewhere. If not given,
+# psgi_return will return an anonymous
+# sub for the PSGI server to call
+#
+# $env->{'qspawn.filter'} - filter callback, receives a string as input,
+# undef on EOF
+#
+# $limiter - the Limiter object to use (uses the def_limiter if not given)
+#
+# $parse_hdr - Initial read function; often for parsing CGI header output.
+# It will be given the return value of sysread from the pipe
+# and a string ref of the current buffer. Returns an arrayref
+# for PSGI responses. 2-element arrays in PSGI mean the
+# body will be streamed, later, via writes (push-based) to
+# psgix.io. 3-element arrays means the body is available
+# immediately (or streamed via ->getline (pull-based)).
sub psgi_return {
my ($self, $env, $limiter, $parse_hdr) = @_;
my ($fh, $rpipe);
@@ -139,8 +188,10 @@ sub psgi_return {
return if !defined($r) && ($!{EINTR} || $!{EAGAIN});
$parse_hdr->($r, \$buf);
};
- my $res = delete $env->{'qspawn.response'};
+
+ my $wcb = delete $env->{'qspawn.wcb'};
my $async = $env->{'pi-httpd.async'};
+
my $cb = sub {
my $r = $rd_hdr->() or return;
$rd_hdr = undef;
@@ -152,16 +203,16 @@ sub psgi_return {
$rpipe->close;
$end->();
}
- $res->($r);
+ $wcb->($r);
} elsif ($async) {
- $fh = $res->($r); # scalar @$r == 2
+ $fh = $wcb->($r); # scalar @$r == 2
$fh = filter_fh($fh, $filter) if $filter;
$async->async_pass($env->{'psgix.io'}, $fh, \$buf);
} else { # for synchronous PSGI servers
require PublicInbox::GetlineBody;
$r->[2] = PublicInbox::GetlineBody->new($rpipe, $end,
$buf, $filter);
- $res->($r);
+ $wcb->($r);
}
};
$limiter ||= $def_limiter ||= PublicInbox::Qspawn::Limiter->new(32);
@@ -175,10 +226,16 @@ sub psgi_return {
}
};
- return $self->start($limiter, $start_cb) if $res;
+ # the caller already captured the PSGI write callback from
+ # the PSGI server, so we can call ->start, here:
+ return $self->start($limiter, $start_cb) if $wcb;
+ # the caller will return this sub to the PSGI server, so
+ # it can set the response callback (that is, for PublicInbox::HTTP,
+ # the chunked_wcb or identity_wcb callback), but other HTTP servers
+ # are supported:
sub {
- ($res) = @_;
+ ($wcb) = @_;
$self->start($limiter, $start_cb);
};
}
diff --git a/lib/PublicInbox/ViewVCS.pm b/lib/PublicInbox/ViewVCS.pm
index 63731e9..1e8c31f 100644
--- a/lib/PublicInbox/ViewVCS.pm
+++ b/lib/PublicInbox/ViewVCS.pm
@@ -54,7 +54,7 @@ sub stream_large_blob ($$$$) {
my $qsp = PublicInbox::Qspawn->new($cmd);
my @cl = ('Content-Length', $size);
my $env = $ctx->{env};
- $env->{'qspawn.response'} = delete $ctx->{-wcb};
+ $env->{'qspawn.wcb'} = delete $ctx->{-wcb};
$qsp->psgi_return($env, undef, sub {
my ($r, $bref) = @_;
if (!defined $r) { # error
--
EW
next prev parent reply other threads:[~2019-01-31 10:51 UTC|newest]
Thread overview: 7+ messages / expand[flat|nested] mbox.gz Atom feed top
2019-01-31 10:51 [PATCH 0/6] more minor cleanups and doc updates Eric Wong
2019-01-31 10:51 ` [PATCH 1/6] inbox: drop psgi.url_scheme requirement from base_url Eric Wong
2019-01-31 10:51 ` Eric Wong [this message]
2019-01-31 10:51 ` [PATCH 3/6] config: tiny cleanup to use _array() sub Eric Wong
2019-01-31 10:51 ` [PATCH 4/6] doc/config: user documentation for limiters Eric Wong
2019-01-31 10:51 ` [PATCH 5/6] doc/config: document "replyto" configuration knob Eric Wong
2019-01-31 10:51 ` [PATCH 6/6] doc: remove completed TODO items Eric Wong
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
List information: http://public-inbox.org/README
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20190131105153.25272-3-e@80x24.org \
--to=e@80x24.org \
--cc=meta@public-inbox.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
Code repositories for project(s) associated with this public inbox
https://80x24.org/public-inbox.git
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).