* [PATCH 06/14] qspawn: implement psgi_qx
2019-01-27 4:03 7% [PATCH 00/14] convert solver to use pi-httpd.async Eric Wong
@ 2019-01-27 4:03 4% ` Eric Wong
0 siblings, 0 replies; 2+ results
From: Eric Wong @ 2019-01-27 4:03 UTC (permalink / raw)
To: meta
This new asynchronous API, will allow us to take
advantage of non-blocking I/O from even small commands;
as those may still need to wait for slow operations.
---
lib/PublicInbox/Qspawn.pm | 89 ++++++++++++++++++++++++++++++++-------
1 file changed, 74 insertions(+), 15 deletions(-)
diff --git a/lib/PublicInbox/Qspawn.pm b/lib/PublicInbox/Qspawn.pm
index 96fbf38..6859a8a 100644
--- a/lib/PublicInbox/Qspawn.pm
+++ b/lib/PublicInbox/Qspawn.pm
@@ -61,6 +61,48 @@ sub start {
}
}
+sub _psgi_finish ($$) {
+ my ($self, $env) = @_;
+ my $err = $self->finish;
+ if ($err && !$env->{'qspawn.quiet'}) {
+ $err = join(' ', @{$self->{args}->[0]}).": $err\n";
+ $env->{'psgi.errors'}->print($err);
+ }
+}
+
+sub psgi_qx {
+ my ($self, $env, $limiter, $qx_cb) = @_;
+ my $qx = PublicInbox::Qspawn::Qx->new;
+ my $end = sub {
+ _psgi_finish($self, $env);
+ eval { $qx_cb->($qx) };
+ $qx = undef;
+ };
+ my $rpipe;
+ my $async = $env->{'pi-httpd.async'};
+ my $cb = sub {
+ my $r = sysread($rpipe, my $buf, 8192);
+ if ($async) {
+ $async->async_pass($env->{'psgix.io'}, $qx, \$buf);
+ } elsif (defined $r) {
+ $r ? $qx->write($buf) : $end->();
+ } else {
+ return if $!{EAGAIN} || $!{EINTR}; # loop again
+ $end->();
+ }
+ };
+ $limiter ||= $def_limiter ||= PublicInbox::Qspawn::Limiter->new(32);
+ $self->start($limiter, sub { # may run later, much later...
+ ($rpipe) = @_;
+ if ($async) {
+ # PublicInbox::HTTPD::Async->new($rpipe, $cb, $end)
+ $async = $async->($rpipe, $cb, $end);
+ } else { # generic PSGI
+ $cb->() while $qx;
+ }
+ });
+}
+
# create a filter for "push"-based streaming PSGI writes used by HTTPD::Async
sub filter_fh ($$) {
my ($fh, $filter) = @_;
@@ -78,11 +120,7 @@ sub psgi_return {
my ($self, $env, $limiter, $parse_hdr) = @_;
my ($fh, $rpipe);
my $end = sub {
- my $err = $self->finish;
- if ($err && !$env->{'qspawn.quiet'}) {
- $err = join(' ', @{$self->{args}->[0]}).": $err\n";
- $env->{'psgi.errors'}->print($err);
- }
+ _psgi_finish($self, $env);
$fh->close if $fh; # async-only
};
@@ -92,7 +130,7 @@ sub psgi_return {
return if !defined($r) && ($!{EINTR} || $!{EAGAIN});
$parse_hdr->($r, \$buf);
};
- my $res;
+ my $res = delete $env->{'qspawn.response'};
my $async = $env->{'pi-httpd.async'};
my $cb = sub {
my $r = $rd_hdr->() or return;
@@ -118,17 +156,21 @@ sub psgi_return {
}
};
$limiter ||= $def_limiter ||= PublicInbox::Qspawn::Limiter->new(32);
+ my $start_cb = sub { # may run later, much later...
+ ($rpipe) = @_;
+ if ($async) {
+ # PublicInbox::HTTPD::Async->new($rpipe, $cb, $end)
+ $async = $async->($rpipe, $cb, $end);
+ } else { # generic PSGI
+ $cb->() while $rd_hdr;
+ }
+ };
+
+ return $self->start($limiter, $start_cb) if $res;
+
sub {
($res) = @_;
- $self->start($limiter, sub { # may run later, much later...
- ($rpipe) = @_;
- if ($async) {
- # PublicInbox::HTTPD::Async->new($rpipe, $cb, $end)
- $async = $async->($rpipe, $cb, $end);
- } else { # generic PSGI
- $cb->() while $rd_hdr;
- }
- });
+ $self->start($limiter, $start_cb);
};
}
@@ -146,4 +188,21 @@ sub new {
}, $class;
}
+# captures everything into a buffer and executes a callback when done
+package PublicInbox::Qspawn::Qx;
+use strict;
+use warnings;
+
+sub new {
+ my ($class) = @_;
+ my $buf = '';
+ bless \$buf, $class;
+}
+
+# called by PublicInbox::HTTPD::Async ($fh->write)
+sub write {
+ ${$_[0]} .= $_[1];
+ undef;
+}
+
1;
--
EW
^ permalink raw reply related [relevance 4%]
* [PATCH 00/14] convert solver to use pi-httpd.async
@ 2019-01-27 4:03 7% Eric Wong
2019-01-27 4:03 4% ` [PATCH 06/14] qspawn: implement psgi_qx Eric Wong
0 siblings, 1 reply; 2+ results
From: Eric Wong @ 2019-01-27 4:03 UTC (permalink / raw)
To: meta
Much of the groundwork for this was laid in the now-abandoned
"repobrowse" branch. The goal was to improves fairness as we no
longer wait synchronously on git (apply|update-index|ls-files)
processes and can requests for other clients.
The end result was slightly (2-3%?) slower with all the
callbacks, but reducing "git apply" invocations by relying on
pathnames (instead of stdin) made the end result ~20% faster for
a large (64) patch series.
Email::Simple (via Email::MIME/PublicInbox::MIME) remains a
performance bottleneck, as it does a lot of unnecessary header
parsing and hash-table populating we don't care about; but I'm
not sure if I'll have time to address that.
Eric Wong (14):
httpd/async: remove needless sysread wrapper
qspawn: implement psgi_return and use it for githttpbackend
qspawn|getlinebody: support streaming filters
qspawn|httpd/async: improve and fix out-of-date comments
httpd/async: stop running command if client disconnects
qspawn: implement psgi_qx
t/qspawn.t: psgi_qx stderr test
view: swap CRLF for LF in HTML output
solver: rewrite to use Qspawn->psgi_qx and pi-httpd.async
solver: hold patches in temporary directory
solver: reduce "git apply" invocations
qspawn: decode $? for user-friendliness
viewvcs: do not show final error message twice
solver: crank up max patches to 9999
lib/PublicInbox/GetlineBody.pm | 16 +-
lib/PublicInbox/Git.pm | 2 +-
lib/PublicInbox/GitHTTPBackend.pm | 64 +---
lib/PublicInbox/HTTPD/Async.pm | 27 +-
lib/PublicInbox/Qspawn.pm | 143 +++++++-
lib/PublicInbox/SolverGit.pm | 532 +++++++++++++++++-------------
lib/PublicInbox/View.pm | 4 +
lib/PublicInbox/ViewVCS.pm | 50 ++-
t/qspawn.t | 12 +-
t/solver_git.t | 22 +-
10 files changed, 543 insertions(+), 329 deletions(-)
--
EW
^ permalink raw reply [relevance 7%]
Results 1-2 of 2 | reverse | options above
-- pct% links below jump to the message on this page, permalinks otherwise --
2019-01-27 4:03 7% [PATCH 00/14] convert solver to use pi-httpd.async Eric Wong
2019-01-27 4:03 4% ` [PATCH 06/14] qspawn: implement psgi_qx Eric Wong
Code repositories for project(s) associated with this public inbox
https://80x24.org/public-inbox.git
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).