* [PATCH 37/43] www: update internal docs
2020-07-05 23:27 7% [PATCH 00/43] www: async git cat-file w/ -httpd Eric Wong
@ 2020-07-05 23:27 6% ` Eric Wong
0 siblings, 0 replies; 2+ results
From: Eric Wong @ 2020-07-05 23:27 UTC (permalink / raw)
To: meta
We no longer favor getline+close for streaming PSGI responses
when using public-inbox-httpd. We still support it for other
PSGI servers, though.
---
Documentation/technical/ds.txt | 4 ++--
lib/PublicInbox/GetlineBody.pm | 4 +---
lib/PublicInbox/GzipFilter.pm | 17 +++++++++++++----
lib/PublicInbox/HTTPD.pm | 5 ++---
lib/PublicInbox/Mbox.pm | 8 ++------
lib/PublicInbox/View.pm | 2 +-
lib/PublicInbox/WwwAtomStream.pm | 6 ++----
lib/PublicInbox/WwwStream.pm | 7 +++----
8 files changed, 26 insertions(+), 27 deletions(-)
diff --git a/Documentation/technical/ds.txt b/Documentation/technical/ds.txt
index cbd06cfb4..a0793ca23 100644
--- a/Documentation/technical/ds.txt
+++ b/Documentation/technical/ds.txt
@@ -64,8 +64,8 @@ Augmented features:
* ->requeue support. An optimization of the AddTimer(0, ...) idiom
for immediately dispatching code at the next event loop iteration.
public-inbox uses this for fairly generating large responses
- iteratively (see PublicInbox::NNTP::long_response or the use of
- ->getline callbacks for generating gigantic gzipped mboxes).
+ iteratively (see PublicInbox::NNTP::long_response or git_async_cat
+ for blob retrievals).
New features
diff --git a/lib/PublicInbox/GetlineBody.pm b/lib/PublicInbox/GetlineBody.pm
index 6becaaf5f..988bc63f4 100644
--- a/lib/PublicInbox/GetlineBody.pm
+++ b/lib/PublicInbox/GetlineBody.pm
@@ -5,9 +5,7 @@
# end callback when the object goes out-of-scope.
# This depends on rpipe being _blocking_ on getline.
#
-# public-inbox-httpd favors "getline" response bodies to take a
-# "pull"-based approach to feeding slow clients (as opposed to a
-# more common "push" model)
+# This is only used by generic PSGI servers and not public-inbox-httpd
package PublicInbox::GetlineBody;
use strict;
use warnings;
diff --git a/lib/PublicInbox/GzipFilter.pm b/lib/PublicInbox/GzipFilter.pm
index 6380f50e9..d72ad3c88 100644
--- a/lib/PublicInbox/GzipFilter.pm
+++ b/lib/PublicInbox/GzipFilter.pm
@@ -1,7 +1,16 @@
# Copyright (C) 2020 all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
-
-# Qspawn filter
+#
+# In public-inbox <=1.5.0, public-inbox-httpd favored "getline"
+# response bodies to take a "pull"-based approach to feeding
+# slow clients (as opposed to a more common "push" model).
+#
+# In newer versions, public-inbox-httpd supports a backpressure-aware
+# pull/push model which also accounts for slow git blob storage.
+# {async_next} callbacks only run when the DS {wbuf} is drained
+# {async_eml} callbacks only run when a blob arrives from git.
+#
+# We continue to support getline+close for generic PSGI servers.
package PublicInbox::GzipFilter;
use strict;
use parent qw(Exporter);
@@ -14,12 +23,12 @@ our @EXPORT_OK = qw(gzf_maybe);
my %OPT = (-WindowBits => 15 + 16, -AppendOutput => 1);
my @GZIP_HDRS = qw(Vary Accept-Encoding Content-Encoding gzip);
-sub new { bless {}, shift }
+sub new { bless {}, shift } # qspawn filter
# for Qspawn if using $env->{'pi-httpd.async'}
sub attach {
my ($self, $http_out) = @_;
- $self->{http_out} = $http_out;
+ $self->{http_out} = $http_out; # PublicInbox::HTTP::{Chunked,Identity}
$self
}
diff --git a/lib/PublicInbox/HTTPD.pm b/lib/PublicInbox/HTTPD.pm
index 331939699..a9f55ff61 100644
--- a/lib/PublicInbox/HTTPD.pm
+++ b/lib/PublicInbox/HTTPD.pm
@@ -36,9 +36,8 @@ sub new {
# XXX unstable API!, only GitHTTPBackend needs
# this to limit git-http-backend(1) parallelism.
- # The rest of our PSGI code is generic, relying
- # on "pull" model using "getline" to prevent
- # over-buffering.
+ # We also check for the truthiness of this to
+ # detect when to use git_async_cat for slow blobs
'pi-httpd.async' => \&pi_httpd_async
);
bless {
diff --git a/lib/PublicInbox/Mbox.pm b/lib/PublicInbox/Mbox.pm
index abdf43c93..8726b9f64 100644
--- a/lib/PublicInbox/Mbox.pm
+++ b/lib/PublicInbox/Mbox.pm
@@ -1,12 +1,8 @@
# Copyright (C) 2015-2020 all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
-# Streaming (via getline) interface for formatting messages as an mboxrd.
-# Used by the PSGI web interface.
-#
-# public-inbox-httpd favors "getline" response bodies to take a
-# "pull"-based approach to feeding slow clients (as opposed to a
-# more common "push" model)
+# Streaming interface for mboxrd HTTP responses
+# See PublicInbox::GzipFilter for details.
package PublicInbox::Mbox;
use strict;
use parent 'PublicInbox::GzipFilter';
diff --git a/lib/PublicInbox/View.pm b/lib/PublicInbox/View.pm
index 895e4f278..60dad6bac 100644
--- a/lib/PublicInbox/View.pm
+++ b/lib/PublicInbox/View.pm
@@ -415,7 +415,7 @@ sub stream_thread ($$) {
PublicInbox::WwwStream::aresponse($ctx, 200, \&stream_thread_i);
}
-# /$INBOX/$MESSAGE_ID/t/
+# /$INBOX/$MSGID/t/ and /$INBOX/$MSGID/T/
sub thread_html {
my ($ctx) = @_;
my $mid = $ctx->{mid};
diff --git a/lib/PublicInbox/WwwAtomStream.pm b/lib/PublicInbox/WwwAtomStream.pm
index 073df1dfa..3b5b133a5 100644
--- a/lib/PublicInbox/WwwAtomStream.pm
+++ b/lib/PublicInbox/WwwAtomStream.pm
@@ -1,10 +1,8 @@
# Copyright (C) 2016-2020 all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
#
-# Atom body stream for which yields getline+close methods
-# public-inbox-httpd favors "getline" response bodies to take a
-# "pull"-based approach to feeding slow clients (as opposed to a
-# more common "push" model)
+# Atom body stream for HTTP responses
+# See PublicInbox::GzipFilter for details.
package PublicInbox::WwwAtomStream;
use strict;
use parent 'PublicInbox::GzipFilter';
diff --git a/lib/PublicInbox/WwwStream.pm b/lib/PublicInbox/WwwStream.pm
index 7d257a191..23b03f0e8 100644
--- a/lib/PublicInbox/WwwStream.pm
+++ b/lib/PublicInbox/WwwStream.pm
@@ -1,11 +1,10 @@
# Copyright (C) 2016-2020 all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
#
-# HTML body stream for which yields getline+close methods
+# HTML body stream for which yields getline+close methods for
+# generic PSGI servers and callbacks for public-inbox-httpd.
#
-# public-inbox-httpd favors "getline" response bodies to take a
-# "pull"-based approach to feeding slow clients (as opposed to a
-# more common "push" model)
+# See PublicInbox::GzipFilter parent class for more info.
package PublicInbox::WwwStream;
use strict;
use parent qw(Exporter PublicInbox::GzipFilter);
^ permalink raw reply related [relevance 6%]
* [PATCH 00/43] www: async git cat-file w/ -httpd
@ 2020-07-05 23:27 7% Eric Wong
2020-07-05 23:27 6% ` [PATCH 37/43] www: update internal docs Eric Wong
0 siblings, 1 reply; 2+ results
From: Eric Wong @ 2020-07-05 23:27 UTC (permalink / raw)
To: meta
This allows -httpd to make better use of time it spends waiting
on git-cat-file to respond. It allows us to deal with
high-latency HDD storage without a client monopolizing the event
loop. Even on a mid-range consumer-grade SSD, this seems to
give a 10+% speed improvement for HTTP responses requiring many
blobs, including all /T/, /t/, and /t.mbox.gz endpoints.
This only benefits indexed inboxes (both v1 and v2); I'm not
sure if anybody still uses unindexed v1 inboxes nowadays.
A new xt/httpd-async-stream.t maintainer test ensures checksums
for responses before and after this series match exactly as
before.
This builds off a branch I started several months ago (but never
published here) to integrate gzip responses into our codebase
and remove our optional dependency on Plack::Middleware::Deflater.
We already gzip a bunch of things independent of
Plack::Middleware::Deflater: manifest.js.gz, altid SQLite3 dumps
and all the *.mbox.gz endpoints; so being able to use gzip on
all of our responses without an extra dependency seemed logical.
Being able to consistently use our GzipFilter API to perform
buffering via ->zmore made it significantly easier to reason
about small response chunks for ghost messages interspersed with
large ones when streaming /$INBOX/$MSGID/t/ endpoints.
I'm not yet maximizing use of ->zmore for all buffering of HTTP
responses, yet; measurements need to happen, first. That may
happen in the 1.7 time frame. In particular, we would need to
ensure the Perl method dispatch and DSO overhead to Zlib.so and
libz.so of making many ->zmore calls doesn't cause performance
regressions compared to the current `.=' use and calling
->zmore/->translate fewer times.
Eric Wong (43):
gzipfilter: minor cleanups
wwwstream: oneshot: perform gzip without middleware
www*stream: gzip ->getline responses
wwwtext: gzip text/plain responses, as well
wwwtext: switch to html_oneshot
www: need: use WwwStream::html_oneshot
wwwlisting: use GzipFilter for HTML
gzipfilter: replace Compress::Raw::Deflate usages
{gzip,noop}filter: ->zmore returns undef, always
mbox: remove html_oneshot import
wwwstatic: support gzipped response directory listings
qspawn: learn to gzip streaming responses
stop auto-loading Plack::Middleware::Deflater
mboxgz: do asynchronous git blob retrievals
mboxgz: reduce object hash depth
mbox: async blob retrieval for "single message" raw mboxrd
wwwatomstream: simplify feed_update callers
wwwatomstream: use PublicInbox::Inbox->modified for feed_updated
wwwatomstream: reuse $ctx as $self
xt/httpd-async-stream: allow more options
wwwatomstream: support asynchronous blob retrievals
wwwstream: reduce object graph depth
wwwstream: reduce blob retrieval paths for ->getline
www: start making gzipfilter the parent response class
remove unused/redundant zlib-related imports
wwwstream: use parent.pm and no warnings
wwwstream: subclass off GzipFilter
view: wire up /$INBOX/$MESSAGE_ID/ permalink to async
view: /$INBOX/$MSGID/t/ reads blobs asynchronously
view: update /$INBOX/$MSGID/T/ to be async
feed: generate_i: eliminate pointless loop
feed: /$INBOX/new.html retrieves blobs asynchronously
ssearchview: /$INBOX/?q=$QUERY&x=t uses async blobs
view: eml_entry: reduce parameters
view: /$INBOX/$MSGID/t/: avoid extra hash lookup in eml case
wwwstream: eliminate ::response, use html_oneshot
www: update internal docs
view: simplify eml_entry callers further
wwwtext: simplify gzf_maybe use
wwwattach: support async blob retrievals
gzipfilter: drop HTTP connection on bugs or data corruption
daemon: warn on missing blobs
gzipfilter: check http->{forward} for client disconnects
Documentation/mknews.perl | 20 +--
Documentation/public-inbox-httpd.pod | 1 -
Documentation/technical/ds.txt | 4 +-
INSTALL | 5 -
MANIFEST | 2 +
ci/deps.perl | 1 -
examples/cgit.psgi | 8 -
examples/newswww.psgi | 8 -
examples/public-inbox.psgi | 9 --
examples/unsubscribe.psgi | 1 -
lib/PublicInbox/CompressNoop.pm | 22 +++
lib/PublicInbox/Feed.pm | 22 ++-
lib/PublicInbox/GetlineBody.pm | 4 +-
lib/PublicInbox/GzipFilter.pm | 168 +++++++++++++++++---
lib/PublicInbox/HTTP.pm | 7 +
lib/PublicInbox/HTTPD.pm | 5 +-
lib/PublicInbox/IMAP.pm | 1 +
lib/PublicInbox/Mbox.pm | 137 +++++++++--------
lib/PublicInbox/MboxGz.pm | 81 ++++------
lib/PublicInbox/NNTP.pm | 1 +
lib/PublicInbox/Qspawn.pm | 6 +-
lib/PublicInbox/SearchView.pm | 40 ++---
lib/PublicInbox/View.pm | 219 ++++++++++++++-------------
lib/PublicInbox/WWW.pm | 9 +-
lib/PublicInbox/WwwAtomStream.pm | 66 ++++----
lib/PublicInbox/WwwAttach.pm | 63 ++++++--
lib/PublicInbox/WwwListing.pm | 24 +--
lib/PublicInbox/WwwStatic.pm | 14 +-
lib/PublicInbox/WwwStream.pm | 110 ++++++++------
lib/PublicInbox/WwwText.pm | 26 ++--
script/public-inbox-httpd | 9 --
script/public-inbox.cgi | 7 -
t/httpd-corner.psgi | 7 +
t/httpd-corner.t | 9 +-
t/plack.t | 4 +
t/psgi_attach.t | 162 +++++++++++---------
t/psgi_text.t | 33 +++-
t/psgi_v2.t | 80 ++++++++--
t/www_listing.t | 8 +-
t/www_static.t | 11 +-
xt/httpd-async-stream.t | 104 +++++++++++++
41 files changed, 964 insertions(+), 554 deletions(-)
create mode 100644 lib/PublicInbox/CompressNoop.pm
create mode 100644 xt/httpd-async-stream.t
^ permalink raw reply [relevance 7%]
Results 1-2 of 2 | reverse | options above
-- pct% links below jump to the message on this page, permalinks otherwise --
2020-07-05 23:27 7% [PATCH 00/43] www: async git cat-file w/ -httpd Eric Wong
2020-07-05 23:27 6% ` [PATCH 37/43] www: update internal docs Eric Wong
Code repositories for project(s) associated with this public inbox
https://80x24.org/public-inbox.git
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).