From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on dcvr.yhbt.net X-Spam-Level: X-Spam-Status: No, score=-4.0 required=3.0 tests=ALL_TRUSTED,BAYES_00 shortcircuit=no autolearn=ham autolearn_force=no version=3.4.2 Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id 0E1701F94F for ; Sun, 5 Jul 2020 23:28:15 +0000 (UTC) From: Eric Wong To: meta@public-inbox.org Subject: [PATCH 37/43] www: update internal docs Date: Sun, 5 Jul 2020 23:27:53 +0000 Message-Id: <20200705232759.3161-38-e@yhbt.net> In-Reply-To: <20200705232759.3161-1-e@yhbt.net> References: <20200705232759.3161-1-e@yhbt.net> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit List-Id: We no longer favor getline+close for streaming PSGI responses when using public-inbox-httpd. We still support it for other PSGI servers, though. --- Documentation/technical/ds.txt | 4 ++-- lib/PublicInbox/GetlineBody.pm | 4 +--- lib/PublicInbox/GzipFilter.pm | 17 +++++++++++++---- lib/PublicInbox/HTTPD.pm | 5 ++--- lib/PublicInbox/Mbox.pm | 8 ++------ lib/PublicInbox/View.pm | 2 +- lib/PublicInbox/WwwAtomStream.pm | 6 ++---- lib/PublicInbox/WwwStream.pm | 7 +++---- 8 files changed, 26 insertions(+), 27 deletions(-) diff --git a/Documentation/technical/ds.txt b/Documentation/technical/ds.txt index cbd06cfb4..a0793ca23 100644 --- a/Documentation/technical/ds.txt +++ b/Documentation/technical/ds.txt @@ -64,8 +64,8 @@ Augmented features: * ->requeue support. An optimization of the AddTimer(0, ...) idiom for immediately dispatching code at the next event loop iteration. public-inbox uses this for fairly generating large responses - iteratively (see PublicInbox::NNTP::long_response or the use of - ->getline callbacks for generating gigantic gzipped mboxes). + iteratively (see PublicInbox::NNTP::long_response or git_async_cat + for blob retrievals). New features diff --git a/lib/PublicInbox/GetlineBody.pm b/lib/PublicInbox/GetlineBody.pm index 6becaaf5f..988bc63f4 100644 --- a/lib/PublicInbox/GetlineBody.pm +++ b/lib/PublicInbox/GetlineBody.pm @@ -5,9 +5,7 @@ # end callback when the object goes out-of-scope. # This depends on rpipe being _blocking_ on getline. # -# public-inbox-httpd favors "getline" response bodies to take a -# "pull"-based approach to feeding slow clients (as opposed to a -# more common "push" model) +# This is only used by generic PSGI servers and not public-inbox-httpd package PublicInbox::GetlineBody; use strict; use warnings; diff --git a/lib/PublicInbox/GzipFilter.pm b/lib/PublicInbox/GzipFilter.pm index 6380f50e9..d72ad3c88 100644 --- a/lib/PublicInbox/GzipFilter.pm +++ b/lib/PublicInbox/GzipFilter.pm @@ -1,7 +1,16 @@ # Copyright (C) 2020 all contributors # License: AGPL-3.0+ - -# Qspawn filter +# +# In public-inbox <=1.5.0, public-inbox-httpd favored "getline" +# response bodies to take a "pull"-based approach to feeding +# slow clients (as opposed to a more common "push" model). +# +# In newer versions, public-inbox-httpd supports a backpressure-aware +# pull/push model which also accounts for slow git blob storage. +# {async_next} callbacks only run when the DS {wbuf} is drained +# {async_eml} callbacks only run when a blob arrives from git. +# +# We continue to support getline+close for generic PSGI servers. package PublicInbox::GzipFilter; use strict; use parent qw(Exporter); @@ -14,12 +23,12 @@ our @EXPORT_OK = qw(gzf_maybe); my %OPT = (-WindowBits => 15 + 16, -AppendOutput => 1); my @GZIP_HDRS = qw(Vary Accept-Encoding Content-Encoding gzip); -sub new { bless {}, shift } +sub new { bless {}, shift } # qspawn filter # for Qspawn if using $env->{'pi-httpd.async'} sub attach { my ($self, $http_out) = @_; - $self->{http_out} = $http_out; + $self->{http_out} = $http_out; # PublicInbox::HTTP::{Chunked,Identity} $self } diff --git a/lib/PublicInbox/HTTPD.pm b/lib/PublicInbox/HTTPD.pm index 331939699..a9f55ff61 100644 --- a/lib/PublicInbox/HTTPD.pm +++ b/lib/PublicInbox/HTTPD.pm @@ -36,9 +36,8 @@ sub new { # XXX unstable API!, only GitHTTPBackend needs # this to limit git-http-backend(1) parallelism. - # The rest of our PSGI code is generic, relying - # on "pull" model using "getline" to prevent - # over-buffering. + # We also check for the truthiness of this to + # detect when to use git_async_cat for slow blobs 'pi-httpd.async' => \&pi_httpd_async ); bless { diff --git a/lib/PublicInbox/Mbox.pm b/lib/PublicInbox/Mbox.pm index abdf43c93..8726b9f64 100644 --- a/lib/PublicInbox/Mbox.pm +++ b/lib/PublicInbox/Mbox.pm @@ -1,12 +1,8 @@ # Copyright (C) 2015-2020 all contributors # License: AGPL-3.0+ -# Streaming (via getline) interface for formatting messages as an mboxrd. -# Used by the PSGI web interface. -# -# public-inbox-httpd favors "getline" response bodies to take a -# "pull"-based approach to feeding slow clients (as opposed to a -# more common "push" model) +# Streaming interface for mboxrd HTTP responses +# See PublicInbox::GzipFilter for details. package PublicInbox::Mbox; use strict; use parent 'PublicInbox::GzipFilter'; diff --git a/lib/PublicInbox/View.pm b/lib/PublicInbox/View.pm index 895e4f278..60dad6bac 100644 --- a/lib/PublicInbox/View.pm +++ b/lib/PublicInbox/View.pm @@ -415,7 +415,7 @@ sub stream_thread ($$) { PublicInbox::WwwStream::aresponse($ctx, 200, \&stream_thread_i); } -# /$INBOX/$MESSAGE_ID/t/ +# /$INBOX/$MSGID/t/ and /$INBOX/$MSGID/T/ sub thread_html { my ($ctx) = @_; my $mid = $ctx->{mid}; diff --git a/lib/PublicInbox/WwwAtomStream.pm b/lib/PublicInbox/WwwAtomStream.pm index 073df1dfa..3b5b133a5 100644 --- a/lib/PublicInbox/WwwAtomStream.pm +++ b/lib/PublicInbox/WwwAtomStream.pm @@ -1,10 +1,8 @@ # Copyright (C) 2016-2020 all contributors # License: AGPL-3.0+ # -# Atom body stream for which yields getline+close methods -# public-inbox-httpd favors "getline" response bodies to take a -# "pull"-based approach to feeding slow clients (as opposed to a -# more common "push" model) +# Atom body stream for HTTP responses +# See PublicInbox::GzipFilter for details. package PublicInbox::WwwAtomStream; use strict; use parent 'PublicInbox::GzipFilter'; diff --git a/lib/PublicInbox/WwwStream.pm b/lib/PublicInbox/WwwStream.pm index 7d257a191..23b03f0e8 100644 --- a/lib/PublicInbox/WwwStream.pm +++ b/lib/PublicInbox/WwwStream.pm @@ -1,11 +1,10 @@ # Copyright (C) 2016-2020 all contributors # License: AGPL-3.0+ # -# HTML body stream for which yields getline+close methods +# HTML body stream for which yields getline+close methods for +# generic PSGI servers and callbacks for public-inbox-httpd. # -# public-inbox-httpd favors "getline" response bodies to take a -# "pull"-based approach to feeding slow clients (as opposed to a -# more common "push" model) +# See PublicInbox::GzipFilter parent class for more info. package PublicInbox::WwwStream; use strict; use parent qw(Exporter PublicInbox::GzipFilter);