diff options
| -rw-r--r-- | MANIFEST | 2 | ||||
| -rw-r--r-- | lib/PublicInbox/CompressNoop.pm | 22 | ||||
| -rw-r--r-- | lib/PublicInbox/GzipFilter.pm | 75 | ||||
| -rw-r--r-- | lib/PublicInbox/Mbox.pm | 90 | ||||
| -rw-r--r-- | lib/PublicInbox/MboxGz.pm | 71 | ||||
| -rw-r--r-- | lib/PublicInbox/NoopFilter.pm | 24 | ||||
| -rw-r--r-- | lib/PublicInbox/WwwAtomStream.pm | 98 | ||||
| -rw-r--r-- | lib/PublicInbox/WwwListing.pm | 3 | ||||
| -rw-r--r-- | lib/PublicInbox/WwwStatic.pm | 3 |
9 files changed, 148 insertions, 240 deletions
@@ -100,6 +100,7 @@ lib/PublicInbox/Admin.pm lib/PublicInbox/AdminEdit.pm lib/PublicInbox/AltId.pm lib/PublicInbox/Cgit.pm +lib/PublicInbox/CompressNoop.pm lib/PublicInbox/Config.pm lib/PublicInbox/ContentHash.pm lib/PublicInbox/DS.pm @@ -159,7 +160,6 @@ lib/PublicInbox/NNTP.pm lib/PublicInbox/NNTPD.pm lib/PublicInbox/NNTPdeflate.pm lib/PublicInbox/NewsWWW.pm -lib/PublicInbox/NoopFilter.pm lib/PublicInbox/Over.pm lib/PublicInbox/OverIdx.pm lib/PublicInbox/ParentPipe.pm diff --git a/lib/PublicInbox/CompressNoop.pm b/lib/PublicInbox/CompressNoop.pm new file mode 100644 index 00000000..fe73c2d1 --- /dev/null +++ b/lib/PublicInbox/CompressNoop.pm @@ -0,0 +1,22 @@ +# Copyright (C) 2020 all contributors <meta@public-inbox.org> +# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt> + +# Provide the same methods as Compress::Raw::Zlib::Deflate but +# does no transformation of outgoing data +package PublicInbox::CompressNoop; +use strict; +use Compress::Raw::Zlib qw(Z_OK); + +sub new { bless \(my $self), __PACKAGE__ } + +sub deflate { # ($self, $input, $output) + $_[2] .= $_[1]; + Z_OK; +} + +sub flush { # ($self, $output, $flags = Z_FINISH) + $_[1] //= ''; + Z_OK; +} + +1; diff --git a/lib/PublicInbox/GzipFilter.pm b/lib/PublicInbox/GzipFilter.pm index 0a6c56a5..99d43cf0 100644 --- a/lib/PublicInbox/GzipFilter.pm +++ b/lib/PublicInbox/GzipFilter.pm @@ -6,6 +6,10 @@ package PublicInbox::GzipFilter; use strict; use parent qw(Exporter); use Compress::Raw::Zlib qw(Z_FINISH Z_OK); +use PublicInbox::CompressNoop; +use PublicInbox::Eml; +use PublicInbox::GitAsyncCat; + our @EXPORT_OK = qw(gzf_maybe); my %OPT = (-WindowBits => 15 + 16, -AppendOutput => 1); my @GZIP_HDRS = qw(Vary Accept-Encoding Content-Encoding gzip); @@ -14,22 +18,41 @@ sub new { bless {}, shift } # for Qspawn if using $env->{'pi-httpd.async'} sub attach { - my ($self, $fh) = @_; - $self->{fh} = $fh; + my ($self, $http_out) = @_; + $self->{http_out} = $http_out; $self } -# returns `0' and not `undef' on failure (see Www*Stream) -sub gzf_maybe ($$) { +sub gz_or_noop { my ($res_hdr, $env) = @_; - return 0 if (($env->{HTTP_ACCEPT_ENCODING}) // '') !~ /\bgzip\b/; - my ($gz, $err) = Compress::Raw::Zlib::Deflate->new(%OPT); - return 0 if $err != Z_OK; + if (($env->{HTTP_ACCEPT_ENCODING} // '') =~ /\bgzip\b/) { + $env->{'plack.skip-deflater'} = 1; + push @$res_hdr, @GZIP_HDRS; + gzip_or_die(); + } else { + PublicInbox::CompressNoop::new(); + } +} - # in case Plack::Middleware::Deflater is loaded: - $env->{'plack.skip-deflater'} = 1; - push @$res_hdr, @GZIP_HDRS; - bless { gz => $gz }, __PACKAGE__; +sub gzf_maybe ($$) { bless { gz => gz_or_noop(@_) }, __PACKAGE__ } + +sub psgi_response { + my ($self, $code, $res_hdr, $next_cb, $eml_cb) = @_; + my $env = $self->{env}; + $self->{gz} //= gz_or_noop($res_hdr, $env); + if ($env->{'pi-httpd.async'}) { + $self->{async_next} = $next_cb; + $self->{async_eml} = $eml_cb; + my $http = $env->{'psgix.io'}; # PublicInbox::HTTP + $http->{forward} = $self; + sub { + my ($wcb) = @_; # -httpd provided write callback + $self->{http_out} = $wcb->([$code, $res_hdr]); + $next_cb->($http); # start stepping + }; + } else { # generic PSGI code path + [ $code, $res_hdr, $self ]; + } } sub qsp_maybe ($$) { @@ -80,7 +103,7 @@ sub translate ($$) { sub write { # my $ret = bytes::length($_[1]); # XXX does anybody care? - $_[0]->{fh}->write(translate($_[0], $_[1])); + $_[0]->{http_out}->write(translate($_[0], $_[1])); } # similar to ->translate; use this when we're sure we know we have @@ -109,9 +132,31 @@ sub zflush ($;$) { sub close { my ($self) = @_; - my $fh = delete $self->{fh}; - $fh->write(zflush($self)); - $fh->close; + if (my $http_out = delete $self->{http_out}) { + $http_out->write(zflush($self)); + $http_out->close; + } +} + +# this is public-inbox-httpd-specific +sub async_blob_cb { # git->cat_async callback + my ($bref, $oid, $type, $size, $self) = @_; + my $http = $self->{env}->{'psgix.io'} or return; # client abort + my $smsg = $self->{smsg} or die 'BUG: no smsg'; + if (!defined($oid)) { + # it's possible to have TOCTOU if an admin runs + # public-inbox-(edit|purge), just move onto the next message + return $http->next_step($self->{async_next}); + } + $smsg->{blob} eq $oid or die "BUG: $smsg->{blob} != $oid"; + $self->{async_eml}->($self, PublicInbox::Eml->new($bref)); + $http->next_step($self->{async_next}); +} + +sub smsg_blob { + my ($self, $smsg) = @_; + git_async_cat($self->{-inbox}->git, $smsg->{blob}, + \&async_blob_cb, $self); } 1; diff --git a/lib/PublicInbox/Mbox.pm b/lib/PublicInbox/Mbox.pm index 895f828c..abdf43c9 100644 --- a/lib/PublicInbox/Mbox.pm +++ b/lib/PublicInbox/Mbox.pm @@ -9,13 +9,11 @@ # more common "push" model) package PublicInbox::Mbox; use strict; -use warnings; +use parent 'PublicInbox::GzipFilter'; use PublicInbox::MID qw/mid_escape/; use PublicInbox::Hval qw/to_filename/; use PublicInbox::Smsg; use PublicInbox::Eml; -use PublicInbox::GitAsyncCat; -use PublicInbox::GzipFilter qw(gzf_maybe); # called by PSGI server as body response # this gets called twice for every message, once to return the header, @@ -25,49 +23,35 @@ sub getline { my $smsg = $ctx->{smsg} or return; my $ibx = $ctx->{-inbox}; my $eml = $ibx->smsg_eml($smsg) or return; - $ctx->{smsg} = $ibx->over->next_by_mid($ctx->{mid}, @{$ctx->{id_prev}}); - msg_hdr($ctx, $eml, $smsg->{mid}) . msg_body($eml); -} - -sub close { !!delete($_[0]->{http_out}) } - -sub mbox_async_step ($) { # public-inbox-httpd-only - my ($ctx) = @_; - if (my $smsg = $ctx->{smsg}) { - git_async_cat($ctx->{-inbox}->git, $smsg->{blob}, - \&mbox_blob_cb, $ctx); - } elsif (my $out = delete $ctx->{http_out}) { - $out->close; + my $n = $ctx->{smsg} = $ibx->over->next_by_mid(@{$ctx->{next_arg}}); + $ctx->zmore(msg_hdr($ctx, $eml, $smsg->{mid})); + if ($n) { + $ctx->translate(msg_body($eml)); + } else { # last message + $ctx->zmore(msg_body($eml)); + $ctx->zflush; } } # called by PublicInbox::DS::write -sub mbox_async_next { +sub async_next { my ($http) = @_; # PublicInbox::HTTP my $ctx = $http->{forward} or return; # client aborted eval { - $ctx->{smsg} = $ctx->{-inbox}->over->next_by_mid( - $ctx->{mid}, @{$ctx->{id_prev}}); - mbox_async_step($ctx); + my $smsg = $ctx->{smsg} or return $ctx->close; + $ctx->smsg_blob($smsg); }; + warn "E: $@" if $@; } -# this is public-inbox-httpd-specific -sub mbox_blob_cb { # git->cat_async callback - my ($bref, $oid, $type, $size, $ctx) = @_; - my $http = $ctx->{env}->{'psgix.io'} or return; # client abort - my $smsg = delete $ctx->{smsg} or die 'BUG: no smsg'; - if (!defined($oid)) { - # it's possible to have TOCTOU if an admin runs - # public-inbox-(edit|purge), just move onto the next message - return $http->next_step(\&mbox_async_next); - } else { - $smsg->{blob} eq $oid or die "BUG: $smsg->{blob} != $oid"; - } - my $eml = PublicInbox::Eml->new($bref); - $ctx->{http_out}->write(msg_hdr($ctx, $eml, $smsg->{mid})); - $ctx->{http_out}->write(msg_body($eml)); - $http->next_step(\&mbox_async_next); +sub async_eml { # ->{async_eml} for async_blob_cb + my ($ctx, $eml) = @_; + my $smsg = delete $ctx->{smsg}; + # next message + $ctx->{smsg} = $ctx->{-inbox}->over->next_by_mid(@{$ctx->{next_arg}}); + + $ctx->zmore(msg_hdr($ctx, $eml, $smsg->{mid})); + $ctx->{http_out}->write($ctx->translate(msg_body($eml))); } sub res_hdr ($$) { @@ -97,41 +81,17 @@ sub no_over_raw ($) { [ msg_hdr($ctx, $eml, $ctx->{mid}) . msg_body($eml) ] ] } -sub stream_raw { # MboxGz response callback - my ($ctx) = @_; - delete($ctx->{smsg}) // - $ctx->{-inbox}->over->next_by_mid($ctx->{mid}, - @{$ctx->{id_prev}}); -} - # /$INBOX/$MESSAGE_ID/raw sub emit_raw { my ($ctx) = @_; - my $env = $ctx->{env}; - $ctx->{base_url} = $ctx->{-inbox}->base_url($env); + $ctx->{base_url} = $ctx->{-inbox}->base_url($ctx->{env}); my $over = $ctx->{-inbox}->over or return no_over_raw($ctx); my ($id, $prev); - my $smsg = $over->next_by_mid($ctx->{mid}, \$id, \$prev) or return; - $ctx->{smsg} = $smsg; + my $mip = $ctx->{next_arg} = [ $ctx->{mid}, \$id, \$prev ]; + my $smsg = $ctx->{smsg} = $over->next_by_mid(@$mip) or return; my $res_hdr = res_hdr($ctx, $smsg->{subject}); - $ctx->{id_prev} = [ \$id, \$prev ]; - - if (my $gzf = gzf_maybe($res_hdr, $env)) { - $ctx->{gz} = delete $gzf->{gz}; - require PublicInbox::MboxGz; - PublicInbox::MboxGz::response($ctx, \&stream_raw, $res_hdr); - } elsif ($env->{'pi-httpd.async'}) { - sub { - my ($wcb) = @_; # -httpd provided write callback - $ctx->{http_out} = $wcb->([200, $res_hdr]); - $ctx->{env}->{'psgix.io'}->{forward} = $ctx; - bless $ctx, __PACKAGE__; - mbox_async_step($ctx); # start stepping - }; - } else { # generic PSGI code path - bless $ctx, __PACKAGE__; # respond to ->getline - [ 200, $res_hdr, $ctx ]; - } + bless $ctx, __PACKAGE__; + $ctx->psgi_response(200, $res_hdr, \&async_next, \&async_eml); } sub msg_hdr ($$;$) { diff --git a/lib/PublicInbox/MboxGz.pm b/lib/PublicInbox/MboxGz.pm index 716bf7b1..fdd16f68 100644 --- a/lib/PublicInbox/MboxGz.pm +++ b/lib/PublicInbox/MboxGz.pm @@ -6,77 +6,32 @@ use parent 'PublicInbox::GzipFilter'; use PublicInbox::Eml; use PublicInbox::Hval qw/to_filename/; use PublicInbox::Mbox; -use PublicInbox::GitAsyncCat; *msg_hdr = \&PublicInbox::Mbox::msg_hdr; *msg_body = \&PublicInbox::Mbox::msg_body; -# this is public-inbox-httpd-specific -sub mboxgz_blob_cb { # git->cat_async callback - my ($bref, $oid, $type, $size, $self) = @_; - my $http = $self->{env}->{'psgix.io'} or return; # client abort - my $smsg = delete $self->{smsg} or die 'BUG: no smsg'; - if (!defined($oid)) { - # it's possible to have TOCTOU if an admin runs - # public-inbox-(edit|purge), just move onto the next message - return $http->next_step(\&mboxgz_async_next); - } else { - $smsg->{blob} eq $oid or die "BUG: $smsg->{blob} != $oid"; - } - my $eml = PublicInbox::Eml->new($bref); - $self->zmore(msg_hdr($self, $eml, $smsg->{mid})); - - # PublicInbox::HTTP::{Chunked,Identity}::write - $self->{http_out}->write($self->translate(msg_body($eml))); - - $http->next_step(\&mboxgz_async_next); -} - -# this is public-inbox-httpd-specific -sub mboxgz_async_step ($) { - my ($self) = @_; - if (my $smsg = $self->{smsg} = $self->{cb}->($self)) { - git_async_cat($self->{-inbox}->git, $smsg->{blob}, - \&mboxgz_blob_cb, $self); - } elsif (my $out = delete $self->{http_out}) { - $out->write($self->zflush); - $out->close; - } -} - -# called by PublicInbox::DS::write -sub mboxgz_async_next { +sub async_next ($) { my ($http) = @_; # PublicInbox::HTTP - mboxgz_async_step($http->{forward}); -} - -# called by PublicInbox::HTTP::close, or any other PSGI server -sub close { !!delete($_[0]->{http_out}) } - -sub response { - my ($self, $cb, $res_hdr) = @_; - $self->{cb} = $cb; - bless $self, __PACKAGE__; - if ($self->{env}->{'pi-httpd.async'}) { - sub { - my ($wcb) = @_; # -httpd provided write callback - $self->{http_out} = $wcb->([200, $res_hdr]); - $self->{env}->{'psgix.io'}->{forward} = $self; - mboxgz_async_step($self); # start stepping - }; - } else { # generic PSGI - [ 200, $res_hdr, $self ]; - } + my $ctx = $http->{forward} or return; + eval { + $ctx->{smsg} = $ctx->{cb}->($ctx) or return $ctx->close; + $ctx->smsg_blob($ctx->{smsg}); + }; + warn "E: $@" if $@; } sub mbox_gz { my ($self, $cb, $fn) = @_; + $self->{cb} = $cb; $self->{base_url} = $self->{-inbox}->base_url($self->{env}); $self->{gz} = PublicInbox::GzipFilter::gzip_or_die(); $fn = to_filename($fn // 'no-subject'); $fn = 'no-subject' if $fn eq ''; # http://www.iana.org/assignments/media-types/application/gzip - response($self, $cb, [ qw(Content-Type application/gzip), - 'Content-Disposition', "inline; filename=$fn.mbox.gz" ]); + bless $self, __PACKAGE__; + my $res_hdr = [ 'Content-Type' => 'application/gzip', + 'Content-Disposition' => "inline; filename=$fn.mbox.gz" ]; + $self->psgi_response(200, $res_hdr, \&async_next, + \&PublicInbox::Mbox::async_eml); } # called by Plack::Util::foreach or similar (generic PSGI) diff --git a/lib/PublicInbox/NoopFilter.pm b/lib/PublicInbox/NoopFilter.pm deleted file mode 100644 index a97dbde6..00000000 --- a/lib/PublicInbox/NoopFilter.pm +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright (C) 2020 all contributors <meta@public-inbox.org> -# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt> - -package PublicInbox::NoopFilter; -use strict; - -sub new { bless \(my $self = ''), __PACKAGE__ } - -# noop workalike for PublicInbox::GzipFilter methods -sub translate { - my $self = $_[0]; - my $ret = $$self .= ($_[1] // ''); - $$self = ''; - $ret; -} - -sub zmore { - ${$_[0]} .= $_[1]; - undef; -} - -sub zflush { translate($_[0], $_[1]) } - -1; diff --git a/lib/PublicInbox/WwwAtomStream.pm b/lib/PublicInbox/WwwAtomStream.pm index 58330922..073df1df 100644 --- a/lib/PublicInbox/WwwAtomStream.pm +++ b/lib/PublicInbox/WwwAtomStream.pm @@ -7,107 +7,59 @@ # more common "push" model) package PublicInbox::WwwAtomStream; use strict; -use warnings; +use parent 'PublicInbox::GzipFilter'; use POSIX qw(strftime); use Digest::SHA qw(sha1_hex); use PublicInbox::Address; use PublicInbox::Hval qw(ascii_html mid_href); use PublicInbox::MsgTime qw(msg_timestamp); -use PublicInbox::GzipFilter qw(gzf_maybe); -use PublicInbox::GitAsyncCat; - -# called by generic PSGI server after getline, -# and also by PublicInbox::HTTP::close -sub close { !!delete($_[0]->{http_out}) } sub new { my ($class, $ctx, $cb) = @_; $ctx->{feed_base_url} = $ctx->{-inbox}->base_url($ctx->{env}); - $ctx->{cb} = $cb || \&close; + $ctx->{cb} = $cb || \&PublicInbox::GzipFilter::close; $ctx->{emit_header} = 1; bless $ctx, $class; } -# called by PublicInbox::DS::write -sub atom_async_next { +sub async_next ($) { my ($http) = @_; # PublicInbox::HTTP - atom_async_step($http->{forward}); -} - -# this is public-inbox-httpd-specific -sub atom_blob_cb { # git->cat_async callback - my ($bref, $oid, $type, $size, $ctx) = @_; - my $http = $ctx->{env}->{'psgix.io'} or return; # client abort - my $smsg = delete $ctx->{smsg} or die 'BUG: no smsg'; - if (!defined($oid)) { - # it's possible to have TOCTOU if an admin runs - # public-inbox-(edit|purge), just move onto the next message - return $http->next_step(\&atom_async_next); - } else { - $smsg->{blob} eq $oid or die "BUG: $smsg->{blob} != $oid"; - } - my $buf = feed_entry($ctx, $smsg, PublicInbox::Eml->new($bref)); - if (my $gzf = $ctx->{gzf}) { - $buf = $gzf->translate($buf); - } - # PublicInbox::HTTP::{Chunked,Identity}::write - $ctx->{http_out}->write($buf); - - $http->next_step(\&atom_async_next); + my $ctx = $http->{forward} or return; + eval { + if (my $smsg = $ctx->{smsg} = $ctx->{cb}->($ctx)) { + $ctx->smsg_blob($smsg); + } else { + $ctx->{http_out}->write($ctx->translate('</feed>')); + $ctx->close; + } + }; + warn "E: $@" if $@; } -sub atom_async_step { # this is public-inbox-httpd-specific - my ($ctx) = @_; - if (my $smsg = $ctx->{smsg} = $ctx->{cb}->($ctx)) { - git_async_cat($ctx->{-inbox}->git, $smsg->{blob}, - \&atom_blob_cb, $ctx); - } elsif (my $out = delete $ctx->{http_out}) { - if (my $gzf = delete $ctx->{gzf}) { - $out->write($gzf->zflush); - } - $out->close; - } +sub async_eml { # ->{async_eml} for async_blob_cb + my ($ctx, $eml) = @_; + my $smsg = delete $ctx->{smsg}; + $ctx->{http_out}->write($ctx->translate(feed_entry($ctx, $smsg, $eml))) } sub response { my ($class, $ctx, $code, $cb) = @_; my $res_hdr = [ 'Content-Type' => 'application/atom+xml' ]; $class->new($ctx, $cb); - $ctx->{gzf} = gzf_maybe($res_hdr, $ctx->{env}); - if ($ctx->{env}->{'pi-httpd.async'}) { - sub { - my ($wcb) = @_; # -httpd provided write callback - $ctx->{http_out} = $wcb->([200, $res_hdr]); - $ctx->{env}->{'psgix.io'}->{forward} = $ctx; - atom_async_step($ctx); # start stepping - }; - } else { - [ $code, $res_hdr, $ctx ]; - } + $ctx->psgi_response($code, $res_hdr, \&async_next, \&async_eml); } # called once for each message by PSGI server sub getline { my ($self) = @_; - my $buf = do { - if (my $middle = $self->{cb}) { - if (my $smsg = $middle->($self)) { - my $eml = $self->{-inbox}->smsg_eml($smsg) or - return ''; - feed_entry($self, $smsg, $eml); - } else { - undef; - } - } - } // (delete($self->{cb}) ? '</feed>' : undef); - - # gzf may be GzipFilter, `undef' or `0' - my $gzf = $self->{gzf} or return $buf; - - return $gzf->translate($buf) if defined $buf; - $self->{gzf} = 0; # next call to ->getline returns $buf (== undef) - $gzf->translate(undef); + my $cb = $self->{cb} or return; + while (my $smsg = $cb->($self)) { + my $eml = $self->{-inbox}->smsg_eml($smsg) or next; + return $self->translate(feed_entry($self, $smsg, $eml)); + } + delete $self->{cb}; + $self->zflush('</feed>'); } # private diff --git a/lib/PublicInbox/WwwListing.pm b/lib/PublicInbox/WwwListing.pm index d641e6d5..5f85e346 100644 --- a/lib/PublicInbox/WwwListing.pm +++ b/lib/PublicInbox/WwwListing.pm @@ -10,7 +10,6 @@ use PublicInbox::Hval qw(ascii_html prurl); use PublicInbox::Linkify; use PublicInbox::View; use PublicInbox::Inbox; -use PublicInbox::NoopFilter; use PublicInbox::GzipFilter qw(gzf_maybe); use bytes (); # bytes::length use HTTP::Date qw(time2str); @@ -108,7 +107,7 @@ sub html ($$) { my ($env, $list) = @_; my $h = [ 'Content-Type', 'text/html; charset=UTF-8', 'Content-Length', undef ]; - my $gzf = gzf_maybe($h, $env) || PublicInbox::NoopFilter::new(); + my $gzf = gzf_maybe($h, $env); $gzf->zmore('<html><head><title>' . 'public-inbox listing</title>' . '</head><body><pre>'); diff --git a/lib/PublicInbox/WwwStatic.pm b/lib/PublicInbox/WwwStatic.pm index d0611949..051d2e03 100644 --- a/lib/PublicInbox/WwwStatic.pm +++ b/lib/PublicInbox/WwwStatic.pm @@ -17,7 +17,6 @@ use HTTP::Date qw(time2str); use HTTP::Status qw(status_message); use Errno qw(EACCES ENOTDIR ENOENT); use URI::Escape qw(uri_escape_utf8); -use PublicInbox::NoopFilter; use PublicInbox::GzipFilter qw(gzf_maybe); use PublicInbox::Hval qw(ascii_html); use Plack::MIME; @@ -313,7 +312,7 @@ sub dir_response ($$$) { my $path_info_html = ascii_html($path_info); my $h = [qw(Content-Type text/html Content-Length), undef]; - my $gzf = gzf_maybe($h, $env) || PublicInbox::NoopFilter::new(); + my $gzf = gzf_maybe($h, $env); $gzf->zmore("<html><head><title>Index of $path_info_html</title>" . ${$self->{style}} . "</head><body><pre>Index of $path_info_html</pre><hr><pre>\n"); |