From 23af251dd607c4e75ab1e68063f2c885c48cc035 Mon Sep 17 00:00:00 2001 From: Eric Wong Date: Sat, 23 Jul 2022 04:41:55 +0000 Subject: imap+nntp: share COMPRESS implementation Their code was nearly identical to begin with, so save some memory in -netd and disk space for all of our tarball/distro users, at least. And I seem to have used multiple inheritance successfully, here, maybe... --- lib/PublicInbox/DS.pm | 4 +- lib/PublicInbox/DSdeflate.pm | 134 ++++++++++++++++++++++++++++++++++++++++ lib/PublicInbox/IMAP.pm | 10 ++- lib/PublicInbox/IMAPD.pm | 2 +- lib/PublicInbox/IMAPdeflate.pm | 126 ------------------------------------- lib/PublicInbox/NNTP.pm | 10 ++- lib/PublicInbox/NNTPD.pm | 2 +- lib/PublicInbox/NNTPdeflate.pm | 137 ----------------------------------------- 8 files changed, 152 insertions(+), 273 deletions(-) create mode 100644 lib/PublicInbox/DSdeflate.pm delete mode 100644 lib/PublicInbox/IMAPdeflate.pm delete mode 100644 lib/PublicInbox/NNTPdeflate.pm (limited to 'lib/PublicInbox') diff --git a/lib/PublicInbox/DS.pm b/lib/PublicInbox/DS.pm index fee31e3d..ef483aac 100644 --- a/lib/PublicInbox/DS.pm +++ b/lib/PublicInbox/DS.pm @@ -648,8 +648,8 @@ sub shutdn ($) { } } -sub zflush {} # overridden by NNTPdeflate and IMAPdeflate - +sub zflush {} # overridden by DSdeflate +sub compressed {} # overridden by DSdeflate sub long_response_done {} # overridden by Net::NNTP sub long_step { diff --git a/lib/PublicInbox/DSdeflate.pm b/lib/PublicInbox/DSdeflate.pm new file mode 100644 index 00000000..b5208e43 --- /dev/null +++ b/lib/PublicInbox/DSdeflate.pm @@ -0,0 +1,134 @@ +# Copyright (C) all contributors +# License: AGPL-3.0+ + +# RFC 8054 NNTP COMPRESS DEFLATE implementation +# RFC 4978 IMAP COMPRESS=DEFLATE extension +# +# RSS usage for 10K idle-but-did-something NNTP clients on 64-bit: +# TLS + DEFLATE[a] : 1.8 GB (MemLevel=9, 1.2 GB with MemLevel=8) +# TLS + DEFLATE[b] : ~300MB +# TLS only : <200MB +# plain : <50MB +# +# [a] - initial implementation using per-client Deflate contexts and buffer +# +# [b] - memory-optimized implementation using a global deflate context. +# It's less efficient in terms of compression, but way more +# efficient in terms of server memory usage. +package PublicInbox::DSdeflate; +use strict; +use v5.10.1; +use Compress::Raw::Zlib; + +my %IN_OPT = ( + -Bufsize => 1024, + -WindowBits => -15, # RFC 1951 + -AppendOutput => 1, +); + +# global deflate context and buffer +my $zbuf = \(my $buf = ''); +my $zout; +{ + my $err; + ($zout, $err) = Compress::Raw::Zlib::Deflate->new( + # nnrpd (INN) and Compress::Raw::Zlib favor MemLevel=9, + # the zlib C library and git use MemLevel=8 as the default + # -MemLevel => 9, + -Bufsize => 65536, # same as nnrpd + -WindowBits => -15, # RFC 1951 + -AppendOutput => 1, + ); + $err == Z_OK or die "Failed to initialize zlib deflate stream: $err"; +} + +sub enable { + my ($class, $self) = @_; + my ($in, $err) = Compress::Raw::Zlib::Inflate->new(%IN_OPT); + if ($err != Z_OK) { + $self->err("Inflate->new failed: $err"); + return; + } + bless $self, $class; + $self->{zin} = $in; +} + +# overrides PublicInbox::DS::compressed +sub compressed { 1 } + +sub do_read ($$$$) { + my ($self, $rbuf, $len, $off) = @_; + + my $zin = $self->{zin} or return; # closed + my $doff; + my $dbuf = delete($self->{dbuf}) // ''; + $doff = length($dbuf); + my $r = PublicInbox::DS::do_read($self, \$dbuf, $len, $doff) or return; + + # Workaround inflate bug appending to OOK scalars: + # + # We only have $off if the client is pipelining, and pipelining + # is where our substr() OOK optimization in event_step makes sense. + if ($off) { + my $copy = $$rbuf; + undef $$rbuf; + $$rbuf = $copy; + } + + # assert(length($$rbuf) == $off) as far as NNTP.pm is concerned + # -ConsumeInput is true, so $dbuf is automatically emptied + my $err = $zin->inflate($dbuf, $rbuf); + if ($err == Z_OK) { + $self->{dbuf} = $dbuf if $dbuf ne ''; + $r = length($$rbuf) and return $r; + # nothing ready, yet, get more, later + $self->requeue; + } else { + delete $self->{zin}; + $self->close; + } + 0; +} + +# override PublicInbox::DS::msg_more +sub msg_more ($$) { + my $self = $_[0]; + + # $_[1] may be a reference or not for ->deflate + my $err = $zout->deflate($_[1], $zbuf); + $err == Z_OK or die "->deflate failed $err"; + 1; +} + +sub zflush ($) { + my ($self) = @_; + + my $deflated = $zbuf; + $zbuf = \(my $next = ''); + + my $err = $zout->flush($deflated, Z_FULL_FLUSH); + $err == Z_OK or die "->flush failed $err"; + + # We can still let the lower socket layer do buffering: + PublicInbox::DS::msg_more($self, $$deflated); +} + +# compatible with PublicInbox::DS::write, so $_[1] may be a reference or not +sub write ($$) { + my $self = $_[0]; + return PublicInbox::DS::write($self, $_[1]) if ref($_[1]) eq 'CODE'; + + my $deflated = $zbuf; + $zbuf = \(my $next = ''); + + # $_[1] may be a reference or not for ->deflate + my $err = $zout->deflate($_[1], $deflated); + $err == Z_OK or die "->deflate failed $err"; + $err = $zout->flush($deflated, Z_FULL_FLUSH); + $err == Z_OK or die "->flush failed $err"; + + # We can still let the socket layer do buffering: + PublicInbox::DS::write($self, $deflated); +} + +1; diff --git a/lib/PublicInbox/IMAP.pm b/lib/PublicInbox/IMAP.pm index ce0dce0f..805f1102 100644 --- a/lib/PublicInbox/IMAP.pm +++ b/lib/PublicInbox/IMAP.pm @@ -1212,8 +1212,6 @@ sub event_step { $self->requeue unless $pending; } -sub compressed { undef } - # RFC 4978 sub cmd_compress ($$$) { my ($self, $tag, $alg) = @_; @@ -1223,7 +1221,9 @@ sub cmd_compress ($$$) { # CRIME made TLS compression obsolete # return "$tag NO [COMPRESSIONACTIVE]\r\n" if $self->tls_compressed; - PublicInbox::IMAPdeflate->enable($self, $tag); + PublicInbox::IMAPdeflate->enable($self) or return + \"$tag BAD failed to activate compression\r\n"; + PublicInbox::DS::write($self, \"$tag OK DEFLATE active\r\n"); $self->requeue; undef } @@ -1269,4 +1269,8 @@ our @ISA = qw(PublicInbox::IMAP); sub logged_in { 0 } +package PublicInbox::IMAPdeflate; +use PublicInbox::DSdeflate; +our @ISA = qw(PublicInbox::DSdeflate PublicInbox::IMAP); + 1; diff --git a/lib/PublicInbox/IMAPD.pm b/lib/PublicInbox/IMAPD.pm index d8814324..b24097a2 100644 --- a/lib/PublicInbox/IMAPD.pm +++ b/lib/PublicInbox/IMAPD.pm @@ -9,7 +9,7 @@ use v5.10.1; use PublicInbox::Config; use PublicInbox::ConfigIter; use PublicInbox::InboxIdle; -use PublicInbox::IMAPdeflate; # loads PublicInbox::IMAP +use PublicInbox::IMAP; use PublicInbox::DummyInbox; my $dummy = bless { uidvalidity => 0 }, 'PublicInbox::DummyInbox'; diff --git a/lib/PublicInbox/IMAPdeflate.pm b/lib/PublicInbox/IMAPdeflate.pm deleted file mode 100644 index d5929ef2..00000000 --- a/lib/PublicInbox/IMAPdeflate.pm +++ /dev/null @@ -1,126 +0,0 @@ -# Copyright (C) 2020-2021 all contributors -# License: AGPL-3.0+ -# TODO: reduce duplication from PublicInbox::NNTPdeflate - -# RFC 4978 -package PublicInbox::IMAPdeflate; -use strict; -use warnings; -use 5.010_001; -use base qw(PublicInbox::IMAP); -use Compress::Raw::Zlib; - -my %IN_OPT = ( - -Bufsize => 1024, - -WindowBits => -15, # RFC 1951 - -AppendOutput => 1, -); - -# global deflate context and buffer -my $zbuf = \(my $buf = ''); -my $zout; -{ - my $err; - ($zout, $err) = Compress::Raw::Zlib::Deflate->new( - # nnrpd (INN) and Compress::Raw::Zlib favor MemLevel=9, - # the zlib C library and git use MemLevel=8 as the default - # -MemLevel => 9, - -Bufsize => 65536, # same as nnrpd - -WindowBits => -15, # RFC 1951 - -AppendOutput => 1, - ); - $err == Z_OK or die "Failed to initialize zlib deflate stream: $err"; -} - -sub enable { - my ($class, $self, $tag) = @_; - my ($in, $err) = Compress::Raw::Zlib::Inflate->new(%IN_OPT); - if ($err != Z_OK) { - $self->err("Inflate->new failed: $err"); - $self->write(\"$tag BAD failed to activate compression\r\n"); - return; - } - $self->write(\"$tag OK DEFLATE active\r\n"); - bless $self, $class; - $self->{zin} = $in; -} - -# overrides PublicInbox::NNTP::compressed -sub compressed { 1 } - -sub do_read ($$$$) { - my ($self, $rbuf, $len, $off) = @_; - - my $zin = $self->{zin} or return; # closed - my $doff; - my $dbuf = delete($self->{dbuf}) // ''; - $doff = length($dbuf); - my $r = PublicInbox::DS::do_read($self, \$dbuf, $len, $doff) or return; - - # Workaround inflate bug appending to OOK scalars: - # - # We only have $off if the client is pipelining, and pipelining - # is where our substr() OOK optimization in event_step makes sense. - if ($off) { - my $copy = $$rbuf; - undef $$rbuf; - $$rbuf = $copy; - } - - # assert(length($$rbuf) == $off) as far as NNTP.pm is concerned - # -ConsumeInput is true, so $dbuf is automatically emptied - my $err = $zin->inflate($dbuf, $rbuf); - if ($err == Z_OK) { - $self->{dbuf} = $dbuf if $dbuf ne ''; - $r = length($$rbuf) and return $r; - # nothing ready, yet, get more, later - $self->requeue; - } else { - delete $self->{zin}; - $self->close; - } - 0; -} - -# override PublicInbox::DS::msg_more -sub msg_more ($$) { - my $self = $_[0]; - - # $_[1] may be a reference or not for ->deflate - my $err = $zout->deflate($_[1], $zbuf); - $err == Z_OK or die "->deflate failed $err"; - 1; -} - -sub zflush ($) { - my ($self) = @_; - - my $deflated = $zbuf; - $zbuf = \(my $next = ''); - - my $err = $zout->flush($deflated, Z_FULL_FLUSH); - $err == Z_OK or die "->flush failed $err"; - - # We can still let the lower socket layer do buffering: - PublicInbox::DS::msg_more($self, $$deflated); -} - -# compatible with PublicInbox::DS::write, so $_[1] may be a reference or not -sub write ($$) { - my $self = $_[0]; - return PublicInbox::DS::write($self, $_[1]) if ref($_[1]) eq 'CODE'; - - my $deflated = $zbuf; - $zbuf = \(my $next = ''); - - # $_[1] may be a reference or not for ->deflate - my $err = $zout->deflate($_[1], $deflated); - $err == Z_OK or die "->deflate failed $err"; - $err = $zout->flush($deflated, Z_FULL_FLUSH); - $err == Z_OK or die "->flush failed $err"; - - # We can still let the socket layer do buffering: - PublicInbox::DS::write($self, $deflated); -} - -1; diff --git a/lib/PublicInbox/NNTP.pm b/lib/PublicInbox/NNTP.pm index 3929f817..8ad7adc1 100644 --- a/lib/PublicInbox/NNTP.pm +++ b/lib/PublicInbox/NNTP.pm @@ -883,8 +883,6 @@ sub cmd_xover ($;$) { $self->long_response(\&xover_i, @$r); } -sub compressed { undef } - sub cmd_starttls ($) { my ($self) = @_; my $sock = $self->{sock} or return; @@ -903,7 +901,9 @@ sub cmd_compress ($$) { my ($self, $alg) = @_; return "503 Only DEFLATE is supported\r\n" if uc($alg) ne 'DEFLATE'; return r502 if $self->compressed; - PublicInbox::NNTPdeflate->enable($self); + PublicInbox::NNTPdeflate->enable($self) or return + \"403 Unable to activate compression\r\n"; + PublicInbox::DS::write($self, \"206 Compression active\r\n"); $self->requeue; undef } @@ -985,4 +985,8 @@ sub busy { # for graceful shutdown in PublicInbox::Daemon: defined($self->{rbuf}) || defined($self->{wbuf}) } +package PublicInbox::NNTPdeflate; +use PublicInbox::DSdeflate; +our @ISA = qw(PublicInbox::DSdeflate PublicInbox::NNTP); + 1; diff --git a/lib/PublicInbox/NNTPD.pm b/lib/PublicInbox/NNTPD.pm index 6e79f0be..f31d4381 100644 --- a/lib/PublicInbox/NNTPD.pm +++ b/lib/PublicInbox/NNTPD.pm @@ -9,7 +9,7 @@ use v5.10.1; use Sys::Hostname; use PublicInbox::Config; use PublicInbox::InboxIdle; -use PublicInbox::NNTPdeflate; # loads PublicInbox::NNTP +use PublicInbox::NNTP; sub new { my ($class) = @_; diff --git a/lib/PublicInbox/NNTPdeflate.pm b/lib/PublicInbox/NNTPdeflate.pm deleted file mode 100644 index 352d4842..00000000 --- a/lib/PublicInbox/NNTPdeflate.pm +++ /dev/null @@ -1,137 +0,0 @@ -# Copyright (C) all contributors -# License: AGPL-3.0+ - -# RFC 8054 NNTP COMPRESS DEFLATE implementation -# -# RSS usage for 10K idle-but-did-something NNTP clients on 64-bit: -# TLS + DEFLATE[a] : 1.8 GB (MemLevel=9, 1.2 GB with MemLevel=8) -# TLS + DEFLATE[b] : ~300MB -# TLS only : <200MB -# plain : <50MB -# -# [a] - initial implementation using per-client Deflate contexts and buffer -# -# [b] - memory-optimized implementation using a global deflate context. -# It's less efficient in terms of compression, but way more -# efficient in terms of server memory usage. -package PublicInbox::NNTPdeflate; -use strict; -use 5.010_001; -use parent qw(PublicInbox::NNTP); -use Compress::Raw::Zlib; - -my %IN_OPT = ( - -Bufsize => PublicInbox::NNTP::LINE_MAX, - -WindowBits => -15, # RFC 1951 - -AppendOutput => 1, -); - -# global deflate context and buffer -my $zbuf = \(my $buf = ''); -my $zout; -{ - my $err; - ($zout, $err) = Compress::Raw::Zlib::Deflate->new( - # nnrpd (INN) and Compress::Raw::Zlib favor MemLevel=9, - # the zlib C library and git use MemLevel=8 as the default - # -MemLevel => 9, - -Bufsize => 65536, # same as nnrpd - -WindowBits => -15, # RFC 1951 - -AppendOutput => 1, - ); - $err == Z_OK or die "Failed to initialize zlib deflate stream: $err"; -} - - -sub enable { - my ($class, $self) = @_; - my ($in, $err) = Compress::Raw::Zlib::Inflate->new(%IN_OPT); - if ($err != Z_OK) { - $self->err("Inflate->new failed: $err"); - $self->write(\"403 Unable to activate compression\r\n"); - return; - } - $self->write(\"206 Compression active\r\n"); - bless $self, $class; - $self->{zin} = $in; -} - -# overrides PublicInbox::NNTP::compressed -sub compressed { 1 } - -sub do_read ($$$$) { - my ($self, $rbuf, $len, $off) = @_; - - my $zin = $self->{zin} or return; # closed - my $doff; - my $dbuf = delete($self->{dbuf}) // ''; - $doff = length($dbuf); - my $r = PublicInbox::DS::do_read($self, \$dbuf, $len, $doff) or return; - - # Workaround inflate bug appending to OOK scalars: - # - # We only have $off if the client is pipelining, and pipelining - # is where our substr() OOK optimization in event_step makes sense. - if ($off) { - my $copy = $$rbuf; - undef $$rbuf; - $$rbuf = $copy; - } - - # assert(length($$rbuf) == $off) as far as NNTP.pm is concerned - # -ConsumeInput is true, so $dbuf is automatically emptied - my $err = $zin->inflate($dbuf, $rbuf); - if ($err == Z_OK) { - $self->{dbuf} = $dbuf if $dbuf ne ''; - $r = length($$rbuf) and return $r; - # nothing ready, yet, get more, later - $self->requeue; - } else { - delete $self->{zin}; - $self->close; - } - 0; -} - -# override PublicInbox::DS::msg_more -sub msg_more ($$) { - my $self = $_[0]; - - # $_[1] may be a reference or not for ->deflate - my $err = $zout->deflate($_[1], $zbuf); - $err == Z_OK or die "->deflate failed $err"; - 1; -} - -sub zflush ($) { - my ($self) = @_; - - my $deflated = $zbuf; - $zbuf = \(my $next = ''); - - my $err = $zout->flush($deflated, Z_FULL_FLUSH); - $err == Z_OK or die "->flush failed $err"; - - # We can still let the lower socket layer do buffering: - PublicInbox::DS::msg_more($self, $$deflated); -} - -# compatible with PublicInbox::DS::write, so $_[1] may be a reference or not -sub write ($$) { - my $self = $_[0]; - return PublicInbox::DS::write($self, $_[1]) if ref($_[1]) eq 'CODE'; - - my $deflated = $zbuf; - $zbuf = \(my $next = ''); - - # $_[1] may be a reference or not for ->deflate - my $err = $zout->deflate($_[1], $deflated); - $err == Z_OK or die "->deflate failed $err"; - $err = $zout->flush($deflated, Z_FULL_FLUSH); - $err == Z_OK or die "->flush failed $err"; - - # We can still let the socket layer do buffering: - PublicInbox::DS::write($self, $deflated); -} - -1; -- cgit v1.2.3-24-ge0c7