From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on dcvr.yhbt.net X-Spam-Level: X-Spam-ASN: X-Spam-Status: No, score=-4.2 required=3.0 tests=ALL_TRUSTED,BAYES_00, DKIM_SIGNED,DKIM_VALID,DKIM_VALID_AU,DKIM_VALID_EF shortcircuit=no autolearn=ham autolearn_force=no version=3.4.2 Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id CF8771F623 for ; Sat, 23 Jul 2022 04:41:57 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=80x24.org; s=selector1; t=1658551317; bh=XyTPEzHz0fJPxTmee4wcbaBRqiUp0Em/nnHxet268Ng=; h=From:To:Subject:Date:In-Reply-To:References:From; b=xu/EXOtbrQWgStSW5zwZSf9eEr3HMLWXOH58VhHbs2Vd96sBwd6TZcTsfUPfMsAR2 PxzguF7udiMJsTnE0FrfI/5f1hNbfxncb8g0tGvKR4hcqeSVuBZDN+yYDHGsdPTxYV uBsPdvM5hh0o3LPK6biTl5/YJOq7X78Y5ACHctCM= From: Eric Wong To: meta@public-inbox.org Subject: [PATCH 11/11] imap+nntp: share COMPRESS implementation Date: Sat, 23 Jul 2022 04:41:55 +0000 Message-Id: <20220723044155.3733949-12-e@80x24.org> In-Reply-To: <20220723044155.3733949-1-e@80x24.org> References: <20220723044155.3733949-1-e@80x24.org> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit List-Id: Their code was nearly identical to begin with, so save some memory in -netd and disk space for all of our tarball/distro users, at least. And I seem to have used multiple inheritance successfully, here, maybe... --- MANIFEST | 3 +- lib/PublicInbox/DS.pm | 4 +- .../{NNTPdeflate.pm => DSdeflate.pm} | 13 +- lib/PublicInbox/IMAP.pm | 10 +- lib/PublicInbox/IMAPD.pm | 2 +- lib/PublicInbox/IMAPdeflate.pm | 126 ------------------ lib/PublicInbox/NNTP.pm | 10 +- lib/PublicInbox/NNTPD.pm | 2 +- xt/mem-imapd-tls.t | 4 +- 9 files changed, 26 insertions(+), 148 deletions(-) rename lib/PublicInbox/{NNTPdeflate.pm => DSdeflate.pm} (92%) delete mode 100644 lib/PublicInbox/IMAPdeflate.pm diff --git a/MANIFEST b/MANIFEST index 923f5147..10547351 100644 --- a/MANIFEST +++ b/MANIFEST @@ -169,6 +169,7 @@ lib/PublicInbox/ContentHash.pm lib/PublicInbox/DS.pm lib/PublicInbox/DSKQXS.pm lib/PublicInbox/DSPoll.pm +lib/PublicInbox/DSdeflate.pm lib/PublicInbox/Daemon.pm lib/PublicInbox/DirIdle.pm lib/PublicInbox/DummyInbox.pm @@ -206,7 +207,6 @@ lib/PublicInbox/IMAP.pm lib/PublicInbox/IMAPClient.pm lib/PublicInbox/IMAPD.pm lib/PublicInbox/IMAPTracker.pm -lib/PublicInbox/IMAPdeflate.pm lib/PublicInbox/IMAPsearchqp.pm lib/PublicInbox/IPC.pm lib/PublicInbox/IdxStack.pm @@ -295,7 +295,6 @@ lib/PublicInbox/Msgmap.pm lib/PublicInbox/MultiGit.pm lib/PublicInbox/NNTP.pm lib/PublicInbox/NNTPD.pm -lib/PublicInbox/NNTPdeflate.pm lib/PublicInbox/NetNNTPSocks.pm lib/PublicInbox/NetReader.pm lib/PublicInbox/NetWriter.pm diff --git a/lib/PublicInbox/DS.pm b/lib/PublicInbox/DS.pm index fee31e3d..ef483aac 100644 --- a/lib/PublicInbox/DS.pm +++ b/lib/PublicInbox/DS.pm @@ -648,8 +648,8 @@ sub shutdn ($) { } } -sub zflush {} # overridden by NNTPdeflate and IMAPdeflate - +sub zflush {} # overridden by DSdeflate +sub compressed {} # overridden by DSdeflate sub long_response_done {} # overridden by Net::NNTP sub long_step { diff --git a/lib/PublicInbox/NNTPdeflate.pm b/lib/PublicInbox/DSdeflate.pm similarity index 92% rename from lib/PublicInbox/NNTPdeflate.pm rename to lib/PublicInbox/DSdeflate.pm index 352d4842..b5208e43 100644 --- a/lib/PublicInbox/NNTPdeflate.pm +++ b/lib/PublicInbox/DSdeflate.pm @@ -2,6 +2,7 @@ # License: AGPL-3.0+ # RFC 8054 NNTP COMPRESS DEFLATE implementation +# RFC 4978 IMAP COMPRESS=DEFLATE extension # # RSS usage for 10K idle-but-did-something NNTP clients on 64-bit: # TLS + DEFLATE[a] : 1.8 GB (MemLevel=9, 1.2 GB with MemLevel=8) @@ -14,14 +15,13 @@ # [b] - memory-optimized implementation using a global deflate context. # It's less efficient in terms of compression, but way more # efficient in terms of server memory usage. -package PublicInbox::NNTPdeflate; +package PublicInbox::DSdeflate; use strict; -use 5.010_001; -use parent qw(PublicInbox::NNTP); +use v5.10.1; use Compress::Raw::Zlib; my %IN_OPT = ( - -Bufsize => PublicInbox::NNTP::LINE_MAX, + -Bufsize => 1024, -WindowBits => -15, # RFC 1951 -AppendOutput => 1, ); @@ -42,21 +42,18 @@ my $zout; $err == Z_OK or die "Failed to initialize zlib deflate stream: $err"; } - sub enable { my ($class, $self) = @_; my ($in, $err) = Compress::Raw::Zlib::Inflate->new(%IN_OPT); if ($err != Z_OK) { $self->err("Inflate->new failed: $err"); - $self->write(\"403 Unable to activate compression\r\n"); return; } - $self->write(\"206 Compression active\r\n"); bless $self, $class; $self->{zin} = $in; } -# overrides PublicInbox::NNTP::compressed +# overrides PublicInbox::DS::compressed sub compressed { 1 } sub do_read ($$$$) { diff --git a/lib/PublicInbox/IMAP.pm b/lib/PublicInbox/IMAP.pm index ce0dce0f..805f1102 100644 --- a/lib/PublicInbox/IMAP.pm +++ b/lib/PublicInbox/IMAP.pm @@ -1212,8 +1212,6 @@ sub event_step { $self->requeue unless $pending; } -sub compressed { undef } - # RFC 4978 sub cmd_compress ($$$) { my ($self, $tag, $alg) = @_; @@ -1223,7 +1221,9 @@ sub cmd_compress ($$$) { # CRIME made TLS compression obsolete # return "$tag NO [COMPRESSIONACTIVE]\r\n" if $self->tls_compressed; - PublicInbox::IMAPdeflate->enable($self, $tag); + PublicInbox::IMAPdeflate->enable($self) or return + \"$tag BAD failed to activate compression\r\n"; + PublicInbox::DS::write($self, \"$tag OK DEFLATE active\r\n"); $self->requeue; undef } @@ -1269,4 +1269,8 @@ our @ISA = qw(PublicInbox::IMAP); sub logged_in { 0 } +package PublicInbox::IMAPdeflate; +use PublicInbox::DSdeflate; +our @ISA = qw(PublicInbox::DSdeflate PublicInbox::IMAP); + 1; diff --git a/lib/PublicInbox/IMAPD.pm b/lib/PublicInbox/IMAPD.pm index d8814324..b24097a2 100644 --- a/lib/PublicInbox/IMAPD.pm +++ b/lib/PublicInbox/IMAPD.pm @@ -9,7 +9,7 @@ use v5.10.1; use PublicInbox::Config; use PublicInbox::ConfigIter; use PublicInbox::InboxIdle; -use PublicInbox::IMAPdeflate; # loads PublicInbox::IMAP +use PublicInbox::IMAP; use PublicInbox::DummyInbox; my $dummy = bless { uidvalidity => 0 }, 'PublicInbox::DummyInbox'; diff --git a/lib/PublicInbox/IMAPdeflate.pm b/lib/PublicInbox/IMAPdeflate.pm deleted file mode 100644 index d5929ef2..00000000 --- a/lib/PublicInbox/IMAPdeflate.pm +++ /dev/null @@ -1,126 +0,0 @@ -# Copyright (C) 2020-2021 all contributors -# License: AGPL-3.0+ -# TODO: reduce duplication from PublicInbox::NNTPdeflate - -# RFC 4978 -package PublicInbox::IMAPdeflate; -use strict; -use warnings; -use 5.010_001; -use base qw(PublicInbox::IMAP); -use Compress::Raw::Zlib; - -my %IN_OPT = ( - -Bufsize => 1024, - -WindowBits => -15, # RFC 1951 - -AppendOutput => 1, -); - -# global deflate context and buffer -my $zbuf = \(my $buf = ''); -my $zout; -{ - my $err; - ($zout, $err) = Compress::Raw::Zlib::Deflate->new( - # nnrpd (INN) and Compress::Raw::Zlib favor MemLevel=9, - # the zlib C library and git use MemLevel=8 as the default - # -MemLevel => 9, - -Bufsize => 65536, # same as nnrpd - -WindowBits => -15, # RFC 1951 - -AppendOutput => 1, - ); - $err == Z_OK or die "Failed to initialize zlib deflate stream: $err"; -} - -sub enable { - my ($class, $self, $tag) = @_; - my ($in, $err) = Compress::Raw::Zlib::Inflate->new(%IN_OPT); - if ($err != Z_OK) { - $self->err("Inflate->new failed: $err"); - $self->write(\"$tag BAD failed to activate compression\r\n"); - return; - } - $self->write(\"$tag OK DEFLATE active\r\n"); - bless $self, $class; - $self->{zin} = $in; -} - -# overrides PublicInbox::NNTP::compressed -sub compressed { 1 } - -sub do_read ($$$$) { - my ($self, $rbuf, $len, $off) = @_; - - my $zin = $self->{zin} or return; # closed - my $doff; - my $dbuf = delete($self->{dbuf}) // ''; - $doff = length($dbuf); - my $r = PublicInbox::DS::do_read($self, \$dbuf, $len, $doff) or return; - - # Workaround inflate bug appending to OOK scalars: - # - # We only have $off if the client is pipelining, and pipelining - # is where our substr() OOK optimization in event_step makes sense. - if ($off) { - my $copy = $$rbuf; - undef $$rbuf; - $$rbuf = $copy; - } - - # assert(length($$rbuf) == $off) as far as NNTP.pm is concerned - # -ConsumeInput is true, so $dbuf is automatically emptied - my $err = $zin->inflate($dbuf, $rbuf); - if ($err == Z_OK) { - $self->{dbuf} = $dbuf if $dbuf ne ''; - $r = length($$rbuf) and return $r; - # nothing ready, yet, get more, later - $self->requeue; - } else { - delete $self->{zin}; - $self->close; - } - 0; -} - -# override PublicInbox::DS::msg_more -sub msg_more ($$) { - my $self = $_[0]; - - # $_[1] may be a reference or not for ->deflate - my $err = $zout->deflate($_[1], $zbuf); - $err == Z_OK or die "->deflate failed $err"; - 1; -} - -sub zflush ($) { - my ($self) = @_; - - my $deflated = $zbuf; - $zbuf = \(my $next = ''); - - my $err = $zout->flush($deflated, Z_FULL_FLUSH); - $err == Z_OK or die "->flush failed $err"; - - # We can still let the lower socket layer do buffering: - PublicInbox::DS::msg_more($self, $$deflated); -} - -# compatible with PublicInbox::DS::write, so $_[1] may be a reference or not -sub write ($$) { - my $self = $_[0]; - return PublicInbox::DS::write($self, $_[1]) if ref($_[1]) eq 'CODE'; - - my $deflated = $zbuf; - $zbuf = \(my $next = ''); - - # $_[1] may be a reference or not for ->deflate - my $err = $zout->deflate($_[1], $deflated); - $err == Z_OK or die "->deflate failed $err"; - $err = $zout->flush($deflated, Z_FULL_FLUSH); - $err == Z_OK or die "->flush failed $err"; - - # We can still let the socket layer do buffering: - PublicInbox::DS::write($self, $deflated); -} - -1; diff --git a/lib/PublicInbox/NNTP.pm b/lib/PublicInbox/NNTP.pm index 3929f817..8ad7adc1 100644 --- a/lib/PublicInbox/NNTP.pm +++ b/lib/PublicInbox/NNTP.pm @@ -883,8 +883,6 @@ sub cmd_xover ($;$) { $self->long_response(\&xover_i, @$r); } -sub compressed { undef } - sub cmd_starttls ($) { my ($self) = @_; my $sock = $self->{sock} or return; @@ -903,7 +901,9 @@ sub cmd_compress ($$) { my ($self, $alg) = @_; return "503 Only DEFLATE is supported\r\n" if uc($alg) ne 'DEFLATE'; return r502 if $self->compressed; - PublicInbox::NNTPdeflate->enable($self); + PublicInbox::NNTPdeflate->enable($self) or return + \"403 Unable to activate compression\r\n"; + PublicInbox::DS::write($self, \"206 Compression active\r\n"); $self->requeue; undef } @@ -985,4 +985,8 @@ sub busy { # for graceful shutdown in PublicInbox::Daemon: defined($self->{rbuf}) || defined($self->{wbuf}) } +package PublicInbox::NNTPdeflate; +use PublicInbox::DSdeflate; +our @ISA = qw(PublicInbox::DSdeflate PublicInbox::NNTP); + 1; diff --git a/lib/PublicInbox/NNTPD.pm b/lib/PublicInbox/NNTPD.pm index 6e79f0be..f31d4381 100644 --- a/lib/PublicInbox/NNTPD.pm +++ b/lib/PublicInbox/NNTPD.pm @@ -9,7 +9,7 @@ use v5.10.1; use Sys::Hostname; use PublicInbox::Config; use PublicInbox::InboxIdle; -use PublicInbox::NNTPdeflate; # loads PublicInbox::NNTP +use PublicInbox::NNTP; sub new { my ($class) = @_; diff --git a/xt/mem-imapd-tls.t b/xt/mem-imapd-tls.t index 8992a6fc..d728ce32 100644 --- a/xt/mem-imapd-tls.t +++ b/xt/mem-imapd-tls.t @@ -1,5 +1,5 @@ #!perl -w -# Copyright (C) 2020-2021 all contributors +# Copyright (C) all contributors # License: AGPL-3.0+ # Idle client memory usage test, particularly after EXAMINE when # Message Sequence Numbers are loaded @@ -221,7 +221,7 @@ package IMAPCdeflate; use strict; our @ISA; use Compress::Raw::Zlib; -use PublicInbox::IMAPdeflate; +use PublicInbox::IMAP; my %ZIN_OPT; BEGIN { @ISA = qw(IMAPC);