From mboxrd@z Thu Jan  1 00:00:00 1970
Return-Path: <e@80x24.org>
X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on dcvr.yhbt.net
X-Spam-Level: 
X-Spam-ASN:  
X-Spam-Status: No, score=-4.0 required=3.0 tests=ALL_TRUSTED,BAYES_00
	shortcircuit=no autolearn=ham autolearn_force=no version=3.4.2
Received: from localhost (dcvr.yhbt.net [127.0.0.1])
	by dcvr.yhbt.net (Postfix) with ESMTP id 574581F4BA
	for <meta@public-inbox.org>; Thu, 27 Jun 2019 22:51:49 +0000 (UTC)
From: Eric Wong <e@80x24.org>
To: meta@public-inbox.org
Subject: [PATCH 4/4] nntp: reduce syscalls for ARTICLE and BODY
Date: Thu, 27 Jun 2019 22:51:48 +0000
Message-Id: <20190627225148.9657-5-e@80x24.org>
In-Reply-To: <20190627225148.9657-1-e@80x24.org>
References: <20190627225148.9657-1-e@80x24.org>
MIME-Version: 1.0
Content-Transfer-Encoding: 8bit
List-Id: <meta.public-inbox.org>

Chances are we already have extra buffer space following the
expensive LF => CRLF conversion that we can safely append an
extra CRLF in those places without incurring a copy of the
full string buffer.

While we're at it, document where our pain points are in terms
of memory usage, since tracking/controlling memory use isn't
exactly obvious in high-level languages.

Perhaps we should start storing messages in git as CRLF...
---
 lib/PublicInbox/NNTP.pm | 22 ++++++++++++----------
 1 file changed, 12 insertions(+), 10 deletions(-)

diff --git a/lib/PublicInbox/NNTP.pm b/lib/PublicInbox/NNTP.pm
index 30d3dab6..5a886a3c 100644
--- a/lib/PublicInbox/NNTP.pm
+++ b/lib/PublicInbox/NNTP.pm
@@ -521,10 +521,12 @@ found:
 
 sub msg_body_write ($$) {
 	my ($self, $msg) = @_;
+
+	# these can momentarily double the memory consumption :<
 	$$msg =~ s/^\./../smg;
-	$$msg =~ s/(?<!\r)\n/\r\n/sg;
+	$$msg =~ s/(?<!\r)\n/\r\n/sg; # Alpine barfs without this
+	$$msg .= "\r\n" unless $$msg =~ /\r\n\z/s;
 	msg_more($self, $$msg);
-	msg_more($self, "\r\n") unless $$msg =~ /\r\n\z/s;
 	'.'
 }
 
@@ -533,18 +535,19 @@ sub set_art {
 	$self->{article} = $art if defined $art && $art =~ /\A[0-9]+\z/;
 }
 
-sub _header ($) {
-	my $hdr = $_[0]->as_string;
+sub msg_hdr_write ($$$) {
+	my ($self, $hdr, $body_follows) = @_;
+	$hdr = $hdr->as_string;
 	utf8::encode($hdr);
-	$hdr =~ s/(?<!\r)\n/\r\n/sg;
+	$hdr =~ s/(?<!\r)\n/\r\n/sg; # Alpine barfs without this
 
 	# for leafnode compatibility, we need to ensure Message-ID headers
 	# are only a single line.  We can't subclass Email::Simple::Header
 	# and override _default_fold_at in here, either; since that won't
 	# affect messages already in the archive.
 	$hdr =~ s/^(Message-ID:)[ \t]*\r\n[ \t]+([^\r]+)\r\n/$1 $2\r\n/igsm;
-
-	$hdr
+	$hdr .= "\r\n" if $body_follows;
+	msg_more($self, $hdr);
 }
 
 sub cmd_article ($;$) {
@@ -554,8 +557,7 @@ sub cmd_article ($;$) {
 	my ($n, $mid, $msg, $hdr) = @$r;
 	set_art($self, $art);
 	more($self, "220 $n <$mid> article retrieved - head and body follow");
-	msg_more($self, _header($hdr));
-	msg_more($self, "\r\n");
+	msg_hdr_write($self, $hdr, 1);
 	msg_body_write($self, $msg);
 }
 
@@ -566,7 +568,7 @@ sub cmd_head ($;$) {
 	my ($n, $mid, undef, $hdr) = @$r;
 	set_art($self, $art);
 	more($self, "221 $n <$mid> article retrieved - head follows");
-	msg_more($self, _header($hdr));
+	msg_hdr_write($self, $hdr, 0);
 	'.'
 }
 
-- 
EW