[PATCH 34/57] allow use of PerlIO layers for filesystem writes

[Eric Wong <e@80x24.org>]

It may make sense to use PerlIO::mmap or PerlIO::scalar for
DS write buffering with IO::Socket::SSL or similar (since we can't
use MSG_MORE), so that means we need to go through buffering
in userspace for the common case; while still being easily
compatible with slow clients.

And it also simplifies GitHTTPBackend slightly.

Maybe it can make sense for HTTP input buffering, too...
---
 lib/PublicInbox/DS.pm             | 32 ++++++++++++-------------------
 lib/PublicInbox/GitHTTPBackend.pm | 18 ++++++++---------
 lib/PublicInbox/HTTP.pm           | 24 ++++++++++++++++++-----
 3 files changed, 39 insertions(+), 35 deletions(-)

diff --git a/lib/PublicInbox/DS.pm b/lib/PublicInbox/DS.pm
index 8735e888..486af40e 100644
--- a/lib/PublicInbox/DS.pm
+++ b/lib/PublicInbox/DS.pm
@@ -21,7 +21,7 @@ use IO::Handle qw();
 use Fcntl qw(FD_CLOEXEC F_SETFD F_GETFD SEEK_SET);
 use Time::HiRes qw(clock_gettime CLOCK_MONOTONIC);
 use parent qw(Exporter);
-our @EXPORT_OK = qw(now msg_more write_in_full);
+our @EXPORT_OK = qw(now msg_more);
 use warnings;
 use 5.010_001;
 
@@ -422,8 +422,8 @@ sub close {
 sub psendfile ($$$) {
     my ($sock, $fh, $off) = @_;
 
-    sysseek($fh, $$off, SEEK_SET) or return;
-    defined(my $to_write = sysread($fh, my $buf, 16384)) or return;
+    seek($fh, $$off, SEEK_SET) or return;
+    defined(my $to_write = read($fh, my $buf, 16384)) or return;
     my $written = 0;
     while ($to_write > 0) {
         if (defined(my $w = syswrite($sock, $buf, $to_write, $written))) {
@@ -482,29 +482,18 @@ sub do_read ($$$$) {
     $! == EAGAIN ? $self->watch_in1 : $self->close;
 }
 
-sub write_in_full ($$$$) {
-    my ($fh, $bref, $len, $off) = @_;
-    my $rv = 0;
-    while ($len > 0) {
-        my $w = syswrite($fh, $$bref, $len, $off);
-        return ($rv ? $rv : $w) unless $w; # undef or 0
-        $rv += $w;
-        $len -= $w;
-        $off += $w;
-    }
-    $rv
-}
-
+# n.b.: use ->write/->read for this buffer to allow compatibility with
+# PerlIO::mmap or PerlIO::scalar if needed
 sub tmpbuf ($$) {
     my ($bref, $off) = @_;
     # open(my $fh, '+>>', undef) doesn't set O_APPEND
     my ($fh, $path) = tempfile('wbuf-XXXXXXX', TMPDIR => 1);
     open $fh, '+>>', $path or die "open: $!";
+    $fh->autoflush(1);
     unlink $path;
     my $to_write = bytes::length($$bref) - $off;
-    my $w = write_in_full($fh, $bref, $to_write, $off);
-    die "write_in_full ($to_write): $!" unless defined $w;
-    $w == $to_write ? $fh : die("short write $w < $to_write");
+    $fh->write($$bref, $to_write, $off) or die "write ($to_write): $!";
+    $fh;
 }
 
 =head2 C<< $obj->write( $data ) >>
@@ -534,7 +523,10 @@ sub write {
         } else {
             my $last = $wbuf->[-1];
             if (ref($last) eq 'GLOB') { # append to tmp file buffer
-                write_in_full($last, $bref, bytes::length($$bref), 0);
+                unless ($last->print($$bref)) {
+                    warn "error buffering: $!";
+                    return $self->close;
+                }
             } else {
                 push @$wbuf, tmpbuf($bref, 0);
             }
diff --git a/lib/PublicInbox/GitHTTPBackend.pm b/lib/PublicInbox/GitHTTPBackend.pm
index a2a81f8e..303d5073 100644
--- a/lib/PublicInbox/GitHTTPBackend.pm
+++ b/lib/PublicInbox/GitHTTPBackend.pm
@@ -231,18 +231,16 @@ sub input_prepare {
 			return;
 		}
 		last if $r == 0;
-		my $off = 0;
-		while ($r > 0) {
-			my $w = syswrite($in, $buf, $r, $off);
-			if (defined $w) {
-				$r -= $w;
-				$off += $w;
-			} else {
-				err($env, "error writing temporary file: $!");
-				return;
-			}
+		unless (print $in $buf) {
+			err($env, "error writing temporary file: $!");
+			return;
 		}
 	}
+	# ensure it's visible to git-http-backend(1):
+	unless ($in->flush) {
+		err($env, "error writing temporary file: $!");
+		return;
+	}
 	unless (defined(sysseek($in, 0, SEEK_SET))) {
 		err($env, "error seeking temporary file: $!");
 		return;
diff --git a/lib/PublicInbox/HTTP.pm b/lib/PublicInbox/HTTP.pm
index 7697ac5c..a1cb4aca 100644
--- a/lib/PublicInbox/HTTP.pm
+++ b/lib/PublicInbox/HTTP.pm
@@ -19,7 +19,7 @@ use HTTP::Status qw(status_message);
 use HTTP::Date qw(time2str);
 use IO::Handle;
 require PublicInbox::EvCleanup;
-PublicInbox::DS->import(qw(msg_more write_in_full));
+PublicInbox::DS->import(qw(msg_more));
 use PublicInbox::Syscall qw(EPOLLIN EPOLLONESHOT);
 use constant {
 	CHUNK_START => -1,   # [a-f0-9]+\r\n
@@ -102,6 +102,15 @@ sub rbuf_process {
 	$len ? read_input($self) : app_dispatch($self);
 }
 
+# IO::Handle::write returns boolean, this returns bytes written:
+sub xwrite ($$$) {
+	my ($fh, $rbuf, $max) = @_;
+	my $w = bytes::length($$rbuf);
+	$w = $max if $w > $max;
+	$fh->write($$rbuf, $w) or return;
+	$w;
+}
+
 sub read_input ($) {
 	my ($self) = @_;
 	my $env = $self->{env};
@@ -116,7 +125,7 @@ sub read_input ($) {
 
 	while ($len > 0) {
 		if ($$rbuf ne '') {
-			my $w = write_in_full($input, $rbuf, $len, 0);
+			my $w = xwrite($input, $rbuf, $len);
 			return write_err($self, $len) unless $w;
 			$len -= $w;
 			die "BUG: $len < 0 (w=$w)" if $len < 0;
@@ -306,6 +315,11 @@ sub response_write {
 	}
 }
 
+sub input_tmpfile ($) {
+	open($_[0], '+>', undef);
+	$_[0]->autoflush(1);
+}
+
 sub input_prepare {
 	my ($self, $env) = @_;
 	my $input;
@@ -315,10 +329,10 @@ sub input_prepare {
 			quit($self, 413);
 			return;
 		}
-		open($input, '+>', undef);
+		input_tmpfile($input);
 	} elsif (env_chunked($env)) {
 		$len = CHUNK_START;
-		open($input, '+>', undef);
+		input_tmpfile($input);
 	} else {
 		$input = $null_io;
 	}
@@ -399,7 +413,7 @@ sub read_input_chunked { # unlikely...
 		# drain the current chunk
 		until ($len <= 0) {
 			if ($$rbuf ne '') {
-				my $w = write_in_full($input, $rbuf, $len, 0);
+				my $w = xwrite($input, $rbuf, $len);
 				return write_err($self, "$len chunk") if !$w;
 				$len -= $w;
 				if ($len == 0) {
-- 
EW