[PATCH 3/5] lei q: demangle and quiet curl output

[Eric Wong <e@80x24.org>]

curl(1) writes to stderr one byte-at-a-time (presumably for the
progress bar).  This ends up being unreadable on my terminal
when parallel processes are trying to write error messages.

So instead, we'll capture the output to a file and run
'tail -f' on it if --verbose is enabled.

Since HTTP 404s from non-existent results are a common response,
we'll ignore them and stay silent, matching behavior of local
searches.
---
 lib/PublicInbox/LeiXSearch.pm | 45 ++++++++++++++++++++++++++---------
 t/lei.t                       |  2 +-
 2 files changed, 35 insertions(+), 12 deletions(-)

diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index fb608d00..68be8ada 100644
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -14,8 +14,9 @@ use PublicInbox::Import;
 use File::Temp 0.19 (); # 0.19 for ->newdir
 use File::Spec ();
 use PublicInbox::Search qw(xap_terms);
-use PublicInbox::Spawn qw(popen_rd);
+use PublicInbox::Spawn qw(popen_rd spawn);
 use PublicInbox::MID qw(mids);
+use Fcntl qw(SEEK_SET F_SETFL O_APPEND O_RDWR);
 
 sub new {
 	my ($class) = @_;
@@ -176,6 +177,13 @@ sub each_eml { # callback for MboxReader->mboxrd
 	$each_smsg->($smsg, undef, $eml);
 }
 
+# PublicInbox::OnDestroy callback
+sub kill_reap {
+	my ($pid) = @_;
+	kill('KILL', $pid); # spawn() blocks other signals
+	waitpid($pid, 0);
+}
+
 sub query_remote_mboxrd {
 	my ($self, $lei, $uris) = @_;
 	local $0 = "$0 query_remote_mboxrd";
@@ -186,7 +194,20 @@ sub query_remote_mboxrd {
 	push(@qform, t => 1) if $opt->{thread};
 	my @cmd = (qw(curl -sSf -d), '');
 	my $verbose = $opt->{verbose};
-	push @cmd, '-v' if $verbose;
+	my $reap;
+	my $cerr = File::Temp->new(TEMPLATE => 'curl.err-XXXX', TMPDIR => 1);
+	fcntl($cerr, F_SETFL, O_APPEND|O_RDWR) or warn "set O_APPEND: $!";
+	my $rdr = { 2 => $cerr };
+	my $coff = 0;
+	if ($verbose) {
+		# spawn a process to force line-buffering, otherwise curl
+		# will write 1 character at-a-time and parallel outputs
+		# mmmaaayyy llloookkk llliiikkkeee ttthhhiiisss
+		push @cmd, '-v';
+		my $o = { 1 => $lei->{2}, 2 => $lei->{2} };
+		my $pid = spawn(['tail', '-f', $cerr->filename], undef, $o);
+		$reap = PublicInbox::OnDestroy->new(\&kill_reap, $pid);
+	}
 	for my $o ($lei->curl_opt) {
 		$o =~ s/\|[a-z0-9]\b//i; # remove single char short option
 		if ($o =~ s/=[is]@\z//) {
@@ -213,21 +234,23 @@ sub query_remote_mboxrd {
 		}
 		$lei->err("# @$cmd") if $verbose;
 		$? = 0;
-		my $fh = popen_rd($cmd, $env, { 2 => $lei->{2} });
+		my $fh = popen_rd($cmd, $env, $rdr);
 		$fh = IO::Uncompress::Gunzip->new($fh);
 		eval {
 			PublicInbox::MboxReader->mboxrd($fh, \&each_eml, $self,
 							$lei, $each_smsg);
 		};
 		return $lei->fail("E: @$cmd: $@") if $@;
-		if (($? >> 8) == 22) { # HTTP 404 from curl(1)
-			$uri->query_form(q => $lei->{mset_opt}->{qstr});
-			$lei->err('# no results from '.$uri->as_string);
-		} elsif ($?) {
-			$uri->query_form(q => $lei->{mset_opt}->{qstr});
-			$lei->err('E: '.$uri->as_string);
-			$lei->child_error($?);
-		}
+		next unless $?;
+		seek($cerr, $coff, SEEK_SET) or warn "seek(curl stderr): $!\n";
+		my $e = do { local $/; <$cerr> } //
+				die "read(curl stderr): $!\n";
+		$coff += length($e);
+		next if (($? >> 8) == 22 && $e =~ /\b404\b/);
+		$lei->child_error($?);
+		$uri->query_form(q => $lei->{mset_opt}->{qstr});
+		# --verbose already showed the error via tail(1)
+		$lei->err("E: $uri \$?=$?\n", $verbose ? () : $e);
 	}
 	undef $each_smsg;
 	$lei->{ovv}->ovv_atexit_child($lei);
diff --git a/t/lei.t b/t/lei.t
index f826a966..69338257 100644
--- a/t/lei.t
+++ b/t/lei.t
@@ -179,7 +179,7 @@ SKIP: {
 	my $res = $json->decode($out);
 	is($res->[0]->{'m'}, "<$mid>", "got expected mid from $url");
 	ok($lei->('q', "m:$mid", 'd:..20101002'), 'no results, no error');
-	like($err, qr/404/, 'noted 404');
+	is($err, '', 'no output on 404, matching local FS behavior');
 	is($out, "[null]\n", 'got null results');
 	$lei->('forget-external', $url);
 } # /SKIP