From 0df6f43c81c72924020b8f71c696784f154d93fa Mon Sep 17 00:00:00 2001 From: Eric Wong Date: Sun, 7 Feb 2021 23:05:09 -1000 Subject: lei q: improve remote mboxrd UX + MUA For early MUA spawners using lock-free outputs, we we need to on the startq pipe to silence progress reporting. For --augment users, we can start the MUA even earlier by creating Maildirs in the pre-augment phase. To improve progress reporting for non-MUA (or late-MUA) spawners, we'll no longer blindly append "--compressed" to the curl(1) command when POST-ing for the gzipped mboxrd. Furthermore, we'll overload stringify ('""') in LeiCurl to ensure the empty -d '' string shows up properly. v2: fix startq waiting with --threads mset_progress is never shown with early MUA spawning, The plan is to still show progress when augmenting and deduping. This fixes all local search cases. A leftover debug bit is dropped, too --- lib/PublicInbox/LeiMirror.pm | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'lib/PublicInbox/LeiMirror.pm') diff --git a/lib/PublicInbox/LeiMirror.pm b/lib/PublicInbox/LeiMirror.pm index 5ba69287..c5153148 100644 --- a/lib/PublicInbox/LeiMirror.pm +++ b/lib/PublicInbox/LeiMirror.pm @@ -31,7 +31,7 @@ sub try_scrape { my $uri = URI->new($self->{src}); my $lei = $self->{lei}; my $curl = $self->{curl} //= PublicInbox::LeiCurl->new($lei) or return; - my $cmd = $curl->for_uri($lei, $uri); + my $cmd = $curl->for_uri($lei, $uri, '--compressed'); my $opt = { 0 => $lei->{0}, 2 => $lei->{2} }; my $fh = popen_rd($cmd, $lei->{env}, $opt); my $html = do { local $/; <$fh> } // die "read(curl $uri): $!"; @@ -93,8 +93,7 @@ sub _try_config { my $path = $uri->path; chop($path) eq '/' or die "BUG: $uri not canonicalized"; $uri->path($path . '/_/text/config/raw'); - my $cmd = $self->{curl}->for_uri($lei, $uri); - push @$cmd, '--compressed'; # curl decompresses for us + my $cmd = $self->{curl}->for_uri($lei, $uri, '--compressed'); my $ce = "$dst/inbox.config.example"; my $f = "$ce-$$.tmp"; open(my $fh, '+>', $f) or return $lei->err("open $f: $! (non-fatal)"); -- cgit v1.2.3-24-ge0c7