[PATCH 00/11] lei: misc fixes, more lcat color support

[PATCH 00/11] lei: misc fixes, more lcat color support

[Eric Wong]

I'm seeing some odd test failures in "make check-run"
(but not "make check") that I haven't diagnosed, yet.
So there's some diagnostic changes and some golfing
to make internals more consistent.

Eric Wong (11):
  t/lei-p2q: add diagnostics
  tests: restore CWD with "lei -C" and run_script
  view_diff: minor coding style fixes
  lei_p2q: add _lei_wq_eof callback
  lei-daemon: note FD count mismatch to client
  t/run.perl: add (GNU) tail and strace support
  lei: quiet down Eml-related warnings consistently
  lei: simple WQ workers use {wq1} field
  lei_view_text: improve attachment display
  lei_view_text: translate background colors from git
  lei (lcat|q): support --no-color and --color

 lib/PublicInbox/LEI.pm         | 30 ++++++++++++++++++++---------
 lib/PublicInbox/LeiBlob.pm     |  9 +--------
 lib/PublicInbox/LeiConvert.pm  |  3 +--
 lib/PublicInbox/LeiImport.pm   | 10 +++-------
 lib/PublicInbox/LeiLsSearch.pm |  2 +-
 lib/PublicInbox/LeiMirror.pm   |  4 ++--
 lib/PublicInbox/LeiP2q.pm      |  6 ++----
 lib/PublicInbox/LeiStore.pm    |  1 -
 lib/PublicInbox/LeiTag.pm      |  9 ++-------
 lib/PublicInbox/LeiViewText.pm | 35 +++++++++++++++++++++++-----------
 lib/PublicInbox/LeiXSearch.pm  |  1 -
 lib/PublicInbox/TestCommon.pm  | 12 ++++++++----
 lib/PublicInbox/ViewDiff.pm    |  9 ++++-----
 script/lei                     |  4 ++--
 t/lei-externals.t              |  5 -----
 t/lei-p2q.t                    |  4 ++--
 t/lei-q-save.t                 |  4 ----
 t/run.perl                     |  7 +++++++
 t/solver_git.t                 |  5 -----
 19 files changed, 80 insertions(+), 80 deletions(-)

[PATCH 01/11] t/lei-p2q: add diagnostics

[Eric Wong]

Maybe this helps fix occasional problems in daemon mode,
but I haven't seen anything on failure, yet...
---
 t/lei-p2q.t | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/t/lei-p2q.t b/t/lei-p2q.t
index be2d437c..f8b073cf 100644
--- a/t/lei-p2q.t
+++ b/t/lei-p2q.t
@@ -9,10 +9,10 @@ test_lei(sub {
 	ok(!lei(qw(p2q this-better-cause-format-patch-to-fail)),
 		'p2q fails on bogus arg');
 	lei_ok(qw(p2q -w dfpost t/data/0001.patch));
-	is($lei_out, "dfpost:6e006fd73b1d\n", 'pathname');
+	is($lei_out, "dfpost:6e006fd73b1d\n", 'pathname') or diag $lei_err;
 	open my $fh, '+<', 't/data/0001.patch' or xbail "open: $!";
 	lei_ok([qw(p2q -w dfpost -)], undef, { %$lei_opt, 0 => $fh });
-	is($lei_out, "dfpost:6e006fd73b1d\n", '--stdin');
+	is($lei_out, "dfpost:6e006fd73b1d\n", '--stdin') or diag $lei_err;
 
 	lei_ok(qw(p2q --uri t/data/0001.patch -w), 'dfpost,dfn');
 	is($lei_out, "dfpost%3A6e006fd73b1d+".

[PATCH 02/11] tests: restore CWD with "lei -C" and run_script

[Eric Wong]

This simplifies test_lei users in t/*.t
---
 lib/PublicInbox/TestCommon.pm | 12 ++++++++----
 t/lei-externals.t             |  5 -----
 t/lei-q-save.t                |  4 ----
 t/solver_git.t                |  5 -----
 4 files changed, 8 insertions(+), 18 deletions(-)

diff --git a/lib/PublicInbox/TestCommon.pm b/lib/PublicInbox/TestCommon.pm
index 09256852..460c9da0 100644
--- a/lib/PublicInbox/TestCommon.pm
+++ b/lib/PublicInbox/TestCommon.pm
@@ -12,6 +12,7 @@ use IO::Socket::INET;
 use File::Spec;
 our @EXPORT;
 my $lei_loud = $ENV{TEST_LEI_ERR_LOUD};
+our ($lei_opt, $lei_out, $lei_err, $lei_cwdfh);
 BEGIN {
 	@EXPORT = qw(tmpdir tcp_server tcp_connect require_git require_mods
 		run_script start_script key2sub xsys xsys_e xqx eml_load tick
@@ -306,14 +307,16 @@ sub run_script ($;$$) {
 		local %SIG = %SIG;
 		local $0 = join(' ', @$cmd);
 		my $orig_io = _prepare_redirects($fhref);
-		my $cwdfh;
+		my $cwdfh = $lei_cwdfh;
 		if (my $d = $opt->{'-C'}) {
-			opendir $cwdfh, '.' or die "opendir .: $!";
+			unless ($cwdfh) {
+				opendir $cwdfh, '.' or die "opendir .: $!";
+			}
 			chdir $d or die "chdir $d: $!";
 		}
 		_run_sub($sub, $key, \@argv);
 		eval { PublicInbox::Inbox::cleanup_task() };
-		die "chdir(restore): $!" if $cwdfh && !chdir($cwdfh);
+		die "fchdir(restore): $!" if $cwdfh && !chdir($cwdfh);
 		_undo_redirects($orig_io);
 		select STDOUT;
 	}
@@ -469,7 +472,6 @@ sub have_xapian_compact () {
 	PublicInbox::Spawn::which($ENV{XAPIAN_COMPACT} || 'xapian-compact');
 }
 
-our ($lei_opt, $lei_out, $lei_err);
 # favor lei() or lei_ok() over $lei for new code
 sub lei (@) {
 	my ($cmd, $env, $xopt) = @_;
@@ -515,6 +517,8 @@ sub test_lei {
 SKIP: {
 	my ($cb) = pop @_;
 	my $test_opt = shift // {};
+	local $lei_cwdfh;
+	opendir $lei_cwdfh, '.' or xbail "opendir .: $!";
 	require_git(2.6, 1) or skip('git 2.6+ required for lei test', 2);
 	require_mods(qw(json DBD::SQLite Search::Xapian), 2);
 	require PublicInbox::Config;
diff --git a/t/lei-externals.t b/t/lei-externals.t
index 2291dd99..16241e02 100644
--- a/t/lei-externals.t
+++ b/t/lei-externals.t
@@ -4,7 +4,6 @@
 use strict; use v5.10.1; use PublicInbox::TestCommon;
 use Fcntl qw(SEEK_SET);
 use PublicInbox::Spawn qw(which);
-use PublicInbox::OnDestroy;
 require_git 2.6;
 require_mods(qw(json DBD::SQLite Search::Xapian));
 use POSIX qw(WTERMSIG WIFSIGNALED SIGPIPE);
@@ -266,10 +265,6 @@ test_lei(sub {
 	{
 		skip 'TEST_LEI_DAEMON_PERSIST_DIR in use', 1 if
 					$ENV{TEST_LEI_DAEMON_PERSIST_DIR};
-		opendir my $dh, '.' or BAIL_OUT "opendir(.) $!";
-		my $od = PublicInbox::OnDestroy->new($$, sub {
-			chdir $dh or BAIL_OUT "chdir: $!"
-		});
 		my @q = qw(q -o mboxcl2:rel.mboxcl2 bye);
 		lei_ok('-C', $home, @q);
 		is(unlink("$home/rel.mboxcl2"), 1, '-C works before q');
diff --git a/t/lei-q-save.t b/t/lei-q-save.t
index 170f7ce5..9f65e4a2 100644
--- a/t/lei-q-save.t
+++ b/t/lei-q-save.t
@@ -36,11 +36,9 @@ test_lei(sub {
 	# ensure "lei up" works, since it compliments "lei q --save"
 	$in = $doc2->as_string;
 	lei_ok [qw(import -q -F eml -)], undef, { 0 => \$in, %$lei_opt };
-	opendir my $dh, '.' or xbail "opendir .: $!";
 	lei_ok qw(up -q md -C), $home;
 	lei_ok qw(up -q . -C), "$home/md";
 	lei_ok qw(up -q), "/$home/md";
-	chdir($dh) or xbail "fchdir . $!";
 	my %after = map { $_ => 1 } glob("$home/md/cur/*");
 	is(delete $after{(keys(%before))[0]}, 1, 'original message kept');
 	is(scalar(keys %after), 1, 'one new message added');
@@ -89,7 +87,6 @@ test_lei(sub {
 	print $mb $pre_existing;
 	close $mb or xbail "close: $!";
 	lei_ok(qw(q --save -o mboxrd:mbrd m:qp@example.com -C), $home);
-	chdir($dh) or xbail "fchdir . $!";
 	open $mb, '<', "$home/mbrd" or xbail "open $!";
 	is_deeply([grep(/pre-existing/, <$mb>)], [],
 		'pre-existing messsage gone w/o augment');
@@ -103,7 +100,6 @@ test_lei(sub {
 	print $mb $pre_existing;
 	close $mb or xbail "close: $!";
 	lei_ok(qw(q -a --save -o mboxrd:mbrd-aug m:qp@example.com -C), $home);
-	chdir($dh) or xbail "fchdir . $!";
 	open $mb, '<', "$home/mbrd-aug" or xbail "open $!";
 	$mb = do { local $/; <$mb> };
 	like($mb, qr/pre-existing/, 'pre-existing message preserved w/ -a');
diff --git a/t/solver_git.t b/t/solver_git.t
index 8acf907f..6875e26b 100644
--- a/t/solver_git.t
+++ b/t/solver_git.t
@@ -50,11 +50,6 @@ test_lei({tmpdir => $tmpdir}, sub {
 			'non-existent blob fails');
 	SKIP: {
 		skip '/.git exists', 1 if -e '/.git';
-		require PublicInbox::OnDestroy;
-		opendir my $dh, '.' or xbail "opendir: $!";
-		my $end = PublicInbox::OnDestroy->new($$, sub {
-			chdir $dh or xbail "chdir: $!";
-		});
 		lei_ok(qw(-C / blob 69df7d5 -I), $ibx->{inboxdir},
 			"--git-dir=$git_dir");
 		is($lei_out, $prev, '--git-dir works');

[PATCH 03/11] view_diff: minor coding style fixes

[Eric Wong]

Prefer "use v5.10", s/base/parent/, rely on "perl -w" for warnings.
We also pass a regexp to the split perlop rather than literal
SV, since split() will compile a new RE every time.
---
 lib/PublicInbox/ViewDiff.pm | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/lib/PublicInbox/ViewDiff.pm b/lib/PublicInbox/ViewDiff.pm
index e9a7bf69..d2c5fabe 100644
--- a/lib/PublicInbox/ViewDiff.pm
+++ b/lib/PublicInbox/ViewDiff.pm
@@ -7,10 +7,9 @@
 # (or reconstruct) blobs.
 
 package PublicInbox::ViewDiff;
-use 5.010_001;
 use strict;
-use warnings;
-use base qw(Exporter);
+use v5.10.1;
+use parent qw(Exporter);
 our @EXPORT_OK = qw(flush_diff);
 use URI::Escape qw(uri_escape_utf8);
 use PublicInbox::Hval qw(ascii_html to_attr);
@@ -122,8 +121,8 @@ sub diff_header ($$$) {
 	my $dctx = { spfx => $spfx };
 
 	# get rid of leading "a/" or "b/" (or whatever --{src,dst}-prefix are)
-	$pa = (split('/', git_unquote($pa), 2))[1] if $pa ne '/dev/null';
-	$pb = (split('/', git_unquote($pb), 2))[1] if $pb ne '/dev/null';
+	$pa = (split(m'/', git_unquote($pa), 2))[1] if $pa ne '/dev/null';
+	$pb = (split(m'/', git_unquote($pb), 2))[1] if $pb ne '/dev/null';
 	if ($pa eq $pb && $pb ne '/dev/null') {
 		$dctx->{Q} = "?b=".uri_escape_utf8($pb, UNSAFE);
 	} else {

[PATCH 04/11] lei_p2q: add _lei_wq_eof callback

[Eric Wong]

This may help with some error diagnostics.
---
 lib/PublicInbox/LeiP2q.pm | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/lib/PublicInbox/LeiP2q.pm b/lib/PublicInbox/LeiP2q.pm
index 3248afd7..07357e32 100644
--- a/lib/PublicInbox/LeiP2q.pm
+++ b/lib/PublicInbox/LeiP2q.pm
@@ -203,4 +203,10 @@ sub ipc_atfork_child {
 	$self->SUPER::ipc_atfork_child;
 }
 
+sub _lei_wq_eof { # EOF callback for main daemon
+	my ($lei) = @_;
+	my $p2q = delete $lei->{p2q} // return $lei->dclose;
+	$p2q->wq_wait_old($lei->can('wq_done_wait'), $lei);
+}
+
 1;

[PATCH 05/11] lei-daemon: note FD count mismatch to client

[Eric Wong]

This should help in some error diagnostics
---
 lib/PublicInbox/LEI.pm | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index ef72758c..a949ae3e 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -998,7 +998,7 @@ sub accept_dispatch { # Listener {post_accept} callback
 			open($self->{$i++}, '+<&=', $fd) and next;
 			send($sock, "open(+<&=$fd) (FD=$i): $!", MSG_EOR);
 		}
-		return if scalar(@fds) != 4;
+		$i == 4 or return send($sock, 'not enough FDs='.($i-1), MSG_EOR)
 	}
 	$self->{2}->autoflush(1); # keep stdout buffered until x_it|DESTROY
 	# $ENV_STR = join('', map { "\0$_=$ENV{$_}" } keys %ENV);

[PATCH 06/11] t/run.perl: add (GNU) tail and strace support

[Eric Wong]

This may help track down some "make check-run" failures
that don't seem to manifest under "make check".
---
 t/run.perl | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/t/run.perl b/t/run.perl
index f4e235b7..2fbe4033 100755
--- a/t/run.perl
+++ b/t/run.perl
@@ -52,6 +52,13 @@ if (!$ENV{TEST_LEI_DAEMON_PERSIST_DIR} &&
 	chomp $lei_daemon_pid;
 	$lei_daemon_pid =~ /\A[0-9]+\z/ or die "no daemon pid: $lei_daemon_pid";
 	kill(0, $lei_daemon_pid) or die "kill $lei_daemon_pid: $!";
+	if (my $t = $ENV{GNU_TAIL}) {
+		system("$t --pid=$lei_daemon_pid -F " .
+			"$lei_env->{XDG_RUNTIME_DIR}/lei/errors.log >&2 &");
+	}
+	if (my $strace_cmd = $ENV{STRACE_CMD}) {
+		system("$strace_cmd -p $lei_daemon_pid &");
+	}
 	$owner_pid = $$;
 }
 

[PATCH 07/11] lei: quiet down Eml-related warnings consistently

[Eric Wong]

"lei import" is probably the only place where it users
might care about warnings.
---
 lib/PublicInbox/LEI.pm        | 3 +++
 lib/PublicInbox/LeiBlob.pm    | 1 -
 lib/PublicInbox/LeiConvert.pm | 1 -
 lib/PublicInbox/LeiImport.pm  | 1 +
 lib/PublicInbox/LeiP2q.pm     | 4 +---
 lib/PublicInbox/LeiStore.pm   | 1 -
 lib/PublicInbox/LeiXSearch.pm | 1 -
 7 files changed, 5 insertions(+), 7 deletions(-)

diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index a949ae3e..cfbf12f0 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -23,6 +23,7 @@ use PublicInbox::Sigfd;
 use PublicInbox::DS qw(now dwaitpid);
 use PublicInbox::Spawn qw(spawn popen_rd);
 use PublicInbox::Lock;
+use PublicInbox::Eml;
 use Time::HiRes qw(stat); # ctime comparisons for config cache
 use File::Path qw(mkpath);
 use File::Spec;
@@ -509,6 +510,8 @@ sub _lei_atfork_child {
 	%PATH2CFG = ();
 	undef $errors_log;
 	$quit = \&CORE::exit;
+	$self->{-eml_noisy} or # only "lei import" sets this atm
+		$SIG{__WARN__} = PublicInbox::Eml::warn_ignore_cb();
 	$current_lei = $persist ? undef : $self; # for SIG{__WARN__}
 }
 
diff --git a/lib/PublicInbox/LeiBlob.pm b/lib/PublicInbox/LeiBlob.pm
index 0b96bd04..ff079e65 100644
--- a/lib/PublicInbox/LeiBlob.pm
+++ b/lib/PublicInbox/LeiBlob.pm
@@ -167,7 +167,6 @@ EOM
 sub ipc_atfork_child {
 	my ($self) = @_;
 	$self->{lei}->_lei_atfork_child;
-	$SIG{__WARN__} = PublicInbox::Eml::warn_ignore_cb();
 	$self->SUPER::ipc_atfork_child;
 }
 
diff --git a/lib/PublicInbox/LeiConvert.pm b/lib/PublicInbox/LeiConvert.pm
index 0c324169..14bed901 100644
--- a/lib/PublicInbox/LeiConvert.pm
+++ b/lib/PublicInbox/LeiConvert.pm
@@ -68,7 +68,6 @@ sub ipc_atfork_child {
 		$net->{mics_cached} = $net->imap_common_init($lei);
 		$net->{nn_cached} = $net->nntp_common_init($lei);
 	}
-	$SIG{__WARN__} = PublicInbox::Eml::warn_ignore_cb();
 	$l2m->pre_augment($lei);
 	$l2m->do_augment($lei);
 	$l2m->post_augment($lei);
diff --git a/lib/PublicInbox/LeiImport.pm b/lib/PublicInbox/LeiImport.pm
index e0d899cc..f2a0c95a 100644
--- a/lib/PublicInbox/LeiImport.pm
+++ b/lib/PublicInbox/LeiImport.pm
@@ -93,6 +93,7 @@ sub lei_import { # the main "lei import" method
 	my $ops = {};
 	$lei->{auth}->op_merge($ops, $self) if $lei->{auth};
 	$self->{-wq_nr_workers} = $j // 1; # locked
+	$lei->{-eml_noisy} = 1;
 	(my $op_c, $ops) = $lei->workers_start($self, 'lei-import', $j, $ops);
 	$lei->{imp} = $self;
 	net_merge_complete($self) unless $lei->{auth};
diff --git a/lib/PublicInbox/LeiP2q.pm b/lib/PublicInbox/LeiP2q.pm
index 07357e32..deb31974 100644
--- a/lib/PublicInbox/LeiP2q.pm
+++ b/lib/PublicInbox/LeiP2q.pm
@@ -197,9 +197,7 @@ sub lei_p2q { # the "lei patch-to-query" entry point
 
 sub ipc_atfork_child {
 	my ($self) = @_;
-	my $lei = $self->{lei};
-	$lei->_lei_atfork_child;
-	$SIG{__WARN__} = PublicInbox::Eml::warn_ignore_cb();
+	$self->{lei}->_lei_atfork_child;
 	$self->SUPER::ipc_atfork_child;
 }
 
diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm
index 1cf7ffc1..fcc9224d 100644
--- a/lib/PublicInbox/LeiStore.pm
+++ b/lib/PublicInbox/LeiStore.pm
@@ -411,7 +411,6 @@ sub ipc_atfork_child {
 		close $err->[0];
 		$self->{-err_wr} = $err->[1];
 	}
-	$SIG{__WARN__} = PublicInbox::Eml::warn_ignore_cb();
 	$self->SUPER::ipc_atfork_child;
 }
 
diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index 018b60f9..b3fd79d0 100644
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -443,7 +443,6 @@ sub incr_start_query { # called whenever an l2m shard starts do_post_auth
 sub ipc_atfork_child {
 	my ($self) = @_;
 	$self->{lei}->_lei_atfork_child;
-	$SIG{__WARN__} = PublicInbox::Eml::warn_ignore_cb();
 	$self->SUPER::ipc_atfork_child;
 }
 

[PATCH 08/11] lei: simple WQ workers use {wq1} field

[Eric Wong]

This lets us share more code and reduces cognitive overhead when
it comes to picking names (because {lsss} was ridiculous).

We'll need to ensure the first error set in lei is the actual
error we exit with, otherwise things can get confusing and
errors may get lost.
---
 lib/PublicInbox/LEI.pm         | 16 ++++++++++++----
 lib/PublicInbox/LeiBlob.pm     |  8 +-------
 lib/PublicInbox/LeiConvert.pm  |  2 +-
 lib/PublicInbox/LeiImport.pm   |  9 ++-------
 lib/PublicInbox/LeiLsSearch.pm |  2 +-
 lib/PublicInbox/LeiMirror.pm   |  4 ++--
 lib/PublicInbox/LeiP2q.pm      |  8 +-------
 lib/PublicInbox/LeiTag.pm      |  9 ++-------
 script/lei                     |  4 ++--
 9 files changed, 24 insertions(+), 38 deletions(-)

diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index cfbf12f0..403f9ed8 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -382,7 +382,7 @@ my %CONFIG_KEYS = (
 	'leistore.dir' => 'top-level storage location',
 );
 
-my @WQ_KEYS = qw(lxs l2m imp mrr cnv p2q tag sol lsss); # internal workers
+my @WQ_KEYS = qw(lxs l2m wq1); # internal workers
 
 sub _drop_wq {
 	my ($self) = @_;
@@ -542,7 +542,7 @@ sub workers_start {
 		'child_error' => [ \&child_error, $lei ],
 		($ops ? %$ops : ()),
 	};
-	$ops->{''} //= [ $wq->can('_lei_wq_eof') || \&dclose, $lei ];
+	$ops->{''} //= [ $wq->can('_lei_wq_eof') || \&wq_eof, $lei ];
 	my $end = $lei->pkt_op_pair;
 	$wq->wq_workers_start($ident, $jobs, $lei->oldset, { lei => $lei });
 	delete $lei->{pkt_op_p};
@@ -1237,9 +1237,17 @@ sub DESTROY {
 
 sub wq_done_wait { # dwaitpid callback
 	my ($arg, $pid) = @_;
-	my ($wq, $lei, $e) = @$arg;
-	$? and $lei->child_error($?, $e ? "$e errors during $lei->{cmd}" : ());
+	my ($wq, $lei) = @$arg;
+	my $err_type = $lei->{-err_type};
+	$? and $lei->child_error($?,
+			$err_type ? "$err_type errors during $lei->{cmd}" : ());
 	$lei->dclose;
 }
 
+sub wq_eof { # EOF callback for main daemon
+	my ($lei) = @_;
+	my $wq1 = delete $lei->{wq1} // return $lei->fail; # already failed
+	$wq1->wq_wait_old(\&wq_done_wait, $lei);
+}
+
 1;
diff --git a/lib/PublicInbox/LeiBlob.pm b/lib/PublicInbox/LeiBlob.pm
index ff079e65..0a957358 100644
--- a/lib/PublicInbox/LeiBlob.pm
+++ b/lib/PublicInbox/LeiBlob.pm
@@ -10,12 +10,6 @@ use parent qw(PublicInbox::IPC);
 use PublicInbox::Spawn qw(spawn popen_rd which);
 use PublicInbox::DS;
 
-sub _lei_wq_eof { # EOF callback for main daemon
-	my ($lei) = @_;
-	my $sol = delete $lei->{sol} // return $lei->dclose; # already failed
-	$sol->wq_wait_old($lei->can('wq_done_wait'), $lei);
-}
-
 sub get_git_dir ($$) {
 	my ($lei, $d) = @_;
 	return $d if -d "$d/objects" && -d "$d/refs" && -e "$d/HEAD";
@@ -158,7 +152,7 @@ EOM
 	require PublicInbox::SolverGit;
 	my $self = bless { lxs => $lxs, oid_b => $blob }, __PACKAGE__;
 	my ($op_c, $ops) = $lei->workers_start($self, 'lei-blob', 1);
-	$lei->{sol} = $self;
+	$lei->{wq1} = $self;
 	$self->wq_io_do('do_solve_blob', []);
 	$self->wq_close(1);
 	$op_c->op_wait_event($ops);
diff --git a/lib/PublicInbox/LeiConvert.pm b/lib/PublicInbox/LeiConvert.pm
index 14bed901..cefcaf65 100644
--- a/lib/PublicInbox/LeiConvert.pm
+++ b/lib/PublicInbox/LeiConvert.pm
@@ -53,7 +53,7 @@ sub lei_convert { # the main "lei convert" method
 	$lei->{opt}->{augment} = 1 if $devfd < 0;
 	$self->prepare_inputs($lei, \@inputs) or return;
 	my ($op_c, $ops) = $lei->workers_start($self, 'lei-convert', 1);
-	$lei->{cnv} = $self;
+	$lei->{wq1} = $self;
 	$self->wq_io_do('process_inputs', []);
 	$self->wq_close(1);
 	$op_c->op_wait_event($ops);
diff --git a/lib/PublicInbox/LeiImport.pm b/lib/PublicInbox/LeiImport.pm
index f2a0c95a..26127ece 100644
--- a/lib/PublicInbox/LeiImport.pm
+++ b/lib/PublicInbox/LeiImport.pm
@@ -53,12 +53,6 @@ sub input_nntp_cb { # nntp_each
 	input_eml_cb($self, $eml, $self->{-import_kw} ? { kw => $kw } : undef);
 }
 
-sub _lei_wq_eof { # EOF callback for main daemon
-	my ($lei) = @_;
-	my $imp = delete $lei->{imp} // return $lei->fail('BUG: {imp} gone');
-	$imp->wq_wait_old($lei->can('wq_done_wait'), $lei, 'non-fatal');
-}
-
 sub net_merge_complete { # callback used by LeiAuth
 	my ($self) = @_;
 	$self->wq_io_do('process_inputs');
@@ -95,7 +89,8 @@ sub lei_import { # the main "lei import" method
 	$self->{-wq_nr_workers} = $j // 1; # locked
 	$lei->{-eml_noisy} = 1;
 	(my $op_c, $ops) = $lei->workers_start($self, 'lei-import', $j, $ops);
-	$lei->{imp} = $self;
+	$lei->{wq1} = $self;
+	$lei->{-err_type} = 'non-fatal';
 	net_merge_complete($self) unless $lei->{auth};
 	$op_c->op_wait_event($ops);
 }
diff --git a/lib/PublicInbox/LeiLsSearch.pm b/lib/PublicInbox/LeiLsSearch.pm
index 9ac4870f..a00e78fc 100644
--- a/lib/PublicInbox/LeiLsSearch.pm
+++ b/lib/PublicInbox/LeiLsSearch.pm
@@ -73,7 +73,7 @@ sub bg_worker ($$$) {
 	my ($lei, $pfx, $json) = @_;
 	my $self = bless { -wq_nr_workers => 1, json => $json }, __PACKAGE__;
 	my ($op_c, $ops) = $lei->workers_start($self, 'ls-search', 1);
-	$lei->{lsss} = $self;
+	$lei->{wq1} = $self;
 	$self->wq_io_do('do_ls_search_long', [], $pfx);
 	$self->wq_close(1);
 	$op_c->op_wait_event($ops);
diff --git a/lib/PublicInbox/LeiMirror.pm b/lib/PublicInbox/LeiMirror.pm
index 50ab4c85..db97b98c 100644
--- a/lib/PublicInbox/LeiMirror.pm
+++ b/lib/PublicInbox/LeiMirror.pm
@@ -26,7 +26,7 @@ sub do_finish_mirror { # dwaitpid callback
 
 sub _lei_wq_eof { # EOF callback for main daemon
 	my ($lei) = @_;
-	my $mrr = delete $lei->{mrr} or return;
+	my $mrr = delete $lei->{wq1} or return $lei->fail;
 	$mrr->wq_wait_old(\&do_finish_mirror, $lei);
 }
 
@@ -283,7 +283,7 @@ sub start {
 	require PublicInbox::Admin;
 	require PublicInbox::InboxWritable;
 	my ($op, $ops) = $lei->workers_start($self, 'lei_mirror', 1);
-	$lei->{mrr} = $self;
+	$lei->{wq1} = $self;
 	$self->wq_io_do('do_mirror', []);
 	$self->wq_close(1);
 	$op->op_wait_event($ops);
diff --git a/lib/PublicInbox/LeiP2q.pm b/lib/PublicInbox/LeiP2q.pm
index deb31974..b4893489 100644
--- a/lib/PublicInbox/LeiP2q.pm
+++ b/lib/PublicInbox/LeiP2q.pm
@@ -189,7 +189,7 @@ sub lei_p2q { # the "lei patch-to-query" entry point
 		$self->{input} = $input;
 	}
 	my ($op, $ops) = $lei->workers_start($self, 'lei-p2q', 1);
-	$lei->{p2q} = $self;
+	$lei->{wq1} = $self;
 	$self->wq_io_do('do_p2q', []);
 	$self->wq_close(1);
 	$op->op_wait_event($ops);
@@ -201,10 +201,4 @@ sub ipc_atfork_child {
 	$self->SUPER::ipc_atfork_child;
 }
 
-sub _lei_wq_eof { # EOF callback for main daemon
-	my ($lei) = @_;
-	my $p2q = delete $lei->{p2q} // return $lei->dclose;
-	$p2q->wq_wait_old($lei->can('wq_done_wait'), $lei);
-}
-
 1;
diff --git a/lib/PublicInbox/LeiTag.pm b/lib/PublicInbox/LeiTag.pm
index 3cda2eca..989a6954 100644
--- a/lib/PublicInbox/LeiTag.pm
+++ b/lib/PublicInbox/LeiTag.pm
@@ -19,12 +19,6 @@ sub input_eml_cb { # used by PublicInbox::LeiInput::input_fh
 
 sub input_mbox_cb { input_eml_cb($_[1], $_[0]) }
 
-sub _lei_wq_eof { # EOF callback for main daemon
-	my ($lei) = @_;
-	my $tag = delete $lei->{tag} // return $lei->dclose;
-	$tag->wq_wait_old($lei->can('wq_done_wait'), $lei, 'non-fatal');
-}
-
 sub net_merge_complete { # callback used by LeiAuth
 	my ($self) = @_;
 	$self->wq_io_do('process_inputs');
@@ -57,7 +51,8 @@ sub lei_tag { # the "lei tag" method
 	$self->{vmd_mod} = $vmd_mod;
 	my $j = $self->{-wq_nr_workers} = 1; # locked for now
 	(my $op_c, $ops) = $lei->workers_start($self, 'lei-tag', $j, $ops);
-	$lei->{tag} = $self;
+	$lei->{wq1} = $self;
+	$lei->{-err_type} = 'non-fatal';
 	net_merge_complete($self) unless $lei->{auth};
 	$op_c->op_wait_event($ops);
 }
diff --git a/script/lei b/script/lei
index db302422..90a93839 100755
--- a/script/lei
+++ b/script/lei
@@ -116,10 +116,10 @@ Falling back to (slow) one-shot mode
 		} elsif ($buf eq '-WINCH') {
 			kill($buf, @parent); # for MUA
 		} elsif ($buf =~ /\Ax_it ([0-9]+)\z/) {
-			$x_it_code = $1 + 0;
+			$x_it_code ||= $1 + 0;
 			last;
 		} elsif ($buf =~ /\Achild_error ([0-9]+)\z/) {
-			$x_it_code = $1 + 0;
+			$x_it_code ||= $1 + 0;
 		} else {
 			$sigchld->();
 			die $buf;

[PATCH 09/11] lei_view_text: improve attachment display

[Eric Wong]

Support setting a color to distinguish from user-supplied text.
We'll also put the $BLOB:$IDX identifier on a separate line and
just put the entire corresponding lei command in the form of:
"[-- lei blob $BLOB:$IDX --]" to teach users how to access it.
---
 lib/PublicInbox/LeiViewText.pm | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/lib/PublicInbox/LeiViewText.pm b/lib/PublicInbox/LeiViewText.pm
index 6f5fca49..5d649840 100644
--- a/lib/PublicInbox/LeiViewText.pm
+++ b/lib/PublicInbox/LeiViewText.pm
@@ -25,6 +25,7 @@ my %DEFAULT_COLOR = (
 	quoted => 'blue',
 	hdrdefault => 'cyan',
 	status => 'bright_cyan', # smsg stuff
+	attachment => 'bright_red',
 
 	# git names and defaults, falls back to ~/.gitconfig
 	new => 'green',
@@ -113,20 +114,21 @@ sub hdr_buf ($$) {
 sub attach_note ($$$$;$) {
 	my ($self, $ct, $p, $fn, $err) = @_;
 	my ($part, $depth, $idx) = @$p;
-	my $obuf = $self->{obuf};
 	my $nl = $idx eq '1' ? '' : "\n"; # like join("\n", ...)
-	$$obuf .= <<EOF if $err;
+	my $abuf = $err ? <<EOF : '';
 [-- Warning: decoded text below may be mangled, UTF-8 assumed --]
 EOF
-	my $blob = $self->{-smsg}->{blob} // '';
-	$blob .= ':' if $blob ne '';
-	$$obuf .= "[-- Attachment $blob$idx ";
+	$abuf .= "[-- Attachment #$idx: ";
 	_xs($ct);
 	my $size = length($part->body);
 	my $ts = "Type: $ct, Size: $size bytes";
 	my $d = $part->header('Content-Description') // $fn // '';
 	_xs($d);
-	$$obuf .= $d eq '' ? "$ts --]\n" : "$d --]\n[-- $ts --]\n";
+	$abuf .= $d eq '' ? "$ts --]\n" : "$d --]\n[-- $ts --]\n";
+	if (my $blob = $self->{-smsg}->{blob}) {
+		$abuf .= "[-- lei blob $blob:$idx --]\n";
+	}
+	$self->{-colored}->($self, 'attachment', $abuf);
 	hdr_buf($self, $part) if $part->{is_submsg};
 }
 

[PATCH 10/11] lei_view_text: translate background colors from git

[Eric Wong]

This seems to work with or without attributes.  We'll deal with
256-color terminal colors when/if somebody cares for it, but the
usual 16 ought to be more than enough.
---
 lib/PublicInbox/LeiViewText.pm | 19 +++++++++++++++----
 1 file changed, 15 insertions(+), 4 deletions(-)

diff --git a/lib/PublicInbox/LeiViewText.pm b/lib/PublicInbox/LeiViewText.pm
index 5d649840..e0d62c0d 100644
--- a/lib/PublicInbox/LeiViewText.pm
+++ b/lib/PublicInbox/LeiViewText.pm
@@ -36,8 +36,11 @@ my %DEFAULT_COLOR = (
 	context => undef,
 );
 
+my $COLOR = qr/(?:bright)?
+		(?:normal|black|red|green|yellow|blue|magenta|cyan|white)/x;
+
 sub my_colored {
-	my ($self, $slot) = @_; # $_[2] = buffer
+	my ($self, $slot, $buf) = @_;
 	my $val = $self->{"color.$slot"} //=
 			$self->{-leicfg}->{"color.$slot"} //
 			$self->{-gitcfg}->{"color.diff.$slot"} //
@@ -45,11 +48,19 @@ sub my_colored {
 			$DEFAULT_COLOR{$slot};
 	$val = $val->[-1] if ref($val) eq 'ARRAY';
 	if (defined $val) {
+		$val = lc $val;
 		# git doesn't use "_", Term::ANSIColor does
-		$val =~ s/\Abright([^_])/bright_$1/i;
-		${$self->{obuf}} .= Term::ANSIColor::colored($_[2], lc $val);
+		$val =~ s/\Abright([^_])/bright_$1/ig;
+
+		# git: "green black" => T::A: "green on_black"
+		$val =~ s/($COLOR)(.+?)($COLOR)/$1$2on_$3/;
+
+		# FIXME: convert git #XXXXXX to T::A-compatible colors
+		# for 256-color terminals
+
+		${$self->{obuf}} .= colored($buf, $val);
 	} else {
-		${$self->{obuf}} .= $_[2];
+		${$self->{obuf}} .= $buf;
 	}
 }
 

[PATCH 11/11] lei (lcat|q): support --no-color and --color

[Eric Wong]

This should be familiar to git users who wish to force color
when writing to pipes or disable color.
---
 lib/PublicInbox/LEI.pm         | 9 +++++----
 lib/PublicInbox/LeiViewText.pm | 2 +-
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 403f9ed8..7ffcf163 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -144,8 +144,8 @@ our %CMD = ( # sorted in order of importance/use:
 	@lxs_opt,
 	qw(save output|mfolder|o=s format|f=s dedupe|d=s threads|t+
 	sort|s=s reverse|r offset=i pretty jobs|j=s globoff|g augment|a
-	import-before! lock=s@ rsyncable alert=s@ mua=s verbose|v+), @c_opt,
-	opt_dash('limit|n=i', '[0-9]+') ],
+	import-before! lock=s@ rsyncable alert=s@ mua=s verbose|v+
+	color!), @c_opt, opt_dash('limit|n=i', '[0-9]+') ],
 
 'up' => [ 'OUTPUT|--all', 'update saved search',
 	qw(jobs|j=s lock=s@ alert=s@ mua=s verbose|v+ all:s), @c_opt ],
@@ -155,8 +155,8 @@ our %CMD = ( # sorted in order of importance/use:
 	# some of these options are ridiculous for lcat
 	@lxs_opt, qw(output|mfolder|o=s format|f=s dedupe|d=s threads|t+
 	sort|s=s reverse|r offset=i jobs|j=s globoff|g augment|a
-	import-before! lock=s@ rsyncable alert=s@ mua=s verbose|v+), @c_opt,
-	opt_dash('limit|n=i', '[0-9]+') ],
+	import-before! lock=s@ rsyncable alert=s@ mua=s verbose|v+
+	color!), @c_opt, opt_dash('limit|n=i', '[0-9]+') ],
 
 'blob' => [ 'OID', 'show a git blob, reconstructing from mail if necessary',
 	qw(git-dir=s@ cwd! verbose|v+ mail! oid-a|A=s path-a|a=s path-b|b=s),
@@ -268,6 +268,7 @@ my %OPTDESC = (
 'incremental!	import' => 'import already seen IMAP and NNTP articles',
 'globoff|g' => "do not match locations using '*?' wildcards ".
 		"and\xa0'[]'\x{a0}ranges",
+'color!' => 'disable color (for --format=text)',
 'verbose|v+' => 'be more verbose',
 'external!' => 'do not use externals',
 'mail!' => 'do not look in mail storage for OID',
diff --git a/lib/PublicInbox/LeiViewText.pm b/lib/PublicInbox/LeiViewText.pm
index e0d62c0d..d0f8b7f4 100644
--- a/lib/PublicInbox/LeiViewText.pm
+++ b/lib/PublicInbox/LeiViewText.pm
@@ -69,7 +69,7 @@ sub uncolored { ${$_[0]->{obuf}} .= $_[2] }
 sub new {
 	my ($cls, $lei) = @_;
 	my $self = bless { %{$lei->{opt}}, -colored => \&uncolored }, $cls;
-	return $self unless $self->{color} || -t $lei->{1};
+	return $self unless $self->{color} //= -t $lei->{1};
 	my $cmd = [ qw(git config -z --includes -l) ];
 	my ($r, $pid) = popen_rd($cmd, undef, { 2 => $lei->{2} });
 	my $cfg = PublicInbox::Config::config_fh_parse($r, "\0", "\n");