[PATCH 00/11] refining lei up+inspect

[PATCH 00/11] refining lei up+inspect

[Eric Wong]

"lei up" gains some flexibility in dealing with offline
situations.  "inspect" is slightly nicer-to-use

11/11 is preparation for "lei fsck"...

Eric Wong (11):
  test_common: lazy-require AutoReap
  lei up: prefix `remote' and `local' with `o_'
  lei: use die for external and query handling
  lei up: propagate redispatch_all failure via exit code
  lei: conditionally add "\n" to error messages
  lei up: support --exclude=, --no-(external|remote|local)
  lei: remove unused ->busy time arg
  doc: lei: describe lei-daemon-kill and upgrades
  lei inspect: add atfork hook
  lei inspect: show ISO8601 {rt} and {dt}, too
  lei_mail_sync: show non-matching SHA

 Documentation/lei-daemon-kill.pod | 29 +++++++++++++++++--
 Documentation/lei-overview.pod    |  8 +++++-
 Documentation/lei-up.pod          | 28 ++++++++++++++----
 lib/PublicInbox/LEI.pm            | 15 +++++-----
 lib/PublicInbox/LeiExternal.pm    | 15 ++++++++--
 lib/PublicInbox/LeiInspect.pm     | 11 +++++++
 lib/PublicInbox/LeiMailSync.pm    |  9 ++++--
 lib/PublicInbox/LeiOverview.pm    |  6 ++--
 lib/PublicInbox/LeiQuery.pm       | 27 ++++++-----------
 lib/PublicInbox/LeiSavedSearch.pm |  8 +++---
 lib/PublicInbox/LeiUp.pm          | 48 ++++++++++++++++++++++---------
 lib/PublicInbox/LeiXSearch.pm     |  7 +++--
 lib/PublicInbox/TestCommon.pm     |  3 +-
 13 files changed, 149 insertions(+), 65 deletions(-)

[PATCH 01/11] test_common: lazy-require AutoReap

[Eric Wong]

This might speed up non-daemon-using tests.
---
 lib/PublicInbox/TestCommon.pm | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/PublicInbox/TestCommon.pm b/lib/PublicInbox/TestCommon.pm
index 835779996d56..fb0d5a4e6ede 100644
--- a/lib/PublicInbox/TestCommon.pm
+++ b/lib/PublicInbox/TestCommon.pm
@@ -6,7 +6,6 @@ package PublicInbox::TestCommon;
 use strict;
 use parent qw(Exporter);
 use v5.10.1;
-use PublicInbox::AutoReap;
 use Fcntl qw(FD_CLOEXEC F_SETFD F_GETFD :seek);
 use POSIX qw(dup2);
 use IO::Socket::INET;
@@ -430,6 +429,7 @@ sub tail_f (@) {
 	require PublicInbox::Spawn;
 	my $pid = PublicInbox::Spawn::spawn($cmd, undef, { 1 => 2 });
 	wait_for_tail($pid, scalar @_);
+	require PublicInbox::AutoReap;
 	PublicInbox::AutoReap->new($pid, \&wait_for_tail);
 }
 
@@ -493,6 +493,7 @@ sub start_script {
 			die "FAIL: ",join(' ', $key, @argv), ": $!\n";
 		}
 	}
+	require PublicInbox::AutoReap;
 	my $td = PublicInbox::AutoReap->new($pid);
 	$td->{-extra} = $tail;
 	$td;

[PATCH 02/11] lei up: prefix `remote' and `local' with `o_'

[Eric Wong]

This will help distinguish between mail outputs and external
public-inboxes.
---
 lib/PublicInbox/LeiUp.pm | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/lib/PublicInbox/LeiUp.pm b/lib/PublicInbox/LeiUp.pm
index 396041771ff9..c35b2e42d49f 100644
--- a/lib/PublicInbox/LeiUp.pm
+++ b/lib/PublicInbox/LeiUp.pm
@@ -46,7 +46,7 @@ sub up1 ($$) {
 
 sub redispatch_all ($$) {
 	my ($self, $lei) = @_;
-	my $upq = [ (@{$self->{local} // []}, @{$self->{remote} // []}) ];
+	my $upq = [ (@{$self->{o_local} // []}, @{$self->{o_remote} // []}) ];
 	return up1($lei, $upq->[0]) if @$upq == 1; # just one, may start MUA
 
 	# FIXME: this is also used per-query, see lei->_start_query
@@ -81,12 +81,12 @@ sub lei_up {
 			$lei->fail('--all and --mua= are incompatible');
 		@outs = PublicInbox::LeiSavedSearch::list($lei);
 		if ($all eq 'local') {
-			$self->{local} = [ grep(!/$REMOTE_RE/, @outs) ];
+			$self->{o_local} = [ grep(!/$REMOTE_RE/, @outs) ];
 		} elsif ($all eq 'remote') {
-			$self->{remote} = [ grep(/$REMOTE_RE/, @outs) ];
+			$self->{o_remote} = [ grep(/$REMOTE_RE/, @outs) ];
 		} elsif ($all eq '') {
-			$self->{remote} = [ grep(/$REMOTE_RE/, @outs) ];
-			$self->{local} = [ grep(!/$REMOTE_RE/, @outs) ];
+			$self->{o_remote} = [ grep(/$REMOTE_RE/, @outs) ];
+			$self->{o_local} = [ grep(!/$REMOTE_RE/, @outs) ];
 		} else {
 			$lei->fail("only --all=$all not understood");
 		}
@@ -94,16 +94,16 @@ sub lei_up {
 		scalar(@outs) == 1 or die "BUG: lse set w/ >1 out[@outs]";
 		return up1($lei, $outs[0]);
 	} else {
-		$self->{remote} = [ grep(/$REMOTE_RE/, @outs) ];
-		$self->{local} = [ grep(!/$REMOTE_RE/, @outs) ];
+		$self->{o_remote} = [ grep(/$REMOTE_RE/, @outs) ];
+		$self->{o_local} = [ grep(!/$REMOTE_RE/, @outs) ];
 	}
 	$lei->{lse} = $lei->_lei_store(1)->write_prepare($lei)->search;
-	((@{$self->{local} // []} + @{$self->{remote} // []}) > 1 &&
+	((@{$self->{o_local} // []} + @{$self->{o_remote} // []}) > 1 &&
 		defined($opt->{mua})) and return $lei->fail(<<EOM);
 multiple outputs and --mua= are incompatible
 EOM
-	if ($self->{remote}) { # setup lei->{auth}
-		$self->prepare_inputs($lei, $self->{remote}) or return;
+	if ($self->{o_remote}) { # setup lei->{auth}
+		$self->prepare_inputs($lei, $self->{o_remote}) or return;
 	}
 	if ($lei->{auth}) { # start auth worker
 		require PublicInbox::NetWriter;

[PATCH 03/11] lei: use die for external and query handling

[Eric Wong]

This allows "lei up" to continue processing unrelated externals
if on output fails.
---
 lib/PublicInbox/LeiExternal.pm    |  4 ++--
 lib/PublicInbox/LeiQuery.pm       | 12 +++++-------
 lib/PublicInbox/LeiSavedSearch.pm |  8 ++++----
 lib/PublicInbox/LeiUp.pm          |  4 ++--
 lib/PublicInbox/LeiXSearch.pm     |  7 ++++---
 5 files changed, 17 insertions(+), 18 deletions(-)

diff --git a/lib/PublicInbox/LeiExternal.pm b/lib/PublicInbox/LeiExternal.pm
index 701d1ad53adf..851715d7099c 100644
--- a/lib/PublicInbox/LeiExternal.pm
+++ b/lib/PublicInbox/LeiExternal.pm
@@ -101,9 +101,9 @@ sub get_externals {
 		return (ext_canonicalize($loc));
 	}
 	if (scalar(@m) == 0) {
-		$self->fail("`$loc' is unknown");
+		die "`$loc' is unknown\n";
 	} else {
-		$self->fail("`$loc' is ambiguous:\n", map { "\t$_\n" } @m);
+		die("`$loc' is ambiguous:\n", map { "\t$_\n" } @m, "\n");
 	}
 	();
 }
diff --git a/lib/PublicInbox/LeiQuery.pm b/lib/PublicInbox/LeiQuery.pm
index ec2ece051492..56b82acc8b5b 100644
--- a/lib/PublicInbox/LeiQuery.pm
+++ b/lib/PublicInbox/LeiQuery.pm
@@ -18,17 +18,15 @@ sub _start_query { # used by "lei q" and "lei up"
 	PublicInbox::LeiOverview->new($self) or return;
 	my $opt = $self->{opt};
 	my ($xj, $mj) = split(/,/, $opt->{jobs} // '');
-	if (defined($xj) && $xj ne '' && $xj !~ /\A[1-9][0-9]*\z/) {
-		return $self->fail("`$xj' search jobs must be >= 1");
-	}
+	(defined($xj) && $xj ne '' && $xj !~ /\A[1-9][0-9]*\z/) and
+		die "`$xj' search jobs must be >= 1\n";
 	my $lxs = $self->{lxs};
 	$xj ||= $lxs->concurrency($opt); # allow: "--jobs ,$WRITER_ONLY"
 	my $nproc = $lxs->detect_nproc || 1; # don't memoize, schedtool(1) exists
 	$xj = $nproc if $xj > $nproc;
 	$lxs->{-wq_nr_workers} = $xj;
-	if (defined($mj) && $mj !~ /\A[1-9][0-9]*\z/) {
-		return $self->fail("`$mj' writer jobs must be >= 1");
-	}
+	(defined($mj) && $mj !~ /\A[1-9][0-9]*\z/) and
+		die "`$mj' writer jobs must be >= 1\n";
 	my $l2m = $self->{l2m};
 	# we use \1 (a ref) to distinguish between default vs. user-supplied
 	if ($l2m && grep { $opt->{$_} //= \1 } (qw(mail-sync import-remote
@@ -112,7 +110,7 @@ sub lxs_prepare {
 		}
 	}
 	($lxs->locals || $lxs->remotes) ? ($self->{lxs} = $lxs) :
-		$self->fail('no local or remote inboxes to search');
+		die("no local or remote inboxes to search\n");
 }
 
 # the main "lei q SEARCH_TERMS" method
diff --git a/lib/PublicInbox/LeiSavedSearch.pm b/lib/PublicInbox/LeiSavedSearch.pm
index 3e10f780ad02..b2f1ad1027d3 100644
--- a/lib/PublicInbox/LeiSavedSearch.pm
+++ b/lib/PublicInbox/LeiSavedSearch.pm
@@ -96,7 +96,7 @@ sub translate_dedupe ($$) {
 	my $dd = $lei->{opt}->{dedupe} // 'content';
 	return 1 if $dd eq 'content'; # the default
 	return $self->{"-dedupe_$dd"} = 1 if ($dd eq 'oid' || $dd eq 'mid');
-	$lei->fail("--dedupe=$dd requires --no-save");
+	die("--dedupe=$dd requires --no-save\n");
 }
 
 sub up { # updating existing saved search via "lei up"
@@ -105,9 +105,9 @@ sub up { # updating existing saved search via "lei up"
 	my $self = bless { ale => $lei->ale }, $cls;
 	my $dir = $dst;
 	output2lssdir($self, $lei, \$dir, \$f) or
-		return $lei->fail("--no-save was used with $dst cwd=".
-					$lei->rel2abs('.'));
-	$self->{-cfg} = $lei->cfg_dump($f) // return $lei->fail;
+		return die("--no-save was used with $dst cwd=".
+					$lei->rel2abs('.')."\n");
+	$self->{-cfg} = $lei->cfg_dump($f) // return $lei->child_error;
 	$self->{-ovf} = "$dir/over.sqlite3";
 	$self->{'-f'} = $f;
 	$self->{lock_path} = "$self->{-f}.flock";
diff --git a/lib/PublicInbox/LeiUp.pm b/lib/PublicInbox/LeiUp.pm
index c35b2e42d49f..4fd0290c7612 100644
--- a/lib/PublicInbox/LeiUp.pm
+++ b/lib/PublicInbox/LeiUp.pm
@@ -19,7 +19,7 @@ sub up1 ($$) {
 	my $f = $lss->{'-f'};
 	my $mset_opt = $lei->{mset_opt} = { relevance => -2 };
 	my $q = $mset_opt->{q_raw} = $lss->{-cfg}->{'lei.q'} //
-				return $lei->fail("lei.q unset in $f");
+				die("lei.q unset in $f (out=$out)\n");
 	my $lse = $lei->{lse} // die 'BUG: {lse} missing';
 	if (ref($q)) {
 		$mset_opt->{qstr} = $lse->query_argv_to_string($lse->git, $q);
@@ -36,7 +36,7 @@ sub up1 ($$) {
 		$lei->{opt}->{$k} //= $v;
 	}
 	my $o = $lei->{opt}->{output} // '';
-	return $lei->fail("lei.q.output unset in $f (out=$out)") if $o eq '';
+	return die("lei.q.output unset in $f (out=$out)\n") if $o eq '';
 	$lss->translate_dedupe($lei) or return;
 	$lei->{lss} = $lss; # for LeiOverview->new and query_remote_mboxrd
 	my $lxs = $lei->lxs_prepare or return;
diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index 8ab84b15c00b..119070a289de 100644
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -142,11 +142,11 @@ sub wait_startq ($) {
 				delete $lei->{opt}->{verbose};
 				delete $lei->{-progress};
 			} else {
-				$lei->fail("$$ WTF `$do_augment_done'");
+				die "BUG: do_augment_done=`$do_augment_done'";
 			}
 			return;
 		}
-		return $lei->fail("$$ wait_startq: $!") unless $!{EINTR};
+		die "wait_startq: $!" unless $!{EINTR};
 	}
 }
 
@@ -473,7 +473,8 @@ sub do_post_augment {
 		$lei->fail("$err");
 	}
 	if (!$err && delete $lei->{early_mua}) { # non-augment case
-		$lei->start_mua;
+		eval { $lei->start_mua };
+		$lei->fail($@) if $@;
 	}
 	close(delete $lei->{au_done}); # triggers wait_startq in lei_xsearch
 }

[PATCH 04/11] lei up: propagate redispatch_all failure via exit code

[Eric Wong]

We can still continue with some local externals, maybe;
but the error needs to be propagated to the calling process
for scripting purposes.
---
 lib/PublicInbox/LeiUp.pm | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/PublicInbox/LeiUp.pm b/lib/PublicInbox/LeiUp.pm
index 4fd0290c7612..fcdd535dc118 100644
--- a/lib/PublicInbox/LeiUp.pm
+++ b/lib/PublicInbox/LeiUp.pm
@@ -124,7 +124,7 @@ sub net_merge_all_done {
 	$lei->{net} = delete($self->{-net_new}) if $self->{-net_new};
 	$self->wq_close;
 	eval { redispatch_all($self, $lei) };
-	warn "E: $@" if $@;
+	$lei->child_error(0, "E: $@") if $@;
 }
 
 sub _complete_up { # lei__complete hook

[PATCH 05/11] lei: conditionally add "\n" to error messages

[Eric Wong]

Some error messages already include "\n" (w/ file+line info),
so don't add another one.  (`warn' will automatically add its
caller location unless there's a final "\n").
---
 lib/PublicInbox/LEI.pm | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 6b989b33647e..553379e404fc 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -510,10 +510,10 @@ sub sigpipe_handler { # handles SIGPIPE from @WQ_KEYS workers
 }
 
 sub fail ($$;$) {
-	my ($self, $buf, $exit_code) = @_;
+	my ($self, $msg, $exit_code) = @_;
 	local $current_lei = $self;
 	$self->{failed}++;
-	warn($buf, "\n") if defined $buf;
+	warn(substr($msg, -1, 1) eq "\n" ? $msg : "$msg\n") if defined $msg;
 	$self->{pkt_op_p}->pkt_do('fail_handler') if $self->{pkt_op_p};
 	x_it($self, ($exit_code // 1) << 8);
 	undef;
@@ -534,7 +534,7 @@ sub child_error { # passes non-fatal curl exit codes to user
 	my ($self, $child_error, $msg) = @_; # child_error is $?
 	local $current_lei = $self;
 	$child_error ||= 1 << 8;
-	warn($msg, "\n") if defined $msg;
+	warn(substr($msg, -1, 1) eq "\n" ? $msg : "$msg\n") if defined $msg;
 	if ($self->{pkt_op_p}) { # to top lei-daemon
 		$self->{pkt_op_p}->pkt_do('child_error', $child_error);
 	} elsif ($self->{sock}) { # to lei(1) client

[PATCH 06/11] lei up: support --exclude=, --no-(external|remote|local)

[Eric Wong]

These can be used to temporarily disable  using certain
externals in case of temporary network failure or mount point
unavailability.
---
 Documentation/lei-up.pod       | 28 +++++++++++++++++++++++-----
 lib/PublicInbox/LEI.pm         |  4 ++--
 lib/PublicInbox/LeiExternal.pm | 11 ++++++++++-
 lib/PublicInbox/LeiQuery.pm    | 15 ++++-----------
 lib/PublicInbox/LeiUp.pm       | 22 ++++++++++++++++++++++
 5 files changed, 61 insertions(+), 19 deletions(-)

diff --git a/Documentation/lei-up.pod b/Documentation/lei-up.pod
index f06ee5eb62b7..8fba0953b1ef 100644
--- a/Documentation/lei-up.pod
+++ b/Documentation/lei-up.pod
@@ -37,11 +37,26 @@ e.g C<1.hour> or C<3.days>
 
 Default: 2.days
 
-=back
+=item --no-external
 
-The following options, described in L<lei-q(1)>, are supported.
+=item --no-local
 
-=over
+=item --no-remote
+
+These disable the use of all externals, local externals, or
+remote externals respectively.  They are useful during
+temporary network or mount-point outages.
+
+Unlike C<lei q>, these switches override the original C<lei q --only>
+options saved as C<lei.q.only>.
+
+The combination C<--all=remote --no-remote> is supported for
+offline use in case a user is updating an IMAP folder on localhost.
+
+=item --exclude=LOCATION
+
+As with L<lei-q(1)>, but may also exclude externals originally
+specified via C<lei q --only>.
 
 =item --lock=METHOD
 
@@ -49,7 +64,10 @@ The following options, described in L<lei-q(1)>, are supported.
 
 =item --mua=CMD
 
-This option is incompatible with C<--all>.
+C<--lock>, C<--alert>, and C<--mua> are all supported and
+documented in L<lei-q(1)>.
+
+C<--mua> is incompatible with C<--all>.
 
 =back
 
@@ -62,7 +80,7 @@ and L<http://4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/meta
 
 =head1 COPYRIGHT
 
-Copyright 2021 all contributors L<mailto:meta@public-inbox.org>
+Copyright all contributors L<mailto:meta@public-inbox.org>
 
 License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>
 
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 553379e404fc..5b726f71382e 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -178,8 +178,8 @@ our %CMD = ( # sorted in order of importance/use:
 	shared color! mail-sync!), @c_opt, opt_dash('limit|n=i', '[0-9]+') ],
 
 'up' => [ 'OUTPUT...|--all', 'update saved search',
-	qw(jobs|j=s lock=s@ alert=s@ mua=s verbose|v+
-	remote-fudge-time=s all:s), @c_opt ],
+	qw(jobs|j=s lock=s@ alert=s@ mua=s verbose|v+ exclude=s@
+	remote-fudge-time=s all:s remote! local! external!), @c_opt ],
 
 'lcat' => [ '--stdin|MSGID_OR_URL...', 'display local copy of message(s)',
 	'stdin|', # /|\z/ must be first for lone dash
diff --git a/lib/PublicInbox/LeiExternal.pm b/lib/PublicInbox/LeiExternal.pm
index 851715d7099c..30bb1a4579c7 100644
--- a/lib/PublicInbox/LeiExternal.pm
+++ b/lib/PublicInbox/LeiExternal.pm
@@ -105,7 +105,16 @@ sub get_externals {
 	} else {
 		die("`$loc' is ambiguous:\n", map { "\t$_\n" } @m, "\n");
 	}
-	();
+}
+
+sub canonicalize_excludes {
+	my ($lei, $excludes) = @_;
+	my %x;
+	for my $loc (@$excludes) {
+		my @l = get_externals($lei, $loc, 1);
+		$x{$_} = 1 for @l;
+	}
+	\%x;
 }
 
 # returns an anonymous sub which returns an array of potential results
diff --git a/lib/PublicInbox/LeiQuery.pm b/lib/PublicInbox/LeiQuery.pm
index 56b82acc8b5b..effc572f6aaf 100644
--- a/lib/PublicInbox/LeiQuery.pm
+++ b/lib/PublicInbox/LeiQuery.pm
@@ -95,18 +95,11 @@ sub lxs_prepare {
 		}
 		# --external is enabled by default, but allow --no-external
 		if ($opt->{external} //= 1) {
-			my %x;
-			for my $loc (@{$opt->{exclude} // []}) {
-				my @l = $self->get_externals($loc, 1) or return;
-				$x{$_} = 1 for @l;
-			}
-			my $ne = $self->externals_each(\&prep_ext, $lxs, \%x);
+			my $ex = $self->canonicalize_excludes($opt->{exclude});
+			$self->externals_each(\&prep_ext, $lxs, $ex);
 			$opt->{remote} //= !($lxs->locals - $opt->{'local'});
-			if ($opt->{'local'}) {
-				$lxs->{remotes} = \@iremotes if !$opt->{remote};
-			} else {
-				$lxs->{locals} = \@ilocals;
-			}
+			$lxs->{locals} = \@ilocals if !$opt->{'local'};
+			$lxs->{remotes} = \@iremotes if !$opt->{remote};
 		}
 	}
 	($lxs->locals || $lxs->remotes) ? ($self->{lxs} = $lxs) :
diff --git a/lib/PublicInbox/LeiUp.pm b/lib/PublicInbox/LeiUp.pm
index fcdd535dc118..dac0fc287885 100644
--- a/lib/PublicInbox/LeiUp.pm
+++ b/lib/PublicInbox/LeiUp.pm
@@ -15,6 +15,14 @@ my $REMOTE_RE = qr!\A(?:imap|http)s?://!i; # http(s) will be for JMAP
 
 sub up1 ($$) {
 	my ($lei, $out) = @_;
+	# precedence note for CLI switches between lei q and up:
+	# `lei q --only' > `lei q --no-(remote|local|external)'
+	# `lei up --no-(remote|local|external)' > `lei.q.only' in saved search
+	my %no = map {
+		my $v = $lei->{opt}->{$_}; # set by CLI
+		(defined($v) && !$v) ? ($_ => 1) : ();
+	} qw(remote local external);
+	my $cli_exclude = delete $lei->{opt}->{exclude};
 	my $lss = PublicInbox::LeiSavedSearch->up($lei, $out) or return;
 	my $f = $lss->{'-f'};
 	my $mset_opt = $lei->{mset_opt} = { relevance => -2 };
@@ -31,6 +39,20 @@ sub up1 ($$) {
 		my $v = $lss->{-cfg}->get_all("lei.q.$k") // next;
 		$lei->{opt}->{$k} //= $v;
 	}
+
+	# --no-(local|remote) CLI flags overrided saved `lei.q.only'
+	my $only = $lei->{opt}->{only};
+	@$only = map { $lei->get_externals($_) } @$only if $only;
+	if (scalar keys %no && $only) {
+		@$only = grep(!m!\Ahttps?://!i, @$only) if $no{remote};
+		@$only = grep(m!\Ahttps?://!i, @$only) if $no{'local'};
+	}
+	if ($cli_exclude) {
+		my $ex = $lei->canonicalize_excludes($cli_exclude);
+		@$only = grep { !$ex->{$_} } @$only if $only;
+		push @{$lei->{opt}->{exclude}}, @$cli_exclude;
+	}
+	delete $lei->{opt}->{only} if $no{external} || ($only && !@$only);
 	for my $k ($lss->BOOL_FIELDS, $lss->SINGLE_FIELDS) {
 		my $v = $lss->{-cfg}->get_1("lei.q.$k") // next;
 		$lei->{opt}->{$k} //= $v;

[PATCH 07/11] lei: remove unused ->busy time arg

[Eric Wong]

Our graceful shutdown doesn't time out clients.
---
 lib/PublicInbox/LEI.pm | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 5b726f71382e..c1f28f7b31bf 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -19,7 +19,7 @@ use IO::Handle ();
 use Fcntl qw(SEEK_SET);
 use PublicInbox::Config;
 use PublicInbox::Syscall qw(EPOLLIN);
-use PublicInbox::DS qw(now dwaitpid);
+use PublicInbox::DS qw(dwaitpid);
 use PublicInbox::Spawn qw(spawn popen_rd);
 use PublicInbox::Lock;
 use PublicInbox::Eml;
@@ -1315,11 +1315,10 @@ sub lazy_start {
 			$quit->();
 		}
 		return 1 if defined($path);
-		my $now = now();
 		my $n = 0;
 		for my $s (values %$dmap) {
 			$s->can('busy') or next;
-			if ($s->busy($now)) {
+			if ($s->busy) {
 				++$n;
 			} else {
 				$s->close;

[PATCH 08/11] doc: lei: describe lei-daemon-kill and upgrades

[Eric Wong]

While we're at it, start dropping copyright years
since it seems acceptable to not have them:

  https://www.linuxfoundation.org/blog/copyright-notices-in-open-source-software-projects/

Copyright years are also a noisy to update every year (maybe,
just maybe, we'll make it to 2022...)
---
 Documentation/lei-daemon-kill.pod | 29 ++++++++++++++++++++++++++---
 Documentation/lei-overview.pod    |  8 +++++++-
 2 files changed, 33 insertions(+), 4 deletions(-)

diff --git a/Documentation/lei-daemon-kill.pod b/Documentation/lei-daemon-kill.pod
index 7fb0fb25ad0e..48c237b8d3a6 100644
--- a/Documentation/lei-daemon-kill.pod
+++ b/Documentation/lei-daemon-kill.pod
@@ -8,7 +8,30 @@ lei daemon-kill [-SIGNAL | -s SIGNAL | --signal SIGNAL]
 
 =head1 DESCRIPTION
 
-Send a signal to the lei-daemon.  C<SIGNAL> defaults to C<TERM>.
+Send a signal to the L<lei-daemon(8)>.  C<SIGNAL> defaults to C<TERM>.
+
+This command should be run after updating the code of lei.
+
+=head1 SIGNALS
+
+=over 8
+
+=item SIGTERM
+
+Send a graceful termination signal.  L<lei-daemon(8)> will exit
+when all currently running lei commands are done.  The listen
+socket will be released as soon as the signal is processed
+so another L<lei-daemon(8)> process can take its place.
+
+=item SIGKILL
+
+Kills L<lei-daemon(8)> immediately.  Some worker processes may
+remain running after a short while after this takes effect.
+
+=back
+
+=for comment
+SIGQUIT and SIGINT currently do what SIGTERM do, may change...
 
 =head1 CONTACT
 
@@ -19,10 +42,10 @@ L<http://4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/meta/>
 
 =head1 COPYRIGHT
 
-Copyright 2021 all contributors L<mailto:meta@public-inbox.org>
+Copyright all contributors L<mailto:meta@public-inbox.org>
 
 License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>
 
 =head1 SEE ALSO
 
-L<lei-daemon-pid(1)>
+L<lei-daemon-pid(1)>, L<lei-daemon(8)>
diff --git a/Documentation/lei-overview.pod b/Documentation/lei-overview.pod
index 40a7b0aadd04..bb2fe50f7cd9 100644
--- a/Documentation/lei-overview.pod
+++ b/Documentation/lei-overview.pod
@@ -131,6 +131,12 @@ C<contrib/completion/>.  Contributions adding support for other
 shells, as well as improvements to the existing Bash completion, are
 welcome.
 
+=head1 UPGRADING
+
+Since lei runs as a daemon, L<lei-daemon-kill(1)> is required to kill
+the daemon so it can load new code.  It will be restarted with the
+next invocation of any lei command.
+
 =head1 CONTACT
 
 Feedback welcome via plain-text mail to L<mailto:meta@public-inbox.org>
@@ -140,6 +146,6 @@ L<http://4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/meta/>
 
 =head1 COPYRIGHT
 
-Copyright 2021 all contributors L<mailto:meta@public-inbox.org>
+Copyright all contributors L<mailto:meta@public-inbox.org>
 
 License: AGPL-3.0+ L<https://www.gnu.org/licenses/agpl-3.0.txt>

[PATCH 09/11] lei inspect: add atfork hook

[Eric Wong]

This is necessary for in case an inspect command is run
in a parallel with other commands.
---
 lib/PublicInbox/LeiInspect.pm | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/lib/PublicInbox/LeiInspect.pm b/lib/PublicInbox/LeiInspect.pm
index 05b6e21d298d..38ef3ad96df2 100644
--- a/lib/PublicInbox/LeiInspect.pm
+++ b/lib/PublicInbox/LeiInspect.pm
@@ -289,4 +289,10 @@ sub _complete_inspect {
 	# TODO: message-ids?, blobs? could get expensive...
 }
 
+sub input_only_atfork_child {
+	my ($self) = @_;
+	$self->{lei}->_lei_atfork_child;
+	$self->SUPER::ipc_atfork_child;
+}
+
 1;

[PATCH 10/11] lei inspect: show ISO8601 {rt} and {dt}, too

[Eric Wong]

While inspect is intended for debugging, the Unix epoch in
seconds requires extra steps for human consumption; just
steal what we used for "lei q -f json" output.
---
 lib/PublicInbox/LeiInspect.pm  | 5 +++++
 lib/PublicInbox/LeiOverview.pm | 6 +++---
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/lib/PublicInbox/LeiInspect.pm b/lib/PublicInbox/LeiInspect.pm
index 38ef3ad96df2..5ea32ccb7e66 100644
--- a/lib/PublicInbox/LeiInspect.pm
+++ b/lib/PublicInbox/LeiInspect.pm
@@ -12,10 +12,15 @@ use parent qw(PublicInbox::IPC);
 use PublicInbox::Config;
 use PublicInbox::MID qw(mids);
 use PublicInbox::NetReader qw(imap_uri nntp_uri);
+use POSIX qw(strftime);
+use PublicInbox::LeiOverview;
+*iso8601 = \&PublicInbox::LeiOverview::iso8601;
 
 sub _json_prep ($) {
 	my ($smsg) = @_;
 	$smsg->{$_} += 0 for qw(bytes lines); # integerize
+	$smsg->{dt} = iso8601($smsg->{ds}) if defined($smsg->{ds});
+	$smsg->{rt} = iso8601($smsg->{ts}) if defined($smsg->{ts});
 	+{ %$smsg } # unbless and scalarize
 }
 
diff --git a/lib/PublicInbox/LeiOverview.pm b/lib/PublicInbox/LeiOverview.pm
index 1b9dc9701c95..2d3db9f4ab92 100644
--- a/lib/PublicInbox/LeiOverview.pm
+++ b/lib/PublicInbox/LeiOverview.pm
@@ -21,7 +21,7 @@ use PublicInbox::LeiToMail;
 # cf. https://en.wikipedia.org/wiki/JSON_streaming
 my $JSONL = 'ldjson|ndjson|jsonl'; # 3 names for the same thing
 
-sub _iso8601 ($) { strftime('%Y-%m-%dT%H:%M:%SZ', gmtime($_[0])) }
+sub iso8601 ($) { strftime('%Y-%m-%dT%H:%M:%SZ', gmtime($_[0])) }
 
 # we open this in the parent process before ->wq_io_do handoff
 sub ovv_out_lk_init ($) {
@@ -139,8 +139,8 @@ sub _unbless_smsg {
 	# num/tid are nonsensical with multi-inbox search,
 	# lines/bytes are not generally useful
 	delete @$smsg{qw(num tid lines bytes)};
-	$smsg->{rt} = _iso8601(delete $smsg->{ts}); # JMAP receivedAt
-	$smsg->{dt} = _iso8601(delete $smsg->{ds}); # JMAP UTCDate
+	$smsg->{rt} = iso8601(delete $smsg->{ts}); # JMAP receivedAt
+	$smsg->{dt} = iso8601(delete $smsg->{ds}); # JMAP UTCDate
 	$smsg->{pct} = get_pct($mitem) if $mitem;
 	if (my $r = delete $smsg->{references}) {
 		$smsg->{refs} = [ map { $_ } ($r =~ m/$MID_EXTRACT/go) ];

[PATCH 11/11] lei_mail_sync: show non-matching SHA

[Eric Wong]

It could prove useful for diagnosing bugs (either on our
end or an MUA's), or storage device failures.
---
 lib/PublicInbox/LeiMailSync.pm | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/lib/PublicInbox/LeiMailSync.pm b/lib/PublicInbox/LeiMailSync.pm
index f2f1e3ed2658..e70cb5de2b6b 100644
--- a/lib/PublicInbox/LeiMailSync.pm
+++ b/lib/PublicInbox/LeiMailSync.pm
@@ -339,9 +339,12 @@ WHERE b.oidbin = ?
 			next unless -s $fh;
 			local $/;
 			my $raw = <$fh>;
-			if ($vrfy && git_sha(1, \$raw)->hexdigest ne $oidhex) {
-				warn "$f changed $oidhex\n";
-				next;
+			if ($vrfy) {
+				my $got = git_sha(1, \$raw)->hexdigest;
+				if ($got ne $oidhex) {
+					warn "$f changed $oidhex => $got\n";
+					next;
+				}
 			}
 			return \$raw;
 		}