From: Eric Wong <e@80x24.org>
To: meta@public-inbox.org
Subject: [PATCH 15/16] lei q: tidy up progress reporting
Date: Tue, 2 Feb 2021 11:47:01 +0000 [thread overview]
Message-ID: <20210202114702.29886-16-e@80x24.org> (raw)
In-Reply-To: <20210202114702.29886-1-e@80x24.org>
We won't be reporting progress when output is going to stdout
since it can clutter up the terminal unless stderr != stdout,
which probably isn't worth checking.
We'll also use a more agnostic mset_progress which may
make it easier to support worker-less invocations.
---
lib/PublicInbox/LEI.pm | 1 +
lib/PublicInbox/LeiOverview.pm | 2 ++
lib/PublicInbox/LeiXSearch.pm | 34 +++++++++++++++++++---------------
3 files changed, 22 insertions(+), 15 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 44afced3..2c512c5e 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -871,6 +871,7 @@ sub accept_dispatch { # Listener {post_accept} callback
sub dclose {
my ($self) = @_;
+ delete $self->{-progress};
for my $f (qw(lxs l2m)) {
my $wq = delete $self->{$f} or next;
if ($wq->wq_kill) {
diff --git a/lib/PublicInbox/LeiOverview.pm b/lib/PublicInbox/LeiOverview.pm
index ff15d295..52da225d 100644
--- a/lib/PublicInbox/LeiOverview.pm
+++ b/lib/PublicInbox/LeiOverview.pm
@@ -90,6 +90,8 @@ sub new {
} else {
ovv_out_lk_init($self);
}
+ } elsif (!$opt->{quiet}) {
+ $lei->{-progress} = 1;
}
if ($json) {
$lei->{dedupe} //= PublicInbox::LeiDedupe->new($lei);
diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index e207f0fc..57a18075 100644
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -107,6 +107,19 @@ sub wait_startq ($) {
read($startq, my $query_prepare_done, 1);
}
+sub mset_progress {
+ my $lei = shift;
+ return unless $lei->{-progress};
+ if ($lei->{pkt_op}) { # called via pkt_op/pkt_do from workers
+ pkt_do($lei->{pkt_op}, 'mset_progress', @_);
+ } else { # single lei-daemon consumer
+ my @args = ref($_[-1]) eq 'ARRAY' ? @{$_[-1]} : @_;
+ my ($desc, $mset_size, $mset_total_est) = @args;
+ $lei->{-mset_total} += $mset_size;
+ $lei->err("# $desc $mset_size/$mset_total_est");
+ }
+}
+
sub query_thread_mset { # for --thread
my ($self, $lei, $ibxish) = @_;
local $0 = "$0 query_thread_mset";
@@ -121,7 +134,7 @@ sub query_thread_mset { # for --thread
my $each_smsg = $lei->{ovv}->ovv_each_smsg_cb($lei, $ibxish);
do {
$mset = $srch->mset($mo->{qstr}, $mo);
- pkt_do($lei->{pkt_op}, 'mset_progress', $desc, $mset->size,
+ mset_progress($lei, $desc, $mset->size,
$mset->get_matches_estimated);
my $ids = $srch->mset_to_artnums($mset, $mo);
my $ctx = { ids => $ids };
@@ -154,7 +167,7 @@ sub query_mset { # non-parallel for non-"--thread" users
my $each_smsg = $lei->{ovv}->ovv_each_smsg_cb($lei, $self);
do {
$mset = $self->mset($mo->{qstr}, $mo);
- pkt_do($lei->{pkt_op}, 'mset_progress', 'xsearch',
+ mset_progress($lei, 'xsearch', $mset->size,
$mset->size, $mset->get_matches_estimated);
for my $mitem ($mset->items) {
my $smsg = smsg_for($self, $mitem) or next;
@@ -174,8 +187,8 @@ sub each_eml { # callback for MboxReader->mboxrd
$smsg->{$_} //= '' for qw(from to cc ds subject references mid);
delete @$smsg{qw(From Subject -ds -ts)};
if (my $startq = delete($lei->{startq})) { wait_startq($startq) }
- ++$lei->{-nr_remote_eml};
- if (!$lei->{opt}->{quiet}) {
+ if ($lei->{-progress}) {
+ ++$lei->{-nr_remote_eml};
my $now = now();
my $next = $lei->{-next_progress} //= ($now + 1);
if ($now > $next) {
@@ -261,8 +274,7 @@ sub query_remote_mboxrd {
return $lei->fail("E: @$cmd: $@") if $@;
if ($? == 0) {
my $nr = $lei->{-nr_remote_eml};
- pkt_do($lei->{pkt_op}, 'mset_progress',
- $lei->{-current_url}, $nr, $nr);
+ mset_progress($lei, $lei->{-current_url}, $nr, $nr);
next;
}
seek($cerr, $coff, SEEK_SET) or warn "seek(curl stderr): $!\n";
@@ -318,19 +330,11 @@ Error closing $lei->{ovv}->{dst}: $!
}
$lei->start_mua;
}
- $lei->{opt}->{quiet} or
+ $lei->{-progress} and
$lei->err('# ', $lei->{-mset_total} // 0, " matches");
$lei->dclose;
}
-sub mset_progress { # called via pkt_op/pkt_do from workers
- my ($lei, $pargs) = @_;
- my ($desc, $mset_size, $mset_total_est) = @$pargs;
- return if $lei->{opt}->{quiet};
- $lei->{-mset_total} += $mset_size;
- $lei->err("# $desc $mset_size/$mset_total_est");
-}
-
sub do_post_augment {
my ($lei, $zpipe, $au_done) = @_;
my $l2m = $lei->{l2m} or die 'BUG: no {l2m}';
next prev parent reply other threads:[~2021-02-02 11:47 UTC|newest]
Thread overview: 17+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-02-02 11:46 [PATCH 00/16] lei: -I/--include and more Eric Wong
2021-02-02 11:46 ` [PATCH 01/16] lei: switch to use SEQPACKET socketpair instead of pipe Eric Wong
2021-02-02 11:46 ` [PATCH 02/16] lei_query: default to 10000 messages as documented Eric Wong
2021-02-02 11:46 ` [PATCH 03/16] lei q: emit progress and counting via PktOp Eric Wong
2021-02-02 11:46 ` [PATCH 04/16] lei q: support --only, --include and --exclude Eric Wong
2021-02-02 11:46 ` [PATCH 05/16] lei: complete: do not complete non-arg options w/ help text Eric Wong
2021-02-02 11:46 ` [PATCH 06/16] lei: q: shell completion for --(include|exclude|only) Eric Wong
2021-02-02 11:46 ` [PATCH 07/16] lei_xsearch: truncate curl stderr after reading it Eric Wong
2021-02-02 11:46 ` [PATCH 08/16] lib: explicitly distinguish oneshot use Eric Wong
2021-02-02 11:46 ` [PATCH 09/16] lei q: do not leave temporary files after oneshot exit Eric Wong
2021-02-02 11:46 ` [PATCH 10/16] cmd_ipc4: fix comments and formatting Eric Wong
2021-02-02 11:46 ` [PATCH 11/16] pktop: fix potential undefined var Eric Wong
2021-02-02 11:46 ` [PATCH 12/16] lei_xsearch: ensure curl.err and tail(1) cleanup happens Eric Wong
2021-02-02 11:46 ` [PATCH 13/16] doc: lei-q: note "-a" and link to Xapian QueryParser Eric Wong
2021-02-02 11:47 ` [PATCH 14/16] lei_overview: avoid unnecessary {l2m} delete Eric Wong
2021-02-02 11:47 ` Eric Wong [this message]
2021-02-02 11:47 ` [PATCH 16/16] lei q: support --jobs [SEARCHERS],[WRITERS] Eric Wong
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
List information: https://public-inbox.org/README
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20210202114702.29886-16-e@80x24.org \
--to=e@80x24.org \
--cc=meta@public-inbox.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
Code repositories for project(s) associated with this public inbox
https://80x24.org/public-inbox.git
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).