From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on dcvr.yhbt.net X-Spam-Level: X-Spam-Status: No, score=-4.0 required=3.0 tests=ALL_TRUSTED,BAYES_00 shortcircuit=no autolearn=ham autolearn_force=no version=3.4.2 Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id E286F20087 for ; Tue, 2 Feb 2021 11:47:03 +0000 (UTC) From: Eric Wong To: meta@public-inbox.org Subject: [PATCH 15/16] lei q: tidy up progress reporting Date: Tue, 2 Feb 2021 11:47:01 +0000 Message-Id: <20210202114702.29886-16-e@80x24.org> In-Reply-To: <20210202114702.29886-1-e@80x24.org> References: <20210202114702.29886-1-e@80x24.org> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit List-Id: We won't be reporting progress when output is going to stdout since it can clutter up the terminal unless stderr != stdout, which probably isn't worth checking. We'll also use a more agnostic mset_progress which may make it easier to support worker-less invocations. --- lib/PublicInbox/LEI.pm | 1 + lib/PublicInbox/LeiOverview.pm | 2 ++ lib/PublicInbox/LeiXSearch.pm | 34 +++++++++++++++++++--------------- 3 files changed, 22 insertions(+), 15 deletions(-) diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm index 44afced3..2c512c5e 100644 --- a/lib/PublicInbox/LEI.pm +++ b/lib/PublicInbox/LEI.pm @@ -871,6 +871,7 @@ sub accept_dispatch { # Listener {post_accept} callback sub dclose { my ($self) = @_; + delete $self->{-progress}; for my $f (qw(lxs l2m)) { my $wq = delete $self->{$f} or next; if ($wq->wq_kill) { diff --git a/lib/PublicInbox/LeiOverview.pm b/lib/PublicInbox/LeiOverview.pm index ff15d295..52da225d 100644 --- a/lib/PublicInbox/LeiOverview.pm +++ b/lib/PublicInbox/LeiOverview.pm @@ -90,6 +90,8 @@ sub new { } else { ovv_out_lk_init($self); } + } elsif (!$opt->{quiet}) { + $lei->{-progress} = 1; } if ($json) { $lei->{dedupe} //= PublicInbox::LeiDedupe->new($lei); diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm index e207f0fc..57a18075 100644 --- a/lib/PublicInbox/LeiXSearch.pm +++ b/lib/PublicInbox/LeiXSearch.pm @@ -107,6 +107,19 @@ sub wait_startq ($) { read($startq, my $query_prepare_done, 1); } +sub mset_progress { + my $lei = shift; + return unless $lei->{-progress}; + if ($lei->{pkt_op}) { # called via pkt_op/pkt_do from workers + pkt_do($lei->{pkt_op}, 'mset_progress', @_); + } else { # single lei-daemon consumer + my @args = ref($_[-1]) eq 'ARRAY' ? @{$_[-1]} : @_; + my ($desc, $mset_size, $mset_total_est) = @args; + $lei->{-mset_total} += $mset_size; + $lei->err("# $desc $mset_size/$mset_total_est"); + } +} + sub query_thread_mset { # for --thread my ($self, $lei, $ibxish) = @_; local $0 = "$0 query_thread_mset"; @@ -121,7 +134,7 @@ sub query_thread_mset { # for --thread my $each_smsg = $lei->{ovv}->ovv_each_smsg_cb($lei, $ibxish); do { $mset = $srch->mset($mo->{qstr}, $mo); - pkt_do($lei->{pkt_op}, 'mset_progress', $desc, $mset->size, + mset_progress($lei, $desc, $mset->size, $mset->get_matches_estimated); my $ids = $srch->mset_to_artnums($mset, $mo); my $ctx = { ids => $ids }; @@ -154,7 +167,7 @@ sub query_mset { # non-parallel for non-"--thread" users my $each_smsg = $lei->{ovv}->ovv_each_smsg_cb($lei, $self); do { $mset = $self->mset($mo->{qstr}, $mo); - pkt_do($lei->{pkt_op}, 'mset_progress', 'xsearch', + mset_progress($lei, 'xsearch', $mset->size, $mset->size, $mset->get_matches_estimated); for my $mitem ($mset->items) { my $smsg = smsg_for($self, $mitem) or next; @@ -174,8 +187,8 @@ sub each_eml { # callback for MboxReader->mboxrd $smsg->{$_} //= '' for qw(from to cc ds subject references mid); delete @$smsg{qw(From Subject -ds -ts)}; if (my $startq = delete($lei->{startq})) { wait_startq($startq) } - ++$lei->{-nr_remote_eml}; - if (!$lei->{opt}->{quiet}) { + if ($lei->{-progress}) { + ++$lei->{-nr_remote_eml}; my $now = now(); my $next = $lei->{-next_progress} //= ($now + 1); if ($now > $next) { @@ -261,8 +274,7 @@ sub query_remote_mboxrd { return $lei->fail("E: @$cmd: $@") if $@; if ($? == 0) { my $nr = $lei->{-nr_remote_eml}; - pkt_do($lei->{pkt_op}, 'mset_progress', - $lei->{-current_url}, $nr, $nr); + mset_progress($lei, $lei->{-current_url}, $nr, $nr); next; } seek($cerr, $coff, SEEK_SET) or warn "seek(curl stderr): $!\n"; @@ -318,19 +330,11 @@ Error closing $lei->{ovv}->{dst}: $! } $lei->start_mua; } - $lei->{opt}->{quiet} or + $lei->{-progress} and $lei->err('# ', $lei->{-mset_total} // 0, " matches"); $lei->dclose; } -sub mset_progress { # called via pkt_op/pkt_do from workers - my ($lei, $pargs) = @_; - my ($desc, $mset_size, $mset_total_est) = @$pargs; - return if $lei->{opt}->{quiet}; - $lei->{-mset_total} += $mset_size; - $lei->err("# $desc $mset_size/$mset_total_est"); -} - sub do_post_augment { my ($lei, $zpipe, $au_done) = @_; my $l2m = $lei->{l2m} or die 'BUG: no {l2m}';