From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on dcvr.yhbt.net X-Spam-Level: X-Spam-ASN: X-Spam-Status: No, score=-4.0 required=3.0 tests=ALL_TRUSTED,BAYES_00 shortcircuit=no autolearn=ham autolearn_force=no version=3.4.2 Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id 7553D1F9E5 for ; Sat, 1 May 2021 06:21:17 +0000 (UTC) From: Eric Wong To: meta@public-inbox.org Subject: [PATCH 2/5] lei : distinguish between mset and l2m counts Date: Sat, 1 May 2021 06:21:14 +0000 Message-Id: <20210501062117.21617-3-e@80x24.org> In-Reply-To: <20210501062117.21617-1-e@80x24.org> References: <20210501062117.21617-1-e@80x24.org> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit List-Id: The number of messages we write to --output is usually different than the mset count due to deduplication from combining multiple sources. This change makes the stderr output of "lei up --all=local" way more useful IMHO. --- lib/PublicInbox/LeiToMail.pm | 36 ++++++++++++++++++++--------------- lib/PublicInbox/LeiXSearch.pm | 18 ++++++++++++++---- 2 files changed, 35 insertions(+), 19 deletions(-) diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm index eda4701c..a546ab42 100644 --- a/lib/PublicInbox/LeiToMail.pm +++ b/lib/PublicInbox/LeiToMail.pm @@ -196,9 +196,13 @@ sub _mbox_write_cb ($$) { return if $dedupe->is_dup($eml, $smsg); $lse->xsmsg_vmd($smsg) if $lse; $buf = $eml2mbox->($eml, $smsg); - return atomic_append($lei, $buf) if $atomic_append; - my $lk = $ovv->lock_for_scope; - $lei->out($$buf); + if ($atomic_append) { + atomic_append($lei, $buf); + } else { + my $lk = $ovv->lock_for_scope; + $lei->out($$buf); + } + ++$lei->{-nr_write}; } } @@ -273,15 +277,13 @@ sub _maildir_write_cb ($$) { my $dst = $lei->{ovv}->{dst}; my $lse = $lei->{lse}; # may be undef sub { # for git_to_mail - my ($buf, $smsg, $eml) = @_; + my ($bref, $smsg, $eml) = @_; $dst // return $lei->fail; # dst may be undef-ed in last run - $buf //= \($eml->as_string); + return if $dedupe && $dedupe->is_dup($eml // + PublicInbox::Eml->new($$bref)); $lse->xsmsg_vmd($smsg) if $lse; - return _buf2maildir($dst, $buf, $smsg) if !$dedupe; - $eml //= PublicInbox::Eml->new($$buf); # copy buf - return if $dedupe->is_dup($eml, $smsg); - undef $eml; - _buf2maildir($dst, $buf, $smsg); + _buf2maildir($dst, $bref // \($eml->as_string), $smsg); + ++$lei->{-nr_write}; } } @@ -296,16 +298,15 @@ sub _imap_write_cb ($$) { sub { # for git_to_mail my ($bref, $smsg, $eml) = @_; $mic // return $lei->fail; # mic may be undef-ed in last run - if ($dedupe) { - $eml //= PublicInbox::Eml->new($$bref); # copy bref - return if $dedupe->is_dup($eml, $smsg); - } + return if $dedupe && $dedupe->is_dup($eml // + PublicInbox::Eml->new($$bref)); $lse->xsmsg_vmd($smsg) if $lse; eval { $imap_append->($mic, $folder, $bref, $smsg, $eml) }; if (my $err = $@) { undef $mic; die $err; } + ++$lei->{-nr_write}; } } @@ -659,7 +660,12 @@ sub write_mail { # via ->wq_io_do sub wq_atexit_child { my ($self) = @_; delete $self->{wcb}; - $self->{lei}->{ale}->git->async_wait_all; + my $lei = $self->{lei}; + $lei->{ale}->git->async_wait_all; + my $nr = delete($lei->{-nr_write}) or return; + return if $lei->{early_mua} || !$lei->{-progress} || !$lei->{pkt_op_p}; + require PublicInbox::PktOp; + PublicInbox::PktOp::pkt_do($lei->{pkt_op_p}, 'l2m_progress', $nr); } # called in top-level lei-daemon when LeiAuth is done diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm index b3fd79d0..e04af0dc 100644 --- a/lib/PublicInbox/LeiXSearch.pm +++ b/lib/PublicInbox/LeiXSearch.pm @@ -144,6 +144,11 @@ sub mset_progress { } } +sub l2m_progress { + my ($lei, $nr) = @_; + $lei->{-nr_write} += $nr; +} + sub query_one_mset { # for --threads and l2m w/o sort my ($self, $ibxish) = @_; local $0 = "$0 query_one_mset"; @@ -354,7 +359,7 @@ sub query_done { # EOF callback for main daemon } my $wait = $lei->{sto} ? $lei->{sto}->ipc_do('done') : undef; $lei->{ovv}->ovv_end($lei); - my @out; + my (@out, $start_mua); if ($l2m) { # close() calls LeiToMail reap_compress @out = (" in $lei->{ovv}->{dst}"); if (my $out = delete $lei->{old_1}) { @@ -370,11 +375,15 @@ Error closing $lei->{ovv}->{dst}: $! $lei->poke_mua; } else { # mbox users delete $l2m->{mbl}; # drop dotlock - $lei->start_mua; + $start_mua = 1; } } - $lei->{-progress} and - $lei->err('# ', $lei->{-mset_total} // 0, " matches", @out); + if ($lei->{-progress}) { + $lei->qerr('# ', $lei->{-mset_total} // 0, " matches", @out); + my $nr = $lei->{-nr_write} // 0; + $lei->qerr("# $nr written to $lei->{ovv}->{dst}") if $l2m; + } + $lei->start_mua if $start_mua; $lei->dclose; } @@ -456,6 +465,7 @@ sub do_query { '+' => [ \&incr_post_augment, $lei ], '' => [ \&query_done, $lei ], 'mset_progress' => [ \&mset_progress, $lei ], + 'l2m_progress' => [ \&l2m_progress, $lei ], 'x_it' => [ $lei->can('x_it'), $lei ], 'child_error' => [ $lei->can('child_error'), $lei ], 'incr_start_query' => [ \&incr_start_query, $self, $l2m ],