From: Eric Wong <e@80x24.org>
To: meta@public-inbox.org
Subject: [PATCH 2/5] lei <q|up>: distinguish between mset and l2m counts
Date: Sat, 1 May 2021 06:21:14 +0000 [thread overview]
Message-ID: <20210501062117.21617-3-e@80x24.org> (raw)
In-Reply-To: <20210501062117.21617-1-e@80x24.org>
The number of messages we write to --output is usually different
than the mset count due to deduplication from combining multiple
sources.
This change makes the stderr output of "lei up --all=local" way
more useful IMHO.
---
lib/PublicInbox/LeiToMail.pm | 36 ++++++++++++++++++++---------------
lib/PublicInbox/LeiXSearch.pm | 18 ++++++++++++++----
2 files changed, 35 insertions(+), 19 deletions(-)
diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm
index eda4701c..a546ab42 100644
--- a/lib/PublicInbox/LeiToMail.pm
+++ b/lib/PublicInbox/LeiToMail.pm
@@ -196,9 +196,13 @@ sub _mbox_write_cb ($$) {
return if $dedupe->is_dup($eml, $smsg);
$lse->xsmsg_vmd($smsg) if $lse;
$buf = $eml2mbox->($eml, $smsg);
- return atomic_append($lei, $buf) if $atomic_append;
- my $lk = $ovv->lock_for_scope;
- $lei->out($$buf);
+ if ($atomic_append) {
+ atomic_append($lei, $buf);
+ } else {
+ my $lk = $ovv->lock_for_scope;
+ $lei->out($$buf);
+ }
+ ++$lei->{-nr_write};
}
}
@@ -273,15 +277,13 @@ sub _maildir_write_cb ($$) {
my $dst = $lei->{ovv}->{dst};
my $lse = $lei->{lse}; # may be undef
sub { # for git_to_mail
- my ($buf, $smsg, $eml) = @_;
+ my ($bref, $smsg, $eml) = @_;
$dst // return $lei->fail; # dst may be undef-ed in last run
- $buf //= \($eml->as_string);
+ return if $dedupe && $dedupe->is_dup($eml //
+ PublicInbox::Eml->new($$bref));
$lse->xsmsg_vmd($smsg) if $lse;
- return _buf2maildir($dst, $buf, $smsg) if !$dedupe;
- $eml //= PublicInbox::Eml->new($$buf); # copy buf
- return if $dedupe->is_dup($eml, $smsg);
- undef $eml;
- _buf2maildir($dst, $buf, $smsg);
+ _buf2maildir($dst, $bref // \($eml->as_string), $smsg);
+ ++$lei->{-nr_write};
}
}
@@ -296,16 +298,15 @@ sub _imap_write_cb ($$) {
sub { # for git_to_mail
my ($bref, $smsg, $eml) = @_;
$mic // return $lei->fail; # mic may be undef-ed in last run
- if ($dedupe) {
- $eml //= PublicInbox::Eml->new($$bref); # copy bref
- return if $dedupe->is_dup($eml, $smsg);
- }
+ return if $dedupe && $dedupe->is_dup($eml //
+ PublicInbox::Eml->new($$bref));
$lse->xsmsg_vmd($smsg) if $lse;
eval { $imap_append->($mic, $folder, $bref, $smsg, $eml) };
if (my $err = $@) {
undef $mic;
die $err;
}
+ ++$lei->{-nr_write};
}
}
@@ -659,7 +660,12 @@ sub write_mail { # via ->wq_io_do
sub wq_atexit_child {
my ($self) = @_;
delete $self->{wcb};
- $self->{lei}->{ale}->git->async_wait_all;
+ my $lei = $self->{lei};
+ $lei->{ale}->git->async_wait_all;
+ my $nr = delete($lei->{-nr_write}) or return;
+ return if $lei->{early_mua} || !$lei->{-progress} || !$lei->{pkt_op_p};
+ require PublicInbox::PktOp;
+ PublicInbox::PktOp::pkt_do($lei->{pkt_op_p}, 'l2m_progress', $nr);
}
# called in top-level lei-daemon when LeiAuth is done
diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index b3fd79d0..e04af0dc 100644
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -144,6 +144,11 @@ sub mset_progress {
}
}
+sub l2m_progress {
+ my ($lei, $nr) = @_;
+ $lei->{-nr_write} += $nr;
+}
+
sub query_one_mset { # for --threads and l2m w/o sort
my ($self, $ibxish) = @_;
local $0 = "$0 query_one_mset";
@@ -354,7 +359,7 @@ sub query_done { # EOF callback for main daemon
}
my $wait = $lei->{sto} ? $lei->{sto}->ipc_do('done') : undef;
$lei->{ovv}->ovv_end($lei);
- my @out;
+ my (@out, $start_mua);
if ($l2m) { # close() calls LeiToMail reap_compress
@out = (" in $lei->{ovv}->{dst}");
if (my $out = delete $lei->{old_1}) {
@@ -370,11 +375,15 @@ Error closing $lei->{ovv}->{dst}: $!
$lei->poke_mua;
} else { # mbox users
delete $l2m->{mbl}; # drop dotlock
- $lei->start_mua;
+ $start_mua = 1;
}
}
- $lei->{-progress} and
- $lei->err('# ', $lei->{-mset_total} // 0, " matches", @out);
+ if ($lei->{-progress}) {
+ $lei->qerr('# ', $lei->{-mset_total} // 0, " matches", @out);
+ my $nr = $lei->{-nr_write} // 0;
+ $lei->qerr("# $nr written to $lei->{ovv}->{dst}") if $l2m;
+ }
+ $lei->start_mua if $start_mua;
$lei->dclose;
}
@@ -456,6 +465,7 @@ sub do_query {
'+' => [ \&incr_post_augment, $lei ],
'' => [ \&query_done, $lei ],
'mset_progress' => [ \&mset_progress, $lei ],
+ 'l2m_progress' => [ \&l2m_progress, $lei ],
'x_it' => [ $lei->can('x_it'), $lei ],
'child_error' => [ $lei->can('child_error'), $lei ],
'incr_start_query' => [ \&incr_start_query, $self, $l2m ],
next prev parent reply other threads:[~2021-05-01 6:21 UTC|newest]
Thread overview: 7+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-05-01 6:21 [PATCH 0/5] lei: more UI/UX tweaks Eric Wong
2021-05-01 6:21 ` [PATCH 1/5] xt/lei-onion-convert: test for NNTP+IMAP onions Eric Wong
2021-05-01 6:21 ` Eric Wong [this message]
2021-05-01 6:21 ` [PATCH 3/5] lei_saved_search: fix excess indent for first lei.q entry Eric Wong
2021-05-01 6:21 ` [PATCH 4/5] lei: rename ls-sync to ls-mail-sync Eric Wong
2021-05-01 19:29 ` [PATCH 6/5] lei import: fix --mail-sync handling in LeiInput Eric Wong
2021-05-01 6:21 ` [PATCH 5/5] lei edit-search: support relocating lei.q.output Eric Wong
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
List information: https://public-inbox.org/README
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20210501062117.21617-3-e@80x24.org \
--to=e@80x24.org \
--cc=meta@public-inbox.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
Code repositories for project(s) associated with this public inbox
https://80x24.org/public-inbox.git
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).