* [PATCH 08/21] lei: get rid of l2m_progress PktOp callback
2023-10-04 3:49 7% [PATCH 00/21] lei + IPC related stuff Eric Wong
@ 2023-10-04 3:49 5% ` Eric Wong
0 siblings, 0 replies; 2+ results
From: Eric Wong @ 2023-10-04 3:49 UTC (permalink / raw)
To: meta
We already have an ->incr callback we can enhance to support
multiple counters with a single request. Furthermore, we can
just flatten the object graph by storing counters directly in
the $lei object itself to reduce hash lookups.
---
lib/PublicInbox/LEI.pm | 13 ++++++-------
lib/PublicInbox/LeiConvert.pm | 7 ++++---
lib/PublicInbox/LeiTag.pm | 6 +++---
lib/PublicInbox/LeiToMail.pm | 20 ++++++++++----------
lib/PublicInbox/LeiXSearch.pm | 15 ++++-----------
t/lei-tag.t | 3 +++
6 files changed, 30 insertions(+), 34 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 3408551b..fba4edf3 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -628,8 +628,8 @@ sub pkt_op_pair {
}
sub incr {
- my ($self, $field, $nr) = @_;
- $self->{counters}->{$field} += $nr;
+ my $lei = shift;
+ while (my ($f, $n) = splice(@_, 0, 2)) { $lei->{$f} += $n }
}
sub pkt_ops {
@@ -1418,11 +1418,10 @@ sub busy { 1 } # prevent daemon-shutdown if client is connected
# can immediately reread it
sub DESTROY {
my ($self) = @_;
- if (my $counters = delete $self->{counters}) {
- for my $k (sort keys %$counters) {
- my $nr = $counters->{$k};
- $self->child_error(0, "$nr $k messages");
- }
+ for my $k (sort(grep(/\A-nr_/, keys %$self))) {
+ my $nr = $self->{$k};
+ substr($k, 0, length('-nr_'), '');
+ $self->child_error(0, "$nr $k messages");
}
$self->{1}->autoflush(1) if $self->{1};
stop_pager($self);
diff --git a/lib/PublicInbox/LeiConvert.pm b/lib/PublicInbox/LeiConvert.pm
index 1acd4558..22aba81a 100644
--- a/lib/PublicInbox/LeiConvert.pm
+++ b/lib/PublicInbox/LeiConvert.pm
@@ -34,9 +34,10 @@ sub process_inputs { # via wq_do
$self->SUPER::process_inputs;
my $lei = $self->{lei};
delete $lei->{1};
+ my $l2m = delete $self->{l2m};
delete $self->{wcb}; # commit
- my $nr_w = delete($lei->{-nr_write}) // 0;
- my $d = (delete($lei->{-nr_seen}) // 0) - $nr_w;
+ my $nr_w = delete($l2m->{-nr_write}) // 0;
+ my $d = (delete($l2m->{-nr_seen}) // 0) - $nr_w;
$d = $d ? " ($d duplicates)" : '';
$lei->qerr("# converted $nr_w messages$d");
}
@@ -64,7 +65,7 @@ sub ipc_atfork_child {
my ($self) = @_;
my $lei = $self->{lei};
$lei->_lei_atfork_child;
- my $l2m = delete $lei->{l2m};
+ my $l2m = $lei->{l2m};
if (my $net = $lei->{net}) { # may prompt user once
$net->{mics_cached} = $net->imap_common_init($lei);
$net->{nn_cached} = $net->nntp_common_init($lei);
diff --git a/lib/PublicInbox/LeiTag.pm b/lib/PublicInbox/LeiTag.pm
index 76bd2d70..320b0355 100644
--- a/lib/PublicInbox/LeiTag.pm
+++ b/lib/PublicInbox/LeiTag.pm
@@ -15,7 +15,7 @@ sub input_eml_cb { # used by PublicInbox::LeiInput::input_fh
$self->{lei}->{sto}->wq_do('update_xvmd', $xoids, $eml,
$self->{lei}->{vmd_mod});
} else {
- ++$self->{unimported};
+ ++$self->{-nr_unimported};
}
}
@@ -40,8 +40,8 @@ sub lei_tag { # the "lei tag" method
sub note_unimported {
my ($self) = @_;
- my $n = $self->{unimported} or return;
- $self->{lei}->{pkt_op_p}->pkt_do('incr', 'unimported', $n);
+ my $n = $self->{-nr_unimported} or return;
+ $self->{lei}->{pkt_op_p}->pkt_do('incr', -nr_unimported => $n);
}
sub ipc_atfork_child {
diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm
index b9f28ee4..f239da82 100644
--- a/lib/PublicInbox/LeiToMail.pm
+++ b/lib/PublicInbox/LeiToMail.pm
@@ -195,7 +195,7 @@ sub _mbox_write_cb ($$) {
sub { # for git_to_mail
my ($buf, $smsg, $eml) = @_;
$eml //= PublicInbox::Eml->new($buf);
- ++$lei->{-nr_seen};
+ ++$self->{-nr_seen};
return if $dedupe->is_dup($eml, $smsg);
$lse->xsmsg_vmd($smsg) if $lse;
$smsg->{-recent} = 1 if $set_recent;
@@ -206,7 +206,7 @@ sub _mbox_write_cb ($$) {
my $lk = $ovv->lock_for_scope;
$lei->out($$buf);
}
- ++$lei->{-nr_write};
+ ++$self->{-nr_write};
}
}
@@ -291,7 +291,7 @@ sub _maildir_write_cb ($$) {
my ($bref, $smsg, $eml) = @_;
$dst // return $lei->fail; # dst may be undef-ed in last run
- ++$lei->{-nr_seen};
+ ++$self->{-nr_seen};
return if $dedupe && $dedupe->is_dup($eml //
PublicInbox::Eml->new($$bref),
$smsg);
@@ -299,7 +299,7 @@ sub _maildir_write_cb ($$) {
my $n = _buf2maildir($dst, $bref // \($eml->as_string),
$smsg, $dir);
$lms->set_src($smsg->oidbin, $out, $n) if $lms;
- ++$lei->{-nr_write};
+ ++$self->{-nr_write};
}
}
@@ -322,7 +322,7 @@ EOM
my ($bref, $smsg, $eml) = @_;
$mic // return $lei->fail; # mic may be undef-ed in last run
- ++$lei->{-nr_seen};
+ ++$self->{-nr_seen};
return if $dedupe && $dedupe->is_dup($eml //
PublicInbox::Eml->new($$bref),
$smsg);
@@ -335,7 +335,7 @@ EOM
# imap_append returns UID if IMAP server has UIDPLUS extension
($lms && $uid =~ /\A[0-9]+\z/) and
$lms->set_src($smsg->oidbin, $$uri, $uid + 0);
- ++$lei->{-nr_write};
+ ++$self->{-nr_write};
}
}
@@ -366,10 +366,10 @@ sub _v2_write_cb ($$) {
sub { # for git_to_mail
my ($bref, $smsg, $eml) = @_;
$eml //= PublicInbox::Eml->new($bref);
- ++$lei->{-nr_seen};
+ ++$self->{-nr_seen};
return if $dedupe && $dedupe->is_dup($eml, $smsg);
$lei->{v2w}->wq_do('add', $eml); # V2Writable->add
- ++$lei->{-nr_write};
+ ++$self->{-nr_write};
}
}
@@ -796,11 +796,11 @@ sub wq_atexit_child {
local $PublicInbox::DS::in_loop = 0; # waitpid synchronously
my $lei = $self->{lei};
$lei->{ale}->git->async_wait_all;
- my ($nr_w, $nr_s) = delete(@$lei{qw(-nr_write -nr_seen)});
+ my ($nr_w, $nr_s) = delete(@$self{qw(-nr_write -nr_seen)});
delete $self->{wcb};
(($nr_w //= 0) + ($nr_s //= 0)) or return;
return if $lei->{early_mua} || !$lei->{-progress} || !$lei->{pkt_op_p};
- $lei->{pkt_op_p}->pkt_do('l2m_progress', $nr_w, $nr_s);
+ $lei->{pkt_op_p}->pkt_do('incr', -nr_write => $nr_w, -nr_seen => $nr_s)
}
# runs on a 1s timer in lei-daemon
diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index 8f63149e..1caa9d06 100644
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -155,12 +155,6 @@ sub mset_progress {
}
}
-sub l2m_progress {
- my ($lei, $nr_write, $nr_seen) = @_;
- $lei->{-nr_write} += $nr_write;
- $lei->{-nr_seen} += $nr_seen;
-}
-
sub query_one_mset { # for --threads and l2m w/o sort
my ($self, $ibxish) = @_;
local $0 = "$0 query_one_mset";
@@ -354,7 +348,6 @@ sub query_remote_mboxrd {
$self->{import_sto} = $lei->{sto} if $lei->{opt}->{'import-remote'};
for my $uri (@$uris) {
$lei->{-current_url} = $uri->as_string;
- $lei->{-nr_remote_eml} = 0;
my $start = time;
my ($q, $key) = fudge_qstr_time($lei, $uri, $qstr);
$uri->query_form(@qform, q => $q);
@@ -369,9 +362,9 @@ sub query_remote_mboxrd {
my $wait = $self->{import_sto}->wq_do('done');
}
$reap_curl->join;
+ my $nr = delete $lei->{-nr_remote_eml} // 0;
if ($? == 0) {
# don't update if no results, maybe MTA is down
- my $nr = $lei->{-nr_remote_eml};
$lei->{lss}->cfg_set($key, $start) if $key && $nr;
mset_progress($lei, $lei->{-current_url}, $nr, $nr);
next;
@@ -433,8 +426,8 @@ Error closing $lei->{ovv}->{dst}: \$!=$! \$?=$?
}
if ($lei->{-progress}) {
my $tot = $lei->{-mset_total} // 0;
- my $nr_w = $lei->{-nr_write} // 0;
- my $d = ($lei->{-nr_seen} // 0) - $nr_w;
+ my $nr_w = delete($lei->{-nr_write}) // 0;
+ my $d = (delete($lei->{-nr_seen}) // 0) - $nr_w;
my $x = "$tot matches";
$x .= ", $d duplicates" if $d;
if ($l2m) {
@@ -532,7 +525,7 @@ sub do_query {
incr_post_augment => [ \&incr_post_augment, $lei ],
'' => [ \&query_done, $lei ],
mset_progress => [ \&mset_progress, $lei ],
- l2m_progress => [ \&l2m_progress, $lei ],
+ incr => [ $lei ],
x_it => [ $lei ],
child_error => [ $lei ],
incr_start_query => [ \&incr_start_query, $lei, $self ],
diff --git a/t/lei-tag.t b/t/lei-tag.t
index 822677a7..cccf0af6 100644
--- a/t/lei-tag.t
+++ b/t/lei-tag.t
@@ -101,5 +101,8 @@ test_lei(sub {
if (0) { # TODO label+kw search w/ externals
lei_ok(qw(q L:qp), "mid:$mid", '--only', "$ro_home/t2");
}
+ lei_ok qw(tag +L:nope -F eml t/data/binary.patch);
+ like $lei_err, qr/\b1 unimported messages/, 'noted unimported'
+ or diag $lei_err;
});
done_testing;
^ permalink raw reply related [relevance 5%]
* [PATCH 00/21] lei + IPC related stuff
@ 2023-10-04 3:49 7% Eric Wong
2023-10-04 3:49 5% ` [PATCH 08/21] lei: get rid of l2m_progress PktOp callback Eric Wong
0 siblings, 1 reply; 2+ results
From: Eric Wong @ 2023-10-04 3:49 UTC (permalink / raw)
To: meta
More work coming to make internal IPC stuff simpler
and better layered for future enhancements
(FUSE, increase xap_helper usage internally, etc).
Eric Wong (21):
lei: drop stores explicitly at daemon shutdown
ds: hoist out close_non_busy
ds: don't pass FD map to post_loop_do callback
move all non-test @post_loop_do into named subs
lei: close DirIdle (inotify) early at daemon shutdown
input_pipe: {args} is never undefined
lei: do_env combines fchdir and local
lei: get rid of l2m_progress PktOp callback
t/lei_to_mail: modernize and document test
lei: reuse PublicInbox::Config::noop
lei: keep signals blocked on daemon shutdown
mbox_lock: retry on EINTR and use autodie
lock: retry on EINTR, improve error reporting
treewide: use PublicInbox::Lock->new
gcf2: use PublicInbox::Lock
spawn: use autodie and PublicInbox::Lock
xap_helper: retry flock on EINTR
XapHelper.pm: use EINTR-aware recv_cmd wrapper
spawn: drop checks for directory writability
lei: document and local-ize $OPT hashref
searchidx: fix redundant `in' in warning message
lib/PublicInbox/DS.pm | 17 +++--
lib/PublicInbox/Daemon.pm | 47 ++++++-------
lib/PublicInbox/DirIdle.pm | 12 +++-
lib/PublicInbox/Gcf2.pm | 8 ++-
lib/PublicInbox/IPC.pm | 4 +-
lib/PublicInbox/InputPipe.pm | 11 ++-
lib/PublicInbox/LEI.pm | 127 ++++++++++++++++------------------
lib/PublicInbox/LeiAuth.pm | 4 +-
lib/PublicInbox/LeiConfig.pm | 25 +++----
lib/PublicInbox/LeiConvert.pm | 7 +-
lib/PublicInbox/LeiInspect.pm | 28 ++++----
lib/PublicInbox/LeiLcat.pm | 17 +++--
lib/PublicInbox/LeiMirror.pm | 2 +-
lib/PublicInbox/LeiQuery.pm | 19 +++--
lib/PublicInbox/LeiTag.pm | 6 +-
lib/PublicInbox/LeiToMail.pm | 20 +++---
lib/PublicInbox/LeiXSearch.pm | 113 +++++++++++++-----------------
lib/PublicInbox/Lock.pm | 52 ++++++++------
lib/PublicInbox/MboxLock.pm | 49 ++++++-------
lib/PublicInbox/PktOp.pm | 15 ++--
lib/PublicInbox/SearchIdx.pm | 2 +-
lib/PublicInbox/Spawn.pm | 32 ++++-----
lib/PublicInbox/TestCommon.pm | 7 +-
lib/PublicInbox/Watch.pm | 4 +-
lib/PublicInbox/XapHelper.pm | 11 ++-
lib/PublicInbox/xap_helper.h | 6 +-
t/lei-tag.t | 3 +
t/lei_to_mail.t | 37 +++++-----
t/solver_git.t | 3 +-
t/v2mirror.t | 2 +-
30 files changed, 339 insertions(+), 351 deletions(-)
^ permalink raw reply [relevance 7%]
Results 1-2 of 2 | reverse | options above
-- pct% links below jump to the message on this page, permalinks otherwise --
2023-10-04 3:49 7% [PATCH 00/21] lei + IPC related stuff Eric Wong
2023-10-04 3:49 5% ` [PATCH 08/21] lei: get rid of l2m_progress PktOp callback Eric Wong
Code repositories for project(s) associated with this public inbox
https://80x24.org/public-inbox.git
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).