* [PATCH 05/11] config: flatten each_inbox and iterate_start args
2020-09-09 6:26 7% [PATCH 00/11] httpd: further reduce event loop monopolization Eric Wong
@ 2020-09-09 6:26 6% ` Eric Wong
0 siblings, 0 replies; 2+ results
From: Eric Wong @ 2020-09-09 6:26 UTC (permalink / raw)
To: meta
In Perl, we can simplify callers by passing a single array
all the way down the stack instead of a single array ref which
needs to be expanded every call.
---
lib/PublicInbox/Config.pm | 12 ++++++------
lib/PublicInbox/ExtMsg.pm | 7 +++----
lib/PublicInbox/Watch.pm | 13 ++++++-------
lib/PublicInbox/WwwListing.pm | 13 +++++--------
4 files changed, 20 insertions(+), 25 deletions(-)
diff --git a/lib/PublicInbox/Config.pm b/lib/PublicInbox/Config.pm
index ae9ad8de..f78115b6 100644
--- a/lib/PublicInbox/Config.pm
+++ b/lib/PublicInbox/Config.pm
@@ -90,29 +90,29 @@ sub lookup_name ($$) {
}
sub each_inbox {
- my ($self, $cb, $arg) = @_;
+ my ($self, $cb, @arg) = @_;
# may auto-vivify if config file is non-existent:
foreach my $section (@{$self->{-section_order}}) {
next if $section !~ m!\Apublicinbox\.([^/]+)\z!;
my $ibx = lookup_name($self, $1) or next;
- $cb->($ibx, $arg);
+ $cb->($ibx, @arg);
}
}
sub iterate_start {
- my ($self, $cb, $arg) = @_;
+ my ($self, $cb, @arg) = @_;
my $i = 0;
- $self->{-iter} = [ \$i, $cb, $arg ];
+ $self->{-iter} = [ \$i, $cb, @arg ];
}
# for PublicInbox::DS::next_tick, we only call this is if
# PublicInbox::DS is already loaded
sub event_step {
my ($self) = @_;
- my ($i, $cb, $arg) = @{$self->{-iter}};
+ my ($i, $cb, @arg) = @{$self->{-iter}};
my $section = $self->{-section_order}->[$$i++];
delete($self->{-iter}) unless defined($section);
- eval { $cb->($self, $section, $arg) };
+ eval { $cb->($self, $section, @arg) };
warn "E: $@ in ${self}::event_step" if $@;
PublicInbox::DS::requeue($self) if defined($section);
}
diff --git a/lib/PublicInbox/ExtMsg.pm b/lib/PublicInbox/ExtMsg.pm
index 5dffc65c..929737f1 100644
--- a/lib/PublicInbox/ExtMsg.pm
+++ b/lib/PublicInbox/ExtMsg.pm
@@ -74,8 +74,7 @@ sub search_partial ($$) {
}
sub ext_msg_i {
- my ($other, $arg) = @_;
- my ($cur, $mid, $ibxs, $found) = @$arg;
+ my ($other, $cur, $mid, $ibxs, $found) = @_;
return if $other->{name} eq $cur->{name} || !$other->base_url;
@@ -101,9 +100,9 @@ sub ext_msg {
eval { require PublicInbox::Msgmap };
my $ibxs = [];
my $found = [];
- my $arg = [ $cur, $mid, $ibxs, $found ];
- $ctx->{www}->{pi_config}->each_inbox(\&ext_msg_i, $arg);
+ $ctx->{www}->{pi_config}->each_inbox(\&ext_msg_i,
+ $cur, $mid, $ibxs, $found);
return exact($ctx, $found, $mid) if @$found;
diff --git a/lib/PublicInbox/Watch.pm b/lib/PublicInbox/Watch.pm
index 17786377..0f41dff2 100644
--- a/lib/PublicInbox/Watch.pm
+++ b/lib/PublicInbox/Watch.pm
@@ -133,8 +133,7 @@ sub _done_for_now {
}
sub remove_eml_i { # each_inbox callback
- my ($ibx, $arg) = @_;
- my ($self, $eml, $loc) = @$arg;
+ my ($ibx, $self, $eml, $loc) = @_;
eval {
# try to avoid taking a lock or unnecessary spawning
@@ -176,7 +175,7 @@ sub _remove_spam {
$path =~ /:2,[A-R]*S[T-Za-z]*\z/ or return;
my $eml = eml_from_path($path) or return;
local $SIG{__WARN__} = warn_ignore_cb();
- $self->{config}->each_inbox(\&remove_eml_i, [ $self, $eml, $path ]);
+ $self->{config}->each_inbox(\&remove_eml_i, $self, $eml, $path);
}
sub import_eml ($$$) {
@@ -419,8 +418,8 @@ sub imap_import_msg ($$$$$) {
if ($flags =~ /\\Seen\b/) {
local $SIG{__WARN__} = warn_ignore_cb();
my $eml = PublicInbox::Eml->new($raw);
- my $arg = [ $self, $eml, "$url UID:$uid" ];
- $self->{config}->each_inbox(\&remove_eml_i, $arg);
+ $self->{config}->each_inbox(\&remove_eml_i,
+ $self, $eml, "$url UID:$uid");
}
} else {
die "BUG: destination unknown $inboxes";
@@ -967,8 +966,8 @@ sub nntp_fetch_all ($$$) {
}
} elsif ($inboxes eq 'watchspam') {
my $eml = PublicInbox::Eml->new(\$raw);
- my $arg = [ $self, $eml, "$url ARTICLE $art" ];
- $self->{config}->each_inbox(\&remove_eml_i, $arg);
+ $self->{config}->each_inbox(\&remove_eml_i,
+ $self, $eml, "$url ARTICLE $art");
} else {
die "BUG: destination unknown $inboxes";
}
diff --git a/lib/PublicInbox/WwwListing.pm b/lib/PublicInbox/WwwListing.pm
index 12b0d9ad..a7c7cbc1 100644
--- a/lib/PublicInbox/WwwListing.pm
+++ b/lib/PublicInbox/WwwListing.pm
@@ -12,21 +12,19 @@ use PublicInbox::ManifestJsGz;
use bytes (); # bytes::length
sub list_all_i {
- my ($ibx, $arg) = @_;
- my ($list, $hide_key) = @$arg;
+ my ($ibx, $list, $hide_key) = @_;
push @$list, $ibx unless $ibx->{-hide}->{$hide_key};
}
sub list_all ($$$) {
my ($self, $env, $hide_key) = @_;
my $list = [];
- $self->{pi_config}->each_inbox(\&list_all_i, [ $list, $hide_key ]);
+ $self->{pi_config}->each_inbox(\&list_all_i, $list, $hide_key);
$list;
}
sub list_match_domain_i {
- my ($ibx, $arg) = @_;
- my ($list, $hide_key, $re) = @$arg;
+ my ($ibx, $list, $hide_key, $re) = @_;
if (!$ibx->{-hide}->{$hide_key} && grep(/$re/, @{$ibx->{url}})) {
push @$list, $ibx;
}
@@ -37,9 +35,8 @@ sub list_match_domain ($$$) {
my $list = [];
my $host = $env->{HTTP_HOST} // $env->{SERVER_NAME};
$host =~ s/:[0-9]+\z//;
- my $arg = [ $list, $hide_key,
- qr!\A(?:https?:)?//\Q$host\E(?::[0-9]+)?/!i ];
- $self->{pi_config}->each_inbox(\&list_match_domain_i, $arg);
+ $self->{pi_config}->each_inbox(\&list_match_domain_i, $list, $hide_key,
+ qr!\A(?:https?:)?//\Q$host\E(?::[0-9]+)?/!i);
$list;
}
^ permalink raw reply related [relevance 6%]
* [PATCH 00/11] httpd: further reduce event loop monopolization
@ 2020-09-09 6:26 7% Eric Wong
2020-09-09 6:26 6% ` [PATCH 05/11] config: flatten each_inbox and iterate_start args Eric Wong
0 siblings, 1 reply; 2+ results
From: Eric Wong @ 2020-09-09 6:26 UTC (permalink / raw)
To: meta
A couple more things to mitigate the effects of slow storage
with many inboxes. Mostly solver-related, and still more to
come... (Hoping the electrical grid stays up and dust bunny
removal solved overheating problems).
Eric Wong (11):
xt/solver: test with public-inbox-httpd, too
solver: drop warnings, modernize use v5.10.1, use SEEK_SET
use "\&" where possible when referring to subroutines
www: manifest.js.gz generation no longer hogs event loop
config: flatten each_inbox and iterate_start args
config: split out iterator into separate object
t/cgi.t: show stderr on failures
extmsg: prevent cross-inbox matches from hogging event loop
wwwlisting: avoid hogging event loop
solver: check one git coderepo and inbox at a time
solver: break apart inbox blob retrieval
MANIFEST | 2 +
lib/PublicInbox/Cgit.pm | 5 +-
lib/PublicInbox/Config.pm | 22 +--
lib/PublicInbox/ConfigIter.pm | 40 +++++
lib/PublicInbox/ExtMsg.pm | 102 ++++++++----
lib/PublicInbox/IMAPD.pm | 6 +-
lib/PublicInbox/Inbox.pm | 2 +-
lib/PublicInbox/ManifestJsGz.pm | 135 ++++++++++++++++
lib/PublicInbox/SolverGit.pm | 190 +++++++++++++---------
lib/PublicInbox/TestCommon.pm | 4 +-
lib/PublicInbox/WWW.pm | 21 +--
lib/PublicInbox/Watch.pm | 13 +-
lib/PublicInbox/WwwListing.pm | 279 ++++++++------------------------
t/cgi.t | 2 +-
t/replace.t | 8 +-
t/solver_git.t | 7 +-
t/www_listing.t | 7 +-
xt/msgtime_cmp.t | 2 +-
xt/solver.t | 31 +++-
19 files changed, 499 insertions(+), 379 deletions(-)
create mode 100644 lib/PublicInbox/ConfigIter.pm
create mode 100644 lib/PublicInbox/ManifestJsGz.pm
^ permalink raw reply [relevance 7%]
Results 1-2 of 2 | reverse | options above
-- pct% links below jump to the message on this page, permalinks otherwise --
2020-09-09 6:26 7% [PATCH 00/11] httpd: further reduce event loop monopolization Eric Wong
2020-09-09 6:26 6% ` [PATCH 05/11] config: flatten each_inbox and iterate_start args Eric Wong
Code repositories for project(s) associated with this public inbox
https://80x24.org/public-inbox.git
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).