* [PATCH 08/12] nntp: move LIST iterators to long_response
2020-11-27 9:52 7% [PATCH 00/12] some NNTP-related fixes + speedups Eric Wong
@ 2020-11-27 9:52 6% ` Eric Wong
0 siblings, 0 replies; 2+ results
From: Eric Wong @ 2020-11-27 9:52 UTC (permalink / raw)
To: meta
Iterating through many newsgroups can hog the event loop if many
random seeks are required. Avoid monopolizing the event loop in
that case by using the long_response API.
For now, we can still rely on grep() since it seems to work
reasonably well with 50K test newsgroup names.
---
lib/PublicInbox/NNTP.pm | 77 +++++++++++++++++++++++++----------------
1 file changed, 48 insertions(+), 29 deletions(-)
diff --git a/lib/PublicInbox/NNTP.pm b/lib/PublicInbox/NNTP.pm
index eb2c0b38..af40b86d 100644
--- a/lib/PublicInbox/NNTP.pm
+++ b/lib/PublicInbox/NNTP.pm
@@ -31,9 +31,9 @@ use Errno qw(EAGAIN);
my $ONE_MSGID = qr/\A$MID_EXTRACT\z/;
my @OVERVIEW = qw(Subject From Date Message-ID References);
my $OVERVIEW_FMT = join(":\r\n", @OVERVIEW, qw(Bytes Lines), '') .
- "Xref:full\r\n";
+ "Xref:full\r\n.";
my $LIST_HEADERS = join("\r\n", @OVERVIEW,
- qw(:bytes :lines Xref To Cc)) . "\r\n";
+ qw(:bytes :lines Xref To Cc)) . "\r\n.";
my $CAPABILITIES = <<"";
101 Capability list:\r
VERSION 2\r
@@ -120,46 +120,66 @@ sub cmd_xgtitle ($;$) {
my ($self, $wildmat) = @_;
more($self, '282 list of groups and descriptions follows');
list_newsgroups($self, $wildmat);
- '.'
}
-sub list_overview_fmt ($) {
- my ($self) = @_;
- $self->msg_more($OVERVIEW_FMT);
-}
+sub list_overview_fmt ($) { $OVERVIEW_FMT }
-sub list_headers ($;$) {
- my ($self) = @_;
- $self->msg_more($LIST_HEADERS);
-}
+sub list_headers ($;$) { $LIST_HEADERS }
-sub list_active ($;$) {
- my ($self, $wildmat) = @_;
- wildmat2re($wildmat);
- my $groups = $self->{nntpd}->{groups};
- for my $ngname (grep(/$wildmat/, @{$self->{nntpd}->{groupnames}})) {
- group_line($self, $groups->{$ngname});
+sub list_active_i { # "LIST ACTIVE" and also just "LIST" (no args)
+ my ($self, $groupnames) = @_;
+ my @window = splice(@$groupnames, 0, 100) or return 0;
+ my $ibx;
+ my $groups = $self->{nntpd}->{pi_config}->{-by_newsgroup};
+ for my $ngname (@window) {
+ $ibx = $groups->{$ngname} and group_line($self, $ibx);
}
+ scalar(@$groupnames); # continue if there's more
}
-sub list_active_times ($;$) {
+sub list_active ($;$) { # called by cmd_list
my ($self, $wildmat) = @_;
wildmat2re($wildmat);
- my $groups = $self->{nntpd}->{groups};
- for my $ngname (grep(/$wildmat/, @{$self->{nntpd}->{groupnames}})) {
- my $ibx = $groups->{$ngname};
+ long_response($self, \&list_active_i, [
+ grep(/$wildmat/, @{$self->{nntpd}->{groupnames}}) ]);
+}
+
+sub list_active_times_i {
+ my ($self, $groupnames) = @_;
+ my @window = splice(@$groupnames, 0, 100) or return 0;
+ my $groups = $self->{nntpd}->{pi_config}->{-by_newsgroup};
+ for my $ngname (@window) {
+ my $ibx = $groups->{$ngname} or next;
my $c = eval { $ibx->uidvalidity } // time;
more($self, "$ngname $c <$ibx->{-primary_address}>");
}
+ scalar(@$groupnames); # continue if there's more
}
-sub list_newsgroups ($;$) {
+sub list_active_times ($;$) { # called by cmd_list
my ($self, $wildmat) = @_;
wildmat2re($wildmat);
- my $groups = $self->{nntpd}->{groups};
- for my $ngname (grep(/$wildmat/, @{$self->{nntpd}->{groupnames}})) {
- more($self, "$ngname ".$groups->{$ngname}->description);
+ long_response($self, \&list_active_times_i, [
+ grep(/$wildmat/, @{$self->{nntpd}->{groupnames}}) ]);
+}
+
+sub list_newsgroups_i {
+ my ($self, $groupnames) = @_;
+ my @window = splice(@$groupnames, 0, 100) or return 0;
+ my $groups = $self->{nntpd}->{pi_config}->{-by_newsgroup};
+ my $ibx;
+ for my $ngname (@window) {
+ $ibx = $groups->{$ngname} and
+ more($self, "$ngname ".$ibx->description);
}
+ scalar(@$groupnames); # continue if there's more
+}
+
+sub list_newsgroups ($;$) { # called by cmd_list
+ my ($self, $wildmat) = @_;
+ wildmat2re($wildmat);
+ long_response($self, \&list_newsgroups_i, [
+ grep(/$wildmat/, @{$self->{nntpd}->{groupnames}}) ]);
}
# LIST SUBSCRIPTIONS, DISTRIB.PATS are not supported
@@ -168,6 +188,7 @@ sub cmd_list ($;$$) {
if (scalar @args) {
my $arg = shift @args;
$arg =~ tr/A-Z./a-z_/;
+ my $ret = $arg eq 'active';
$arg = "list_$arg";
$arg = $self->can($arg);
return r501 unless $arg && args_ok($arg, scalar @args);
@@ -175,11 +196,9 @@ sub cmd_list ($;$$) {
$arg->($self, @args);
} else {
more($self, '215 list of newsgroups follows');
- foreach my $ng (@{$self->{nntpd}->{grouplist}}) {
- group_line($self, $ng);
- }
+ long_response($self, \&list_active_i, [ # copy array
+ @{$self->{nntpd}->{groupnames}} ]);
}
- '.'
}
sub listgroup_range_i {
^ permalink raw reply related [relevance 6%]
* [PATCH 00/12] some NNTP-related fixes + speedups
@ 2020-11-27 9:52 7% Eric Wong
2020-11-27 9:52 6% ` [PATCH 08/12] nntp: move LIST iterators to long_response Eric Wong
0 siblings, 1 reply; 2+ results
From: Eric Wong @ 2020-11-27 9:52 UTC (permalink / raw)
To: meta
A few minor bugfixes and some major speedups for targeting
thousands/millions of inboxes based on stuff learned during
IMAP server development.
[PATCH 6/12] "miscsearch: implement ->newsgroup_matches"
may not be necessary, after all, but maybe it could also
be used for WWW.
Still a lot more to do...
Eric Wong (12):
nntp: use Inbox->uidvalidity instead of ->mm->created_at
nntpd: share {groups} hash with {-by_newsgroup} in Config
mm: min/max: return 0 instead of undef
nntp: use grep operation for wildmat matching
nntp: NEWNEWS: speed up filtering
miscsearch: implement ->newsgroup_matches
nntp: LIST ACTIVE.TIMES use angle brackets around address
nntp: move LIST iterators to long_response
t/extsearch: show a more realistic case
nntp: some minor golfing
nntp: xref: simplify sub signature
nntp: xref: use ->ALL extindex if available
lib/PublicInbox/Config.pm | 4 +-
lib/PublicInbox/ExtSearch.pm | 56 ++++++++++
lib/PublicInbox/IMAPD.pm | 2 +-
lib/PublicInbox/MiscSearch.pm | 40 +++++++
lib/PublicInbox/Msgmap.pm | 7 +-
lib/PublicInbox/NNTP.pm | 200 +++++++++++++++++++---------------
lib/PublicInbox/NNTPD.pm | 25 +++--
lib/PublicInbox/OverIdx.pm | 5 +
lib/PublicInbox/V2Writable.pm | 4 +-
t/extsearch.t | 54 ++++++++-
t/msgmap.t | 2 +-
t/nntp.t | 7 +-
12 files changed, 291 insertions(+), 115 deletions(-)
^ permalink raw reply [relevance 7%]
Results 1-2 of 2 | reverse | options above
-- pct% links below jump to the message on this page, permalinks otherwise --
2020-11-27 9:52 7% [PATCH 00/12] some NNTP-related fixes + speedups Eric Wong
2020-11-27 9:52 6% ` [PATCH 08/12] nntp: move LIST iterators to long_response Eric Wong
Code repositories for project(s) associated with this public inbox
https://80x24.org/public-inbox.git
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).