From: Eric Wong <e@80x24.org>
To: meta@public-inbox.org
Subject: [PATCH] nntp: reduce timers for weakening
Date: Mon, 25 Apr 2016 07:55:48 +0000 [thread overview]
Message-ID: <20160425075548.31842-1-e@80x24.org> (raw)
Danga::Socket timers are not cheap, so avoid creating up
to 3 timers per-newsgroup by batching resource weakening.
This lets us reduce resource consumption for scheduing
additional resource consumption reduction :)
---
lib/PublicInbox/NNTP.pm | 19 ++++++++++++++++++-
lib/PublicInbox/NewsGroup.pm | 30 ++++++------------------------
2 files changed, 24 insertions(+), 25 deletions(-)
diff --git a/lib/PublicInbox/NNTP.pm b/lib/PublicInbox/NNTP.pm
index c715e8f..e143d15 100644
--- a/lib/PublicInbox/NNTP.pm
+++ b/lib/PublicInbox/NNTP.pm
@@ -38,6 +38,8 @@ my %DISABLED; # = map { $_ => 1 } qw(xover list_overview_fmt newnews xhdr);
my $EXPMAP; # fd -> [ idle_time, $self ]
my $EXPTIMER;
our $EXPTIME = 180; # 3 minutes
+my $WEAKEN = {}; # string(nntpd) -> nntpd
+my $WEAKTIMER;
sub update_idle_time ($) {
my ($self) = @_;
@@ -46,6 +48,16 @@ sub update_idle_time ($) {
defined $tmp and $EXPMAP->{$tmp} = [ now(), $self ];
}
+# reduce FD pressure by closing some "git cat-file --batch" processes
+# and unused FDs for msgmap and Xapian indices
+sub weaken_groups () {
+ $WEAKTIMER = undef;
+ foreach my $nntpd (values %$WEAKEN) {
+ $_->weaken_all foreach (@{$nntpd->{grouplist}});
+ }
+ $WEAKEN = {};
+}
+
sub expire_old () {
my $now = now();
my $exp = $EXPTIME;
@@ -69,11 +81,15 @@ sub expire_old () {
$next -= $now;
$next = 0 if $next < 0;
$EXPTIMER = Danga::Socket->AddTimer($next, *expire_old);
+ weaken_groups();
} else {
$EXPTIMER = undef;
- # noop to kick outselves out of the loop so descriptors
+ # noop to kick outselves out of the loop ASAP so descriptors
# really get closed
Danga::Socket->AddTimer(0, sub {});
+
+ # grace period for reaping resources
+ $WEAKTIMER ||= Danga::Socket->AddTimer(30, *weaken_groups);
}
}
@@ -87,6 +103,7 @@ sub new ($$$) {
$self->{rbuf} = '';
$self->watch_read(1);
update_idle_time($self);
+ $WEAKEN->{"$nntpd"} = $nntpd;
$EXPTIMER ||= Danga::Socket->AddTimer($EXPTIME, *expire_old);
$self;
}
diff --git a/lib/PublicInbox/NewsGroup.pm b/lib/PublicInbox/NewsGroup.pm
index b20180e..adac919 100644
--- a/lib/PublicInbox/NewsGroup.pm
+++ b/lib/PublicInbox/NewsGroup.pm
@@ -24,20 +24,14 @@ sub new {
$self;
}
-sub defer_weaken {
- my ($self, $field) = @_;
- Danga::Socket->AddTimer(30, sub { weaken($self->{$field}) });
+sub weaken_all {
+ my ($self) = @_;
+ weaken($self->{$_}) foreach qw(gcf mm search);
}
sub gcf {
my ($self) = @_;
- $self->{gcf} ||= eval {
- my $gcf = PublicInbox::Git->new($self->{git_dir});
-
- # git repos may be repacked and old packs unlinked
- defer_weaken($self, 'gcf');
- $gcf;
- };
+ $self->{gcf} ||= eval { PublicInbox::Git->new($self->{git_dir}) };
}
sub usable {
@@ -50,24 +44,12 @@ sub usable {
sub mm {
my ($self) = @_;
- $self->{mm} ||= eval {
- my $mm = PublicInbox::Msgmap->new($self->{git_dir});
-
- # may be needed if we run low on handles
- defer_weaken($self, 'mm');
- $mm;
- };
+ $self->{mm} ||= eval { PublicInbox::Msgmap->new($self->{git_dir}) };
}
sub search {
my ($self) = @_;
- $self->{search} ||= eval {
- my $search = PublicInbox::Search->new($self->{git_dir});
-
- # may be needed if we run low on handles
- defer_weaken($self, 'search');
- $search;
- };
+ $self->{search} ||= eval { PublicInbox::Search->new($self->{git_dir}) };
}
sub description {
--
EW
reply other threads:[~2016-04-25 7:55 UTC|newest]
Thread overview: [no followups] expand[flat|nested] mbox.gz Atom feed
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
List information: https://public-inbox.org/README
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20160425075548.31842-1-e@80x24.org \
--to=e@80x24.org \
--cc=meta@public-inbox.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
Code repositories for project(s) associated with this public inbox
https://80x24.org/public-inbox.git
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).