From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on dcvr.yhbt.net X-Spam-Level: X-Spam-ASN: X-Spam-Status: No, score=-4.2 required=3.0 tests=ALL_TRUSTED,BAYES_00, DKIM_SIGNED,DKIM_VALID,DKIM_VALID_AU,DKIM_VALID_EF, T_SCC_BODY_TEXT_LINE shortcircuit=no autolearn=ham autolearn_force=no version=3.4.2 Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id D249C1F54E for ; Fri, 5 Aug 2022 08:29:54 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=80x24.org; s=selector1; t=1659688194; bh=3y80GXmCjeF46OSiyTjT0p1zhlL7qjVHMmuZ+FUMrFE=; h=Date:From:To:Subject:From; b=BmtPQXWU+aLCBN32MqIio3giIFVFJUYQXilT7wtJHC8mWN2HwcGdQunWrS1EOsiFN CnAAQksjKmaNGKAvCiKLQH2B2fdtEmqxXrKSJg4fSuoumQzadI4HSwl647Neus0qo5 6bug59EzsB46+mpS3Dip7T5nq3uXXMTC2WzjuxqA= Date: Fri, 5 Aug 2022 08:29:54 +0000 From: Eric Wong To: meta@public-inbox.org Subject: [PATCH] daemon: dedupe PublicInbox::Config objects by pathname Message-ID: <20220805082954.GA14626@dcvr> MIME-Version: 1.0 Content-Type: text/plain; charset=us-ascii Content-Disposition: inline List-Id: This means all Inbox, Git, Over, Msgmap, Search objects also get deduplicated if they belong to the same config file, reducing memory and FD usage. This helps save memory and improve cache hit rates in -netd setups where NNTP, IMAP, HTTP, and POP3 servers run in the same process. InboxIdle was the only bit which needed adjustment, but there may be other bugs lurking despite all tests passing. --- Pored over the read-only server code a bit to see if there's any problems lurking, but I haven't noticed any... lib/PublicInbox/Config.pm | 9 ++++++++- lib/PublicInbox/Daemon.pm | 3 +++ lib/PublicInbox/InboxIdle.pm | 4 ++-- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/lib/PublicInbox/Config.pm b/lib/PublicInbox/Config.pm index a31b5b74..ad8b8e9d 100644 --- a/lib/PublicInbox/Config.pm +++ b/lib/PublicInbox/Config.pm @@ -13,6 +13,7 @@ use v5.10.1; use PublicInbox::Inbox; use PublicInbox::Spawn qw(popen_rd); our $LD_PRELOAD = $ENV{LD_PRELOAD}; # only valid at startup +our $DEDUPE; # set to {} to dedupe or clear cache sub _array ($) { ref($_[0]) eq 'ARRAY' ? $_[0] : [ $_[0] ] } @@ -22,11 +23,17 @@ sub new { my ($class, $file, $errfh) = @_; $file //= default_file(); my $self; + my $set_dedupe; if (ref($file) eq 'SCALAR') { # used by some tests open my $fh, '<', $file or die; # PerlIO::scalar $self = config_fh_parse($fh, "\n", '='); bless $self, $class; } else { + if (-f $file && $DEDUPE) { + $file = rel2abs_collapsed($file); + $self = $DEDUPE->{$file} and return $self; + $set_dedupe = 1; + } $self = git_config_dump($class, $file, $errfh); $self->{'-f'} = $file; } @@ -62,7 +69,7 @@ sub new { if (my $css = delete $self->{'publicinbox.css'}) { $self->{css} = _array($css); } - + $DEDUPE->{$file} = $self if $set_dedupe; $self; } diff --git a/lib/PublicInbox/Daemon.pm b/lib/PublicInbox/Daemon.pm index 86234771..b6f4f9ed 100644 --- a/lib/PublicInbox/Daemon.pm +++ b/lib/PublicInbox/Daemon.pm @@ -22,6 +22,7 @@ use PublicInbox::Sigfd; use PublicInbox::Git; use PublicInbox::GitAsyncCat; use PublicInbox::Eml; +use PublicInbox::Config; our $SO_ACCEPTFILTER = 0x1000; my @CMD; my ($set_user, $oldset); @@ -653,8 +654,10 @@ sub defer_accept ($$) { sub daemon_loop ($) { my ($xnetd) = @_; + local $PublicInbox::Config::DEDUPE = {}; # enable dedupe cache my $refresh = sub { my ($sig) = @_; + %$PublicInbox::Config::DEDUPE = (); # clear cache for my $xn (values %$xnetd) { delete $xn->{tlsd}->{ssl_ctx}; # PublicInbox::TLS::start eval { $xn->{refresh}->($sig) }; diff --git a/lib/PublicInbox/InboxIdle.pm b/lib/PublicInbox/InboxIdle.pm index 2781b3e1..ffbbfea7 100644 --- a/lib/PublicInbox/InboxIdle.pm +++ b/lib/PublicInbox/InboxIdle.pm @@ -30,9 +30,9 @@ sub in2_arm ($$) { # PublicInbox::Config::each_inbox callback my $old_ibx = $cur->[0]; $cur->[0] = $ibx; if ($old_ibx) { - $ibx->{unlock_subs} and - die "BUG: $dir->{unlock_subs} should not exist"; + my $u = $ibx->{unlock_subs}; $ibx->{unlock_subs} = $old_ibx->{unlock_subs}; + %{$ibx->{unlock_subs}} = (%$u, %{$ibx->{unlock_subs}}) if $u; # Linux::Inotify2::Watch::name matches if watches are the # same, no point in replacing a watch of the same name