From ab243aa2328e2fc4cf895c99c68345e57cc4653c Mon Sep 17 00:00:00 2001 From: Eric Wong Date: Sun, 20 Dec 2020 06:30:12 +0000 Subject: inboxidle: remove needless check for {inboxdir} ->each_inbox will never attempt to iterate an object without {inboxdir}, and simplify + short-circuit the corresponding code --- lib/PublicInbox/InboxIdle.pm | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) (limited to 'lib/PublicInbox/InboxIdle.pm') diff --git a/lib/PublicInbox/InboxIdle.pm b/lib/PublicInbox/InboxIdle.pm index 2737bbbd..f1cbc012 100644 --- a/lib/PublicInbox/InboxIdle.pm +++ b/lib/PublicInbox/InboxIdle.pm @@ -7,7 +7,6 @@ package PublicInbox::InboxIdle; use strict; use parent qw(PublicInbox::DS); -use Cwd qw(abs_path); use PublicInbox::Syscall qw(EPOLLIN EPOLLET); my $IN_MODIFY = 0x02; # match Linux inotify my $ino_cls; @@ -22,11 +21,7 @@ require PublicInbox::In2Tie if $ino_cls; sub in2_arm ($$) { # PublicInbox::Config::each_inbox callback my ($ibx, $self) = @_; - my $dir = abs_path($ibx->{inboxdir}); - if (!defined($dir)) { - warn "W: $ibx->{inboxdir} not watched: $!\n"; - return; - } + my $dir = $ibx->{inboxdir}; my $inot = $self->{inot}; my $cur = $self->{pathmap}->{$dir} //= []; -- cgit v1.2.3-24-ge0c7 From 5e05c2eb58a450849f1826f3d02ed62b814b6617 Mon Sep 17 00:00:00 2001 From: Eric Wong Date: Sat, 26 Dec 2020 05:59:22 +0000 Subject: inboxidle: clue users into resolving ENOSPC from inotify It may not be obvious to users a ENOSPC error is from hitting a (tunable) kernel-imposed limit on inotify watches, and not some storage device running out of space. Give them a hint here to reduce our own support burden. --- lib/PublicInbox/InboxIdle.pm | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'lib/PublicInbox/InboxIdle.pm') diff --git a/lib/PublicInbox/InboxIdle.pm b/lib/PublicInbox/InboxIdle.pm index f1cbc012..84b6d26f 100644 --- a/lib/PublicInbox/InboxIdle.pm +++ b/lib/PublicInbox/InboxIdle.pm @@ -39,6 +39,11 @@ sub in2_arm ($$) { # PublicInbox::Config::each_inbox callback $self->{on_unlock}->{$w->name} = $ibx; } else { warn "E: ".ref($inot)."->watch($lock, IN_MODIFY) failed: $!\n"; + if ($!{ENOSPC} && $^O eq 'linux') { + warn <<""; +I: consider increasing /proc/sys/fs/inotify/max_user_watches + + } } # TODO: detect deleted packs (and possibly other files) -- cgit v1.2.3-24-ge0c7 From 10bf54305da8422d9ece6b809996092c1c4b1786 Mon Sep 17 00:00:00 2001 From: Eric Wong Date: Sat, 26 Dec 2020 09:34:39 +0000 Subject: inboxidle: avoid needless syscalls on refresh We don't have to replace a bunch of existing watches with identical new ones. On Linux with Linux::Inotify2 installed, this avoids a storm of inotify_add_watch(2) and inotify_rm_watch(2) syscalls on SIGHUP with -imapd and "-extindex --watch" --- lib/PublicInbox/InboxIdle.pm | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) (limited to 'lib/PublicInbox/InboxIdle.pm') diff --git a/lib/PublicInbox/InboxIdle.pm b/lib/PublicInbox/InboxIdle.pm index 84b6d26f..508007d7 100644 --- a/lib/PublicInbox/InboxIdle.pm +++ b/lib/PublicInbox/InboxIdle.pm @@ -24,17 +24,26 @@ sub in2_arm ($$) { # PublicInbox::Config::each_inbox callback my $dir = $ibx->{inboxdir}; my $inot = $self->{inot}; my $cur = $self->{pathmap}->{$dir} //= []; + my $lock = "$dir/".($ibx->version >= 2 ? 'inbox.lock' : 'ssoma.lock'); # transfer old subscriptions to the current inbox, cancel the old watch - if (my $old_ibx = $cur->[0]) { + my $old_ibx = $cur->[0]; + $cur->[0] = $ibx; + if ($old_ibx) { $ibx->{unlock_subs} and die "BUG: $dir->{unlock_subs} should not exist"; $ibx->{unlock_subs} = $old_ibx->{unlock_subs}; + + # Linux::Inotify2::Watch::name matches if watches are the + # same, no point in replacing a watch of the same name + if ($cur->[1]->name eq $lock) { + $self->{on_unlock}->{$lock} = $ibx; + return; + } + # rare, name changed (v1 inbox converted to v2) $cur->[1]->cancel; # Linux::Inotify2::Watch::cancel } - $cur->[0] = $ibx; - my $lock = "$dir/".($ibx->version >= 2 ? 'inbox.lock' : 'ssoma.lock'); if (my $w = $cur->[1] = $inot->watch($lock, $IN_MODIFY)) { $self->{on_unlock}->{$w->name} = $ibx; } else { -- cgit v1.2.3-24-ge0c7 From 1d96509a3f59c38394d2f3ac4323dc54c74dc202 Mon Sep 17 00:00:00 2001 From: Eric Wong Date: Sat, 26 Dec 2020 01:44:37 +0000 Subject: extindex: --watch for inotify-based updates This reuses existing InboxIdle infrastructure to update external indices based on per-inbox updates. This is an alternative to auto-updating external indices via the -index command and also works with existing uses of -mda and public-inbox-watch. Using inotify (or EVFILT_VNODE) allows watching thousands of inboxes without having to scan every single one at every invocation. This is especially beneficial in cases where an external index is not writable to the users writing to per-inbox indices. --- lib/PublicInbox/InboxIdle.pm | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'lib/PublicInbox/InboxIdle.pm') diff --git a/lib/PublicInbox/InboxIdle.pm b/lib/PublicInbox/InboxIdle.pm index 508007d7..35aed696 100644 --- a/lib/PublicInbox/InboxIdle.pm +++ b/lib/PublicInbox/InboxIdle.pm @@ -63,6 +63,9 @@ sub refresh { $pi_cfg->each_inbox(\&in2_arm, $self); } +# internal API for ease-of-use +sub watch_inbox { in2_arm($_[1], $_[0]) }; + sub new { my ($class, $pi_cfg) = @_; my $self = bless {}, $class; @@ -78,7 +81,7 @@ sub new { $self->{inot} = $inot; $self->{pathmap} = {}; # inboxdir => [ ibx, watch1, watch2, watch3...] $self->{on_unlock} = {}; # lock path => ibx - refresh($self, $pi_cfg); + refresh($self, $pi_cfg) if $pi_cfg; PublicInbox::FakeInotify::poll_once($self) if !$ino_cls; $self; } @@ -89,7 +92,8 @@ sub event_step { my @events = $self->{inot}->read; # Linux::Inotify2::read my $on_unlock = $self->{on_unlock}; for my $ev (@events) { - if (my $ibx = $on_unlock->{$ev->fullname}) { + my $fn = $ev->fullname // next; # cancelled + if (my $ibx = $on_unlock->{$fn}) { $ibx->on_unlock; } } -- cgit v1.2.3-24-ge0c7