From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on dcvr.yhbt.net X-Spam-Level: X-Spam-Status: No, score=-4.0 required=3.0 tests=ALL_TRUSTED,BAYES_00 shortcircuit=no autolearn=ham autolearn_force=no version=3.4.2 Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id EFC761F8EE for ; Sat, 27 Jun 2020 10:04:01 +0000 (UTC) From: Eric Wong To: meta@public-inbox.org Subject: [PATCH 09/34] kqnotify|fake_inotify: detect Maildir write ops Date: Sat, 27 Jun 2020 10:03:35 +0000 Message-Id: <20200627100400.9871-10-e@yhbt.net> In-Reply-To: <20200627100400.9871-1-e@yhbt.net> References: <20200627100400.9871-1-e@yhbt.net> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit List-Id: We need to detect link(2) and rename(2) in other apps writing to the Maildir. We'll be removing the Filesys::Notify::Simple from -watch in favor of using IO::KQueue or Linux::Inotify2 directly. Ensure non-inotify emulations can support everything we expect for Maildir writers. --- MANIFEST | 2 ++ lib/PublicInbox/FakeInotify.pm | 46 ++++++++++++++++++++++++++++------ lib/PublicInbox/KQNotify.pm | 38 +++++++++++++++++++++++----- t/fake_inotify.t | 45 +++++++++++++++++++++++++++++++++ t/kqnotify.t | 41 ++++++++++++++++++++++++++++++ 5 files changed, 159 insertions(+), 13 deletions(-) create mode 100644 t/fake_inotify.t create mode 100644 t/kqnotify.t diff --git a/MANIFEST b/MANIFEST index 161b6cddbe0..9d1a4e4a8b1 100644 --- a/MANIFEST +++ b/MANIFEST @@ -253,6 +253,7 @@ t/eml_content_disposition.t t/eml_content_type.t t/epoll.t t/fail-bin/spamc +t/fake_inotify.t t/feed.t t/filter_base-junk.eml t/filter_base-xhtml.eml @@ -286,6 +287,7 @@ t/indexlevels-mirror-v1.t t/indexlevels-mirror.t t/init.t t/iso-2202-jp.eml +t/kqnotify.t t/linkify.t t/main-bin/spamc t/mda-mime.eml diff --git a/lib/PublicInbox/FakeInotify.pm b/lib/PublicInbox/FakeInotify.pm index b077d63a4b4..df63173f083 100644 --- a/lib/PublicInbox/FakeInotify.pm +++ b/lib/PublicInbox/FakeInotify.pm @@ -6,10 +6,13 @@ package PublicInbox::FakeInotify; use strict; use Time::HiRes qw(stat); +use PublicInbox::DS; my $IN_CLOSE = 0x08 | 0x10; # match Linux inotify +# my $IN_MOVED_TO = 0x80; +# my $IN_CREATE = 0x100; +sub MOVED_TO_OR_CREATE () { 0x80 | 0x100 } my $poll_intvl = 2; # same as Filesys::Notify::Simple -my $for_cancel = bless \(my $x), 'PublicInbox::FakeInotify::Watch'; sub poll_once { my ($self) = @_; @@ -30,8 +33,22 @@ sub new { sub watch { my ($self, $path, $mask, $cb) = @_; my @st = stat($path) or return; - $self->{watch}->{"$path\0$mask"} = [ @st, $cb ]; - $for_cancel; + my $k = "$path\0$mask"; + $self->{watch}->{$k} = [ $st[10], $cb ]; # 10 - ctime + bless [ $self->{watch}, $k ], 'PublicInbox::FakeInotify::Watch'; +} + +sub on_new_files ($$$$) { + my ($dh, $cb, $path, $old_ctime) = @_; + while (defined(my $base = readdir($dh))) { + next if $base =~ /\A\.\.?\z/; + my $full = "$path/$base"; + my @st = stat($full); + if (@st && $st[10] > $old_ctime) { + bless \$full, 'PublicInbox::FakeInotify::Event'; + eval { $cb->(\$full) }; + } + } } # behaves like non-blocking Linux::Inotify2->poll @@ -43,17 +60,32 @@ sub poll { my @now = stat($path) or next; my $prv = $watch->{$x}; my $cb = $prv->[-1]; - # 10: ctime, 7: size - if ($prv->[10] != $now[10]) { + my $old_ctime = $prv->[0]; + if ($old_ctime != $now[10]) { if (($mask & $IN_CLOSE) == $IN_CLOSE) { eval { $cb->() }; + } elsif ($mask & MOVED_TO_OR_CREATE) { + opendir(my $dh, $path) or do { + warn "W: opendir $path: $!\n"; + next; + }; + on_new_files($dh, $cb, $path, $old_ctime); } } - @$prv = (@now, $cb); + @$prv = ($now[10], $cb); } } package PublicInbox::FakeInotify::Watch; -sub cancel {} # noop +use strict; + +sub cancel { + my ($self) = @_; + delete $self->[0]->{$self->[1]}; +} + +package PublicInbox::FakeInotify::Event; +use strict; +sub fullname { ${$_[0]} } 1; diff --git a/lib/PublicInbox/KQNotify.pm b/lib/PublicInbox/KQNotify.pm index 110594cc02c..9673b44290a 100644 --- a/lib/PublicInbox/KQNotify.pm +++ b/lib/PublicInbox/KQNotify.pm @@ -7,6 +7,11 @@ package PublicInbox::KQNotify; use strict; use IO::KQueue; use PublicInbox::DSKQXS; # wraps IO::KQueue for fork-safe DESTROY +use PublicInbox::FakeInotify; +use Time::HiRes qw(stat); + +# NOTE_EXTEND detects rename(2), NOTE_WRITE detects link(2) +sub MOVED_TO_OR_CREATE () { NOTE_EXTEND|NOTE_WRITE } sub new { my ($class) = @_; @@ -15,19 +20,28 @@ sub new { sub watch { my ($self, $path, $mask, $cb) = @_; - open(my $fh, '<', $path) or return; + my ($fh, $cls, @extra); + if (-d $path) { + opendir($fh, $path) or return; + my @st = stat($fh); + @extra = ($path, $st[10]); # 10: ctime + $cls = 'PublicInbox::KQNotify::Watchdir'; + } else { + open($fh, '<', $path) or return; + $cls = 'PublicInbox::KQNotify::Watch'; + } my $ident = fileno($fh); $self->{dskq}->{kq}->EV_SET($ident, # ident EVFILT_VNODE, # filter EV_ADD | EV_CLEAR, # flags $mask, # fflags 0, 0); # data, udata - if ($mask == NOTE_WRITE) { - $self->{watch}->{$ident} = [ $fh, $cb ]; + if ($mask == NOTE_WRITE || $mask == MOVED_TO_OR_CREATE) { + $self->{watch}->{$ident} = [ $fh, $cb, @extra ]; } else { die "TODO Not implemented: $mask"; } - bless \$fh, 'PublicInbox::KQNotify::Watch'; + bless \$fh, $cls; } # emulate Linux::Inotify::fileno @@ -48,8 +62,15 @@ sub poll { for my $kev (@kevents) { my $ident = $kev->[KQ_IDENT]; my $mask = $kev->[KQ_FFLAGS]; - if (($mask & NOTE_WRITE) == NOTE_WRITE) { - eval { $self->{watch}->{$ident}->[1]->() }; + my ($dh, $cb, $path, $old_ctime) = @{$self->{watch}->{$ident}}; + if (!defined($path) && ($mask & NOTE_WRITE) == NOTE_WRITE) { + eval { $cb->() }; + } elsif ($mask & MOVED_TO_OR_CREATE) { + my @new_st = stat($path) or next; + $self->{watch}->{$ident}->[3] = $new_st[10]; # ctime + rewinddir($dh); + PublicInbox::FakeInotify::on_new_files($dh, $cb, + $path, $old_ctime); } } } @@ -59,4 +80,9 @@ use strict; sub cancel { close ${$_[0]} or die "close: $!" } +package PublicInbox::KQNotify::Watchdir; +use strict; + +sub cancel { closedir ${$_[0]} or die "closedir: $!" } + 1; diff --git a/t/fake_inotify.t b/t/fake_inotify.t new file mode 100644 index 00000000000..f0db0cb58ec --- /dev/null +++ b/t/fake_inotify.t @@ -0,0 +1,45 @@ +#!perl -w +# Copyright (C) 2020 all contributors +# License: AGPL-3.0+ +# +# Ensure FakeInotify can pick up rename(2) and link(2) operations +# used by Maildir writing tools +use strict; +use Test::More; +use PublicInbox::TestCommon; +use_ok 'PublicInbox::FakeInotify'; +my $MIN_FS_TICK = 0.011; # for low-res CONFIG_HZ=100 systems +my ($tmpdir, $for_destroy) = tmpdir(); +mkdir "$tmpdir/new" or BAIL_OUT "mkdir: $!"; +open my $fh, '>', "$tmpdir/tst" or BAIL_OUT "open: $!"; +close $fh or BAIL_OUT "close: $!"; + +my $fi = PublicInbox::FakeInotify->new; +my $mask = PublicInbox::FakeInotify::MOVED_TO_OR_CREATE(); +my $hit = []; +my $cb = sub { push @$hit, map { $_->fullname } @_ }; +my $w = $fi->watch("$tmpdir/new", $mask, $cb); + +select undef, undef, undef, $MIN_FS_TICK; +rename("$tmpdir/tst", "$tmpdir/new/tst") or BAIL_OUT "rename: $!"; +$fi->poll; +is_deeply($hit, ["$tmpdir/new/tst"], 'rename(2) detected'); + +@$hit = (); +select undef, undef, undef, $MIN_FS_TICK; +open $fh, '>', "$tmpdir/tst" or BAIL_OUT "open: $!"; +close $fh or BAIL_OUT "close: $!"; +link("$tmpdir/tst", "$tmpdir/new/link") or BAIL_OUT "link: $!"; +$fi->poll; +is_deeply($hit, ["$tmpdir/new/link"], 'link(2) detected'); + +$w->cancel; +@$hit = (); +select undef, undef, undef, $MIN_FS_TICK; +link("$tmpdir/new/tst", "$tmpdir/new/link2") or BAIL_OUT "link: $!"; +$fi->poll; +is_deeply($hit, [], 'link(2) not detected after cancel'); + +PublicInbox::DS->Reset; + +done_testing; diff --git a/t/kqnotify.t b/t/kqnotify.t new file mode 100644 index 00000000000..b3414b8ae33 --- /dev/null +++ b/t/kqnotify.t @@ -0,0 +1,41 @@ +#!perl -w +# Copyright (C) 2020 all contributors +# License: AGPL-3.0+ +# +# Ensure KQNotify can pick up rename(2) and link(2) operations +# used by Maildir writing tools +use strict; +use Test::More; +use PublicInbox::TestCommon; +plan skip_all => 'KQNotify is only for *BSD systems' if $^O !~ /bsd/; +require_mods('IO::KQueue'); +use_ok 'PublicInbox::KQNotify'; +my ($tmpdir, $for_destroy) = tmpdir(); +mkdir "$tmpdir/new" or BAIL_OUT "mkdir: $!"; +open my $fh, '>', "$tmpdir/tst" or BAIL_OUT "open: $!"; +close $fh or BAIL_OUT "close: $!"; + +my $kqn = PublicInbox::KQNotify->new; +my $mask = PublicInbox::KQNotify::MOVED_TO_OR_CREATE(); +my $hit = []; +my $cb = sub { push @$hit, map { $_->fullname } @_ }; +my $w = $kqn->watch("$tmpdir/new", $mask, $cb); + +rename("$tmpdir/tst", "$tmpdir/new/tst") or BAIL_OUT "rename: $!"; +$kqn->poll; +is_deeply($hit, ["$tmpdir/new/tst"], 'rename(2) detected (via NOTE_EXTEND)'); + +@$hit = (); +open $fh, '>', "$tmpdir/tst" or BAIL_OUT "open: $!"; +close $fh or BAIL_OUT "close: $!"; +link("$tmpdir/tst", "$tmpdir/new/link") or BAIL_OUT "link: $!"; +$kqn->poll; +is_deeply($hit, ["$tmpdir/new/link"], 'link(2) detected (via NOTE_WRITE)'); + +$w->cancel; +@$hit = (); +link("$tmpdir/new/tst", "$tmpdir/new/link2") or BAIL_OUT "link: $!"; +$kqn->poll; +is_deeply($hit, [], 'link(2) not detected after cancel'); + +done_testing;