* [PATCH 09/34] kqnotify|fake_inotify: detect Maildir write ops
2020-06-27 10:03 7% [PATCH 00/34] watch: add IMAP and NNTP support Eric Wong
@ 2020-06-27 10:03 5% ` Eric Wong
0 siblings, 0 replies; 2+ results
From: Eric Wong @ 2020-06-27 10:03 UTC (permalink / raw)
To: meta
We need to detect link(2) and rename(2) in other apps
writing to the Maildir.
We'll be removing the Filesys::Notify::Simple from -watch
in favor of using IO::KQueue or Linux::Inotify2 directly.
Ensure non-inotify emulations can support everything we
expect for Maildir writers.
---
MANIFEST | 2 ++
lib/PublicInbox/FakeInotify.pm | 46 ++++++++++++++++++++++++++++------
lib/PublicInbox/KQNotify.pm | 38 +++++++++++++++++++++++-----
t/fake_inotify.t | 45 +++++++++++++++++++++++++++++++++
t/kqnotify.t | 41 ++++++++++++++++++++++++++++++
5 files changed, 159 insertions(+), 13 deletions(-)
create mode 100644 t/fake_inotify.t
create mode 100644 t/kqnotify.t
diff --git a/MANIFEST b/MANIFEST
index 161b6cddbe0..9d1a4e4a8b1 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -253,6 +253,7 @@ t/eml_content_disposition.t
t/eml_content_type.t
t/epoll.t
t/fail-bin/spamc
+t/fake_inotify.t
t/feed.t
t/filter_base-junk.eml
t/filter_base-xhtml.eml
@@ -286,6 +287,7 @@ t/indexlevels-mirror-v1.t
t/indexlevels-mirror.t
t/init.t
t/iso-2202-jp.eml
+t/kqnotify.t
t/linkify.t
t/main-bin/spamc
t/mda-mime.eml
diff --git a/lib/PublicInbox/FakeInotify.pm b/lib/PublicInbox/FakeInotify.pm
index b077d63a4b4..df63173f083 100644
--- a/lib/PublicInbox/FakeInotify.pm
+++ b/lib/PublicInbox/FakeInotify.pm
@@ -6,10 +6,13 @@
package PublicInbox::FakeInotify;
use strict;
use Time::HiRes qw(stat);
+use PublicInbox::DS;
my $IN_CLOSE = 0x08 | 0x10; # match Linux inotify
+# my $IN_MOVED_TO = 0x80;
+# my $IN_CREATE = 0x100;
+sub MOVED_TO_OR_CREATE () { 0x80 | 0x100 }
my $poll_intvl = 2; # same as Filesys::Notify::Simple
-my $for_cancel = bless \(my $x), 'PublicInbox::FakeInotify::Watch';
sub poll_once {
my ($self) = @_;
@@ -30,8 +33,22 @@ sub new {
sub watch {
my ($self, $path, $mask, $cb) = @_;
my @st = stat($path) or return;
- $self->{watch}->{"$path\0$mask"} = [ @st, $cb ];
- $for_cancel;
+ my $k = "$path\0$mask";
+ $self->{watch}->{$k} = [ $st[10], $cb ]; # 10 - ctime
+ bless [ $self->{watch}, $k ], 'PublicInbox::FakeInotify::Watch';
+}
+
+sub on_new_files ($$$$) {
+ my ($dh, $cb, $path, $old_ctime) = @_;
+ while (defined(my $base = readdir($dh))) {
+ next if $base =~ /\A\.\.?\z/;
+ my $full = "$path/$base";
+ my @st = stat($full);
+ if (@st && $st[10] > $old_ctime) {
+ bless \$full, 'PublicInbox::FakeInotify::Event';
+ eval { $cb->(\$full) };
+ }
+ }
}
# behaves like non-blocking Linux::Inotify2->poll
@@ -43,17 +60,32 @@ sub poll {
my @now = stat($path) or next;
my $prv = $watch->{$x};
my $cb = $prv->[-1];
- # 10: ctime, 7: size
- if ($prv->[10] != $now[10]) {
+ my $old_ctime = $prv->[0];
+ if ($old_ctime != $now[10]) {
if (($mask & $IN_CLOSE) == $IN_CLOSE) {
eval { $cb->() };
+ } elsif ($mask & MOVED_TO_OR_CREATE) {
+ opendir(my $dh, $path) or do {
+ warn "W: opendir $path: $!\n";
+ next;
+ };
+ on_new_files($dh, $cb, $path, $old_ctime);
}
}
- @$prv = (@now, $cb);
+ @$prv = ($now[10], $cb);
}
}
package PublicInbox::FakeInotify::Watch;
-sub cancel {} # noop
+use strict;
+
+sub cancel {
+ my ($self) = @_;
+ delete $self->[0]->{$self->[1]};
+}
+
+package PublicInbox::FakeInotify::Event;
+use strict;
+sub fullname { ${$_[0]} }
1;
diff --git a/lib/PublicInbox/KQNotify.pm b/lib/PublicInbox/KQNotify.pm
index 110594cc02c..9673b44290a 100644
--- a/lib/PublicInbox/KQNotify.pm
+++ b/lib/PublicInbox/KQNotify.pm
@@ -7,6 +7,11 @@ package PublicInbox::KQNotify;
use strict;
use IO::KQueue;
use PublicInbox::DSKQXS; # wraps IO::KQueue for fork-safe DESTROY
+use PublicInbox::FakeInotify;
+use Time::HiRes qw(stat);
+
+# NOTE_EXTEND detects rename(2), NOTE_WRITE detects link(2)
+sub MOVED_TO_OR_CREATE () { NOTE_EXTEND|NOTE_WRITE }
sub new {
my ($class) = @_;
@@ -15,19 +20,28 @@ sub new {
sub watch {
my ($self, $path, $mask, $cb) = @_;
- open(my $fh, '<', $path) or return;
+ my ($fh, $cls, @extra);
+ if (-d $path) {
+ opendir($fh, $path) or return;
+ my @st = stat($fh);
+ @extra = ($path, $st[10]); # 10: ctime
+ $cls = 'PublicInbox::KQNotify::Watchdir';
+ } else {
+ open($fh, '<', $path) or return;
+ $cls = 'PublicInbox::KQNotify::Watch';
+ }
my $ident = fileno($fh);
$self->{dskq}->{kq}->EV_SET($ident, # ident
EVFILT_VNODE, # filter
EV_ADD | EV_CLEAR, # flags
$mask, # fflags
0, 0); # data, udata
- if ($mask == NOTE_WRITE) {
- $self->{watch}->{$ident} = [ $fh, $cb ];
+ if ($mask == NOTE_WRITE || $mask == MOVED_TO_OR_CREATE) {
+ $self->{watch}->{$ident} = [ $fh, $cb, @extra ];
} else {
die "TODO Not implemented: $mask";
}
- bless \$fh, 'PublicInbox::KQNotify::Watch';
+ bless \$fh, $cls;
}
# emulate Linux::Inotify::fileno
@@ -48,8 +62,15 @@ sub poll {
for my $kev (@kevents) {
my $ident = $kev->[KQ_IDENT];
my $mask = $kev->[KQ_FFLAGS];
- if (($mask & NOTE_WRITE) == NOTE_WRITE) {
- eval { $self->{watch}->{$ident}->[1]->() };
+ my ($dh, $cb, $path, $old_ctime) = @{$self->{watch}->{$ident}};
+ if (!defined($path) && ($mask & NOTE_WRITE) == NOTE_WRITE) {
+ eval { $cb->() };
+ } elsif ($mask & MOVED_TO_OR_CREATE) {
+ my @new_st = stat($path) or next;
+ $self->{watch}->{$ident}->[3] = $new_st[10]; # ctime
+ rewinddir($dh);
+ PublicInbox::FakeInotify::on_new_files($dh, $cb,
+ $path, $old_ctime);
}
}
}
@@ -59,4 +80,9 @@ use strict;
sub cancel { close ${$_[0]} or die "close: $!" }
+package PublicInbox::KQNotify::Watchdir;
+use strict;
+
+sub cancel { closedir ${$_[0]} or die "closedir: $!" }
+
1;
diff --git a/t/fake_inotify.t b/t/fake_inotify.t
new file mode 100644
index 00000000000..f0db0cb58ec
--- /dev/null
+++ b/t/fake_inotify.t
@@ -0,0 +1,45 @@
+#!perl -w
+# Copyright (C) 2020 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+#
+# Ensure FakeInotify can pick up rename(2) and link(2) operations
+# used by Maildir writing tools
+use strict;
+use Test::More;
+use PublicInbox::TestCommon;
+use_ok 'PublicInbox::FakeInotify';
+my $MIN_FS_TICK = 0.011; # for low-res CONFIG_HZ=100 systems
+my ($tmpdir, $for_destroy) = tmpdir();
+mkdir "$tmpdir/new" or BAIL_OUT "mkdir: $!";
+open my $fh, '>', "$tmpdir/tst" or BAIL_OUT "open: $!";
+close $fh or BAIL_OUT "close: $!";
+
+my $fi = PublicInbox::FakeInotify->new;
+my $mask = PublicInbox::FakeInotify::MOVED_TO_OR_CREATE();
+my $hit = [];
+my $cb = sub { push @$hit, map { $_->fullname } @_ };
+my $w = $fi->watch("$tmpdir/new", $mask, $cb);
+
+select undef, undef, undef, $MIN_FS_TICK;
+rename("$tmpdir/tst", "$tmpdir/new/tst") or BAIL_OUT "rename: $!";
+$fi->poll;
+is_deeply($hit, ["$tmpdir/new/tst"], 'rename(2) detected');
+
+@$hit = ();
+select undef, undef, undef, $MIN_FS_TICK;
+open $fh, '>', "$tmpdir/tst" or BAIL_OUT "open: $!";
+close $fh or BAIL_OUT "close: $!";
+link("$tmpdir/tst", "$tmpdir/new/link") or BAIL_OUT "link: $!";
+$fi->poll;
+is_deeply($hit, ["$tmpdir/new/link"], 'link(2) detected');
+
+$w->cancel;
+@$hit = ();
+select undef, undef, undef, $MIN_FS_TICK;
+link("$tmpdir/new/tst", "$tmpdir/new/link2") or BAIL_OUT "link: $!";
+$fi->poll;
+is_deeply($hit, [], 'link(2) not detected after cancel');
+
+PublicInbox::DS->Reset;
+
+done_testing;
diff --git a/t/kqnotify.t b/t/kqnotify.t
new file mode 100644
index 00000000000..b3414b8ae33
--- /dev/null
+++ b/t/kqnotify.t
@@ -0,0 +1,41 @@
+#!perl -w
+# Copyright (C) 2020 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+#
+# Ensure KQNotify can pick up rename(2) and link(2) operations
+# used by Maildir writing tools
+use strict;
+use Test::More;
+use PublicInbox::TestCommon;
+plan skip_all => 'KQNotify is only for *BSD systems' if $^O !~ /bsd/;
+require_mods('IO::KQueue');
+use_ok 'PublicInbox::KQNotify';
+my ($tmpdir, $for_destroy) = tmpdir();
+mkdir "$tmpdir/new" or BAIL_OUT "mkdir: $!";
+open my $fh, '>', "$tmpdir/tst" or BAIL_OUT "open: $!";
+close $fh or BAIL_OUT "close: $!";
+
+my $kqn = PublicInbox::KQNotify->new;
+my $mask = PublicInbox::KQNotify::MOVED_TO_OR_CREATE();
+my $hit = [];
+my $cb = sub { push @$hit, map { $_->fullname } @_ };
+my $w = $kqn->watch("$tmpdir/new", $mask, $cb);
+
+rename("$tmpdir/tst", "$tmpdir/new/tst") or BAIL_OUT "rename: $!";
+$kqn->poll;
+is_deeply($hit, ["$tmpdir/new/tst"], 'rename(2) detected (via NOTE_EXTEND)');
+
+@$hit = ();
+open $fh, '>', "$tmpdir/tst" or BAIL_OUT "open: $!";
+close $fh or BAIL_OUT "close: $!";
+link("$tmpdir/tst", "$tmpdir/new/link") or BAIL_OUT "link: $!";
+$kqn->poll;
+is_deeply($hit, ["$tmpdir/new/link"], 'link(2) detected (via NOTE_WRITE)');
+
+$w->cancel;
+@$hit = ();
+link("$tmpdir/new/tst", "$tmpdir/new/link2") or BAIL_OUT "link: $!";
+$kqn->poll;
+is_deeply($hit, [], 'link(2) not detected after cancel');
+
+done_testing;
^ permalink raw reply related [relevance 5%]
* [PATCH 00/34] watch: add IMAP and NNTP support
@ 2020-06-27 10:03 7% Eric Wong
2020-06-27 10:03 5% ` [PATCH 09/34] kqnotify|fake_inotify: detect Maildir write ops Eric Wong
0 siblings, 1 reply; 2+ results
From: Eric Wong @ 2020-06-27 10:03 UTC (permalink / raw)
To: meta
Some fairly major changes to -watch. Filesys::Notify::Simple is
no longer used, and -watch now uses inotify, signalfd or kevent
like the read-only daemons.
Credentials are handled via Net::Netrc (Perl standard library)
or "git-credential", so we do no password storage on our own.
NNTP (and non-IDLE IMAP) may allow more parallelization in the
future.
One significant project-wide change is getting rid of "use
fields". It gets in my way more than it helps, and it's
probably alien to a fair amount of Perl hackers. AFAIK, it's
never really been popular outside of Danga::Socket-based
projects.
Eric W. Biederman (1):
IMAPTracker: Add a helper to track our place in reading imap mailboxes
Eric Wong (33):
inboxwritable: ensure ssoma.lock exists on init
inbox: warn on ->on_inbox_unlock exception
imaptracker: use ~/.local/share/public-inbox/imap.sqlite3
watchmaildir: hoist out compile_watchheaders
watchmaildir: fix check for spam vs ham inbox conflicts
URI IMAP support
watch: preliminary IMAP support
kqnotify|fake_inotify: detect Maildir write ops
watch: remove Filesys::Notify::Simple dependency
watch: use signalfd for Maildir watching
ds: remove fields.pm usage
watch: wire up IMAP IDLE reapers to DS
watch: support IMAP polling
config: support ->urlmatch method for -watch
watch: stop importers before forking
watch: use UID SEARCH to avoid empty UID FETCH
ds: add_timer: allow passing arg to callback.
imaptracker: add {url} field to reduce args
imaptracker: drop {dbname} field
watch: avoid long transaction when writing to IMAPTracker
watch: support imap.fetchBatchSize parameter
watch: imap: be quieter about disconnecting on quit
watch: support multiple watch: directives per-inbox
watch: remove {mdir} array
watch: just use ->urlmatch
testcommon: $ENV{TAIL} supports non-@ARGV redirects
watch: add NNTP support
watch: show user-specified URL consistently.
watch: enable autoflush for STDOUT and STDERR
watch: use our own "git credential" wrapper
watch: support ~/.netrc via Net::Netrc
imaptracker: use flock(2) around writes
watch: simplify internal structures
Documentation/public-inbox-watch.pod | 3 +-
INSTALL | 8 -
MANIFEST | 11 +
Makefile.PL | 4 -
ci/deps.perl | 1 -
lib/PublicInbox/Config.pm | 21 +-
lib/PublicInbox/DS.pm | 29 +-
lib/PublicInbox/Daemon.pm | 19 +-
lib/PublicInbox/DirIdle.pm | 49 ++
lib/PublicInbox/FakeInotify.pm | 56 +-
lib/PublicInbox/GitAsyncCat.pm | 4 +-
lib/PublicInbox/GitCredential.pm | 55 ++
lib/PublicInbox/HTTP.pm | 23 +-
lib/PublicInbox/HTTPD/Async.pm | 22 +-
lib/PublicInbox/IMAP.pm | 19 +-
lib/PublicInbox/IMAPTracker.pm | 82 +++
lib/PublicInbox/In2Tie.pm | 13 +
lib/PublicInbox/Inbox.pm | 1 +
lib/PublicInbox/InboxIdle.pm | 20 +-
lib/PublicInbox/InboxWritable.pm | 3 +
lib/PublicInbox/KQNotify.pm | 38 +-
lib/PublicInbox/Listener.pm | 8 +-
lib/PublicInbox/NNTP.pm | 12 +-
lib/PublicInbox/NNTPdeflate.pm | 5 +-
lib/PublicInbox/ParentPipe.pm | 8 +-
lib/PublicInbox/Sigfd.pm | 21 +-
lib/PublicInbox/TestCommon.pm | 40 +-
lib/PublicInbox/URIimap.pm | 113 +++
lib/PublicInbox/WatchMaildir.pm | 998 +++++++++++++++++++++++----
script/public-inbox-watch | 33 +-
t/config.t | 18 +
t/dir_idle.t | 6 +
t/fake_inotify.t | 45 ++
t/imap_tracker.t | 54 ++
t/imapd.t | 74 ++
t/kqnotify.t | 41 ++
t/nntpd.t | 52 ++
t/uri_imap.t | 65 ++
t/watch_filter_rubylang.t | 2 +-
t/watch_imap.t | 21 +
t/watch_maildir.t | 96 ++-
t/watch_maildir_v2.t | 4 +-
t/watch_multiple_headers.t | 2 +-
t/watch_nntp.t | 17 +
xt/mem-imapd-tls.t | 18 +-
45 files changed, 1944 insertions(+), 290 deletions(-)
create mode 100644 lib/PublicInbox/DirIdle.pm
create mode 100644 lib/PublicInbox/GitCredential.pm
create mode 100644 lib/PublicInbox/IMAPTracker.pm
create mode 100644 lib/PublicInbox/URIimap.pm
create mode 100644 t/dir_idle.t
create mode 100644 t/fake_inotify.t
create mode 100644 t/imap_tracker.t
create mode 100644 t/kqnotify.t
create mode 100644 t/uri_imap.t
create mode 100644 t/watch_imap.t
create mode 100644 t/watch_nntp.t
^ permalink raw reply [relevance 7%]
Results 1-2 of 2 | reverse | options above
-- pct% links below jump to the message on this page, permalinks otherwise --
2020-06-27 10:03 7% [PATCH 00/34] watch: add IMAP and NNTP support Eric Wong
2020-06-27 10:03 5% ` [PATCH 09/34] kqnotify|fake_inotify: detect Maildir write ops Eric Wong
Code repositories for project(s) associated with this public inbox
https://80x24.org/public-inbox.git
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).