* [PATCH 14/34] watch: support IMAP polling
2020-06-27 10:03 7% [PATCH 00/34] watch: add IMAP and NNTP support Eric Wong
@ 2020-06-27 10:03 4% ` Eric Wong
0 siblings, 0 replies; 2+ results
From: Eric Wong @ 2020-06-27 10:03 UTC (permalink / raw)
To: meta
Not all IMAP servers support IDLE, and IDLE may be prohibitively
expensive for some IMAP servers with many inboxes. So allow
configuring a imap.$IMAP_URL.pollInterval=SECONDS to poll
mailboxes.
We'll also need to poll for NNTP servers in the future.
---
lib/PublicInbox/WatchMaildir.pm | 64 ++++++++++++++++++++++++++++++---
t/imapd.t | 39 ++++++++++++++++++--
2 files changed, 95 insertions(+), 8 deletions(-)
diff --git a/lib/PublicInbox/WatchMaildir.pm b/lib/PublicInbox/WatchMaildir.pm
index 431350be277..ac980d9b0f1 100644
--- a/lib/PublicInbox/WatchMaildir.pm
+++ b/lib/PublicInbox/WatchMaildir.pm
@@ -202,8 +202,9 @@ sub quit {
if (my $imap_pid = $self->{-imap_pid}) {
kill('QUIT', $imap_pid);
}
- if (my $idle_pids = $self->{idle_pids}) {
- kill('QUIT', $_) for (keys %$idle_pids);
+ for (qw(idle_pids poll_pids)) {
+ my $pids = $self->{$_} or next;
+ kill('QUIT', $_) for (keys %$pids);
}
if (my $idle_mic = $self->{idle_mic}) {
eval { $idle_mic->done };
@@ -237,12 +238,12 @@ sub imap_section ($) {
sub cfg_intvl ($$) {
my ($cfg, $key) = @_;
defined(my $v = $cfg->{lc($key)}) or return;
- $v =~ /\A[0-9]+\z/s and return $v + 0;
+ $v =~ /\A[0-9]+(?:\.[0-9]+)?\z/s and return $v + 0;
if (ref($v) eq 'ARRAY') {
$v = join(', ', @$v);
warn "W: $key has multiple values: $v\nW: $key ignored\n";
} else {
- warn "W: $key=$v is not an integer value in seconds\n";
+ warn "W: $key=$v is not a numeric value in seconds\n";
}
}
@@ -460,6 +461,7 @@ sub watch_imap_idle_1 ($$$) {
sub watch_atfork_child ($) {
my ($self) = @_;
delete $self->{idle_pids};
+ delete $self->{poll_pids};
PublicInbox::DS->Reset;
PublicInbox::Sigfd::sig_setmask($self->{oldset});
%SIG = (%SIG, %{$self->{sig}});
@@ -504,6 +506,52 @@ sub event_step {
goto(&fs_scan_step) if $self->{mdre};
}
+sub watch_imap_fetch_all ($$) {
+ my ($self, $uris) = @_;
+ for my $uri (@$uris) {
+ my $sec = imap_section($uri);
+ my $mic_arg = $self->{mic_arg}->{$sec} or
+ die "BUG: no Mail::IMAPClient->new arg for $sec";
+ my $mic = PublicInbox::IMAPClient->new(%$mic_arg) or next;
+ my $err = imap_fetch_all($self, $mic, $uri);
+ last if $self->{quit};
+ warn $err, "\n" if $err;
+ }
+}
+
+sub imap_fetch_fork ($$$) {
+ my ($self, $intvl, $uris) = @_;
+ return if $self->{quit};
+ $self->{mics} = {}; # going to be forking, so disconnect
+ defined(my $pid = fork) or die "fork: $!";
+ if ($pid == 0) {
+ watch_atfork_child($self);
+ watch_imap_fetch_all($self, $uris);
+ _exit(0);
+ }
+ $self->{poll_pids}->{$pid} = [ $intvl, $uris ];
+ PublicInbox::DS::dwaitpid($pid, \&imap_fetch_reap, $self);
+}
+
+sub imap_fetch_cb ($$$) {
+ my ($self, $intvl, $uris) = @_;
+ sub { imap_fetch_fork($self, $intvl, $uris) };
+}
+
+sub imap_fetch_reap { # PublicInbox::DS::dwaitpid callback
+ my ($self, $pid) = @_;
+ my $intvl_uris = delete $self->{poll_pids}->{$pid} or
+ die "BUG: PID=$pid (unknown) reaped: \$?=$?\n";
+ return if $self->{quit};
+ my ($intvl, $uris) = @$intvl_uris;
+ if ($?) {
+ warn "W: PID=$pid died: \$?=$?\n",
+ map { $_->as_string."\n" } @$uris;
+ }
+ warn('I: will check ', $_->as_string, " in ${intvl}s\n") for @$uris;
+ PublicInbox::DS::add_timer($intvl, imap_fetch_cb($self, $intvl, $uris));
+}
+
sub watch_imap_init ($) {
my ($self) = @_;
eval { require PublicInbox::IMAPClient } or
@@ -542,7 +590,13 @@ sub watch_imap_init ($) {
$self->{idle_todo} = $idle;
PublicInbox::DS::requeue($self); # ->event_step to fork
}
- # TODO: polling
+ return unless scalar keys %$poll;
+ $self->{poll_pids} = {};
+
+ # poll all URIs for a given interval sequentially
+ while (my ($intvl, $uris) = each %$poll) {
+ PublicInbox::DS::requeue(imap_fetch_cb($self, $intvl, $uris));
+ }
}
sub watch {
diff --git a/t/imapd.t b/t/imapd.t
index cc87a127851..ee3a3b26767 100644
--- a/t/imapd.t
+++ b/t/imapd.t
@@ -443,6 +443,7 @@ ok($mic->logout, 'logged out');
{
use_ok 'PublicInbox::WatchMaildir';
use_ok 'PublicInbox::InboxIdle';
+ my $old_env = { HOME => $ENV{HOME} };
my $home = "$tmpdir/watch_home";
mkdir $home or BAIL_OUT $!;
mkdir "$home/.public-inbox" or BAIL_OUT $!;
@@ -464,13 +465,45 @@ ok($mic->logout, 'logged out');
my $cb = sub { PublicInbox::DS->SetPostLoopCallback(sub {}) };
my $obj = bless \$cb, 'PublicInbox::TestCommon::InboxWakeup';
$cfg->each_inbox(sub { $_[0]->subscribe_unlock('ident', $obj) });
- open my $err, '+>', undef or BAIL_OUT $!;
- my $w = start_script(['-watch'], undef, { 2 => $err });
+ my $watcherr = "$tmpdir/watcherr";
+ open my $err_wr, '>', $watcherr or BAIL_OUT $!;
+ open my $err, '<', $watcherr or BAIL_OUT $!;
+ my $w = start_script(['-watch'], undef, { 2 => $err_wr });
+
+ diag 'waiting for initial fetch...';
+ PublicInbox::DS->EventLoop;
+ diag 'inbox unlocked on initial fetch, waiting for IDLE';
+
+ tick until (grep(/I: \S+ idling/, <$err>));
+ open my $fh, '<', 't/iso-2202-jp.eml' or BAIL_OUT $!;
+ $old_env->{ORIGINAL_RECIPIENT} = $addr;
+ ok(run_script([qw(-mda --no-precheck)], $old_env, { 0 => $fh }),
+ 'delivered a message for IDLE to kick -watch');
+ diag 'waiting for IMAP IDLE wakeup';
+ PublicInbox::DS->SetPostLoopCallback(undef);
+ PublicInbox::DS->EventLoop;
+ diag 'inbox unlocked on IDLE wakeup';
+
+ # try again with polling
+ xsys(qw(git config), "--file=$home/.public-inbox/config",
+ "imap.imap://$ihost:$iport.PollInterval", 0.11) == 0
+ or BAIL_OUT "git config $?";
+ $w->kill('HUP');
+ diag 'waiting for -watch reload + initial fetch';
+ tick until (grep(/I: will check/, <$err>));
+
+ open $fh, '<', 't/psgi_attach.eml' or BAIL_OUT $!;
+ ok(run_script([qw(-mda --no-precheck)], $old_env, { 0 => $fh }),
+ 'delivered a message for -watch PollInterval');
+
+ diag 'waiting for PollInterval wakeup';
+ PublicInbox::DS->SetPostLoopCallback(undef);
PublicInbox::DS->EventLoop;
- diag 'inbox unlocked';
+ diag 'inbox unlocked (poll)';
$w->kill;
$w->join;
is($?, 0, 'no error in exited -watch process');
+
$cfg->each_inbox(sub { shift->unsubscribe_unlock('ident') });
$ii->close;
PublicInbox::DS->Reset;
^ permalink raw reply related [relevance 4%]
* [PATCH 00/34] watch: add IMAP and NNTP support
@ 2020-06-27 10:03 7% Eric Wong
2020-06-27 10:03 4% ` [PATCH 14/34] watch: support IMAP polling Eric Wong
0 siblings, 1 reply; 2+ results
From: Eric Wong @ 2020-06-27 10:03 UTC (permalink / raw)
To: meta
Some fairly major changes to -watch. Filesys::Notify::Simple is
no longer used, and -watch now uses inotify, signalfd or kevent
like the read-only daemons.
Credentials are handled via Net::Netrc (Perl standard library)
or "git-credential", so we do no password storage on our own.
NNTP (and non-IDLE IMAP) may allow more parallelization in the
future.
One significant project-wide change is getting rid of "use
fields". It gets in my way more than it helps, and it's
probably alien to a fair amount of Perl hackers. AFAIK, it's
never really been popular outside of Danga::Socket-based
projects.
Eric W. Biederman (1):
IMAPTracker: Add a helper to track our place in reading imap mailboxes
Eric Wong (33):
inboxwritable: ensure ssoma.lock exists on init
inbox: warn on ->on_inbox_unlock exception
imaptracker: use ~/.local/share/public-inbox/imap.sqlite3
watchmaildir: hoist out compile_watchheaders
watchmaildir: fix check for spam vs ham inbox conflicts
URI IMAP support
watch: preliminary IMAP support
kqnotify|fake_inotify: detect Maildir write ops
watch: remove Filesys::Notify::Simple dependency
watch: use signalfd for Maildir watching
ds: remove fields.pm usage
watch: wire up IMAP IDLE reapers to DS
watch: support IMAP polling
config: support ->urlmatch method for -watch
watch: stop importers before forking
watch: use UID SEARCH to avoid empty UID FETCH
ds: add_timer: allow passing arg to callback.
imaptracker: add {url} field to reduce args
imaptracker: drop {dbname} field
watch: avoid long transaction when writing to IMAPTracker
watch: support imap.fetchBatchSize parameter
watch: imap: be quieter about disconnecting on quit
watch: support multiple watch: directives per-inbox
watch: remove {mdir} array
watch: just use ->urlmatch
testcommon: $ENV{TAIL} supports non-@ARGV redirects
watch: add NNTP support
watch: show user-specified URL consistently.
watch: enable autoflush for STDOUT and STDERR
watch: use our own "git credential" wrapper
watch: support ~/.netrc via Net::Netrc
imaptracker: use flock(2) around writes
watch: simplify internal structures
Documentation/public-inbox-watch.pod | 3 +-
INSTALL | 8 -
MANIFEST | 11 +
Makefile.PL | 4 -
ci/deps.perl | 1 -
lib/PublicInbox/Config.pm | 21 +-
lib/PublicInbox/DS.pm | 29 +-
lib/PublicInbox/Daemon.pm | 19 +-
lib/PublicInbox/DirIdle.pm | 49 ++
lib/PublicInbox/FakeInotify.pm | 56 +-
lib/PublicInbox/GitAsyncCat.pm | 4 +-
lib/PublicInbox/GitCredential.pm | 55 ++
lib/PublicInbox/HTTP.pm | 23 +-
lib/PublicInbox/HTTPD/Async.pm | 22 +-
lib/PublicInbox/IMAP.pm | 19 +-
lib/PublicInbox/IMAPTracker.pm | 82 +++
lib/PublicInbox/In2Tie.pm | 13 +
lib/PublicInbox/Inbox.pm | 1 +
lib/PublicInbox/InboxIdle.pm | 20 +-
lib/PublicInbox/InboxWritable.pm | 3 +
lib/PublicInbox/KQNotify.pm | 38 +-
lib/PublicInbox/Listener.pm | 8 +-
lib/PublicInbox/NNTP.pm | 12 +-
lib/PublicInbox/NNTPdeflate.pm | 5 +-
lib/PublicInbox/ParentPipe.pm | 8 +-
lib/PublicInbox/Sigfd.pm | 21 +-
lib/PublicInbox/TestCommon.pm | 40 +-
lib/PublicInbox/URIimap.pm | 113 +++
lib/PublicInbox/WatchMaildir.pm | 998 +++++++++++++++++++++++----
script/public-inbox-watch | 33 +-
t/config.t | 18 +
t/dir_idle.t | 6 +
t/fake_inotify.t | 45 ++
t/imap_tracker.t | 54 ++
t/imapd.t | 74 ++
t/kqnotify.t | 41 ++
t/nntpd.t | 52 ++
t/uri_imap.t | 65 ++
t/watch_filter_rubylang.t | 2 +-
t/watch_imap.t | 21 +
t/watch_maildir.t | 96 ++-
t/watch_maildir_v2.t | 4 +-
t/watch_multiple_headers.t | 2 +-
t/watch_nntp.t | 17 +
xt/mem-imapd-tls.t | 18 +-
45 files changed, 1944 insertions(+), 290 deletions(-)
create mode 100644 lib/PublicInbox/DirIdle.pm
create mode 100644 lib/PublicInbox/GitCredential.pm
create mode 100644 lib/PublicInbox/IMAPTracker.pm
create mode 100644 lib/PublicInbox/URIimap.pm
create mode 100644 t/dir_idle.t
create mode 100644 t/fake_inotify.t
create mode 100644 t/imap_tracker.t
create mode 100644 t/kqnotify.t
create mode 100644 t/uri_imap.t
create mode 100644 t/watch_imap.t
create mode 100644 t/watch_nntp.t
^ permalink raw reply [relevance 7%]
Results 1-2 of 2 | reverse | options above
-- pct% links below jump to the message on this page, permalinks otherwise --
2020-06-27 10:03 7% [PATCH 00/34] watch: add IMAP and NNTP support Eric Wong
2020-06-27 10:03 4% ` [PATCH 14/34] watch: support IMAP polling Eric Wong
Code repositories for project(s) associated with this public inbox
https://80x24.org/public-inbox.git
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).