* [PATCH 1/5] inbox_writable: fix import_maildir
@ 2021-06-09 7:47 5% ` Eric Wong
0 siblings, 0 replies; 3+ results
From: Eric Wong @ 2021-06-09 7:47 UTC (permalink / raw)
To: meta
I'm not sure if anybody uses this, but it exists. It'll likely
be dropped in the future.
Fixes: fa3f0cbcd1af5008 ("use MdirReader in -watch and InboxWritable")
---
lib/PublicInbox/InboxWritable.pm | 15 ++++++---------
1 file changed, 6 insertions(+), 9 deletions(-)
diff --git a/lib/PublicInbox/InboxWritable.pm b/lib/PublicInbox/InboxWritable.pm
index 45d8cdc7..65539781 100644
--- a/lib/PublicInbox/InboxWritable.pm
+++ b/lib/PublicInbox/InboxWritable.pm
@@ -131,13 +131,9 @@ sub eml_from_path ($) {
}
}
-sub _each_maildir_fn {
- my ($fn, $im, $self) = @_;
- if ($fn =~ /:2,([A-Za-z]*)\z/) {
- my $fl = $1;
- return if $fl =~ /[DT]/; # no Drafts or Trash for public
- }
- my $eml = eml_from_path($fn) or return;
+sub _each_maildir_eml {
+ my ($fn, $kw, $eml, $im, $self) = @_;
+ return if grep(/\Adraft\z/, @$kw);
if ($self && (my $filter = $self->filter($im))) {
my $ret = $filter->scrub($eml) or return;
return if $ret == REJECT();
@@ -146,6 +142,7 @@ sub _each_maildir_fn {
$im->add($eml);
}
+# XXX does anybody use this?
sub import_maildir {
my ($self, $dir) = @_;
foreach my $sub (qw(cur new tmp)) {
@@ -154,8 +151,8 @@ sub import_maildir {
my $im = $self->importer(1);
my @self = $self->filter($im) ? ($self) : ();
require PublicInbox::MdirReader;
- PublicInbox::MdirReader->new->maildir_each_file(\&_each_maildir_fn,
- $im, @self);
+ PublicInbox::MdirReader->new->maildir_each_eml($dir,
+ \&_each_maildir_eml, $im, @self);
$im->done;
}
^ permalink raw reply related [relevance 5%]
* [PATCH 07/11] use MdirReader in -watch and InboxWritable
2021-02-09 8:09 7% [PATCH 00/11] Maildir code consolidation, test updates Eric Wong
@ 2021-02-09 8:09 5% ` Eric Wong
0 siblings, 0 replies; 3+ results
From: Eric Wong @ 2021-02-09 8:09 UTC (permalink / raw)
To: meta
MdirReader now handles files in "$MAILDIR/new" properly and
is stricter about what it accepts. eml_from_path is also
made robust against FIFOs while eliminating TOCTOU races with
between stat(2) and open(2) calls.
---
MANIFEST | 1 +
lib/PublicInbox/InboxWritable.pm | 55 +++++++++++++-------------------
lib/PublicInbox/MdirReader.pm | 22 +++++++++++--
lib/PublicInbox/Watch.pm | 6 ++--
t/mdir_reader.t | 22 +++++++++++++
5 files changed, 69 insertions(+), 37 deletions(-)
create mode 100644 t/mdir_reader.t
diff --git a/MANIFEST b/MANIFEST
index 6b3fc812..f8ee6998 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -376,6 +376,7 @@ t/mbox_reader.t
t/mda-mime.eml
t/mda.t
t/mda_filter_rubylang.t
+t/mdir_reader.t
t/mid.t
t/mime.t
t/miscsearch.t
diff --git a/lib/PublicInbox/InboxWritable.pm b/lib/PublicInbox/InboxWritable.pm
index 3a4012cd..c3acc4f9 100644
--- a/lib/PublicInbox/InboxWritable.pm
+++ b/lib/PublicInbox/InboxWritable.pm
@@ -10,6 +10,7 @@ use PublicInbox::Import;
use PublicInbox::Filter::Base qw(REJECT);
use Errno qw(ENOENT);
our @EXPORT_OK = qw(eml_from_path);
+use Fcntl qw(O_RDONLY O_NONBLOCK);
use constant {
PERM_UMASK => 0,
@@ -118,25 +119,10 @@ sub filter {
undef;
}
-sub is_maildir_basename ($) {
- my ($bn) = @_;
- return 0 if $bn !~ /\A[a-zA-Z0-9][\-\w:,=\.]+\z/;
- if ($bn =~ /:2,([A-Z]+)\z/i) {
- my $flags = $1;
- return 0 if $flags =~ /[DT]/; # no [D]rafts or [T]rashed mail
- }
- 1;
-}
-
-sub is_maildir_path ($) {
- my ($path) = @_;
- my @p = split(m!/+!, $path);
- (is_maildir_basename($p[-1]) && -f $path) ? 1 : 0;
-}
-
sub eml_from_path ($) {
my ($path) = @_;
- if (open my $fh, '<', $path) {
+ if (sysopen(my $fh, $path, O_RDONLY|O_NONBLOCK)) {
+ return unless -f $fh; # no FIFOs or directories
my $str = do { local $/; <$fh> } or return;
PublicInbox::Eml->new(\$str);
} else { # ENOENT is common with Maildir
@@ -145,27 +131,30 @@ sub eml_from_path ($) {
}
}
+sub _each_maildir_fn {
+ my ($fn, $im, $self) = @_;
+ if ($fn =~ /:2,([A-Za-z]*)\z/) {
+ my $fl = $1;
+ return if $fl =~ /[DT]/; # no Drafts or Trash for public
+ }
+ my $eml = eml_from_path($fn) or return;
+ if ($self && (my $filter = $self->filter($im))) {
+ my $ret = $filter->scrub($eml) or return;
+ return if $ret == REJECT();
+ $eml = $ret;
+ }
+ $im->add($eml);
+}
+
sub import_maildir {
my ($self, $dir) = @_;
- my $im = $self->importer(1);
-
foreach my $sub (qw(cur new tmp)) {
-d "$dir/$sub" or die "$dir is not a Maildir (missing $sub)\n";
}
- foreach my $sub (qw(cur new)) {
- opendir my $dh, "$dir/$sub" or die "opendir $dir/$sub: $!\n";
- while (defined(my $fn = readdir($dh))) {
- next unless is_maildir_basename($fn);
- my $mime = eml_from_path("$dir/$fn") or next;
-
- if (my $filter = $self->filter($im)) {
- my $ret = $filter->scrub($mime) or return;
- return if $ret == REJECT();
- $mime = $ret;
- }
- $im->add($mime);
- }
- }
+ my $im = $self->importer(1);
+ my @self = $self->filter($im) ? ($self) : ();
+ PublicInbox::MdirReader::maildir_each_file(\&_each_maildir_fn,
+ $im, @self);
$im->done;
}
diff --git a/lib/PublicInbox/MdirReader.pm b/lib/PublicInbox/MdirReader.pm
index c6a0e7a8..e0ff676d 100644
--- a/lib/PublicInbox/MdirReader.pm
+++ b/lib/PublicInbox/MdirReader.pm
@@ -2,18 +2,36 @@
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
# Maildirs for now, MH eventually
+# ref: https://cr.yp.to/proto/maildir.html
+# https://wiki2.dovecot.org/MailboxFormat/Maildir
package PublicInbox::MdirReader;
use strict;
use v5.10.1;
+# returns Maildir flags from a basename ('' for no flags, undef for invalid)
+sub maildir_basename_flags {
+ my (@f) = split(/:/, $_[0], -1);
+ return if (scalar(@f) > 2 || substr($f[0], 0, 1) eq '.');
+ $f[1] // return ''; # "new"
+ $f[1] =~ /\A2,([A-Za-z]*)\z/ ? $1 : undef; # "cur"
+}
+
+# same as above, but for full path name
+sub maildir_path_flags {
+ my ($f) = @_;
+ my $i = rindex($f, '/');
+ $i >= 0 ? maildir_basename_flags(substr($f, $i + 1)) : undef;
+}
+
sub maildir_each_file ($$;@) {
my ($dir, $cb, @arg) = @_;
$dir .= '/' unless substr($dir, -1) eq '/';
for my $d (qw(new/ cur/)) {
my $pfx = $dir.$d;
opendir my $dh, $pfx or next;
- while (defined(my $fn = readdir($dh))) {
- $cb->($pfx.$fn, @arg) if $fn =~ /:2,[A-Za-z]*\z/;
+ while (defined(my $bn = readdir($dh))) {
+ maildir_basename_flags($bn) // next;
+ $cb->($pfx.$bn, @arg);
}
}
}
diff --git a/lib/PublicInbox/Watch.pm b/lib/PublicInbox/Watch.pm
index 1835fa0e..a4302162 100644
--- a/lib/PublicInbox/Watch.pm
+++ b/lib/PublicInbox/Watch.pm
@@ -2,12 +2,13 @@
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
#
# ref: https://cr.yp.to/proto/maildir.html
-# http://wiki2.dovecot.org/MailboxFormat/Maildir
+# httsp://wiki2.dovecot.org/MailboxFormat/Maildir
package PublicInbox::Watch;
use strict;
use v5.10.1;
use PublicInbox::Eml;
use PublicInbox::InboxWritable qw(eml_from_path);
+use PublicInbox::MdirReader;
use PublicInbox::Filter::Base qw(REJECT);
use PublicInbox::Spamcheck;
use PublicInbox::Sigfd;
@@ -207,7 +208,8 @@ sub import_eml ($$$) {
sub _try_path {
my ($self, $path) = @_;
- return unless PublicInbox::InboxWritable::is_maildir_path($path);
+ my $fl = PublicInbox::MdirReader::maildir_path_flags($path) // return;
+ return if $fl =~ /[DT]/; # no Drafts or Trash
if ($path !~ $self->{mdre}) {
warn "unrecognized path: $path\n";
return;
diff --git a/t/mdir_reader.t b/t/mdir_reader.t
new file mode 100644
index 00000000..51b38af4
--- /dev/null
+++ b/t/mdir_reader.t
@@ -0,0 +1,22 @@
+#!perl -w
+# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+use PublicInbox::TestCommon;
+require_ok 'PublicInbox::MdirReader';
+*maildir_basename_flags = \&PublicInbox::MdirReader::maildir_basename_flags;
+*maildir_path_flags = \&PublicInbox::MdirReader::maildir_path_flags;
+
+is(maildir_basename_flags('foo'), '', 'new valid name accepted');
+is(maildir_basename_flags('foo:2,'), '', 'cur valid name accepted');
+is(maildir_basename_flags('foo:2,bar'), 'bar', 'flags name accepted');
+is(maildir_basename_flags('.foo:2,bar'), undef, 'no hidden files');
+is(maildir_basename_flags('fo:o:2,bar'), undef, 'no extra colon');
+is(maildir_path_flags('/path/to/foo:2,S'), 'S', 'flag returned for path');
+is(maildir_path_flags('/path/to/.foo:2,S'), undef, 'no hidden paths');
+is(maildir_path_flags('/path/to/foo:2,'), '', 'no flags in path');
+
+# not sure if there's a better place for eml_from_path
+use_ok 'PublicInbox::InboxWritable', qw(eml_from_path);
+is(eml_from_path('.'), undef, 'eml_from_path fails on directory');
+
+done_testing;
^ permalink raw reply related [relevance 5%]
* [PATCH 00/11] Maildir code consolidation, test updates
@ 2021-02-09 8:09 7% Eric Wong
2021-02-09 8:09 5% ` [PATCH 07/11] use MdirReader in -watch and InboxWritable Eric Wong
0 siblings, 1 reply; 3+ results
From: Eric Wong @ 2021-02-09 8:09 UTC (permalink / raw)
To: meta
MdirReader is the home of Maildir-related code, and
-watch is updated to use some of it, as is the
InboxWritable->import_maildir (which I'm not sure
should live).
"lei q --alert=" is slightly changed for CLI-friendliness
tr/-/:/
And damn, I just misread "Cwd" as "Covid"; /me goes back
to hiding under the bed :<
Eric Wong (11):
t/thread-index-gap.t: avoid unnecessary map
test_common: disable fsync on the command-line where possible
t/cgi.t: modernizations and style updates
git: ->qx: respect caller's $/ in array context
lei: split out MdirReader package, lazy-require earlier
t/run.perl: fix for >128 tests
use MdirReader in -watch and InboxWritable
lei q: prefix --alert ops with ':' instead of '-'
t/run.perl: drop Cwd dependency
lei: replace "I:"-prefixed info messages with "#"
tests: (lei) fixes for TEST_RUN_MODE=0 and oneshot mode
MANIFEST | 2 +
lib/PublicInbox/Git.pm | 1 -
lib/PublicInbox/IPC.pm | 2 +
lib/PublicInbox/Import.pm | 6 +--
lib/PublicInbox/InboxWritable.pm | 55 +++++++++------------
lib/PublicInbox/LEI.pm | 21 ++++----
lib/PublicInbox/LeiImport.pm | 25 ++++++----
lib/PublicInbox/LeiOverview.pm | 2 +-
lib/PublicInbox/LeiToMail.pm | 26 ++++------
lib/PublicInbox/MdirReader.pm | 39 +++++++++++++++
lib/PublicInbox/TestCommon.pm | 10 +++-
lib/PublicInbox/Watch.pm | 6 ++-
t/cgi.t | 84 +++++++++++++-------------------
t/lei-import.t | 5 +-
t/lei-mirror.t | 2 +-
t/lei.t | 2 +-
t/lei_to_mail.t | 19 ++++++--
t/mdir_reader.t | 22 +++++++++
t/run.perl | 22 ++++-----
t/thread-index-gap.t | 10 ++--
20 files changed, 211 insertions(+), 150 deletions(-)
create mode 100644 lib/PublicInbox/MdirReader.pm
create mode 100644 t/mdir_reader.t
^ permalink raw reply [relevance 7%]
Results 1-3 of 3 | reverse | options above
-- pct% links below jump to the message on this page, permalinks otherwise --
2021-02-09 8:09 7% [PATCH 00/11] Maildir code consolidation, test updates Eric Wong
2021-02-09 8:09 5% ` [PATCH 07/11] use MdirReader in -watch and InboxWritable Eric Wong
2021-06-09 7:47 [PATCH 0/5] lei Maildir stuff Eric Wong
2021-06-09 7:47 5% ` [PATCH 1/5] inbox_writable: fix import_maildir Eric Wong
Code repositories for project(s) associated with this public inbox
https://80x24.org/public-inbox.git
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).