From abd0a85b212ce1467ddc94d523152d9a65028960 Mon Sep 17 00:00:00 2001 From: Eric Wong Date: Thu, 31 Dec 2020 13:51:31 +0000 Subject: lei_to_mail: support Maildir, fix+test --augment Maildir should be plenty fine for short-lived output folders. --- t/lei_to_mail.t | 97 +++++++++++++++++++++++++++++++++++++++++++++++++++++---- t/mbox_reader.t | 3 +- 2 files changed, 92 insertions(+), 8 deletions(-) (limited to 't') diff --git a/t/lei_to_mail.t b/t/lei_to_mail.t index f3cc71ad..bd864754 100644 --- a/t/lei_to_mail.t +++ b/t/lei_to_mail.t @@ -7,14 +7,18 @@ use Test::More; use PublicInbox::TestCommon; use PublicInbox::Eml; use Fcntl qw(SEEK_SET); +use PublicInbox::Spawn qw(popen_rd which); +use List::Util qw(shuffle); require_mods(qw(DBD::SQLite)); +require PublicInbox::MboxReader; use_ok 'PublicInbox::LeiToMail'; my $from = "Content-Length: 10\nSubject: x\n\nFrom hell\n"; my $noeol = "Subject: x\n\nFrom hell"; my $crlf = $noeol; $crlf =~ s/\n/\r\n/g; my $kw = [qw(seen answered flagged)]; -for my $mbox (qw(mboxrd mboxo mboxcl mboxcl2)) { +my @MBOX = qw(mboxcl2 mboxrd mboxcl mboxo); +for my $mbox (@MBOX) { my $m = "eml2$mbox"; my $cb = PublicInbox::LeiToMail->can($m); my $s = $cb->(PublicInbox::Eml->new($from), $kw); @@ -75,8 +79,9 @@ Subject: x blah EOM my $fn = "$tmpdir/x.mbox"; +my ($mbox) = shuffle(@MBOX); # pick one, shouldn't matter my $orig = do { - my $wcb = PublicInbox::LeiToMail->write_cb("mboxcl2:$fn", $lei); + my $wcb = PublicInbox::LeiToMail->write_cb("$mbox:$fn", $lei); is(ref $wcb, 'CODE', 'write_cb returned callback'); ok(-f $fn && !-s _, 'empty file created'); $wcb->(\(my $dup = $buf), 'deadbeef', [ qw(seen) ]); @@ -87,7 +92,8 @@ my $orig = do { unlink $fn or BAIL_OUT $!; local $lei->{opt} = { jobs => 2 }; - $wcb = PublicInbox::LeiToMail->write_cb("mboxcl2:$fn", $lei); + $wcb = PublicInbox::LeiToMail->write_cb("$mbox:$fn", $lei); + ok(-f $fn && !-s _, 'truncated mbox destination'); $lei->{dedupe}->prepare_dedupe; $wcb->(\($dup = $buf), 'deadbeef', [ qw(seen) ]); undef $wcb; @@ -103,7 +109,7 @@ for my $zsfx (qw(gz bz2 xz)) { # XXX should we support zst, zz, lzo, lzma? my $dc_cmd = eval { $zsfx2cmd->($zsfx, 1, $lei) }; ok($dc_cmd, "decompressor for .$zsfx"); my $f = "$fn.$zsfx"; - my $dst = "mboxcl2:$f"; + my $dst = "$mbox:$f"; my $wcb = PublicInbox::LeiToMail->write_cb($dst, $lei); $wcb->(\(my $dup = $buf), 'deadbeef', [ qw(seen) ]); undef $wcb; @@ -117,11 +123,37 @@ for my $zsfx (qw(gz bz2 xz)) { # XXX should we support zst, zz, lzo, lzma? $wcb->(\($dup = $buf), 'deadbeef', [ qw(seen) ]); undef $wcb; is(xqx([@$dc_cmd, $f]), $orig, "$zsfx matches with lock"); + + local $lei->{opt} = { augment => 1 }; + $wcb = PublicInbox::LeiToMail->write_cb($dst, $lei); + $lei->{dedupe}->prepare_dedupe; + $wcb->(\($dup = $buf . "\nx\n"), 'deadbeef', [ qw(seen) ]); + undef $wcb; # commit + + my $cat = popen_rd([@$dc_cmd, $f]); + my @raw; + PublicInbox::MboxReader->$mbox($cat, + sub { push @raw, shift->as_string }); + like($raw[1], qr/\nblah\n\nx\n\z/s, "augmented $zsfx"); + like($raw[0], qr/\nblah\n\z/s, "original preserved $zsfx"); + + local $lei->{opt} = { augment => 1, jobs => 2 }; + $wcb = PublicInbox::LeiToMail->write_cb($dst, $lei); + $lei->{dedupe}->prepare_dedupe; + $wcb->(\($dup = $buf . "\ny\n"), 'deadbeef', [ qw(seen) ]); + undef $wcb; # commit + + my @raw3; + $cat = popen_rd([@$dc_cmd, $f]); + PublicInbox::MboxReader->$mbox($cat, + sub { push @raw3, shift->as_string }); + my $y = pop @raw3; + is_deeply(\@raw3, \@raw, 'previous messages preserved'); + like($y, qr/\nblah\n\ny\n\z/s, "augmented $zsfx (atomic)"); } } unlink $fn or BAIL_OUT $!; -require PublicInbox::MboxReader; if ('default deduplication uses content_hash') { my $wcb = PublicInbox::LeiToMail->write_cb("mboxo:$fn", $lei); $wcb->(\(my $x = $buf), 'deadbeef', []) for (1..2); @@ -130,6 +162,17 @@ if ('default deduplication uses content_hash') { open my $fh, '<', $fn or BAIL_OUT $!; PublicInbox::MboxReader->mboxo($fh, sub { $cmp .= shift->as_string }); is($cmp, $buf, 'only one message written'); + + local $lei->{opt} = { augment => 1 }; + $wcb = PublicInbox::LeiToMail->write_cb("mboxo:$fn", $lei); + $wcb->(\($x = $buf . "\nx\n"), 'deadbeef', []) for (1..2); + undef $wcb; # undef to commit changes + open $fh, '<', $fn or BAIL_OUT $!; + my @x; + PublicInbox::MboxReader->mboxo($fh, sub { push @x, shift->as_string }); + is(scalar(@x), 2, 'augmented mboxo'); + is($x[0], $cmp, 'original message preserved'); + is($x[1], $buf . "\nx\n", 'new message appended'); } { # stdout support @@ -145,7 +188,6 @@ if ('default deduplication uses content_hash') { } SKIP: { # FIFO support - use PublicInbox::Spawn qw(popen_rd which); use POSIX qw(mkfifo); my $fn = "$tmpdir/fifo"; mkfifo($fn, 0600) or skip("mkfifo not supported: $!", 1); @@ -158,4 +200,47 @@ SKIP: { # FIFO support is($cmp, $buf, 'message written to FIFO'); } +{ # Maildir support + my $md = "$tmpdir/maildir/"; + my $wcb = PublicInbox::LeiToMail->write_cb("Maildir:$md", $lei); + ok($wcb, 'got Maildir callback'); + $lei->{dedupe}->prepare_dedupe; + $wcb->(\(my $x = $buf), 'badc0ffee', []); + undef $wcb; # commit + + my @f; + PublicInbox::LeiToMail::_maildir_each_file($md, sub { push @f, shift }); + is(scalar(@f), 1, 'wrote one file'); + open my $fh, $f[0] or BAIL_OUT $!; + is(do { local $/; <$fh> }, $buf, 'wrote to Maildir'); + + $wcb = PublicInbox::LeiToMail->write_cb("maildir:$md", $lei); + $lei->{dedupe}->prepare_dedupe; + $wcb->(\($x = $buf."\nx\n"), 'deadcafe', []); + undef $wcb; # commit + + my @x = (); + PublicInbox::LeiToMail::_maildir_each_file($md, sub { push @x, shift }); + is(scalar(@x), 1, 'wrote one new file'); + ok(!-f $f[0], 'old file clobbered'); + open $fh, $x[0] or BAIL_OUT $!; + is(do { local $/; <$fh> }, $buf."\nx\n", 'wrote new file to Maildir'); + + local $lei->{opt}->{augment} = 1; + $wcb = PublicInbox::LeiToMail->write_cb("maildir:$md", $lei); + $lei->{dedupe}->prepare_dedupe; + $wcb->(\($x = $buf."\ny\n"), 'deadcafe', []); + $wcb->(\($x = $buf."\ny\n"), 'b4dc0ffee', []); # skipped by dedupe + undef $wcb; # commit + @f = (); + PublicInbox::LeiToMail::_maildir_each_file($md, sub { push @f, shift }); + is(scalar grep(/\A\Q$x[0]\E\z/, @f), 1, 'old file still there'); + my @new = grep(!/\A\Q$x[0]\E\z/, @f); + is(scalar @new, 1, '1 new file written (b4dc0ffee skipped)'); + open $fh, $x[0] or BAIL_OUT $!; + is(do { local $/; <$fh> }, $buf."\nx\n", 'old file untouched'); + open $fh, $new[0] or BAIL_OUT $!; + is(do { local $/; <$fh> }, $buf."\ny\n", 'new file written'); +} + done_testing; diff --git a/t/mbox_reader.t b/t/mbox_reader.t index 9391dc24..7d5a6ba5 100644 --- a/t/mbox_reader.t +++ b/t/mbox_reader.t @@ -32,7 +32,6 @@ if ($ENV{TEST_EXTRA}) { } my $reader = PublicInbox::MboxReader->new; -my $write_in_full = PublicInbox::LeiToMail->can('write_in_full'); my $check_fmt = sub { my $fmt = shift; my @order = shuffle(keys %raw); @@ -41,7 +40,7 @@ my $check_fmt = sub { for my $k (@order) { my $eml = PublicInbox::Eml->new($raw{$k}); my $buf = $eml2mbox->($eml); - $write_in_full->($fh, $buf, undef); + print $fh $$buf or BAIL_OUT "print $!"; } seek($fh, 0, SEEK_SET) or BAIL_OUT "seek: $!"; $reader->$fmt($fh, sub { -- cgit v1.2.3-24-ge0c7