lei_to_mail: support Maildir, fix+test --augment

Maildir should be plenty fine for short-lived output folders.

2 files changed, 92 insertions, 8 deletions

diff --git a/t/lei_to_mail.t b/t/lei_to_mail.t
index f3cc71ad..bd864754 100644
--- a/t/lei_to_mail.t
+++ b/t/lei_to_mail.t
@@ -7,14 +7,18 @@ use Test::More;
 use PublicInbox::TestCommon;
 use PublicInbox::Eml;
 use Fcntl qw(SEEK_SET);
+use PublicInbox::Spawn qw(popen_rd which);
+use List::Util qw(shuffle);
 require_mods(qw(DBD::SQLite));
+require PublicInbox::MboxReader;
 use_ok 'PublicInbox::LeiToMail';
 my $from = "Content-Length: 10\nSubject: x\n\nFrom hell\n";
 my $noeol = "Subject: x\n\nFrom hell";
 my $crlf = $noeol;
 $crlf =~ s/\n/\r\n/g;
 my $kw = [qw(seen answered flagged)];
-for my $mbox (qw(mboxrd mboxo mboxcl mboxcl2)) {
+my @MBOX = qw(mboxcl2 mboxrd mboxcl mboxo);
+for my $mbox (@MBOX) {
         my $m = "eml2$mbox";
         my $cb = PublicInbox::LeiToMail->can($m);
         my $s = $cb->(PublicInbox::Eml->new($from), $kw);
@@ -75,8 +79,9 @@ Subject: x
 blah
 EOM
 my $fn = "$tmpdir/x.mbox";
+my ($mbox) = shuffle(@MBOX); # pick one, shouldn't matter
 my $orig = do {
-        my $wcb = PublicInbox::LeiToMail->write_cb("mboxcl2:$fn", $lei);
+        my $wcb = PublicInbox::LeiToMail->write_cb("$mbox:$fn", $lei);
         is(ref $wcb, 'CODE', 'write_cb returned callback');
         ok(-f $fn && !-s _, 'empty file created');
         $wcb->(\(my $dup = $buf), 'deadbeef', [ qw(seen) ]);
@@ -87,7 +92,8 @@ my $orig = do {
         unlink $fn or BAIL_OUT $!;
 
         local $lei->{opt} = { jobs => 2 };
-        $wcb = PublicInbox::LeiToMail->write_cb("mboxcl2:$fn", $lei);
+        $wcb = PublicInbox::LeiToMail->write_cb("$mbox:$fn", $lei);
+        ok(-f $fn && !-s _, 'truncated mbox destination');
         $lei->{dedupe}->prepare_dedupe;
         $wcb->(\($dup = $buf), 'deadbeef', [ qw(seen) ]);
         undef $wcb;
@@ -103,7 +109,7 @@ for my $zsfx (qw(gz bz2 xz)) { # XXX should we support zst, zz, lzo, lzma?
                 my $dc_cmd = eval { $zsfx2cmd->($zsfx, 1, $lei) };
                 ok($dc_cmd, "decompressor for .$zsfx");
                 my $f = "$fn.$zsfx";
-                my $dst = "mboxcl2:$f";
+                my $dst = "$mbox:$f";
                 my $wcb = PublicInbox::LeiToMail->write_cb($dst, $lei);
                 $wcb->(\(my $dup = $buf), 'deadbeef', [ qw(seen) ]);
                 undef $wcb;
@@ -117,11 +123,37 @@ for my $zsfx (qw(gz bz2 xz)) { # XXX should we support zst, zz, lzo, lzma?
                 $wcb->(\($dup = $buf), 'deadbeef', [ qw(seen) ]);
                 undef $wcb;
                 is(xqx([@$dc_cmd, $f]), $orig, "$zsfx matches with lock");
+
+                local $lei->{opt} = { augment => 1 };
+                $wcb = PublicInbox::LeiToMail->write_cb($dst, $lei);
+                $lei->{dedupe}->prepare_dedupe;
+                $wcb->(\($dup = $buf . "\nx\n"), 'deadbeef', [ qw(seen) ]);
+                undef $wcb; # commit
+
+                my $cat = popen_rd([@$dc_cmd, $f]);
+                my @raw;
+                PublicInbox::MboxReader->$mbox($cat,
+                        sub { push @raw, shift->as_string });
+                like($raw[1], qr/\nblah\n\nx\n\z/s, "augmented $zsfx");
+                like($raw[0], qr/\nblah\n\z/s, "original preserved $zsfx");
+
+                local $lei->{opt} = { augment => 1, jobs => 2 };
+                $wcb = PublicInbox::LeiToMail->write_cb($dst, $lei);
+                $lei->{dedupe}->prepare_dedupe;
+                $wcb->(\($dup = $buf . "\ny\n"), 'deadbeef', [ qw(seen) ]);
+                undef $wcb; # commit
+
+                my @raw3;
+                $cat = popen_rd([@$dc_cmd, $f]);
+                PublicInbox::MboxReader->$mbox($cat,
+                        sub { push @raw3, shift->as_string });
+                my $y = pop @raw3;
+                is_deeply(\@raw3, \@raw, 'previous messages preserved');
+                like($y, qr/\nblah\n\ny\n\z/s, "augmented $zsfx (atomic)");
         }
 }
 
 unlink $fn or BAIL_OUT $!;
-require PublicInbox::MboxReader;
 if ('default deduplication uses content_hash') {
         my $wcb = PublicInbox::LeiToMail->write_cb("mboxo:$fn", $lei);
         $wcb->(\(my $x = $buf), 'deadbeef', []) for (1..2);
@@ -130,6 +162,17 @@ if ('default deduplication uses content_hash') {
         open my $fh, '<', $fn or BAIL_OUT $!;
         PublicInbox::MboxReader->mboxo($fh, sub { $cmp .= shift->as_string });
         is($cmp, $buf, 'only one message written');
+
+        local $lei->{opt} = { augment => 1 };
+        $wcb = PublicInbox::LeiToMail->write_cb("mboxo:$fn", $lei);
+        $wcb->(\($x = $buf . "\nx\n"), 'deadbeef', []) for (1..2);
+        undef $wcb; # undef to commit changes
+        open $fh, '<', $fn or BAIL_OUT $!;
+        my @x;
+        PublicInbox::MboxReader->mboxo($fh, sub { push @x, shift->as_string });
+        is(scalar(@x), 2, 'augmented mboxo');
+        is($x[0], $cmp, 'original message preserved');
+        is($x[1], $buf . "\nx\n", 'new message appended');
 }
 
 { # stdout support
@@ -145,7 +188,6 @@ if ('default deduplication uses content_hash') {
 }
 
 SKIP: { # FIFO support
-        use PublicInbox::Spawn qw(popen_rd which);
         use POSIX qw(mkfifo);
         my $fn = "$tmpdir/fifo";
         mkfifo($fn, 0600) or skip("mkfifo not supported: $!", 1);
@@ -158,4 +200,47 @@ SKIP: { # FIFO support
         is($cmp, $buf, 'message written to FIFO');
 }
 
+{ # Maildir support
+        my $md = "$tmpdir/maildir/";
+        my $wcb = PublicInbox::LeiToMail->write_cb("Maildir:$md", $lei);
+        ok($wcb, 'got Maildir callback');
+        $lei->{dedupe}->prepare_dedupe;
+        $wcb->(\(my $x = $buf), 'badc0ffee', []);
+        undef $wcb; # commit
+
+        my @f;
+        PublicInbox::LeiToMail::_maildir_each_file($md, sub { push @f, shift });
+        is(scalar(@f), 1, 'wrote one file');
+        open my $fh, $f[0] or BAIL_OUT $!;
+        is(do { local $/; <$fh> }, $buf, 'wrote to Maildir');
+
+        $wcb = PublicInbox::LeiToMail->write_cb("maildir:$md", $lei);
+        $lei->{dedupe}->prepare_dedupe;
+        $wcb->(\($x = $buf."\nx\n"), 'deadcafe', []);
+        undef $wcb; # commit
+
+        my @x = ();
+        PublicInbox::LeiToMail::_maildir_each_file($md, sub { push @x, shift });
+        is(scalar(@x), 1, 'wrote one new file');
+        ok(!-f $f[0], 'old file clobbered');
+        open $fh, $x[0] or BAIL_OUT $!;
+        is(do { local $/; <$fh> }, $buf."\nx\n", 'wrote new file to Maildir');
+
+        local $lei->{opt}->{augment} = 1;
+        $wcb = PublicInbox::LeiToMail->write_cb("maildir:$md", $lei);
+        $lei->{dedupe}->prepare_dedupe;
+        $wcb->(\($x = $buf."\ny\n"), 'deadcafe', []);
+        $wcb->(\($x = $buf."\ny\n"), 'b4dc0ffee', []); # skipped by dedupe
+        undef $wcb; # commit
+        @f = ();
+        PublicInbox::LeiToMail::_maildir_each_file($md, sub { push @f, shift });
+        is(scalar grep(/\A\Q$x[0]\E\z/, @f), 1, 'old file still there');
+        my @new = grep(!/\A\Q$x[0]\E\z/, @f);
+        is(scalar @new, 1, '1 new file written (b4dc0ffee skipped)');
+        open $fh, $x[0] or BAIL_OUT $!;
+        is(do { local $/; <$fh> }, $buf."\nx\n", 'old file untouched');
+        open $fh, $new[0] or BAIL_OUT $!;
+        is(do { local $/; <$fh> }, $buf."\ny\n", 'new file written');
+}
+
 done_testing;
diff --git a/t/mbox_reader.t b/t/mbox_reader.t
index 9391dc24..7d5a6ba5 100644
--- a/t/mbox_reader.t
+++ b/t/mbox_reader.t
@@ -32,7 +32,6 @@ if ($ENV{TEST_EXTRA}) {
 }
 
 my $reader = PublicInbox::MboxReader->new;
-my $write_in_full = PublicInbox::LeiToMail->can('write_in_full');
 my $check_fmt = sub {
         my $fmt = shift;
         my @order = shuffle(keys %raw);
@@ -41,7 +40,7 @@ my $check_fmt = sub {
         for my $k (@order) {
                 my $eml = PublicInbox::Eml->new($raw{$k});
                 my $buf = $eml2mbox->($eml);
-                $write_in_full->($fh, $buf, undef);
+                print $fh $$buf or BAIL_OUT "print $!";
         }
         seek($fh, 0, SEEK_SET) or BAIL_OUT "seek: $!";
         $reader->$fmt($fh, sub {