#!perl -w
# Copyright (C) all contributors
# License: AGPL-3.0+
use v5.12; use PublicInbox::TestCommon;
use PublicInbox::MboxReader;
use PublicInbox::MdirReader;
use PublicInbox::NetReader;
use PublicInbox::Eml;
use IO::Uncompress::Gunzip;
use File::Path qw(remove_tree);
use PublicInbox::Spawn qw(which run_qx);
use File::Compare;
use autodie qw(open);
require_mods(qw(lei -imapd -nntpd Mail::IMAPClient Net::NNTP));
my ($tmpdir, $for_destroy) = tmpdir;
my $sock = tcp_server;
my $cmd = [ '-imapd', '-W0', "--stdout=$tmpdir/i1", "--stderr=$tmpdir/i2" ];
my ($ro_home, $cfg_path) = setup_public_inboxes;
my $env = { PI_CONFIG => $cfg_path };
my $tdi = start_script($cmd, $env, { 3 => $sock }) or BAIL_OUT("-imapd: $?");
my $imap_host_port = tcp_host_port($sock);
$sock = tcp_server;
$cmd = [ '-nntpd', '-W0', "--stdout=$tmpdir/n1", "--stderr=$tmpdir/n2" ];
my $tdn = start_script($cmd, $env, { 3 => $sock }) or BAIL_OUT("-nntpd: $?");
my $nntp_host_port = tcp_host_port($sock);
undef $sock;
test_lei({ tmpdir => $tmpdir }, sub {
my $d = $ENV{HOME};
lei_ok('convert', '-o', "mboxrd:$d/foo.mboxrd",
"imap://$imap_host_port/t.v2.0");
my ($nc0) = ($lei_err =~ /converted (\d+) messages/);
ok(-f "$d/foo.mboxrd", 'mboxrd created from imap://');
lei_ok qw(convert -o), "v2:$d/v2-test", "mboxrd:$d/foo.mboxrd";
my ($nc) = ($lei_err =~ /converted (\d+) messages/);
is $nc, $nc0, 'converted all messages messages';
lei_ok qw(q z:0.. -f jsonl --only), "$d/v2-test";
is(scalar(split(/^/sm, $lei_out)), $nc, 'got all messages in v2-test');
lei_ok qw(convert -o), "mboxrd:$d/from-v2.mboxrd", "$d/v2-test";
like $lei_err, qr/converted $nc messages/;
is(compare("$d/foo.mboxrd", "$d/from-v2.mboxrd"), 0,
'convert mboxrd -> v2 ->mboxrd roundtrip') or
diag run_qx([qw(git diff --no-index),
"$d/foo.mboxrd", "$d/from-v2.mboxrd"]);
lei_ok [qw(convert -F eml -o), "$d/v2-test"], undef,
{ 0 => \<<'EOM', %$lei_opt };
From: f@example.com
To: t@example.com
Subject: append-to-v2-on-convert
Message-ID:
Date: Fri, 02 Oct 1993 00:00:00 +0000
EOM
like $lei_err, qr/converted 1 messages/, 'only one message added';
lei_ok qw(q z:0.. -f jsonl --only), "$d/v2-test";
is(scalar(split(/^/sm, $lei_out)), $nc + 1,
'got expected number of messages after append convert');
like $lei_out, qr/append-to-v2-on-convert/;
lei_ok('convert', '-o', "mboxrd:$d/nntp.mboxrd",
"nntp://$nntp_host_port/t.v2");
ok(-f "$d/nntp.mboxrd", 'mboxrd created from nntp://');
my (@mboxrd, @mboxcl2);
open my $fh, '<', "$d/foo.mboxrd" or BAIL_OUT $!;
PublicInbox::MboxReader->mboxrd($fh, sub { push @mboxrd, shift });
ok(scalar(@mboxrd) > 1, 'got multiple messages');
open $fh, '<', "$d/nntp.mboxrd" or BAIL_OUT $!;
my $i = 0;
PublicInbox::MboxReader->mboxrd($fh, sub {
my ($eml) = @_;
is($eml->body, $mboxrd[$i]->body, "body matches #$i");
$i++;
});
lei_ok('convert', '-o', "mboxcl2:$d/cl2", "mboxrd:$d/foo.mboxrd");
ok(-s "$d/cl2", 'mboxcl2 non-empty') or diag $lei_err;
open $fh, '<', "$d/cl2" or BAIL_OUT $!;
PublicInbox::MboxReader->mboxcl2($fh, sub {
my $eml = shift;
$eml->header_set($_) for (qw(Content-Length Lines));
push @mboxcl2, $eml;
});
is_deeply(\@mboxcl2, \@mboxrd, 'mboxrd and mboxcl2 have same mail');
lei_ok('convert', '-o', "$d/md", "mboxrd:$d/foo.mboxrd");
ok(-d "$d/md", 'Maildir created');
my @md;
PublicInbox::MdirReader->new->maildir_each_eml("$d/md", sub {
push @md, $_[2];
});
is(scalar(@md), scalar(@mboxrd), 'got expected emails in Maildir') or
diag $lei_err;
@md = sort { ${$a->{bdy}} cmp ${$b->{bdy}} } @md;
@mboxrd = sort { ${$a->{bdy}} cmp ${$b->{bdy}} } @mboxrd;
my @rd_nostatus = map {
my $eml = PublicInbox::Eml->new(\($_->as_string));
$eml->header_set('Status');
$eml;
} @mboxrd;
is_deeply(\@md, \@rd_nostatus, 'Maildir output matches mboxrd');
my @bar;
lei_ok('convert', '-o', "mboxrd:$d/bar.mboxrd", "$d/md");
open $fh, '<', "$d/bar.mboxrd" or BAIL_OUT $!;
PublicInbox::MboxReader->mboxrd($fh, sub { push @bar, shift });
@bar = sort { ${$a->{bdy}} cmp ${$b->{bdy}} } @bar;
is_deeply(\@mboxrd, \@bar,
'mboxrd round-tripped through Maildir w/ flags');
open my $in, '<', "$d/foo.mboxrd" or BAIL_OUT;
my $rdr = { 0 => $in, 1 => \(my $out), 2 => \$lei_err };
lei_ok([qw(convert --stdin -F mboxrd -o mboxrd:/dev/stdout)],
undef, $rdr);
open $fh, '<', "$d/foo.mboxrd" or BAIL_OUT;
my $exp = do { local $/; <$fh> };
is($out, $exp, 'stdin => stdout');
lei_ok qw(convert -F eml -o mboxcl2:/dev/fd/1 t/plack-qp.eml);
open $fh, '<', \$lei_out or BAIL_OUT;
@bar = ();
PublicInbox::MboxReader->mboxcl2($fh, sub {
my $eml = shift;
for my $h (qw(Content-Length Lines)) {
ok(defined($eml->header_raw($h)),
"$h defined for mboxcl2");
$eml->header_set($h);
}
push @bar, $eml;
});
my $qp_eml = eml_load('t/plack-qp.eml');
$qp_eml->header_set('Status', 'O');
is_deeply(\@bar, [ $qp_eml ], 'eml => mboxcl2');
lei_ok qw(convert t/plack-qp.eml -o), "mboxrd:$d/qp.gz";
open $fh, '<', "$d/qp.gz" or xbail $!;
ok(-s $fh, 'not empty');
$fh = IO::Uncompress::Gunzip->new($fh, MultiStream => 1);
@bar = ();
PublicInbox::MboxReader->mboxrd($fh, sub { push @bar, shift });
is_deeply(\@bar, [ $qp_eml ], 'wrote gzipped mboxrd');
lei_ok qw(convert -o mboxrd:/dev/stdout), "mboxrd:$d/qp.gz";
open $fh, '<', \$lei_out or xbail;
@bar = ();
PublicInbox::MboxReader->mboxrd($fh, sub { push @bar, shift });
is_deeply(\@bar, [ $qp_eml ], 'readed gzipped mboxrd');
# Status => Maildir flag => Status round trip
$lei_out =~ s/^Status: O/Status: RO/sm or xbail "`seen' Status";
$rdr = { 0 => \($in = $lei_out), %$lei_opt };
lei_ok([qw(convert -F mboxrd -o), "$d/md2"], undef, $rdr);
@md = glob("$d/md2/*/*");
is(scalar(@md), 1, 'one message');
like($md[0], qr/:2,S\z/, "`seen' flag set in Maildir");
lei_ok(qw(convert -o mboxrd:/dev/stdout), "$d/md2");
like($lei_out, qr/^Status: RO/sm, "`seen' flag preserved");
SKIP: {
my $ok;
for my $x (($ENV{GZIP}//''), qw(pigz gzip)) {
$x && (`$x -h 2>&1`//'') =~ /--rsyncable\b/s or next;
$ok = $x;
last;
}
skip 'pigz || gzip do not support --rsyncable', 1 if !$ok;
lei_ok qw(convert --rsyncable), "mboxrd:$d/qp.gz",
'-o', "mboxcl2:$d/qp2.gz";
undef $fh; # necessary to make IO::Uncompress::Gunzip happy
open $fh, '<', "$d/qp2.gz";
$fh = IO::Uncompress::Gunzip->new($fh, MultiStream => 1);
my @tmp;
PublicInbox::MboxReader->mboxcl2($fh, sub {
my ($eml) = @_;
$eml->header_set($_) for qw(Content-Length Lines);
push @tmp, $eml;
});
is_deeply(\@tmp, \@bar, 'read rsyncable-gzipped mboxcl2');
}
my $cp = which('cp') or xbail 'cp(1) not available (WTF?)';
for my $v (1, 2) {
my $ibx_dir = "$ro_home/t$v";
lei_ok qw(convert -f mboxrd), $ibx_dir,
\"dump v$v inbox to mboxrd";
my $out = $lei_out;
lei_ok qw(convert -f mboxrd), "v$v:$ibx_dir",
\"dump v$v inbox to mboxrd w/ v$v:// prefix";
is $out, $lei_out, "v$v:// prefix accepted";
open my $fh, '<', \$out;
my (@mb, @md, @md2);
PublicInbox::MboxReader->mboxrd($fh, sub {
$_[0]->header_set('Status');
push @mb, $_[0]->as_string;
});
undef $out;
ok(scalar(@mb), 'got messages output');
my $mdir = "$d/v$v-mdir";
lei_ok qw(convert -o), $mdir, $ibx_dir,
\"dump v$v inbox to Maildir";
PublicInbox::MdirReader->new->maildir_each_eml($mdir, sub {
push @md, $_[2]->as_string;
});
@md = sort { $a cmp $b } @md;
@mb = sort { $a cmp $b } @mb;
is_deeply(\@mb, \@md, 'got matching inboxes');
xsys_e([$cp, '-Rp', $ibx_dir, "$d/tv$v" ]);
remove_tree($mdir, "$d/tv$v/public-inbox",
glob("$d/tv$v/xap*"));
lei_ok qw(convert -o), $mdir, "$d/tv$v",
\"dump u indexed v$v inbox to Maildir";
PublicInbox::MdirReader->new->maildir_each_eml($mdir, sub {
push @md2, $_[2]->as_string;
});
@md2 = sort { $a cmp $b } @md2;
is_deeply(\@md, \@md2, 'got matching inboxes even unindexed');
}
});
done_testing;