From 68a85e8dd7d01578a10a4bfa24818cfe53200690 Mon Sep 17 00:00:00 2001 From: Eric Wong Date: Thu, 19 May 2016 10:23:28 +0000 Subject: msg_iter: workaround broken Email::MIME versions Email::MIME >= 1.923 and < 1.935 would drop too many newlines in attachments. This would lead to ugly text files without a proper trailing newline if using quoted-printable, 7bit, or 8bit. Attachments encoded with base64 were not affected. These versions of Email::MIME are widely available in Debian 8 (Jessie) and even Ubuntu LTS distros so we will need to support this workaround for a while. --- lib/PublicInbox/MsgIter.pm | 13 +++++++ t/msg_iter.t | 14 ++++--- t/psgi_attach.t | 96 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 118 insertions(+), 5 deletions(-) create mode 100644 t/psgi_attach.t diff --git a/lib/PublicInbox/MsgIter.pm b/lib/PublicInbox/MsgIter.pm index d0dd82fb..e0127ab9 100644 --- a/lib/PublicInbox/MsgIter.pm +++ b/lib/PublicInbox/MsgIter.pm @@ -6,6 +6,16 @@ use strict; use warnings; use base qw(Exporter); our @EXPORT = qw(msg_iter); +use Email::MIME; + +# Workaround Email::MIME versions without +# commit dcef9be66c49ae89c7a5027a789bbbac544499ce +# ("removing all trailing newlines was too much") +# This is necessary for Debian jessie +my $bad = 1.923; +my $good = 1.935; +my $ver = $Email::MIME::VERSION; +my $extra_nl = 1 if ($ver >= $bad && $ver < $good); # Like Email::MIME::walk_parts, but this is: # * non-recursive @@ -25,6 +35,9 @@ sub msg_iter ($$) { @sub = map { [ $_, $depth, @idx, ++$i ] } @sub; @parts = (@sub, @parts); } else { + if ($extra_nl) { + ${$part->{body}} .= $part->{mycrlf}; + } $cb->($p); } } diff --git a/t/msg_iter.t b/t/msg_iter.t index cc58b93f..7ade6e41 100644 --- a/t/msg_iter.t +++ b/t/msg_iter.t @@ -7,14 +7,16 @@ use Email::MIME; use_ok('PublicInbox::MsgIter'); { - my $parts = [ Email::MIME->create(body => 'a'), - Email::MIME->create(body => 'b') ]; + my $parts = [ Email::MIME->create(body => "a\n"), + Email::MIME->create(body => "b\n") ]; my $mime = Email::MIME->create(parts => $parts, header_str => [ From => 'root@localhost' ]); my @parts; msg_iter($mime, sub { my ($part, $level, @ex) = @{$_[0]}; - push @parts, [ $part->body_str, $level, @ex ]; + my $s = $part->body_str; + $s =~ s/\s+//s; + push @parts, [ $s, $level, @ex ]; }); is_deeply(\@parts, [ [ qw(a 1 1) ], [ qw(b 1 2) ] ], 'order is fine'); } @@ -30,9 +32,11 @@ use_ok('PublicInbox::MsgIter'); my @parts; msg_iter($mime, sub { my ($part, $level, @ex) = @{$_[0]}; - push @parts, [ $part->body_str, $level, @ex ]; + my $s = $part->body_str; + $s =~ s/\s+//s; + push @parts, [ $s, $level, @ex ]; }); - is_deeply(\@parts, [ [ qw(a 2 1 1)], [qw(b 2 1 2)], [qw(sig 1 2)] ], + is_deeply(\@parts, [ [qw(a 2 1 1)], [qw(b 2 1 2)], [qw(sig 1 2)] ], 'nested part shows up properly'); } diff --git a/t/psgi_attach.t b/t/psgi_attach.t new file mode 100644 index 00000000..535cd21d --- /dev/null +++ b/t/psgi_attach.t @@ -0,0 +1,96 @@ +# Copyright (C) 2016 all contributors +# License: AGPL-3.0+ +use strict; +use warnings; +use Test::More; +use Email::MIME; +use File::Temp qw/tempdir/; +my $tmpdir = tempdir('psgi-attach-XXXXXX', TMPDIR => 1, CLEANUP => 1); +my $maindir = "$tmpdir/main.git"; +my $addr = 'test-public@example.com'; +my $cfgpfx = "publicinbox.test"; +my @mods = qw(HTTP::Request::Common Plack::Request Plack::Test URI::Escape); +foreach my $mod (@mods) { + eval "require $mod"; + plan skip_all => "$mod missing for plack.t" if $@; +} +use_ok $_ foreach @mods; +use PublicInbox::Import; +use PublicInbox::Git; +use PublicInbox::Config; +use PublicInbox::WWW; +use_ok 'PublicInbox::WwwAttach'; +use Plack::Builder; +my $config = PublicInbox::Config->new({ + "$cfgpfx.address" => $addr, + "$cfgpfx.mainrepo" => $maindir, +}); +is(0, system(qw(git init -q --bare), $maindir), "git init (main)"); +my $git = PublicInbox::Git->new($maindir); +my $im = PublicInbox::Import->new($git, 'test', $addr); + +{ + open my $fh, '<', '/dev/urandom' or die "unable to open urandom: $!\n"; + sysread($fh, my $buf, 8); + is(8, length($buf), 'read some random data'); + my $qp = "abcdef=g\n==blah\n"; + my $b64 = 'b64'.$buf."\n"; + my $txt = "plain\ntext\npass\nthrough\n"; + my $parts = [ + Email::MIME->create( + attributes => { + filename => 'queue-pee', + content_type => 'text/plain', + encoding => 'quoted-printable' + }, + body => $qp), + Email::MIME->create( + attributes => { + filename => 'bayce-sixty-four', + content_type => 'appication/octet-stream', + encoding => 'base64', + }, + body => $b64), + Email::MIME->create( + attributes => { + filename => 'noop', + content_type => 'text/plain', + }, + body => $txt), + ]; + my $mime = Email::MIME->create( + parts => $parts, + header_str => [ From => 'root@z', 'Message-Id' => '', + Subject => 'hi'] + ); + $mime = $mime->as_string; + $mime =~ s/\r\n/\n/g; # normalize to LF only + $mime = Email::MIME->new($mime); + $im->add($mime); + $im->done; + + my $www = PublicInbox::WWW->new($config); + test_psgi(sub { $www->call(@_) }, sub { + my ($cb) = @_; + my $res; + + $res = $cb->(GET('/test/Z%40B/1-a.txt')); + my $qp_res = $res->content; + ok(length($qp_res) >= length($qp), 'QP length is close'); + like($qp_res, qr/\n\z/s, 'trailing newline exists'); + # is(index($qp_res, $qp), 0, 'QP trailing newline is there'); + $qp_res =~ s/\r\n/\n/g; + is(index($qp_res, $qp), 0, 'QP trailing newline is there'); + + $res = $cb->(GET('/test/Z%40B/2-a.txt')); + is(quotemeta($res->content), quotemeta($b64), + 'Base64 matches exactly'); + + $res = $cb->(GET('/test/Z%40B/3-a.txt')); + my $txt_res = $res->content; + ok(length($txt_res) >= length($txt), 'plain text almost matches'); + like($txt_res, qr/\n\z/s, 'trailing newline exists in text'); + is(index($txt_res, $txt), 0, 'plain text not truncated'); + }); +} +done_testing(); -- cgit v1.2.3-24-ge0c7