about summary refs log tree commit homepage
diff options
context:
space:
mode:
authorEric Wong <e@80x24.org>2016-05-19 10:23:28 +0000
committerEric Wong <e@80x24.org>2016-05-19 20:53:21 +0000
commit68a85e8dd7d01578a10a4bfa24818cfe53200690 (patch)
tree9615529d881897a9f12815e0901e78eaf4a4f2e3
parent78042ed9abb2bbe9783adcc485365a8672e4b36f (diff)
downloadpublic-inbox-68a85e8dd7d01578a10a4bfa24818cfe53200690.tar.gz
Email::MIME >= 1.923 and < 1.935 would drop too many newlines
in attachments.  This would lead to ugly text files without
a proper trailing newline if using quoted-printable, 7bit, or
8bit.  Attachments encoded with base64 were not affected.

These versions of Email::MIME are widely available in Debian 8
(Jessie) and even Ubuntu LTS distros so we will need to support
this workaround for a while.
-rw-r--r--lib/PublicInbox/MsgIter.pm13
-rw-r--r--t/msg_iter.t14
-rw-r--r--t/psgi_attach.t96
3 files changed, 118 insertions, 5 deletions
diff --git a/lib/PublicInbox/MsgIter.pm b/lib/PublicInbox/MsgIter.pm
index d0dd82fb..e0127ab9 100644
--- a/lib/PublicInbox/MsgIter.pm
+++ b/lib/PublicInbox/MsgIter.pm
@@ -6,6 +6,16 @@ use strict;
 use warnings;
 use base qw(Exporter);
 our @EXPORT = qw(msg_iter);
+use Email::MIME;
+
+# Workaround Email::MIME versions without
+# commit dcef9be66c49ae89c7a5027a789bbbac544499ce
+# ("removing all trailing newlines was too much")
+# This is necessary for Debian jessie
+my $bad = 1.923;
+my $good = 1.935;
+my $ver = $Email::MIME::VERSION;
+my $extra_nl = 1 if ($ver >= $bad && $ver < $good);
 
 # Like Email::MIME::walk_parts, but this is:
 # * non-recursive
@@ -25,6 +35,9 @@ sub msg_iter ($$) {
                                 @sub = map { [ $_, $depth, @idx, ++$i ] } @sub;
                                 @parts = (@sub, @parts);
                         } else {
+                                if ($extra_nl) {
+                                        ${$part->{body}} .= $part->{mycrlf};
+                                }
                                 $cb->($p);
                         }
                 }
diff --git a/t/msg_iter.t b/t/msg_iter.t
index cc58b93f..7ade6e41 100644
--- a/t/msg_iter.t
+++ b/t/msg_iter.t
@@ -7,14 +7,16 @@ use Email::MIME;
 use_ok('PublicInbox::MsgIter');
 
 {
-        my $parts = [ Email::MIME->create(body => 'a'),
-                        Email::MIME->create(body => 'b') ];
+        my $parts = [ Email::MIME->create(body => "a\n"),
+                        Email::MIME->create(body => "b\n") ];
         my $mime = Email::MIME->create(parts => $parts,
                                 header_str => [ From => 'root@localhost' ]);
         my @parts;
         msg_iter($mime, sub {
                 my ($part, $level, @ex) = @{$_[0]};
-                push @parts, [ $part->body_str, $level, @ex ];
+                my $s = $part->body_str;
+                $s =~ s/\s+//s;
+                push @parts, [ $s, $level, @ex ];
         });
         is_deeply(\@parts, [ [ qw(a 1 1) ], [ qw(b 1 2) ] ], 'order is fine');
 }
@@ -30,9 +32,11 @@ use_ok('PublicInbox::MsgIter');
         my @parts;
         msg_iter($mime, sub {
                 my ($part, $level, @ex) = @{$_[0]};
-                push @parts, [ $part->body_str, $level, @ex ];
+                my $s = $part->body_str;
+                $s =~ s/\s+//s;
+                push @parts, [ $s, $level, @ex ];
         });
-        is_deeply(\@parts, [ [ qw(a 2 1 1)], [qw(b 2 1 2)], [qw(sig 1 2)] ],
+        is_deeply(\@parts, [ [qw(a 2 1 1)], [qw(b 2 1 2)], [qw(sig 1 2)] ],
                 'nested part shows up properly');
 }
 
diff --git a/t/psgi_attach.t b/t/psgi_attach.t
new file mode 100644
index 00000000..535cd21d
--- /dev/null
+++ b/t/psgi_attach.t
@@ -0,0 +1,96 @@
+# Copyright (C) 2016 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+use strict;
+use warnings;
+use Test::More;
+use Email::MIME;
+use File::Temp qw/tempdir/;
+my $tmpdir = tempdir('psgi-attach-XXXXXX', TMPDIR => 1, CLEANUP => 1);
+my $maindir = "$tmpdir/main.git";
+my $addr = 'test-public@example.com';
+my $cfgpfx = "publicinbox.test";
+my @mods = qw(HTTP::Request::Common Plack::Request Plack::Test URI::Escape);
+foreach my $mod (@mods) {
+        eval "require $mod";
+        plan skip_all => "$mod missing for plack.t" if $@;
+}
+use_ok $_ foreach @mods;
+use PublicInbox::Import;
+use PublicInbox::Git;
+use PublicInbox::Config;
+use PublicInbox::WWW;
+use_ok 'PublicInbox::WwwAttach';
+use Plack::Builder;
+my $config = PublicInbox::Config->new({
+        "$cfgpfx.address" => $addr,
+        "$cfgpfx.mainrepo" => $maindir,
+});
+is(0, system(qw(git init -q --bare), $maindir), "git init (main)");
+my $git = PublicInbox::Git->new($maindir);
+my $im = PublicInbox::Import->new($git, 'test', $addr);
+
+{
+        open my $fh, '<', '/dev/urandom' or die "unable to open urandom: $!\n";
+        sysread($fh, my $buf, 8);
+        is(8, length($buf), 'read some random data');
+        my $qp = "abcdef=g\n==blah\n";
+        my $b64 = 'b64'.$buf."\n";
+        my $txt = "plain\ntext\npass\nthrough\n";
+        my $parts = [
+                Email::MIME->create(
+                        attributes => {
+                                filename => 'queue-pee',
+                                content_type => 'text/plain',
+                                encoding => 'quoted-printable'
+                        },
+                        body => $qp),
+                Email::MIME->create(
+                        attributes => {
+                                filename => 'bayce-sixty-four',
+                                content_type => 'appication/octet-stream',
+                                encoding => 'base64',
+                        },
+                        body => $b64),
+                Email::MIME->create(
+                        attributes => {
+                                filename => 'noop',
+                                content_type => 'text/plain',
+                        },
+                        body => $txt),
+        ];
+        my $mime = Email::MIME->create(
+                parts => $parts,
+                header_str => [ From => 'root@z', 'Message-Id' => '<Z@B>',
+                        Subject => 'hi']
+        );
+        $mime = $mime->as_string;
+        $mime =~ s/\r\n/\n/g; # normalize to LF only
+        $mime = Email::MIME->new($mime);
+        $im->add($mime);
+        $im->done;
+
+        my $www = PublicInbox::WWW->new($config);
+        test_psgi(sub { $www->call(@_) }, sub {
+                my ($cb) = @_;
+                my $res;
+
+                $res = $cb->(GET('/test/Z%40B/1-a.txt'));
+                my $qp_res = $res->content;
+                ok(length($qp_res) >= length($qp), 'QP length is close');
+                like($qp_res, qr/\n\z/s, 'trailing newline exists');
+                # is(index($qp_res, $qp), 0, 'QP trailing newline is there');
+                $qp_res =~ s/\r\n/\n/g;
+                is(index($qp_res, $qp), 0, 'QP trailing newline is there');
+
+                $res = $cb->(GET('/test/Z%40B/2-a.txt'));
+                is(quotemeta($res->content), quotemeta($b64),
+                        'Base64 matches exactly');
+
+                $res = $cb->(GET('/test/Z%40B/3-a.txt'));
+                my $txt_res = $res->content;
+                ok(length($txt_res) >= length($txt), 'plain text almost matches');
+                like($txt_res, qr/\n\z/s, 'trailing newline exists in text');
+                is(index($txt_res, $txt), 0, 'plain text not truncated');
+        });
+}
+done_testing();