diff options
Diffstat (limited to 'lib/PublicInbox')
| -rw-r--r-- | lib/PublicInbox/Feed.pm | 78 | ||||
| -rw-r--r-- | lib/PublicInbox/Filter.pm | 57 | ||||
| -rw-r--r-- | lib/PublicInbox/Hval.pm | 20 | ||||
| -rw-r--r-- | lib/PublicInbox/MDA.pm | 9 | ||||
| -rw-r--r-- | lib/PublicInbox/View.pm | 35 |
5 files changed, 90 insertions, 109 deletions
diff --git a/lib/PublicInbox/Feed.pm b/lib/PublicInbox/Feed.pm index abfc0a9a..3bee3a51 100644 --- a/lib/PublicInbox/Feed.pm +++ b/lib/PublicInbox/Feed.pm @@ -4,9 +4,7 @@ package PublicInbox::Feed; use strict; use warnings; use Email::Address; -use Encode qw/find_encoding/; -use Encode::MIME::Header; -use CGI qw(escapeHTML); +use Email::MIME; use Date::Parse qw(strptime str2time); use PublicInbox::Hval; eval { require Git }; # this is GPLv2+, so we are OK to use it @@ -14,9 +12,6 @@ use constant { DATEFMT => '%Y-%m-%dT%H:%M:%SZ', MAX_PER_PAGE => 25, }; -my $enc_utf8 = find_encoding('utf8'); -my $enc_ascii = find_encoding('us-ascii'); -my $enc_mime = find_encoding('MIME-Header'); # FIXME: workaround https://rt.cpan.org/Public/Bug/Display.html?id=22817 @@ -25,7 +20,6 @@ sub generate { my ($class, $args) = @_; require XML::Atom::SimpleFeed; require PublicInbox::View; - require Email::MIME; require POSIX; my $max = $args->{max} || MAX_PER_PAGE; my $top = $args->{top}; # bool @@ -62,22 +56,25 @@ sub generate_html_index { my $top = $args->{top}; # bool local $ENV{GIT_DIR} = $args->{git_dir}; my $feed_opts = get_feedopts($args); - my $title = xs_html($feed_opts->{description} || ""); + + my $title = $feed_opts->{description} || ''; + $title = PublicInbox::Hval->new_oneline($title)->as_html; + my @messages; my $git = try_git_pm($args->{git_dir}); my ($first, $last) = each_recent_blob($args, sub { - my $simple = do_cat_mail($git, 'Email::Simple', $_[0]) + my $mime = do_cat_mail($git, $_[0]) or return 0; - if ($top && ($simple->header("In-Reply-To") || - $simple->header("References"))) { + if ($top && ($mime->header('In-Reply-To') || + $mime->header('References'))) { return 0; } - $simple->body_set(""); # save some memory + $mime->body_set(''); # save some memory - my $t = eval { str2time($simple->header('Date')) }; + my $t = eval { str2time($mime->header('Date')) }; defined($t) or $t = 0; - $simple->header_set('X-PI-Date', $t); - push @messages, $simple; + $mime->header_set('X-PI-Date', $t); + push @messages, $mime; 1; }); @@ -234,13 +231,9 @@ sub get_feedopts { \%rv; } -sub utf8_header { - my ($simple, $name) = @_; - my $val = $simple->header($name); - return "" unless defined $val; - $val =~ tr/\t\n / /s; - $val =~ tr/\r//d; - $enc_utf8->encode($enc_mime->decode($val)); +sub mime_header { + my ($mime, $name) = @_; + PublicInbox::Hval->new_oneline($mime->header($name))->raw; } sub feed_date { @@ -254,27 +247,25 @@ sub feed_date { sub add_to_feed { my ($feed_opts, $feed, $add, $top, $git) = @_; - my $mime = do_cat_mail($git, 'Email::MIME', $add) or return 0; - if ($top && $mime->header("In-Reply-To")) { + my $mime = do_cat_mail($git, $add) or return 0; + if ($top && $mime->header('In-Reply-To')) { return 0; } my $midurl = $feed_opts->{midurl} || 'http://example.com/m/'; my $fullurl = $feed_opts->{fullurl} || 'http://example.com/f/'; - my $mid = $mime->header('Message-ID'); + my $mid = $mime->header_obj->header_raw('Message-ID'); + defined $mid or return 0; $mid = PublicInbox::Hval->new_msgid($mid); my $href = $mid->as_href; my $content = PublicInbox::View->as_feed_entry($mime, "$fullurl$href.html"); defined($content) or return 0; - my $subject = utf8_header($mime, "Subject") || ""; - length($subject) or return 0; - - my $from = $mime->header('From') or return 0; - + my $subject = mime_header($mime, 'Subject') or return 0; + my $from = mime_header($mime, 'From') or return 0; my @from = Email::Address->parse($from); my $name = $from[0]->name; defined $name or $name = ""; @@ -282,9 +273,8 @@ sub add_to_feed { defined $email or $email = ""; my $date = $mime->header('Date'); - $date or return 0; $date = PublicInbox::Hval->new_oneline($date); - $date = feed_date($date->as_utf8) or return 0; + $date = feed_date($date->raw) or return 0; $feed->add_entry( author => { name => $name, email => $email }, title => $subject, @@ -300,36 +290,32 @@ sub dump_html_line { my ($self, $level, $args) = @_; # args => [ $html, $midurl ] if ($self->message) { $args->[0] .= (' ' x $level); - my $simple = $self->message; - my $subj = $simple->header('Subject'); - my $mid = $simple->header('Message-ID'); + my $mime = $self->message; + my $subj = $mime->header('Subject'); + my $mid = $mime->header_obj->header_raw('Message-ID'); $mid = PublicInbox::Hval->new_msgid($mid); my $url = $args->[1] . $mid->as_href; - my $from = utf8_header($simple, "From"); + my $from = mime_header($mime, 'From'); + my @from = Email::Address->parse($from); $from = $from[0]->name; (defined($from) && length($from)) or $from = $from[0]->address; - $from = xs_html($from); - $subj = PublicInbox::Hval->new_oneline($subj); - $subj = $subj->as_html; + + $from = PublicInbox::Hval->new_oneline($from)->as_html; + $subj = PublicInbox::Hval->new_oneline($subj)->as_html; $args->[0] .= "<a href=\"$url.html\">$subj</a> $from\n"; } dump_html_line($self->child, $level+1, $args) if $self->child; dump_html_line($self->next, $level, $args) if $self->next; } -sub xs_html { - $enc_ascii->encode(escapeHTML($enc_utf8->decode($_[0])), - Encode::HTMLCREF); -} - sub try_git_pm { my ($dir) = @_; eval { Git->repository(Directory => $dir) }; }; sub do_cat_mail { - my ($git, $class, $sha1) = @_; + my ($git, $sha1) = @_; my $str; if ($git) { open my $fh, '>', \$str or @@ -342,7 +328,7 @@ sub do_cat_mail { $str = `git cat-file blob $sha1`; return if $? != 0 || length($str) == 0; } - $class->new($str); + Email::MIME->new($str); } 1; diff --git a/lib/PublicInbox/Filter.pm b/lib/PublicInbox/Filter.pm index bd33130c..a1587682 100644 --- a/lib/PublicInbox/Filter.pm +++ b/lib/PublicInbox/Filter.pm @@ -21,9 +21,9 @@ our $MIME_TEXT_ANY = qr!\btext/[a-z0-9\+\._-]+\b!i; # this is highly opinionated delivery # returns 0 only if there is nothing to deliver sub run { - my ($class, $simple) = @_; + my ($class, $mime) = @_; - my $content_type = $simple->header("Content-Type") || "text/plain"; + my $content_type = $mime->header('Content-Type') || 'text/plain'; # kill potentially bad/confusing headers # Note: ssoma already does this, but since we mangle the message, @@ -32,35 +32,35 @@ sub run { # the nature of public-inbox having no real subscribers. foreach my $d (qw(status lines content-length mail-followup-to mail-reply-to reply-to)) { - $simple->header_set($d); + $mime->header_set($d); } if ($content_type =~ m!\btext/plain\b!i) { return 1; # yay, nothing to do } elsif ($content_type =~ $MIME_HTML) { # HTML-only, non-multipart - my $body = $simple->body; + my $body = $mime->body; my $ct_parsed = parse_content_type($content_type); dump_html(\$body, $ct_parsed->{attributes}->{charset}); - replace_body($simple, $body); + replace_body($mime, $body); return 1; } elsif ($content_type =~ m!\bmultipart/!i) { - return strip_multipart($simple, $content_type); + return strip_multipart($mime, $content_type); } else { - replace_body($simple, "$content_type message scrubbed"); + replace_body($mime, "$content_type message scrubbed"); return 0; } } sub replace_part { - my ($simple, $part, $type) = ($_[0], $_[1], $_[3]); + my ($mime, $part, $type) = ($_[0], $_[1], $_[3]); # don't copy $_[2], that's the body (it may be huge) # Email::MIME insists on setting Date:, so just set it consistently # to avoid conflicts to avoid git merge conflicts in a split brain # situation. unless (defined $part->header('Date')) { - my $date = $simple->header('Date') || + my $date = $mime->header('Date') || 'Thu, 01 Jan 1970 00:00:00 +0000'; $part->header_set('Date', $date); } @@ -77,11 +77,11 @@ sub replace_part { # converts one part of a multipart message to text sub html_part_to_text { - my ($simple, $part) = @_; + my ($mime, $part) = @_; my $body = $part->body; my $ct_parsed = parse_content_type($part->content_type); dump_html(\$body, $ct_parsed->{attributes}->{charset}); - replace_part($simple, $part, $body, 'text/plain'); + replace_part($mime, $part, $body, 'text/plain'); } # modifies $_[0] in place @@ -110,8 +110,7 @@ sub dump_html { # unfortunately, too many people send HTML mail and we'll attempt to convert # it to something safer, smaller and harder-to-track. sub strip_multipart { - my ($simple, $content_type) = @_; - my $mime = Email::MIME->new($simple->as_string); + my ($mime, $content_type) = @_; my (@html, @keep); my $rejected = 0; @@ -164,11 +163,11 @@ sub strip_multipart { if ($content_type =~ m!\bmultipart/alternative\b!i) { if (scalar @keep == 1) { - return collapse($simple, $keep[0]); + return collapse($mime, $keep[0]); } } else { # convert HTML parts to plain text foreach my $part (@html) { - html_part_to_text($simple, $part); + html_part_to_text($mime, $part); push @keep, $part; } } @@ -186,34 +185,34 @@ sub strip_multipart { } if (scalar(@html) || $rejected) { $mime->parts_set(\@keep); - $simple->body_set($mime->body_raw); - mark_changed($simple); + $mime->body_set($mime->body_raw); + mark_changed($mime); } # else: no changes return $ok; } sub mark_changed { - my ($simple) = @_; - $simple->header_set("X-Content-Filtered-By", __PACKAGE__ ." $VERSION"); + my ($mime) = @_; + $mime->header_set('X-Content-Filtered-By', __PACKAGE__ ." $VERSION"); } sub collapse { - my ($simple, $part) = @_; - $simple->header_set("Content-Type", $part->content_type); - $simple->body_set($part->body_raw); - mark_changed($simple); + my ($mime, $part) = @_; + $mime->header_set('Content-Type', $part->content_type); + $mime->body_set($part->body_raw); + mark_changed($mime); return 1; } sub replace_body { - my $simple = $_[0]; - $simple->body_set($_[1]); - $simple->header_set("Content-Type", "text/plain"); - if ($simple->header("Content-Transfer-Encoding")) { - $simple->header_set("Content-Transfer-Encoding", undef); + my $mime = $_[0]; + $mime->body_set($_[1]); + $mime->header_set('Content-Type', 'text/plain'); + if ($mime->header('Content-Transfer-Encoding')) { + $mime->header_set('Content-Transfer-Encoding', undef); } - mark_changed($simple); + mark_changed($mime); } # Check for display-able text, no messed up binaries diff --git a/lib/PublicInbox/Hval.pm b/lib/PublicInbox/Hval.pm index 47b41a58..09ae2676 100644 --- a/lib/PublicInbox/Hval.pm +++ b/lib/PublicInbox/Hval.pm @@ -5,12 +5,11 @@ package PublicInbox::Hval; use strict; use warnings; -use fields qw(raw -as_utf8); +use fields qw(raw); use Encode qw(find_encoding); use CGI qw(escapeHTML); use URI::Escape qw(uri_escape); -my $enc_utf8 = find_encoding('utf8'); my $enc_ascii = find_encoding('us-ascii'); sub new { @@ -39,14 +38,17 @@ sub new_oneline { $class->new($raw); } -sub as_utf8 { - my ($self) = @_; - $self->{-as_utf8} ||= $enc_utf8->encode($self->{raw}); -} - sub ascii_html { $enc_ascii->encode(escapeHTML($_[0]), Encode::HTMLCREF) } -sub as_html { ascii_html($_[0]->as_utf8) } -sub as_href { ascii_html(uri_escape($_[0]->as_utf8)) } +sub as_html { ascii_html($_[0]->{raw}) } +sub as_href { ascii_html(uri_escape($_[0]->{raw})) } + +sub raw { + if (defined $_[1]) { + $_[0]->{raw} = $_[1]; + } else { + $_[0]->{raw}; + } +} 1; diff --git a/lib/PublicInbox/MDA.pm b/lib/PublicInbox/MDA.pm index bb14ae50..ed8d74da 100644 --- a/lib/PublicInbox/MDA.pm +++ b/lib/PublicInbox/MDA.pm @@ -4,7 +4,7 @@ package PublicInbox::MDA; use strict; use warnings; use Email::Address; -use Encode qw/decode encode/; +use Encode qw/decode/; use Date::Parse qw(strptime); use constant MAX_SIZE => 1024 * 500; # same as spamc default use constant cmd => qw/ssoma-mda -1/; @@ -71,16 +71,15 @@ sub set_list_headers { # returns a 3-element array: name, email, date sub author_info { - my ($class, $simple) = @_; + my ($class, $mime) = @_; - my $from = decode('MIME-Header', $simple->header('From')); - $from = encode('utf8', $from); + my $from = $mime->header('From'); my @from = Email::Address->parse($from); my $name = $from[0]->name; defined $name or $name = ''; my $email = $from[0]->address; defined $email or $email = ''; - ($name, $email, $simple->header('Date')); + ($name, $email, $mime->header('Date')); } 1; diff --git a/lib/PublicInbox/View.pm b/lib/PublicInbox/View.pm index 811e4703..0a8ab549 100644 --- a/lib/PublicInbox/View.pm +++ b/lib/PublicInbox/View.pm @@ -5,7 +5,6 @@ use strict; use warnings; use PublicInbox::Hval; use URI::Escape qw/uri_escape/; -use CGI qw/escapeHTML/; use Encode qw/find_encoding/; use Encode::MIME::Header; use Email::MIME::ContentType qw/parse_content_type/; @@ -13,8 +12,7 @@ use constant MAX_INLINE_QUOTED => 5; use constant MAX_TRUNC_LEN => 72; *ascii_html = *PublicInbox::Hval::ascii_html; -my $enc_utf8 = find_encoding('utf8'); -my $enc_ascii = find_encoding('us-ascii'); +my $enc_utf8 = find_encoding('UTF-8'); my $enc_mime = find_encoding('MIME-Header'); # public functions: @@ -138,33 +136,30 @@ sub add_text_body_full { } sub headers_to_html_header { - my ($simple, $full_pfx) = @_; + my ($mime, $full_pfx) = @_; my $rv = ""; my @title; foreach my $h (qw(From To Cc Subject Date)) { - my $v = $simple->header($h); - defined $v or next; - $v =~ tr/\n/ /s; - $v =~ tr/\r//d; - my $raw = $enc_mime->decode($v); - $v = ascii_html($raw); - $rv .= "$h: $v\n"; + my $v = $mime->header($h); + defined($v) && length($v) or next; + $v = PublicInbox::Hval->new_oneline($v); + $rv .= "$h: " . $v->as_html . "\n"; if ($h eq 'From') { - my @from = Email::Address->parse($raw); - $raw = $from[0]->name; - unless (defined($raw) && length($raw)) { - $raw = '<' . $from[0]->address . '>'; + my @from = Email::Address->parse($v->raw); + $v = $from[0]->name; + unless (defined($v) && length($v)) { + $v = '<' . $from[0]->address . '>'; } - $title[1] = ascii_html($raw); - + $title[1] = ascii_html($v); } elsif ($h eq 'Subject') { - $title[0] = $v; + $title[0] = $v->as_html; } } - my $mid = $simple->header('Message-ID'); + my $header_obj = $mime->header_obj; + my $mid = $header_obj->header_raw('Message-ID'); if (defined $mid) { $mid = PublicInbox::Hval->new_msgid($mid); $rv .= 'Message-ID: <' . $mid->as_html . '> '; @@ -173,7 +168,7 @@ sub headers_to_html_header { $rv .= "(<a href=\"$href.txt\">original</a>)\n"; } - my $irp = $simple->header('In-Reply-To'); + my $irp = $header_obj->header_raw('In-Reply-To'); if (defined $irp) { $irp = PublicInbox::Hval->new_msgid($irp); my $html = $irp->as_html; |