From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on dcvr.yhbt.net X-Spam-Level: X-Spam-ASN: X-Spam-Status: No, score=-4.0 required=3.0 tests=ALL_TRUSTED,AWL,BAYES_00 shortcircuit=no autolearn=ham autolearn_force=no version=3.4.2 Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id 78F5D1F609 for ; Tue, 4 Jun 2019 11:27:48 +0000 (UTC) From: Eric Wong To: meta@public-inbox.org Subject: [PATCH 01/24] linkify: support Internationalized Domain Names in URLs Date: Tue, 4 Jun 2019 11:27:25 +0000 Message-Id: <20190604112748.23598-2-e@80x24.org> In-Reply-To: <20190604112748.23598-1-e@80x24.org> References: <20190604112748.23598-1-e@80x24.org> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit List-Id: The "\w" character class in Perl matches any word characters in the Unicode database, not just ASCII characters. So we must be prepared for that and generate links to IDNs. --- lib/PublicInbox/Linkify.pm | 5 +++-- t/linkify.t | 12 ++++++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/lib/PublicInbox/Linkify.pm b/lib/PublicInbox/Linkify.pm index d4778e7..84960a9 100644 --- a/lib/PublicInbox/Linkify.pm +++ b/lib/PublicInbox/Linkify.pm @@ -13,6 +13,7 @@ package PublicInbox::Linkify; use strict; use warnings; use Digest::SHA qw/sha1_hex/; +use PublicInbox::Hval qw(ascii_html); my $SALT = rand; my $LINK_RE = qr{([\('!])?\b((?:ftps?|https?|nntps?|gopher):// @@ -61,12 +62,12 @@ sub linkify_1 { $end = ')'; } + $url = ascii_html($url); # for IDN + # salt this, as this could be exploited to show # links in the HTML which don't show up in the raw mail. my $key = sha1_hex($url . $SALT); - # only escape ampersands, others do not match LINK_RE - $url =~ s/&/&/g; $_[0]->{$key} = $url; $beg . 'PI-LINK-'. $key . $end; ^ge; diff --git a/t/linkify.t b/t/linkify.t index fe218b9..c492358 100644 --- a/t/linkify.t +++ b/t/linkify.t @@ -132,4 +132,16 @@ use PublicInbox::Linkify; 'punctuation with unpaired ) OK') } +if ('IDN example: ') { + my $hc = '月'; + my $u = "http://www.\x{6708}.example.com/"; + my $s = $u; + my $l = PublicInbox::Linkify->new; + $s = $l->linkify_1($s); + $s = $l->linkify_2($s); + my $expect = qq{http://www.$hc.example.com/}; + is($s, $expect, 'IDN message escaped properly'); +} + done_testing(); -- EW