[PATCH 01/24] linkify: support Internationalized Domain Names in URLs

[Eric Wong <e@80x24.org>]

The "\w" character class in Perl matches any word characters
in the Unicode database, not just ASCII characters.  So we
must be prepared for that and generate links to IDNs.
---
 lib/PublicInbox/Linkify.pm |  5 +++--
 t/linkify.t                | 12 ++++++++++++
 2 files changed, 15 insertions(+), 2 deletions(-)

diff --git a/lib/PublicInbox/Linkify.pm b/lib/PublicInbox/Linkify.pm
index d4778e7..84960a9 100644
--- a/lib/PublicInbox/Linkify.pm
+++ b/lib/PublicInbox/Linkify.pm
@@ -13,6 +13,7 @@ package PublicInbox::Linkify;
 use strict;
 use warnings;
 use Digest::SHA qw/sha1_hex/;
+use PublicInbox::Hval qw(ascii_html);
 
 my $SALT = rand;
 my $LINK_RE = qr{([\('!])?\b((?:ftps?|https?|nntps?|gopher)://
@@ -61,12 +62,12 @@ sub linkify_1 {
 			$end = ')';
 		}
 
+		$url = ascii_html($url); # for IDN
+
 		# salt this, as this could be exploited to show
 		# links in the HTML which don't show up in the raw mail.
 		my $key = sha1_hex($url . $SALT);
 
-		# only escape ampersands, others do not match LINK_RE
-		$url =~ s/&/&/g;
 		$_[0]->{$key} = $url;
 		$beg . 'PI-LINK-'. $key . $end;
 	^ge;
diff --git a/t/linkify.t b/t/linkify.t
index fe218b9..c492358 100644
--- a/t/linkify.t
+++ b/t/linkify.t
@@ -132,4 +132,16 @@ use PublicInbox::Linkify;
 		'punctuation with unpaired ) OK')
 }
 
+if ('IDN example: <ACDB98F4-178C-43C3-99C4-A1D03DD6A8F5@sb.org>') {
+	my $hc = '&#26376;';
+	my $u = "http://www.\x{6708}.example.com/";
+	my $s = $u;
+	my $l = PublicInbox::Linkify->new;
+	$s = $l->linkify_1($s);
+	$s = $l->linkify_2($s);
+	my $expect = qq{<a
+href="http://www.$hc.example.com/">http://www.$hc.example.com/</a>};
+	is($s, $expect, 'IDN message escaped properly');
+}
+
 done_testing();
-- 
EW