[PATCH 0/4] viewvcs: Atom bugfix + diffstat anchors

[PATCH 0/4] viewvcs: Atom bugfix + diffstat anchors

[Eric Wong]

Oops, I meant to email this out before
https://public-inbox.org/meta/20190201075030.23449-1-e@80x24.org/
("viewvcs: odds and ends"), but I forgot :x

diffstat anchors greatly aid in navigating large, multi-file
patches.  And I noticed Atom feeds were generating bad links
to solver :x

Eric Wong (4):
  hval: routines for attribute escaping
  viewdiff: diffstat links to diff anchors
  view: diffstat anchors for multi-message/attachment views
  view: fix broken hunk header hrefs in Atom feeds

 Documentation/design_www.txt     |  5 ++-
 lib/PublicInbox/Hval.pm          | 35 +++++++++++++++++++--
 lib/PublicInbox/View.pm          | 52 +++++++++++++++++++-----------
 lib/PublicInbox/ViewDiff.pm      | 54 ++++++++++++++++++++++++++------
 lib/PublicInbox/WwwAtomStream.pm |  2 +-
 t/hval.t                         | 14 +++++++--
 6 files changed, 127 insertions(+), 35 deletions(-)

-- 
EW

[PATCH 1/4] hval: routines for attribute escaping

[Eric Wong]

We'll use HTML attributes + anchor links to link to filenames
in coming commits.
---
 lib/PublicInbox/Hval.pm | 35 +++++++++++++++++++++++++++++++++--
 t/hval.t                | 14 ++++++++++++--
 2 files changed, 45 insertions(+), 4 deletions(-)

diff --git a/lib/PublicInbox/Hval.pm b/lib/PublicInbox/Hval.pm
index 53810b3..95a0f70 100644
--- a/lib/PublicInbox/Hval.pm
+++ b/lib/PublicInbox/Hval.pm
@@ -9,8 +9,8 @@ use warnings;
 use Encode qw(find_encoding);
 use PublicInbox::MID qw/mid_clean mid_escape/;
 use base qw/Exporter/;
-our @EXPORT_OK = qw/ascii_html obfuscate_addrs to_filename src_escape/;
-
+our @EXPORT_OK = qw/ascii_html obfuscate_addrs to_filename src_escape
+		to_attr from_attr/;
 my $enc_ascii = find_encoding('us-ascii');
 
 sub new {
@@ -122,4 +122,35 @@ sub to_filename ($) {
 	$s
 }
 
+# convert a filename (or any string) to HTML attribute
+
+my %ESCAPES = map { chr($_) => sprintf('::%02x', $_) } (0..255);
+$ESCAPES{'/'} = ':'; # common
+
+sub to_attr ($) {
+	my ($str) = @_;
+
+	# git would never do this to us:
+	return if index($str, '//') >= 0;
+
+	my $first = '';
+	if ($str =~ s/\A([^A-Ya-z])//ms) { # start with a letter
+		  $first = sprintf('Z%02x', ord($1));
+	}
+	$str =~ s/([^A-Za-z0-9_\.\-])/$ESCAPES{$1}/egms;
+	$first . $str;
+}
+
+# reverse the result of to_attr
+sub from_attr ($) {
+	my ($str) = @_;
+	my $first = '';
+	if ($str =~ s/\AZ([a-f0-9]{2})//ms) {
+		$first = chr(hex($1));
+	}
+	$str =~ s!::([a-f0-9]{2})!chr(hex($1))!egms;
+	$str =~ tr!:!/!;
+	$first . $str;
+}
+
 1;
diff --git a/t/hval.t b/t/hval.t
index bfc9a85..f93d6be 100644
--- a/t/hval.t
+++ b/t/hval.t
@@ -1,9 +1,9 @@
-# Copyright (C) 2017-2018 all contributors <meta@public-inbox.org>
+# Copyright (C) 2017-2019 all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 use strict;
 use warnings;
 use Test::More;
-use_ok 'PublicInbox::Hval';
+use_ok 'PublicInbox::Hval', qw(from_attr to_attr);
 
 my $ibx = {
 	-no_obfuscate_re => qr/(?:example\.com)\z/i,
@@ -47,4 +47,14 @@ my $s = "\0\x07\n";
 PublicInbox::Hval::src_escape($s);
 is($s, "\\0\\a\n", 'src_escape works as intended');
 
+foreach my $s ('Hello/World.pm', 'Zcat', 'hello world.c', 'Eléanor', '$at') {
+	my $attr = to_attr($s);
+	is(from_attr($attr), $s, "$s => $attr => $s round trips");
+}
+
+{
+	my $bad = to_attr('foo//bar');
+	ok(!$bad, 'double-slash rejected');
+}
+
 done_testing();
-- 
EW

[PATCH 2/4] viewdiff: diffstat links to diff anchors

[Eric Wong]

This can be helpful for reviewing larger patches which span
across several files on the permalink (/$MESSAGE_ID/) HTML
page.

More work will be needed to get this working for the /T/ and /t/
pages which show multiple emails, as the filename-based anchors
will conflict at the moment.
---
 lib/PublicInbox/ViewDiff.pm | 51 +++++++++++++++++++++++++++++++------
 1 file changed, 43 insertions(+), 8 deletions(-)

diff --git a/lib/PublicInbox/ViewDiff.pm b/lib/PublicInbox/ViewDiff.pm
index a804568..38cb5a1 100644
--- a/lib/PublicInbox/ViewDiff.pm
+++ b/lib/PublicInbox/ViewDiff.pm
@@ -12,11 +12,11 @@ use warnings;
 use base qw(Exporter);
 our @EXPORT_OK = qw(flush_diff);
 use URI::Escape qw(uri_escape_utf8);
-use PublicInbox::Hval qw(ascii_html);
+use PublicInbox::Hval qw(ascii_html to_attr from_attr);
 use PublicInbox::Git qw(git_unquote);
 
 sub DSTATE_INIT () { 0 }
-sub DSTATE_STAT () { 1 } # TODO
+sub DSTATE_STAT () { 1 }
 sub DSTATE_HEAD () { 2 } # /^diff --git /, /^index /, /^--- /, /^\+\+\+ /
 sub DSTATE_CTX () { 3 } # /^ /
 sub DSTATE_ADD () { 4 } # /^\+/
@@ -75,14 +75,47 @@ sub to_state ($$$) {
 	$$dst .= qq(<span\nclass="$class">);
 }
 
+sub anchor0 ($$$$$) {
+	my ($dst, $anchors, $linkify, $fn, $rest) = @_;
+	if (my $attr = to_attr($fn)) {
+		$anchors->{$attr} = 1;
+		$$dst .= " <a\nhref=#$attr>" .
+			ascii_html($fn) . '</a>'.
+			to_html($linkify, $rest);
+		return 1;
+	}
+	undef;
+}
+
+sub anchor1 ($$$$$) {
+	my ($dst, $anchors, $linkify, $pb, $s) = @_;
+	my $attr = to_attr($pb) or return;
+	my $line = to_html($linkify, $s);
+
+	if (delete $anchors->{$attr} && $line =~ s/^diff //) {
+		$$dst .= "<a\nhref=#ds\nid=$attr>diff</a> ".$line;
+		return 1;
+	}
+	undef
+}
+
 sub flush_diff ($$$$) {
 	my ($dst, $spfx, $linkify, $diff) = @_;
 	my $state = DSTATE_INIT;
 	my $dctx = { Q => '' }; # {}, keys: oid_a, oid_b, path_a, path_b
+	my $anchors = {}; # attr => filename
 
 	foreach my $s (@$diff) {
-		if ($s =~ /^ /) {
-			if ($state2class[$state]) {
+		if ($s =~ /^---$/) {
+			to_state($dst, $state, DSTATE_STAT);
+			$$dst .= "<span\nid=ds>" . $s . '</span>';
+		} elsif ($s =~ /^ /) {
+			# works for common cases, but not weird/long filenames
+			if ($state == DSTATE_STAT &&
+					$s =~ /^ (\S+)(\s+\|.*\z)/s) {
+				anchor0($dst, $anchors, $linkify, $1, $2)
+					and next;
+			} elsif ($state2class[$state]) {
 				to_state($dst, $state, DSTATE_CTX);
 			}
 			$$dst .= to_html($linkify, $s);
@@ -103,6 +136,8 @@ sub flush_diff ($$$$) {
 					$dctx->{Q} .=
 					     "&a=".uri_escape_utf8($pa, UNSAFE);
 				}
+				anchor1($dst, $anchors, $linkify, $pb, $s)
+					and next;
 			}
 			$$dst .= to_html($linkify, $s);
 		} elsif ($s =~ s/^(index $OID_NULL\.\.)($OID_BLOB)\b//o) {
@@ -127,16 +162,16 @@ sub flush_diff ($$$$) {
 		} elsif ($s =~ m!^--- $PATH_A! ||
 		         $s =~ m!^\+{3} $PATH_B!)  {
 			# color only (no oid link)
-			$state == DSTATE_INIT and
+			$state <= DSTATE_STAT and
 				to_state($dst, $state, DSTATE_HEAD);
 			$$dst .= to_html($linkify, $s);
 		} elsif ($s =~ /^\+/) {
-			if ($state != DSTATE_ADD && $state != DSTATE_INIT) {
+			if ($state != DSTATE_ADD && $state > DSTATE_STAT) {
 				to_state($dst, $state, DSTATE_ADD);
 			}
 			$$dst .= to_html($linkify, $s);
 		} elsif ($s =~ /^-/) {
-			if ($state != DSTATE_DEL && $state != DSTATE_INIT) {
+			if ($state != DSTATE_DEL && $state > DSTATE_STAT) {
 				to_state($dst, $state, DSTATE_DEL);
 			}
 			$$dst .= to_html($linkify, $s);
@@ -148,7 +183,7 @@ sub flush_diff ($$$$) {
 			 $s =~ /^(?:dis)?similarity index /) {
 			$$dst .= to_html($linkify, $s);
 		} else {
-			$state == DSTATE_INIT or
+			$state <= DSTATE_STAT or
 				to_state($dst, $state, DSTATE_INIT);
 			$$dst .= to_html($linkify, $s);
 		}
-- 
EW

[PATCH 3/4] view: diffstat anchors for multi-message/attachment views

[Eric Wong]

diffstat <-> ^diff anchors work within the same attachment or
message while in HTML views which display multiple messages.
---
 Documentation/design_www.txt     |  5 ++++-
 lib/PublicInbox/View.pm          | 33 ++++++++++++++++++++------------
 lib/PublicInbox/ViewDiff.pm      | 31 +++++++++++++++---------------
 lib/PublicInbox/WwwAtomStream.pm |  2 +-
 4 files changed, 41 insertions(+), 30 deletions(-)

diff --git a/Documentation/design_www.txt b/Documentation/design_www.txt
index c7d7fcb..cd715b2 100644
--- a/Documentation/design_www.txt
+++ b/Documentation/design_www.txt
@@ -6,12 +6,15 @@ URL and anchor naming
 /$INBOX/new.atom                       -> Atom feed
 
 #### Optional, relies on Search::Xapian
-/$INBOX/$MESSAGE_ID/t/                 -> HTML content of thread
+/$INBOX/$MESSAGE_ID/t/                 -> HTML content of thread (nested)
+/$INBOX/$MESSAGE_ID/T/                 -> HTML content of thread (flat)
 	anchors:
 	#u          location of $MESSAGE_ID in URL
 	#m<SHA-1>   per-message links, where <SHA-1> is of the Message-ID
 	            of each message (stable)
 	#s<NUM>     relative numeric position of message in thread (unstable)
+	#i<...>     diffstat location for patch emails
+	#Z?<...>    per-file diff header location for patch emails
 
 /$INBOX/$MESSAGE_ID/t.atom             -> Atom feed for thread
 /$INBOX/$MESSAGE_ID/t.mbox.gz          -> gzipped mbox of thread
diff --git a/lib/PublicInbox/View.pm b/lib/PublicInbox/View.pm
index ca9b955..8c81253 100644
--- a/lib/PublicInbox/View.pm
+++ b/lib/PublicInbox/View.pm
@@ -37,7 +37,7 @@ sub msg_html {
 		if ($nr == 1) {
 			# $more cannot be true w/o $smsg being defined:
 			my $upfx = $more ? '../'.mid_escape($smsg->mid).'/' : '';
-			$tip . multipart_text_as_html($mime, $upfx, $ibx) .
+			$tip . multipart_text_as_html($mime, $upfx, $ctx) .
 				'</pre><hr>'
 		} elsif ($more && @$more) {
 			++$end;
@@ -90,7 +90,7 @@ sub msg_html_more {
 			my $mime = $smsg->{mime};
 			my $upfx = '../' . mid_escape($smsg->mid) . '/';
 			_msg_html_prepare($mime->header_obj, $ctx, $more, $nr) .
-				multipart_text_as_html($mime, $upfx, $ibx) .
+				multipart_text_as_html($mime, $upfx, $ctx) .
 				'</pre><hr>'
 		} else {
 			'';
@@ -262,7 +262,7 @@ sub index_entry {
 
 	# scan through all parts, looking for displayable text
 	my $ibx = $ctx->{-inbox};
-	msg_iter($mime, sub { $rv .= add_text_body($mhref, $ibx, $_[0]) });
+	msg_iter($mime, sub { $rv .= add_text_body($mhref, $ctx, $_[0]) });
 
 	# add the footer
 	$rv .= "\n<a\nhref=#$id_m\nid=e$id>^</a> ".
@@ -490,11 +490,11 @@ sub thread_html {
 }
 
 sub multipart_text_as_html {
-	my ($mime, $upfx, $ibx) = @_;
+	my ($mime, $upfx, $ctx) = @_;
 	my $rv = "";
 
 	# scan through all parts, looking for displayable text
-	msg_iter($mime, sub { $rv .= add_text_body($upfx, $ibx, $_[0]) });
+	msg_iter($mime, sub { $rv .= add_text_body($upfx, $ctx, $_[0]) });
 	$rv;
 }
 
@@ -547,10 +547,11 @@ sub attach_link ($$$$;$) {
 }
 
 sub add_text_body {
-	my ($upfx, $ibx, $p) = @_;
+	my ($upfx, $ctx, $p) = @_;
+	my $ibx = $ctx->{-inbox};
 	my $obfs_ibx = $ibx->{obfuscate} ? $ibx : undef;
 	# $p - from msg_iter: [ Email::MIME, depth, @idx ]
-	my ($part, $depth) = @$p; # attachment @idx is unused
+	my ($part, $depth, @idx) = @$p;
 	my $ct = $part->content_type || 'text/plain';
 	my $fn = $part->filename;
 	my ($s, $err) = msg_part_text($part, $ct);
@@ -561,9 +562,16 @@ sub add_text_body {
 	# link generation in diffs with the extra '%0D'
 	$s =~ s/\r\n/\n/sg;
 
-	my ($diff, $spfx);
+	# always support diff-highlighting, but we can't linkify hunk
+	# headers for solver unless some coderepo are configured:
+	my $diff;
 	if ($s =~ /^(?:diff|---|\+{3}) /ms) {
-		$diff = [];
+		# diffstat anchors do not link across attachments or messages:
+		$idx[0] = $upfx . $idx[0] if $upfx ne '';
+		$ctx->{-apfx} = join('/', @idx);
+		$ctx->{-anchors} = {}; # attr => filename
+		$ctx->{-diff} = $diff = [];
+		my $spfx;
 		if ($ibx->{-repo_objs}) {
 			my $n_slash = $upfx =~ tr!/!/!;
 			if ($n_slash == 0) {
@@ -574,6 +582,7 @@ sub add_text_body {
 				$spfx = '../../';
 			}
 		}
+		$ctx->{-spfx} = $spfx;
 	};
 
 	my @lines = split(/^/m, $s);
@@ -598,18 +607,18 @@ sub add_text_body {
 				$s .= $l->linkify_2(ascii_html($cur));
 			}
 		} else {
-			flush_diff(\$s, $spfx, $l, $diff) if $diff && @$diff;
+			flush_diff(\$s, $ctx, $l) if $diff && @$diff;
 			push @quot, $cur;
 		}
 	}
 
 	if (@quot) { # ugh, top posted
 		flush_quote(\$s, $l, \@quot);
-		flush_diff(\$s, $spfx, $l, $diff) if $diff && @$diff;
+		flush_diff(\$s, $ctx, $l) if $diff && @$diff;
 		obfuscate_addrs($obfs_ibx, $s) if $obfs_ibx;
 		$s;
 	} else {
-		flush_diff(\$s, $spfx, $l, $diff) if $diff && @$diff;
+		flush_diff(\$s, $ctx, $l) if $diff && @$diff;
 		obfuscate_addrs($obfs_ibx, $s) if $obfs_ibx;
 		if ($s =~ /\n\z/s) { # common, last line ends with a newline
 			$s;
diff --git a/lib/PublicInbox/ViewDiff.pm b/lib/PublicInbox/ViewDiff.pm
index 38cb5a1..2074e12 100644
--- a/lib/PublicInbox/ViewDiff.pm
+++ b/lib/PublicInbox/ViewDiff.pm
@@ -76,10 +76,10 @@ sub to_state ($$$) {
 }
 
 sub anchor0 ($$$$$) {
-	my ($dst, $anchors, $linkify, $fn, $rest) = @_;
-	if (my $attr = to_attr($fn)) {
-		$anchors->{$attr} = 1;
-		$$dst .= " <a\nhref=#$attr>" .
+	my ($dst, $ctx, $linkify, $fn, $rest) = @_;
+	if (my $attr = to_attr($ctx->{-apfx}.$fn)) {
+		$ctx->{-anchors}->{$attr} = 1;
+		$$dst .= " <a\nid=i$attr\nhref=#$attr>" .
 			ascii_html($fn) . '</a>'.
 			to_html($linkify, $rest);
 		return 1;
@@ -88,33 +88,33 @@ sub anchor0 ($$$$$) {
 }
 
 sub anchor1 ($$$$$) {
-	my ($dst, $anchors, $linkify, $pb, $s) = @_;
-	my $attr = to_attr($pb) or return;
+	my ($dst, $ctx, $linkify, $pb, $s) = @_;
+	my $attr = to_attr($ctx->{-apfx}.$pb) or return;
 	my $line = to_html($linkify, $s);
 
-	if (delete $anchors->{$attr} && $line =~ s/^diff //) {
-		$$dst .= "<a\nhref=#ds\nid=$attr>diff</a> ".$line;
+	if (delete $ctx->{-anchors}->{$attr} && $line =~ s/^diff //) {
+		$$dst .= "<a\nhref=#i$attr\nid=$attr>diff</a> ".$line;
 		return 1;
 	}
 	undef
 }
 
-sub flush_diff ($$$$) {
-	my ($dst, $spfx, $linkify, $diff) = @_;
+sub flush_diff ($$$) {
+	my ($dst, $ctx, $linkify) = @_;
+	my $diff = $ctx->{-diff};
+	my $spfx = $ctx->{-spfx};
 	my $state = DSTATE_INIT;
 	my $dctx = { Q => '' }; # {}, keys: oid_a, oid_b, path_a, path_b
-	my $anchors = {}; # attr => filename
 
 	foreach my $s (@$diff) {
 		if ($s =~ /^---$/) {
 			to_state($dst, $state, DSTATE_STAT);
-			$$dst .= "<span\nid=ds>" . $s . '</span>';
+			$$dst .= $s;
 		} elsif ($s =~ /^ /) {
 			# works for common cases, but not weird/long filenames
 			if ($state == DSTATE_STAT &&
 					$s =~ /^ (\S+)(\s+\|.*\z)/s) {
-				anchor0($dst, $anchors, $linkify, $1, $2)
-					and next;
+				anchor0($dst, $ctx, $linkify, $1, $2) and next;
 			} elsif ($state2class[$state]) {
 				to_state($dst, $state, DSTATE_CTX);
 			}
@@ -136,8 +136,7 @@ sub flush_diff ($$$$) {
 					$dctx->{Q} .=
 					     "&a=".uri_escape_utf8($pa, UNSAFE);
 				}
-				anchor1($dst, $anchors, $linkify, $pb, $s)
-					and next;
+				anchor1($dst, $ctx, $linkify, $pb, $s) and next;
 			}
 			$$dst .= to_html($linkify, $s);
 		} elsif ($s =~ s/^(index $OID_NULL\.\.)($OID_BLOB)\b//o) {
diff --git a/lib/PublicInbox/WwwAtomStream.pm b/lib/PublicInbox/WwwAtomStream.pm
index 712c3dc..6d3a936 100644
--- a/lib/PublicInbox/WwwAtomStream.pm
+++ b/lib/PublicInbox/WwwAtomStream.pm
@@ -137,7 +137,7 @@ sub feed_entry {
 		qq{<content\ntype="xhtml">} .
 		qq{<div\nxmlns="http://www.w3.org/1999/xhtml">} .
 		qq(<pre\nstyle="white-space:pre-wrap">) .
-		PublicInbox::View::multipart_text_as_html($mime, $href) .
+		PublicInbox::View::multipart_text_as_html($mime, $href, $ctx) .
 		'</pre></div></content></entry>';
 }
 
-- 
EW

[PATCH 4/4] view: fix broken hunk header hrefs in Atom feeds

[Eric Wong]

We use absolute URLs in the Atom feeds (to ease
syndication/mirroring), so hunk headers need to point to the
solver URLs.
---
 lib/PublicInbox/View.pm | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/lib/PublicInbox/View.pm b/lib/PublicInbox/View.pm
index 8c81253..782e668 100644
--- a/lib/PublicInbox/View.pm
+++ b/lib/PublicInbox/View.pm
@@ -573,13 +573,18 @@ sub add_text_body {
 		$ctx->{-diff} = $diff = [];
 		my $spfx;
 		if ($ibx->{-repo_objs}) {
-			my $n_slash = $upfx =~ tr!/!/!;
-			if ($n_slash == 0) {
-				$spfx = '../';
-			} elsif ($n_slash == 1) {
-				$spfx = '';
-			} else { # nslash == 2
-				$spfx = '../../';
+			if (index($upfx, '//') >= 0) { # absolute URL (Atom feeds)
+				$spfx = $upfx;
+				$spfx =~ s!/([^/]*)/\z!/!;
+			} else {
+				my $n_slash = $upfx =~ tr!/!/!;
+				if ($n_slash == 0) {
+					$spfx = '../';
+				} elsif ($n_slash == 1) {
+					$spfx = '';
+				} else { # nslash == 2
+					$spfx = '../../';
+				}
 			}
 		}
 		$ctx->{-spfx} = $spfx;
-- 
EW