* [PATCH 01/13] www: implement hybrid flat+thread conversation view
2016-06-30 9:21 7% [PATCH 0/13] www: hybrid flat+thread conversation view Eric Wong
@ 2016-06-30 9:21 2% ` Eric Wong
0 siblings, 0 replies; 2+ results
From: Eric Wong @ 2016-06-30 9:21 UTC (permalink / raw)
To: meta
This should be more accessible to readers on narrow terminals
(or giant fonts) while providing a chronological view which
is also aware of message threading relationships.
---
TODO | 2 -
lib/PublicInbox/SearchView.pm | 42 +++----
lib/PublicInbox/View.pm | 248 ++++++++++++++++--------------------------
lib/PublicInbox/WWW.pm | 9 +-
t/plack.t | 2 +-
5 files changed, 115 insertions(+), 188 deletions(-)
diff --git a/TODO b/TODO
index f29f2f0..3b6401f 100644
--- a/TODO
+++ b/TODO
@@ -4,8 +4,6 @@ TODO items for public-inbox
* mailmap support (same as git) for remapping expired email addresses
-* WWW: Hybrid flat view + thread skeleton (requires Xapian)
-
* POP3 server, since some webmail providers support external POP3:
https://public-inbox.org/meta/20160411034104.GA7817@dcvr.yhbt.net/
diff --git a/lib/PublicInbox/SearchView.pm b/lib/PublicInbox/SearchView.pm
index ae875bf..fbef411 100644
--- a/lib/PublicInbox/SearchView.pm
+++ b/lib/PublicInbox/SearchView.pm
@@ -163,44 +163,34 @@ sub tdump {
} else { # order by time (default for threaded view)
$th->order(*PublicInbox::View::sort_ts);
}
+ my $skel = '';
my $state = {
+ -inbox => $ctx->{-inbox},
+ anchor_idx => 1,
ctx => $ctx,
- anchor_idx => 0,
- pct => \%pct,
cur_level => 0,
- -inbox => $ctx->{-inbox},
+ dst => \$skel,
fh => $fh,
+ mapping => {},
+ pct => \%pct,
+ prev_attr => '',
+ prev_level => 0,
+ seen => {},
+ srch => $ctx->{srch},
+ upfx => './',
};
$ctx->{searchview} = 1;
- PublicInbox::View::walk_thread($th, $state, *tdump_ent);
- PublicInbox::View::thread_adj_level($state, 0);
+ PublicInbox::View::walk_thread($th, $state,
+ *PublicInbox::View::pre_thread);
+
+ PublicInbox::View::thread_entry($state, $_, 0) for @m;
- $fh->write(search_nav_bot($mset, $q). "\n\n" .
+ $fh->write(search_nav_bot($mset, $q). "\n\n" . $skel . "\n" .
foot($ctx). '</pre></body></html>');
$fh->close;
}
-sub tdump_ent {
- my ($state, $level, $node) = @_;
- my $mime = $node->message;
-
- if ($mime) {
- # lazy load the full message from mini_mime:
- my $mid = mid_mime($mime);
- $mime = eval { $state->{-inbox}->msg_by_mid($mid) } and
- $mime = Email::MIME->new($mime);
- }
- if ($mime) {
- my $end = PublicInbox::View::thread_adj_level($state, $level);
- PublicInbox::View::index_entry($mime, $level, $state);
- $state->{fh}->write($end) if $end;
- } else {
- my $mid = $node->messageid;
- PublicInbox::View::ghost_flush($state, '', $mid, $level);
- }
-}
-
sub foot {
my ($ctx) = @_;
my $foot = $ctx->{footer} || '';
diff --git a/lib/PublicInbox/View.pm b/lib/PublicInbox/View.pm
index 30339cd..65788db 100644
--- a/lib/PublicInbox/View.pm
+++ b/lib/PublicInbox/View.pm
@@ -89,77 +89,72 @@ sub _hdr_names ($$) {
ascii_html(join(', ', PublicInbox::Address::names($val)));
}
+sub nr_to_s ($$$) {
+ my ($nr, $singular, $plural) = @_;
+ return "0 $plural" if $nr == 0;
+ $nr == 1 ? "$nr $singular" : "$nr $plural";
+}
+
# this is already inside a <pre>
sub index_entry {
my ($mime, $level, $state) = @_;
- my $midx = $state->{anchor_idx}++;
+ $state->{anchor_idx}++;
my $ctx = $state->{ctx};
my $srch = $ctx->{srch};
my $hdr = $mime->header_obj;
my $subj = $hdr->header('Subject');
my $mid_raw = mid_clean(mid_mime($mime));
- my $id = anchor_for($mid_raw);
- my $seen = $state->{seen};
- $seen->{$id} = "#$id"; # save the anchor for children, later
-
+ my $id = id_compress($mid_raw);
+ my $id_m = 'm'.$id;
my $mid = PublicInbox::Hval->new_msgid($mid_raw);
my $root_anchor = $state->{root_anchor} || '';
my $path = $root_anchor ? '../../' : '';
my $href = $mid->as_href;
my $irt = in_reply_to($hdr);
- my $parent_anchor = $seen->{anchor_for($irt)} if defined $irt;
- $subj = ascii_html($subj);
- $subj = "<a\nhref=\"${path}$href/\">$subj</a>";
- $subj = "<u\nid=u>$subj</u>" if $root_anchor eq $id;
+ $subj = '<b>'.ascii_html($subj).'</b>';
+ $subj = "<u\nid=u>$subj</u>" if $root_anchor eq $id_m;
my $ts = _msg_date($hdr);
- my $rv = "<pre\nid=s$midx>";
- $rv .= "<b\nid=$id>$subj</b>\n";
- my $txt = "${path}$href/raw";
- my $fh = $state->{fh};
+ my $rv = "<pre><a\nhref=#e$id\nid=$id_m>#</a> ";
+ $rv .= $subj;
+ my $mhref = $path.$href.'/';
my $from = _hdr_names($hdr, 'From');
- $rv .= "- $from @ $ts UTC (<a\nhref=\"$txt\">raw</a>)\n";
+ $rv .= "\n- $from @ $ts UTC\n";
my @tocc;
foreach my $f (qw(To Cc)) {
my $dst = _hdr_names($hdr, $f);
push @tocc, "$f: $dst" if $dst ne '';
}
$rv .= ' '.join('; +', @tocc) . "\n" if @tocc;
- $fh->write($rv .= "\n");
-
- my $mhref = "${path}$href/";
+ $rv .= "\n";
# scan through all parts, looking for displayable text
- msg_iter($mime, sub { index_walk($fh, $mhref, $_[0]) });
- $rv = "\n" . html_footer($hdr, 0, $ctx, "$path$href/#R");
-
+ msg_iter($mime, sub { $rv .= add_text_body($mhref, $_[0]) });
+ $rv .= "\n<a\nhref=\"$mhref\"\n>permalink</a>" .
+ " / <a\nhref=\"${mhref}raw\">raw</a> / ";
+ my $mapping = $state->{mapping};
+ my $nr_c = $mapping->{$mid_raw} || 0;
+ my $nr_s = 0;
if (defined $irt) {
- unless (defined $parent_anchor) {
- my $v = PublicInbox::Hval->new_msgid($irt, 1);
- $v = $v->as_href;
- $parent_anchor = "${path}$v/";
- }
- $rv .= " <a\nhref=\"$parent_anchor\">parent</a>";
+ $nr_s = ($mapping->{$irt} || 0) - 1;
+ $nr_s = 0 if $nr_s < 0;
+ $irt = anchor_for($irt);
+ $rv .= "<a\nhref=#$irt>#parent</a>,";
+ } else {
+ $rv .= 'root message:';
}
+ $nr_s = nr_to_s($nr_s, 'sibling', 'siblings');
+ $nr_c = nr_to_s($nr_c, 'reply', 'replies');
+ $rv .= " <a\nhref=#r$id\nid=e$id>$nr_s, $nr_c</a>";
+ $rv .= " / <a\nhref=\"${mhref}#R\">reply</a>";
+
if (my $pct = $state->{pct}) { # used by SearchView.pm
$rv .= " [relevance $pct->{$mid_raw}%]";
- } elsif ($srch) {
- my $threaded = 'threaded';
- my $flat = 'flat';
- my $end = '';
- if ($ctx->{flat}) {
- $flat = "<b>$flat</b>";
- $end = "\n"; # for lynx
- } else {
- $threaded = "<b>$threaded</b>";
- }
- $rv .= " [<a\nhref=\"${path}$href/t/#u\">$threaded</a>";
- $rv .= "|<a\nhref=\"${path}$href/T/#u\">$flat</a>]$end";
}
- $fh->write($rv .= '</pre>');
+ $state->{fh}->write($rv .= "\n</pre>"); # '\n' for lynx
}
sub thread_html {
@@ -179,63 +174,62 @@ sub walk_thread {
}
}
+sub pre_thread {
+ my ($state, $level, $node) = @_;
+ my $parent = $node->parent;
+ if ($parent) {
+ my $mid = $parent->messageid;
+ my $m = $state->{mapping};
+ $m->{$mid} ||= 0;
+ $m->{$mid}++;
+ }
+ skel_dump($state, $level, $node);
+}
+
# only private functions below.
sub emit_thread_html {
my ($res, $ctx, $foot, $srch) = @_;
my $mid = $ctx->{mid};
- my $flat = $ctx->{flat};
- my $msgs = load_results($srch->get_thread($mid, { asc => $flat }));
- my $nr = scalar @$msgs;
+ my $sres = $srch->get_thread($mid, { asc => 1 });
+ my $msgs = load_results($sres);
+ my $nr = $sres->{total};
return missing_thread($res, $ctx) if $nr == 0;
- my $seen = {};
+ my $skel = '';
my $state = {
- res => $res,
- ctx => $ctx,
- seen => $seen,
- root_anchor => anchor_for($mid),
anchor_idx => 0,
+ ctx => $ctx,
cur_level => 0,
+ dst => \$skel,
+ mapping => {}, # mid -> reply count
+ prev_attr => '',
+ prev_level => 0,
+ res => $res,
+ root_anchor => anchor_for($mid),
+ seen => {},
+ srch => $ctx->{srch},
+ upfx => '../../',
};
- require PublicInbox::Git;
- $ctx->{git} ||= PublicInbox::Git->new($ctx->{git_dir});
- if ($flat) {
- pre_anchor_entry($seen, $_) for (@$msgs);
- __thread_entry($state, $_, 0) for (@$msgs);
- } else {
- walk_thread(thread_results($msgs), $state, *thread_entry);
- if (my $max = $state->{cur_level}) {
- $state->{fh}->write(
- ('</ul></li>' x ($max - 1)) . '</ul>');
- }
- }
+ walk_thread(thread_results($msgs), $state, *pre_thread);
+
+ thread_entry($state, $_, 0) for @$msgs;
# there could be a race due to a message being deleted in git
# but still being in the Xapian index:
my $fh = delete $state->{fh} or return missing_thread($res, $ctx);
- my $final_anchor = $state->{anchor_idx};
- my $next = "<a\nid=s$final_anchor>";
- $next .= $final_anchor == 1 ? 'only message in' : 'end of';
- $next .= " thread</a>, back to <a\nhref=\"../../\">index</a>";
- $next .= "\ndownload thread: ";
+ my $next = @$msgs == 1 ? 'only message in thread' : 'end of thread';
+ $next .= ", back to <a\nhref=\"../../\">index</a>";
+ $next .= "\n<a\nid=t>$nr+ messages in thread:</a> (download: ";
$next .= "<a\nhref=\"../t.mbox.gz\">mbox.gz</a>";
- $next .= " / follow: <a\nhref=\"../t.atom\">Atom feed</a>";
+ $next .= " / follow: <a\nhref=\"../t.atom\">Atom feed</a>)\n";
+ $next .= $skel;
$fh->write('<hr /><pre>' . $next . "\n\n".
$foot . '</pre></body></html>');
$fh->close;
}
-sub index_walk {
- my ($fh, $upfx, $p) = @_;
- my $s = add_text_body($upfx, $p);
-
- return if $s eq '';
-
- $fh->write($s);
-}
-
sub multipart_text_as_html {
my ($mime, $upfx) = @_;
my $rv = "";
@@ -542,11 +536,7 @@ sub linkify_ref_nosrch {
sub anchor_for {
my ($msgid) = @_;
- my $id = $msgid;
- if ($id !~ /\A[a-f0-9]{40}\z/) {
- $id = id_compress(mid_clean($id), 1);
- }
- 'm' . $id;
+ 'm' . id_compress($msgid, 1);
}
sub thread_html_head {
@@ -563,12 +553,6 @@ sub thread_html_head {
"</head><body>");
}
-sub pre_anchor_entry {
- my ($seen, $mime) = @_;
- my $id = anchor_for(mid_mime($mime));
- $seen->{$id} = "#$id"; # save the anchor for children, later
-}
-
sub ghost_parent {
my ($upfx, $mid) = @_;
# 'subject dummy' is used internally by Mail::Thread
@@ -580,39 +564,7 @@ sub ghost_parent {
qq{[parent not found: <<a\nhref="$upfx$href/">$html</a>>]};
}
-sub thread_adj_level {
- my ($state, $level) = @_;
-
- my $max = $state->{cur_level};
- if ($level <= 0) {
- return '' if $max == 0; # flat output
-
- # reset existing lists
- my $x = $max > 1 ? ('</ul></li>' x ($max - 1)) : '';
- $state->{fh}->write($x . '</ul>');
- $state->{cur_level} = 0;
- return '';
- }
- if ($level == $max) { # continue existing list
- $state->{fh}->write('<li>');
- } elsif ($level < $max) {
- my $x = $max > 1 ? ('</ul></li>' x ($max - $level)) : '';
- $state->{fh}->write($x .= '<li>');
- $state->{cur_level} = $level;
- } else { # ($level > $max) # start a new level
- $state->{cur_level} = $level;
- $state->{fh}->write(($max ? '<li>' : '') . '<ul><li>');
- }
- '</li>';
-}
-
-sub ghost_flush {
- my ($state, $upfx, $mid, $level) = @_;
- my $end = '<pre>'. ghost_parent($upfx, $mid) . '</pre>';
- $state->{fh}->write($end .= thread_adj_level($state, $level));
-}
-
-sub __thread_entry {
+sub thread_entry {
my ($state, $mime, $level) = @_;
# lazy load the full message from mini_mime:
@@ -623,16 +575,7 @@ sub __thread_entry {
$mime = Email::MIME->new($mime);
thread_html_head($mime, $state) if $state->{anchor_idx} == 0;
- if (my $ghost = delete $state->{ghost}) {
- # n.b. ghost messages may only be parents, not children
- foreach my $g (@$ghost) {
- ghost_flush($state, '../../', @$g);
- }
- }
- my $end = thread_adj_level($state, $level);
index_entry($mime, $level, $state);
- $state->{fh}->write($end) if $end;
-
1;
}
@@ -641,23 +584,6 @@ sub indent_for {
INDENT x ($level - 1);
}
-sub __ghost_prepare {
- my ($state, $node, $level) = @_;
- my $ghost = $state->{ghost} ||= [];
- push @$ghost, [ $node->messageid, $level ];
-}
-
-sub thread_entry {
- my ($state, $level, $node) = @_;
- if (my $mime = $node->message) {
- unless (__thread_entry($state, $mime, $level)) {
- __ghost_prepare($state, $node, $level);
- }
- } else {
- __ghost_prepare($state, $node, $level);
- }
-}
-
sub load_results {
my ($sres) = @_;
@@ -738,30 +664,44 @@ sub _skel_header {
$s = $s->as_html;
}
my $m = PublicInbox::Hval->new_msgid($mid);
- $m = $state->{upfx} . $m->as_href . '/';
- $$dst .= "$pfx<a\nhref=\"$m\">";
+ my $id = '';
+ if ($state->{mapping}) {
+ $id = id_compress($mid, 1);
+ $m = '#m'.$id;
+ $id = "\nid=r".$id;
+ } else {
+ $m = $state->{upfx}.$m->as_href.'/';
+ }
+ $$dst .= "$pfx<a\nhref=\"$m\"$id>";
$$dst .= defined($s) ? "$s</a> $f\n" : "$f</a>\n";
}
sub skel_dump {
my ($state, $level, $node) = @_;
if (my $mime = $node->message) {
- my $hdr = $mime->header_obj;
- my $mid = mid_clean($hdr->header_raw('Message-ID'));
- _skel_header($state, $hdr, $level);
+ _skel_header($state, $mime->header_obj, $level);
} else {
my $mid = $node->messageid;
my $dst = $state->{dst};
if ($mid eq 'subject dummy') {
$$dst .= "\t[no common parent]\n";
+ return;
+ }
+ if ($state->{pct}) { # search result
+ $$dst .= ' [irrelevant] ';
} else {
$$dst .= ' [not found] ';
- $$dst .= indent_for($level) . th_pfx($level);
- $mid = PublicInbox::Hval->new_msgid($mid);
- my $href = $state->{upfx} . $mid->as_href . '/';
- my $html = $mid->as_html;
- $$dst .= qq{<<a\nhref="$href">$html</a>>\n};
}
+ $$dst .= indent_for($level) . th_pfx($level);
+ my $upfx = $state->{upfx};
+ my $id = '';
+ if ($state->{mapping}) { # thread index view
+ $id = "\nid=".anchor_for($mid);
+ }
+ $mid = PublicInbox::Hval->new_msgid($mid);
+ my $href = $upfx . $mid->as_href . '/';
+ my $html = $mid->as_html;
+ $$dst .= qq{<<a\nhref="$href"$id>$html</a>>\n};
}
}
diff --git a/lib/PublicInbox/WWW.pm b/lib/PublicInbox/WWW.pm
index d6b07bf..984268e 100644
--- a/lib/PublicInbox/WWW.pm
+++ b/lib/PublicInbox/WWW.pm
@@ -23,7 +23,7 @@ require PublicInbox::Git;
use PublicInbox::GitHTTPBackend;
our $INBOX_RE = qr!\A/([\w\.\-]+)!;
our $MID_RE = qr!([^/]+)!;
-our $END_RE = qr!(T/|t/|t\.mbox(?:\.gz)?|t\.atom|raw|)!;
+our $END_RE = qr!(t/|t\.mbox(?:\.gz)?|t\.atom|raw|)!;
our $ATTACH_RE = qr!(\d[\.\d]*)-([[:alnum:]][\w\.-]+[[:alnum:]])!i;
sub new {
@@ -91,10 +91,9 @@ sub call {
invalid_inbox_mid($self, $ctx, $1, $2) ||
get_attach($ctx, $idx, $fn);
# in case people leave off the trailing slash:
- } elsif ($path_info =~ m!$INBOX_RE/$MID_RE/(T|t)\z!o) {
- my ($inbox, $mid, $suffix) = ($1, $2, $3);
- $suffix .= $suffix =~ /\A[tT]\z/ ? '/#u' : '/';
- r301($ctx, $inbox, $mid, $suffix);
+ } elsif ($path_info =~ m!$INBOX_RE/$MID_RE/(?:T|T/|t)\z!o) {
+ my ($inbox, $mid) = ($1, $2);
+ r301($ctx, $inbox, $mid, 't/#u');
} elsif ($path_info =~ m!$INBOX_RE/$MID_RE/R/?\z!o) {
my ($inbox, $mid) = ($1, $2);
diff --git a/t/plack.t b/t/plack.t
index a4f3245..209c6f9 100644
--- a/t/plack.t
+++ b/t/plack.t
@@ -101,7 +101,7 @@ EOF
my $res = $cb->(GET($u));
is(301, $res->code, "redirect for missing /");
my $location = $res->header('Location');
- like($location, qr!/\Q$t\E/#u\z!,
+ like($location, qr!/t/#u\z!,
'redirected with missing /');
});
}
--
EW
^ permalink raw reply related [relevance 2%]
* [PATCH 0/13] www: hybrid flat+thread conversation view
@ 2016-06-30 9:21 7% Eric Wong
2016-06-30 9:21 2% ` [PATCH 01/13] www: implement " Eric Wong
0 siblings, 1 reply; 2+ results
From: Eric Wong @ 2016-06-30 9:21 UTC (permalink / raw)
To: meta
I've been long-dreaming of this and finally it's at least
publishable (I hope :x). This flat view with thread skeletons
is 100% more usable than the dumb old one, but a little slower
(naturally :<)
I was originally hoping to remove the threaded /t/ endpoint
conversation view entirely to reduce server/caching overheads
but I still find it more usable in some situations.
What I still enjoy is being able to toggle between
[flat|threaded] views.
Eric Wong (13):
www: implement hybrid flat+thread conversation view
www: use WwwStream for dumping thread and search views
view: show thread context in the thread-aware flat view
view: merge $state hash with existing $ctx
feed: add $INBOX/new.html endpoint
view: tweak thread/index header slightly
view: show more nearby messages in flat thread view
www: reinstate old thread view as an option
view: fix up some HTML injection via Message-ID vectors
view: default to flat/hybrid thread display
view: show thread size when linking to summary
view: fixup bad reference to new_msgid
www_stream: add response wrapper sub
TODO | 2 -
lib/PublicInbox/Feed.pm | 51 +++-
lib/PublicInbox/SearchView.pm | 141 +++++-----
lib/PublicInbox/View.pm | 590 +++++++++++++++++++++++-------------------
lib/PublicInbox/WWW.pm | 22 +-
lib/PublicInbox/WwwStream.pm | 20 +-
t/view.t | 3 +-
7 files changed, 464 insertions(+), 365 deletions(-)
^ permalink raw reply [relevance 7%]
Results 1-2 of 2 | reverse | options above
-- pct% links below jump to the message on this page, permalinks otherwise --
2016-06-30 9:21 7% [PATCH 0/13] www: hybrid flat+thread conversation view Eric Wong
2016-06-30 9:21 2% ` [PATCH 01/13] www: implement " Eric Wong
Code repositories for project(s) associated with this public inbox
https://80x24.org/public-inbox.git
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).