* [PATCH 02/13] www: use WwwStream for dumping thread and search views
2016-06-30 9:21 7% [PATCH 0/13] www: hybrid flat+thread conversation view Eric Wong
@ 2016-06-30 9:21 3% ` Eric Wong
0 siblings, 0 replies; 2+ results
From: Eric Wong @ 2016-06-30 9:21 UTC (permalink / raw)
To: meta
This allows us the HTTP server to react to backpressure
from slow clients when writing. As a side effect, this
also makes it easier for us to maintain a consistent
header/footer across our HTML.
---
lib/PublicInbox/Feed.pm | 3 +-
lib/PublicInbox/SearchView.pm | 106 +++++++++++++++++++++---------------------
lib/PublicInbox/View.pm | 103 +++++++++++++++-------------------------
lib/PublicInbox/WWW.pm | 9 ++--
lib/PublicInbox/WwwStream.pm | 14 +++++-
5 files changed, 108 insertions(+), 127 deletions(-)
diff --git a/lib/PublicInbox/Feed.pm b/lib/PublicInbox/Feed.pm
index 8e23306..36802fa 100644
--- a/lib/PublicInbox/Feed.pm
+++ b/lib/PublicInbox/Feed.pm
@@ -168,12 +168,13 @@ sub emit_html_index {
sub emit_index_nosrch {
my ($ctx, $state) = @_;
my $ibx = $ctx->{-inbox};
+ my $fh = $state->{fh};
my (undef, $last) = each_recent_blob($ctx, sub {
my ($path, $commit, $ts, $u, $subj) = @_;
$state->{first} ||= $commit;
my $mime = do_cat_mail($ibx, $path) or return 0;
- PublicInbox::View::index_entry($mime, 0, $state);
+ $fh->write(PublicInbox::View::index_entry($mime, $state));
1;
});
$last;
diff --git a/lib/PublicInbox/SearchView.pm b/lib/PublicInbox/SearchView.pm
index fbef411..488822e 100644
--- a/lib/PublicInbox/SearchView.pm
+++ b/lib/PublicInbox/SearchView.pm
@@ -8,12 +8,14 @@ use warnings;
use PublicInbox::SearchMsg;
use PublicInbox::Hval qw/ascii_html/;
use PublicInbox::View;
-use PublicInbox::MID qw(mid2path mid_mime);
+use PublicInbox::MID qw(mid2path mid_mime mid_clean);
use Email::MIME;
require PublicInbox::Git;
require PublicInbox::Thread;
our $LIM = 50;
+sub noop {}
+
sub sres_top_html {
my ($ctx) = @_;
my $q = PublicInbox::SearchQuery->new($ctx->{qp});
@@ -27,44 +29,46 @@ sub sres_top_html {
relevance => $q->{r},
};
my ($mset, $total);
-
eval {
- $mset = $ctx->{srch}->query($q->{q}, $opts);
+ $mset = $ctx->{srch}->query($q->{'q'}, $opts);
$total = $mset->get_matches_estimated;
};
my $err = $@;
- my $res = html_start($q, $ctx) . '<pre>';
+ ctx_prepare($q, $ctx);
+ my $cb;
if ($err) {
$code = 400;
- $res .= err_txt($ctx, $err) . "</pre><hr /><pre>" . foot($ctx);
+ $ctx->{-html_tip} = '<pre>'.err_txt($ctx, $err).'</pre><hr />';
+ $cb = *noop;
} elsif ($total == 0) {
$code = 404;
- $res .= "\n\n[No results found]</pre><hr /><pre>".foot($ctx);
+ $ctx->{-html_tip} = "<pre>\n[No results found]</pre><hr />";
+ $cb = *noop;
} else {
my $x = $q->{x};
return sub { adump($_[0], $mset, $q, $ctx) } if ($x eq 'A');
- $res .= search_nav_top($mset, $q) . "\n\n";
+ $ctx->{-html_tip} = search_nav_top($mset, $q) . "\n\n";
if ($x eq 't') {
- return sub { tdump($_[0], $res, $mset, $q, $ctx) };
+ $cb = mset_thread($ctx, $mset, $q);
+ } else {
+ $cb = mset_summary($ctx, $mset, $q);
}
- dump_mset(\$res, $mset);
- $res .= '</pre>' . search_nav_bot($mset, $q) .
- "\n\n" . foot($ctx);
}
- $res .= "</pre></body></html>";
- [$code, ['Content-Type'=>'text/html; charset=UTF-8'], [$res]];
+ [ $code, ['Content-Type', 'text/html; charset=UTF-8'],
+ PublicInbox::WwwStream->new($ctx, $cb) ];
}
# display non-threaded search results similar to what users expect from
# regular WWW search engines:
-sub dump_mset {
- my ($res, $mset) = @_;
+sub mset_summary {
+ my ($ctx, $mset, $q) = @_;
my $total = $mset->get_matches_estimated;
my $pad = length("$total");
my $pfx = ' ' x $pad;
+ my $res = \($ctx->{-html_tip});
foreach my $m ($mset->items) {
my $rank = sprintf("%${pad}d", $m->get_rank + 1);
my $pct = $m->get_percent;
@@ -77,6 +81,8 @@ sub dump_mset {
$s . "</a></b>\n";
$$res .= "$pfx - by $f @ $ts UTC [$pct%]\n\n";
}
+ $$res .= search_nav_bot($mset, $q);
+ *noop;
}
sub err_txt {
@@ -85,14 +91,14 @@ sub err_txt {
$u = PublicInbox::Hval::prurl($ctx->{cgi}->{env}, $u);
$err =~ s/^\s*Exception:\s*//; # bad word to show users :P
$err = ascii_html($err);
- "\n\nBad query: <b>$err</b>\n" .
+ "\nBad query: <b>$err</b>\n" .
qq{See <a\nhref="$u">$u</a> for Xapian query syntax};
}
sub search_nav_top {
my ($mset, $q) = @_;
- my $rv = "Search results ordered by [";
+ my $rv = "<pre>Search results ordered by [";
if ($q->{r}) {
my $d = $q->qs_html(r => 0);
$rv .= qq{<a\nhref="?$d">date</a>|<b>relevance</b>};
@@ -122,7 +128,7 @@ sub search_nav_bot {
my $o = $q->{o};
my $end = $o + $nr;
my $beg = $o + 1;
- my $rv = "<hr /><pre>Results $beg-$end of $total";
+ my $rv = "</pre><hr /><pre>Results $beg-$end of $total";
my $n = $o + $LIM;
if ($n < $total) {
@@ -135,13 +141,11 @@ sub search_nav_bot {
my $qs = $q->qs_html(o => ($p > 0 ? $p : 0));
$rv .= qq{<a\nhref="?$qs"\nrel=prev>prev</a>};
}
- $rv;
+ $rv .= '</pre>';
}
-sub tdump {
- my ($cb, $res, $mset, $q, $ctx) = @_;
- my $fh = $cb->([200, ['Content-Type'=>'text/html; charset=UTF-8']]);
- $fh->write($res .= '</pre>');
+sub mset_thread {
+ my ($ctx, $mset, $q) = @_;
my %pct;
my @m = map {
my $i = $_;
@@ -163,14 +167,14 @@ sub tdump {
} else { # order by time (default for threaded view)
$th->order(*PublicInbox::View::sort_ts);
}
- my $skel = '';
+ my $skel = search_nav_bot($mset, $q). "<pre>";
+ my $inbox = $ctx->{-inbox};
my $state = {
- -inbox => $ctx->{-inbox},
+ -inbox => $inbox,
anchor_idx => 1,
ctx => $ctx,
cur_level => 0,
dst => \$skel,
- fh => $fh,
mapping => {},
pct => \%pct,
prev_attr => '',
@@ -179,42 +183,40 @@ sub tdump {
srch => $ctx->{srch},
upfx => './',
};
- $ctx->{searchview} = 1;
+
PublicInbox::View::walk_thread($th, $state,
*PublicInbox::View::pre_thread);
- PublicInbox::View::thread_entry($state, $_, 0) for @m;
-
- $fh->write(search_nav_bot($mset, $q). "\n\n" . $skel . "\n" .
- foot($ctx). '</pre></body></html>');
-
- $fh->close;
-}
-
-sub foot {
- my ($ctx) = @_;
- my $foot = $ctx->{footer} || '';
- qq{Back to <a\nhref=".">index</a>.\n$foot};
+ my $msgs = \@m;
+ my $mime;
+ sub {
+ return unless $msgs;
+ while ($mime = shift @$msgs) {
+ my $mid = mid_clean(mid_mime($mime));
+ $mime = $inbox->msg_by_mid($mid) and last;
+ }
+ if ($mime) {
+ $mime = Email::MIME->new($mime);
+ return PublicInbox::View::index_entry($mime, $state);
+ }
+ $msgs = undef;
+ $skel .= "\n</pre>";
+ };
}
-sub html_start {
+sub ctx_prepare {
my ($q, $ctx) = @_;
my $qh = ascii_html($q->{'q'});
- my $A = $q->qs_html(x => 'A', r => undef);
- my $res = '<html><head>' . PublicInbox::Hval::STYLE .
- "<title>$qh - search results</title>" .
- qq{<link\nrel=alternate\ntitle="Atom feed"\n} .
- qq!href="?$A"\ntype="application/atom+xml"/></head>! .
- qq{<body><form\naction="">} .
- qq{<input\nname=q\nvalue="$qh"\ntype=text />};
-
- $res .= qq{<input\ntype=hidden\nname=r />} if $q->{r};
+ $ctx->{-q_value_html} = $qh;
+ $ctx->{-atom} = '?'.$q->qs_html(x => 'A', r => undef);
+ $ctx->{-title_html} = "$qh - search results";
+ my $extra = '';
+ $extra .= qq{<input\ntype=hidden\nname=r />} if $q->{r};
if (my $x = $q->{x}) {
$x = ascii_html($x);
- $res .= qq{<input\ntype=hidden\nname=x\nvalue="$x" />};
+ $extra .= qq{<input\ntype=hidden\nname=x\nvalue="$x" />};
}
-
- $res .= qq{<input\ntype=submit\nvalue=search /></form>};
+ $ctx->{-extra_form_html} = $extra;
}
sub adump {
diff --git a/lib/PublicInbox/View.pm b/lib/PublicInbox/View.pm
index 65788db..a774feb 100644
--- a/lib/PublicInbox/View.pm
+++ b/lib/PublicInbox/View.pm
@@ -97,8 +97,7 @@ sub nr_to_s ($$$) {
# this is already inside a <pre>
sub index_entry {
- my ($mime, $level, $state) = @_;
- $state->{anchor_idx}++;
+ my ($mime, $state) = @_;
my $ctx = $state->{ctx};
my $srch = $ctx->{srch};
my $hdr = $mime->header_obj;
@@ -118,7 +117,7 @@ sub index_entry {
$subj = "<u\nid=u>$subj</u>" if $root_anchor eq $id_m;
my $ts = _msg_date($hdr);
- my $rv = "<pre><a\nhref=#e$id\nid=$id_m>#</a> ";
+ my $rv = "<a\nhref=#e$id\nid=$id_m>#</a> ";
$rv .= $subj;
my $mhref = $path.$href.'/';
my $from = _hdr_names($hdr, 'From');
@@ -154,13 +153,7 @@ sub index_entry {
if (my $pct = $state->{pct}) { # used by SearchView.pm
$rv .= " [relevance $pct->{$mid_raw}%]";
}
- $state->{fh}->write($rv .= "\n</pre>"); # '\n' for lynx
-}
-
-sub thread_html {
- my ($ctx, $foot, $srch) = @_;
- # $_[0] in sub is the Plack callback
- sub { emit_thread_html($_[0], $ctx, $foot, $srch) }
+ $rv .= "\n\n";
}
sub walk_thread {
@@ -186,25 +179,26 @@ sub pre_thread {
skel_dump($state, $level, $node);
}
-# only private functions below.
-
-sub emit_thread_html {
- my ($res, $ctx, $foot, $srch) = @_;
+sub thread_html {
+ my ($ctx) = @_;
my $mid = $ctx->{mid};
- my $sres = $srch->get_thread($mid, { asc => 1 });
+ my $sres = $ctx->{srch}->get_thread($mid, { asc => 1 });
my $msgs = load_results($sres);
my $nr = $sres->{total};
- return missing_thread($res, $ctx) if $nr == 0;
- my $skel = '';
+ return missing_thread($ctx) if $nr == 0;
+ my $skel = '</pre><hr /><pre>';
+ $skel .= $nr == 1 ? 'only message in thread' : 'end of thread';
+ $skel .= ", back to <a\nhref=\"../../\">index</a>";
+ $skel .= "\n<a\nid=t>$nr+ messages in thread:</a> (download: ";
+ $skel .= "<a\nhref=\"../t.mbox.gz\">mbox.gz</a>";
+ $skel .= " / follow: <a\nhref=\"../t.atom\">Atom feed</a>)\n";
my $state = {
- anchor_idx => 0,
ctx => $ctx,
cur_level => 0,
dst => \$skel,
mapping => {}, # mid -> reply count
prev_attr => '',
prev_level => 0,
- res => $res,
root_anchor => anchor_for($mid),
seen => {},
srch => $ctx->{srch},
@@ -213,21 +207,28 @@ sub emit_thread_html {
walk_thread(thread_results($msgs), $state, *pre_thread);
- thread_entry($state, $_, 0) for @$msgs;
-
- # there could be a race due to a message being deleted in git
- # but still being in the Xapian index:
- my $fh = delete $state->{fh} or return missing_thread($res, $ctx);
-
- my $next = @$msgs == 1 ? 'only message in thread' : 'end of thread';
- $next .= ", back to <a\nhref=\"../../\">index</a>";
- $next .= "\n<a\nid=t>$nr+ messages in thread:</a> (download: ";
- $next .= "<a\nhref=\"../t.mbox.gz\">mbox.gz</a>";
- $next .= " / follow: <a\nhref=\"../t.atom\">Atom feed</a>)\n";
- $next .= $skel;
- $fh->write('<hr /><pre>' . $next . "\n\n".
- $foot . '</pre></body></html>');
- $fh->close;
+ # lazy load the full message from mini_mime:
+ my $inbox = $ctx->{-inbox};
+ my $mime;
+ while ($mime = shift @$msgs) {
+ $mime = $inbox->msg_by_mid(mid_clean(mid_mime($mime))) and last;
+ }
+ $mime = Email::MIME->new($mime);
+ $ctx->{-upfx} = '../../';
+ $ctx->{-title_html} = ascii_html($mime->header('Subject'));
+ $ctx->{-html_tip} = '<pre>'.index_entry($mime, $state);
+ $mime = undef;
+ my $body = PublicInbox::WwwStream->new($ctx, sub {
+ return unless $msgs;
+ while ($mime = shift @$msgs) {
+ $mid = mid_clean(mid_mime($mime));
+ $mime = $inbox->msg_by_mid($mid) and last;
+ }
+ return index_entry(Email::MIME->new($mime), $state) if $mime;
+ $msgs = undef;
+ $skel .= "</pre>";
+ });
+ [ 200, ['Content-Type', 'text/html; charset=UTF-8'], $body ];
}
sub multipart_text_as_html {
@@ -539,20 +540,6 @@ sub anchor_for {
'm' . id_compress($msgid, 1);
}
-sub thread_html_head {
- my ($hdr, $state) = @_;
- my $res = delete $state->{res} or die "BUG: no Plack callback in {res}";
- my $fh = $res->([200, ['Content-Type'=> 'text/html; charset=UTF-8']]);
- $state->{fh} = $fh;
-
- my $s = ascii_html($hdr->header('Subject'));
- $fh->write("<html><head><title>$s</title>".
- qq{<link\nrel=alternate\ntitle="Atom feed"\n} .
- qq!href="../t.atom"\ntype="application/atom+xml"/>! .
- PublicInbox::Hval::STYLE .
- "</head><body>");
-}
-
sub ghost_parent {
my ($upfx, $mid) = @_;
# 'subject dummy' is used internally by Mail::Thread
@@ -564,21 +551,6 @@ sub ghost_parent {
qq{[parent not found: <<a\nhref="$upfx$href/">$html</a>>]};
}
-sub thread_entry {
- my ($state, $mime, $level) = @_;
-
- # lazy load the full message from mini_mime:
- $mime = eval {
- my $mid = mid_clean(mid_mime($mime));
- $state->{ctx}->{-inbox}->msg_by_mid($mid);
- } or return;
- $mime = Email::MIME->new($mime);
-
- thread_html_head($mime, $state) if $state->{anchor_idx} == 0;
- index_entry($mime, $level, $state);
- 1;
-}
-
sub indent_for {
my ($level) = @_;
INDENT x ($level - 1);
@@ -606,10 +578,9 @@ sub thread_results {
}
sub missing_thread {
- my ($res, $ctx) = @_;
+ my ($ctx) = @_;
require PublicInbox::ExtMsg;
-
- $res->(PublicInbox::ExtMsg::ext_msg($ctx))
+ PublicInbox::ExtMsg::ext_msg($ctx);
}
sub _msg_date {
diff --git a/lib/PublicInbox/WWW.pm b/lib/PublicInbox/WWW.pm
index 984268e..196486f 100644
--- a/lib/PublicInbox/WWW.pm
+++ b/lib/PublicInbox/WWW.pm
@@ -233,12 +233,10 @@ sub get_mid_html {
# /$INBOX/$MESSAGE_ID/t/
sub get_thread {
- my ($ctx, $flat) = @_;
- my $srch = searcher($ctx) or return need_search($ctx);
+ my ($ctx) = @_;
+ searcher($ctx) or return need_search($ctx);
require PublicInbox::View;
- my $foot = footer($ctx);
- $ctx->{flat} = $flat;
- PublicInbox::View::thread_html($ctx, $foot, $srch);
+ PublicInbox::View::thread_html($ctx);
}
sub ctx_get {
@@ -414,7 +412,6 @@ sub msg_page {
't.atom' eq $e and return get_thread_atom($ctx);
't.mbox' eq $e and return get_thread_mbox($ctx);
't.mbox.gz' eq $e and return get_thread_mbox($ctx, '.gz');
- 'T/' eq $e and return get_thread($ctx, 1);
'raw' eq $e and return get_mid_txt($ctx);
# legacy, but no redirect for compatibility:
diff --git a/lib/PublicInbox/WwwStream.pm b/lib/PublicInbox/WwwStream.pm
index 34f32c0..d2bf318 100644
--- a/lib/PublicInbox/WwwStream.pm
+++ b/lib/PublicInbox/WwwStream.pm
@@ -22,10 +22,20 @@ sub _html_top ($) {
my $title = $ctx->{-title_html} || $desc;
my $upfx = $ctx->{-upfx} || '';
my $atom = $ctx->{-atom} || $upfx.'new.atom';
+ my $tip = $ctx->{-html_tip} || '';
my $top = "<b>$desc</b> (<a\nhref=\"$atom\">Atom feed</a>)";
if ($obj->search) {
+ my $q_val = $ctx->{-q_value_html};
+ if (defined $q_val && $q_val ne '') {
+ $q_val = qq(\nvalue="$q_val" );
+ } else {
+ $q_val = '';
+ }
+ # XXX gross, for SearchView.pm
+ my $extra = $ctx->{-extra_form_html} || '';
$top = qq{<form\naction="$upfx"><pre>$top} .
- qq{ <input\nname=q\ntype=text />} .
+ qq{ <input\nname=q\ntype=text$q_val/>} .
+ $extra .
qq{<input\ntype=submit\nvalue=search />} .
q{</pre></form>}
} else {
@@ -35,7 +45,7 @@ sub _html_top ($) {
"<link\nrel=alternate\ntitle=\"Atom feed\"\n".
"href=\"$atom\"\ntype=\"application/atom+xml\"/>" .
PublicInbox::Hval::STYLE .
- "</head><body>$top";
+ "</head><body>". $top . $tip;
}
sub _html_end {
--
EW
^ permalink raw reply related [relevance 3%]
* [PATCH 0/13] www: hybrid flat+thread conversation view
@ 2016-06-30 9:21 7% Eric Wong
2016-06-30 9:21 3% ` [PATCH 02/13] www: use WwwStream for dumping thread and search views Eric Wong
0 siblings, 1 reply; 2+ results
From: Eric Wong @ 2016-06-30 9:21 UTC (permalink / raw)
To: meta
I've been long-dreaming of this and finally it's at least
publishable (I hope :x). This flat view with thread skeletons
is 100% more usable than the dumb old one, but a little slower
(naturally :<)
I was originally hoping to remove the threaded /t/ endpoint
conversation view entirely to reduce server/caching overheads
but I still find it more usable in some situations.
What I still enjoy is being able to toggle between
[flat|threaded] views.
Eric Wong (13):
www: implement hybrid flat+thread conversation view
www: use WwwStream for dumping thread and search views
view: show thread context in the thread-aware flat view
view: merge $state hash with existing $ctx
feed: add $INBOX/new.html endpoint
view: tweak thread/index header slightly
view: show more nearby messages in flat thread view
www: reinstate old thread view as an option
view: fix up some HTML injection via Message-ID vectors
view: default to flat/hybrid thread display
view: show thread size when linking to summary
view: fixup bad reference to new_msgid
www_stream: add response wrapper sub
TODO | 2 -
lib/PublicInbox/Feed.pm | 51 +++-
lib/PublicInbox/SearchView.pm | 141 +++++-----
lib/PublicInbox/View.pm | 590 +++++++++++++++++++++++-------------------
lib/PublicInbox/WWW.pm | 22 +-
lib/PublicInbox/WwwStream.pm | 20 +-
t/view.t | 3 +-
7 files changed, 464 insertions(+), 365 deletions(-)
^ permalink raw reply [relevance 7%]
Results 1-2 of 2 | reverse | options above
-- pct% links below jump to the message on this page, permalinks otherwise --
2016-06-30 9:21 7% [PATCH 0/13] www: hybrid flat+thread conversation view Eric Wong
2016-06-30 9:21 3% ` [PATCH 02/13] www: use WwwStream for dumping thread and search views Eric Wong
Code repositories for project(s) associated with this public inbox
https://80x24.org/public-inbox.git
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).