www: use WwwStream for dumping thread and search views

This allows us the HTTP server to react to backpressure
from slow clients when writing.  As a side effect, this
also makes it easier for us to maintain a consistent
header/footer across our HTML.

5 files changed, 108 insertions, 127 deletions

diff --git a/lib/PublicInbox/Feed.pm b/lib/PublicInbox/Feed.pm
index 8e233061..36802fa1 100644
--- a/lib/PublicInbox/Feed.pm
+++ b/lib/PublicInbox/Feed.pm
@@ -168,12 +168,13 @@ sub emit_html_index {
 sub emit_index_nosrch {
         my ($ctx, $state) = @_;
         my $ibx = $ctx->{-inbox};
+        my $fh = $state->{fh};
         my (undef, $last) = each_recent_blob($ctx, sub {
                 my ($path, $commit, $ts, $u, $subj) = @_;
                 $state->{first} ||= $commit;
 
                 my $mime = do_cat_mail($ibx, $path) or return 0;
-                PublicInbox::View::index_entry($mime, 0, $state);
+                $fh->write(PublicInbox::View::index_entry($mime, $state));
                 1;
         });
         $last;
diff --git a/lib/PublicInbox/SearchView.pm b/lib/PublicInbox/SearchView.pm
index fbef4116..488822e6 100644
--- a/lib/PublicInbox/SearchView.pm
+++ b/lib/PublicInbox/SearchView.pm
@@ -8,12 +8,14 @@ use warnings;
 use PublicInbox::SearchMsg;
 use PublicInbox::Hval qw/ascii_html/;
 use PublicInbox::View;
-use PublicInbox::MID qw(mid2path mid_mime);
+use PublicInbox::MID qw(mid2path mid_mime mid_clean);
 use Email::MIME;
 require PublicInbox::Git;
 require PublicInbox::Thread;
 our $LIM = 50;
 
+sub noop {}
+
 sub sres_top_html {
         my ($ctx) = @_;
         my $q = PublicInbox::SearchQuery->new($ctx->{qp});
@@ -27,44 +29,46 @@ sub sres_top_html {
                 relevance => $q->{r},
         };
         my ($mset, $total);
-
         eval {
-                $mset = $ctx->{srch}->query($q->{q}, $opts);
+                $mset = $ctx->{srch}->query($q->{'q'}, $opts);
                 $total = $mset->get_matches_estimated;
         };
         my $err = $@;
-        my $res = html_start($q, $ctx) . '<pre>';
+        ctx_prepare($q, $ctx);
+        my $cb;
         if ($err) {
                 $code = 400;
-                $res .= err_txt($ctx, $err) . "</pre><hr /><pre>" . foot($ctx);
+                $ctx->{-html_tip} = '<pre>'.err_txt($ctx, $err).'</pre><hr />';
+                $cb = *noop;
         } elsif ($total == 0) {
                 $code = 404;
-                $res .= "\n\n[No results found]</pre><hr /><pre>".foot($ctx);
+                $ctx->{-html_tip} = "<pre>\n[No results found]</pre><hr />";
+                $cb = *noop;
         } else {
                 my $x = $q->{x};
                 return sub { adump($_[0], $mset, $q, $ctx) } if ($x eq 'A');
 
-                $res .= search_nav_top($mset, $q) . "\n\n";
+                $ctx->{-html_tip} = search_nav_top($mset, $q) . "\n\n";
                 if ($x eq 't') {
-                        return sub { tdump($_[0], $res, $mset, $q, $ctx) };
+                        $cb = mset_thread($ctx, $mset, $q);
+                } else {
+                        $cb = mset_summary($ctx, $mset, $q);
                 }
-                dump_mset(\$res, $mset);
-                $res .= '</pre>' . search_nav_bot($mset, $q) .
-                        "\n\n" . foot($ctx);
         }
 
-        $res .= "</pre></body></html>";
-        [$code, ['Content-Type'=>'text/html; charset=UTF-8'], [$res]];
+        [ $code, ['Content-Type', 'text/html; charset=UTF-8'],
+                PublicInbox::WwwStream->new($ctx, $cb) ];
 }
 
 # display non-threaded search results similar to what users expect from
 # regular WWW search engines:
-sub dump_mset {
-        my ($res, $mset) = @_;
+sub mset_summary {
+        my ($ctx, $mset, $q) = @_;
 
         my $total = $mset->get_matches_estimated;
         my $pad = length("$total");
         my $pfx = ' ' x $pad;
+        my $res = \($ctx->{-html_tip});
         foreach my $m ($mset->items) {
                 my $rank = sprintf("%${pad}d", $m->get_rank + 1);
                 my $pct = $m->get_percent;
@@ -77,6 +81,8 @@ sub dump_mset {
                         $s . "</a></b>\n";
                 $$res .= "$pfx  - by $f @ $ts UTC [$pct%]\n\n";
         }
+        $$res .= search_nav_bot($mset, $q);
+        *noop;
 }
 
 sub err_txt {
@@ -85,14 +91,14 @@ sub err_txt {
         $u = PublicInbox::Hval::prurl($ctx->{cgi}->{env}, $u);
         $err =~ s/^\s*Exception:\s*//; # bad word to show users :P
         $err = ascii_html($err);
-        "\n\nBad query: <b>$err</b>\n" .
+        "\nBad query: <b>$err</b>\n" .
                 qq{See <a\nhref="$u">$u</a> for Xapian query syntax};
 }
 
 sub search_nav_top {
         my ($mset, $q) = @_;
 
-        my $rv = "Search results ordered by [";
+        my $rv = "<pre>Search results ordered by [";
         if ($q->{r}) {
                 my $d = $q->qs_html(r => 0);
                 $rv .= qq{<a\nhref="?$d">date</a>|<b>relevance</b>};
@@ -122,7 +128,7 @@ sub search_nav_bot {
         my $o = $q->{o};
         my $end = $o + $nr;
         my $beg = $o + 1;
-        my $rv = "<hr /><pre>Results $beg-$end of $total";
+        my $rv = "</pre><hr /><pre>Results $beg-$end of $total";
         my $n = $o + $LIM;
 
         if ($n < $total) {
@@ -135,13 +141,11 @@ sub search_nav_bot {
                 my $qs = $q->qs_html(o => ($p > 0 ? $p : 0));
                 $rv .= qq{<a\nhref="?$qs"\nrel=prev>prev</a>};
         }
-        $rv;
+        $rv .= '</pre>';
 }
 
-sub tdump {
-        my ($cb, $res, $mset, $q, $ctx) = @_;
-        my $fh = $cb->([200, ['Content-Type'=>'text/html; charset=UTF-8']]);
-        $fh->write($res .= '</pre>');
+sub mset_thread {
+        my ($ctx, $mset, $q) = @_;
         my %pct;
         my @m = map {
                 my $i = $_;
@@ -163,14 +167,14 @@ sub tdump {
         } else { # order by time (default for threaded view)
                 $th->order(*PublicInbox::View::sort_ts);
         }
-        my $skel = '';
+        my $skel = search_nav_bot($mset, $q). "<pre>";
+        my $inbox = $ctx->{-inbox};
         my $state = {
-                -inbox => $ctx->{-inbox},
+                -inbox => $inbox,
                 anchor_idx => 1,
                 ctx => $ctx,
                 cur_level => 0,
                 dst => \$skel,
-                fh => $fh,
                 mapping => {},
                 pct => \%pct,
                 prev_attr => '',
@@ -179,42 +183,40 @@ sub tdump {
                 srch => $ctx->{srch},
                 upfx => './',
         };
-        $ctx->{searchview} = 1;
+
         PublicInbox::View::walk_thread($th, $state,
                 *PublicInbox::View::pre_thread);
 
-        PublicInbox::View::thread_entry($state, $_, 0) for @m;
-
-        $fh->write(search_nav_bot($mset, $q). "\n\n" . $skel . "\n" .
-                        foot($ctx). '</pre></body></html>');
-
-        $fh->close;
-}
-
-sub foot {
-        my ($ctx) = @_;
-        my $foot = $ctx->{footer} || '';
-        qq{Back to <a\nhref=".">index</a>.\n$foot};
+        my $msgs = \@m;
+        my $mime;
+        sub {
+                return unless $msgs;
+                while ($mime = shift @$msgs) {
+                        my $mid = mid_clean(mid_mime($mime));
+                        $mime = $inbox->msg_by_mid($mid) and last;
+                }
+                if ($mime) {
+                        $mime = Email::MIME->new($mime);
+                        return PublicInbox::View::index_entry($mime, $state);
+                }
+                $msgs = undef;
+                $skel .= "\n</pre>";
+        };
 }
 
-sub html_start {
+sub ctx_prepare {
         my ($q, $ctx) = @_;
         my $qh = ascii_html($q->{'q'});
-        my $A = $q->qs_html(x => 'A', r => undef);
-        my $res = '<html><head>' . PublicInbox::Hval::STYLE .
-                "<title>$qh - search results</title>" .
-                qq{<link\nrel=alternate\ntitle="Atom feed"\n} .
-                qq!href="?$A"\ntype="application/atom+xml"/></head>! .
-                qq{<body><form\naction="">} .
-                qq{<input\nname=q\nvalue="$qh"\ntype=text />};
-
-        $res .= qq{<input\ntype=hidden\nname=r />} if $q->{r};
+        $ctx->{-q_value_html} = $qh;
+        $ctx->{-atom} = '?'.$q->qs_html(x => 'A', r => undef);
+        $ctx->{-title_html} = "$qh - search results";
+        my $extra = '';
+        $extra .= qq{<input\ntype=hidden\nname=r />} if $q->{r};
         if (my $x = $q->{x}) {
                 $x = ascii_html($x);
-                $res .= qq{<input\ntype=hidden\nname=x\nvalue="$x" />};
+                $extra .= qq{<input\ntype=hidden\nname=x\nvalue="$x" />};
         }
-
-        $res .= qq{<input\ntype=submit\nvalue=search /></form>};
+        $ctx->{-extra_form_html} = $extra;
 }
 
 sub adump {
diff --git a/lib/PublicInbox/View.pm b/lib/PublicInbox/View.pm
index 65788dbe..a774febd 100644
--- a/lib/PublicInbox/View.pm
+++ b/lib/PublicInbox/View.pm
@@ -97,8 +97,7 @@ sub nr_to_s ($$$) {
 
 # this is already inside a <pre>
 sub index_entry {
-        my ($mime, $level, $state) = @_;
-        $state->{anchor_idx}++;
+        my ($mime, $state) = @_;
         my $ctx = $state->{ctx};
         my $srch = $ctx->{srch};
         my $hdr = $mime->header_obj;
@@ -118,7 +117,7 @@ sub index_entry {
         $subj = "<u\nid=u>$subj</u>" if $root_anchor eq $id_m;
 
         my $ts = _msg_date($hdr);
-        my $rv = "<pre><a\nhref=#e$id\nid=$id_m>#</a> ";
+        my $rv = "<a\nhref=#e$id\nid=$id_m>#</a> ";
         $rv .= $subj;
         my $mhref = $path.$href.'/';
         my $from = _hdr_names($hdr, 'From');
@@ -154,13 +153,7 @@ sub index_entry {
         if (my $pct = $state->{pct}) { # used by SearchView.pm
                 $rv .= " [relevance $pct->{$mid_raw}%]";
         }
-        $state->{fh}->write($rv .= "\n</pre>"); # '\n' for lynx
-}
-
-sub thread_html {
-        my ($ctx, $foot, $srch) = @_;
-        # $_[0] in sub is the Plack callback
-        sub { emit_thread_html($_[0], $ctx, $foot, $srch) }
+        $rv .= "\n\n";
 }
 
 sub walk_thread {
@@ -186,25 +179,26 @@ sub pre_thread  {
         skel_dump($state, $level, $node);
 }
 
-# only private functions below.
-
-sub emit_thread_html {
-        my ($res, $ctx, $foot, $srch) = @_;
+sub thread_html {
+        my ($ctx) = @_;
         my $mid = $ctx->{mid};
-        my $sres = $srch->get_thread($mid, { asc => 1 });
+        my $sres = $ctx->{srch}->get_thread($mid, { asc => 1 });
         my $msgs = load_results($sres);
         my $nr = $sres->{total};
-        return missing_thread($res, $ctx) if $nr == 0;
-        my $skel = '';
+        return missing_thread($ctx) if $nr == 0;
+        my $skel = '</pre><hr /><pre>';
+        $skel .= $nr == 1 ? 'only message in thread' : 'end of thread';
+        $skel .= ", back to <a\nhref=\"../../\">index</a>";
+        $skel .= "\n<a\nid=t>$nr+ messages in thread:</a> (download: ";
+        $skel .= "<a\nhref=\"../t.mbox.gz\">mbox.gz</a>";
+        $skel .= " / follow: <a\nhref=\"../t.atom\">Atom feed</a>)\n";
         my $state = {
-                anchor_idx => 0,
                 ctx => $ctx,
                 cur_level => 0,
                 dst => \$skel,
                 mapping => {}, # mid -> reply count
                 prev_attr => '',
                 prev_level => 0,
-                res => $res,
                 root_anchor => anchor_for($mid),
                 seen => {},
                 srch => $ctx->{srch},
@@ -213,21 +207,28 @@ sub emit_thread_html {
 
         walk_thread(thread_results($msgs), $state, *pre_thread);
 
-        thread_entry($state, $_, 0) for @$msgs;
-
-        # there could be a race due to a message being deleted in git
-        # but still being in the Xapian index:
-        my $fh = delete $state->{fh} or return missing_thread($res, $ctx);
-
-        my $next = @$msgs == 1 ? 'only message in thread' : 'end of thread';
-        $next .= ", back to <a\nhref=\"../../\">index</a>";
-        $next .= "\n<a\nid=t>$nr+ messages in thread:</a> (download: ";
-        $next .= "<a\nhref=\"../t.mbox.gz\">mbox.gz</a>";
-        $next .= " / follow: <a\nhref=\"../t.atom\">Atom feed</a>)\n";
-        $next .= $skel;
-        $fh->write('<hr /><pre>' . $next . "\n\n".
-                        $foot .  '</pre></body></html>');
-        $fh->close;
+        # lazy load the full message from mini_mime:
+        my $inbox = $ctx->{-inbox};
+        my $mime;
+        while ($mime = shift @$msgs) {
+                $mime = $inbox->msg_by_mid(mid_clean(mid_mime($mime))) and last;
+        }
+        $mime = Email::MIME->new($mime);
+        $ctx->{-upfx} = '../../';
+        $ctx->{-title_html} = ascii_html($mime->header('Subject'));
+        $ctx->{-html_tip} = '<pre>'.index_entry($mime, $state);
+        $mime = undef;
+        my $body = PublicInbox::WwwStream->new($ctx, sub {
+                return unless $msgs;
+                while ($mime = shift @$msgs) {
+                        $mid = mid_clean(mid_mime($mime));
+                        $mime = $inbox->msg_by_mid($mid) and last;
+                }
+                return index_entry(Email::MIME->new($mime), $state) if $mime;
+                $msgs = undef;
+                $skel .= "</pre>";
+        });
+        [ 200, ['Content-Type', 'text/html; charset=UTF-8'], $body ];
 }
 
 sub multipart_text_as_html {
@@ -539,20 +540,6 @@ sub anchor_for {
         'm' . id_compress($msgid, 1);
 }
 
-sub thread_html_head {
-        my ($hdr, $state) = @_;
-        my $res = delete $state->{res} or die "BUG: no Plack callback in {res}";
-        my $fh = $res->([200, ['Content-Type'=> 'text/html; charset=UTF-8']]);
-        $state->{fh} = $fh;
-
-        my $s = ascii_html($hdr->header('Subject'));
-        $fh->write("<html><head><title>$s</title>".
-                qq{<link\nrel=alternate\ntitle="Atom feed"\n} .
-                qq!href="../t.atom"\ntype="application/atom+xml"/>! .
-                PublicInbox::Hval::STYLE .
-                "</head><body>");
-}
-
 sub ghost_parent {
         my ($upfx, $mid) = @_;
         # 'subject dummy' is used internally by Mail::Thread
@@ -564,21 +551,6 @@ sub ghost_parent {
         qq{[parent not found: &lt;<a\nhref="$upfx$href/">$html</a>&gt;]};
 }
 
-sub thread_entry {
-        my ($state, $mime, $level) = @_;
-
-        # lazy load the full message from mini_mime:
-        $mime = eval {
-                my $mid = mid_clean(mid_mime($mime));
-                $state->{ctx}->{-inbox}->msg_by_mid($mid);
-        } or return;
-        $mime = Email::MIME->new($mime);
-
-        thread_html_head($mime, $state) if $state->{anchor_idx} == 0;
-        index_entry($mime, $level, $state);
-        1;
-}
-
 sub indent_for {
         my ($level) = @_;
         INDENT x ($level - 1);
@@ -606,10 +578,9 @@ sub thread_results {
 }
 
 sub missing_thread {
-        my ($res, $ctx) = @_;
+        my ($ctx) = @_;
         require PublicInbox::ExtMsg;
-
-        $res->(PublicInbox::ExtMsg::ext_msg($ctx))
+        PublicInbox::ExtMsg::ext_msg($ctx);
 }
 
 sub _msg_date {
diff --git a/lib/PublicInbox/WWW.pm b/lib/PublicInbox/WWW.pm
index 984268e9..196486f2 100644
--- a/lib/PublicInbox/WWW.pm
+++ b/lib/PublicInbox/WWW.pm
@@ -233,12 +233,10 @@ sub get_mid_html {
 
 # /$INBOX/$MESSAGE_ID/t/
 sub get_thread {
-        my ($ctx, $flat) = @_;
-        my $srch = searcher($ctx) or return need_search($ctx);
+        my ($ctx) = @_;
+        searcher($ctx) or return need_search($ctx);
         require PublicInbox::View;
-        my $foot = footer($ctx);
-        $ctx->{flat} = $flat;
-        PublicInbox::View::thread_html($ctx, $foot, $srch);
+        PublicInbox::View::thread_html($ctx);
 }
 
 sub ctx_get {
@@ -414,7 +412,6 @@ sub msg_page {
         't.atom' eq $e and return get_thread_atom($ctx);
         't.mbox' eq $e and return get_thread_mbox($ctx);
         't.mbox.gz' eq $e and return get_thread_mbox($ctx, '.gz');
-        'T/' eq $e and return get_thread($ctx, 1);
         'raw' eq $e and return get_mid_txt($ctx);
 
         # legacy, but no redirect for compatibility:
diff --git a/lib/PublicInbox/WwwStream.pm b/lib/PublicInbox/WwwStream.pm
index 34f32c0b..d2bf318b 100644
--- a/lib/PublicInbox/WwwStream.pm
+++ b/lib/PublicInbox/WwwStream.pm
@@ -22,10 +22,20 @@ sub _html_top ($) {
         my $title = $ctx->{-title_html} || $desc;
         my $upfx = $ctx->{-upfx} || '';
         my $atom = $ctx->{-atom} || $upfx.'new.atom';
+        my $tip = $ctx->{-html_tip} || '';
         my $top = "<b>$desc</b> (<a\nhref=\"$atom\">Atom feed</a>)";
         if ($obj->search) {
+                my $q_val = $ctx->{-q_value_html};
+                if (defined $q_val && $q_val ne '') {
+                        $q_val = qq(\nvalue="$q_val" );
+                } else {
+                        $q_val = '';
+                }
+                # XXX gross, for SearchView.pm
+                my $extra = $ctx->{-extra_form_html} || '';
                 $top = qq{<form\naction="$upfx"><pre>$top} .
-                          qq{ <input\nname=q\ntype=text />} .
+                          qq{ <input\nname=q\ntype=text$q_val/>} .
+                          $extra .
                           qq{<input\ntype=submit\nvalue=search />} .
                           q{</pre></form>}
         } else {
@@ -35,7 +45,7 @@ sub _html_top ($) {
                 "<link\nrel=alternate\ntitle=\"Atom feed\"\n".
                 "href=\"$atom\"\ntype=\"application/atom+xml\"/>" .
                 PublicInbox::Hval::STYLE .
-                "</head><body>$top";
+                "</head><body>". $top . $tip;
 }
 
 sub _html_end {