From 1704bdb4aabdbc155eb962edf51498f59b65c839 Mon Sep 17 00:00:00 2001 From: Eric Wong Date: Sat, 22 Aug 2015 00:06:45 +0000 Subject: stream HTML views as much as possible This should allow progressive rendering on the client and reduce memory usage on the server. Unfortunately XML::Atom::SimpleFeed does not yet support streaming, so we may not use it in the future. --- lib/PublicInbox/Feed.pm | 43 +++++++++++++++++++++---------------- lib/PublicInbox/Mbox.pm | 3 ++- lib/PublicInbox/View.pm | 57 +++++++++++++++++++++++++++++++------------------ lib/PublicInbox/WWW.pm | 8 ++----- 4 files changed, 65 insertions(+), 46 deletions(-) (limited to 'lib') diff --git a/lib/PublicInbox/Feed.pm b/lib/PublicInbox/Feed.pm index 40f08ac7..5d122ac6 100644 --- a/lib/PublicInbox/Feed.pm +++ b/lib/PublicInbox/Feed.pm @@ -52,7 +52,15 @@ sub generate { } sub generate_html_index { - my ($class, $ctx) = @_; + my ($ctx) = @_; + sub { emit_html_index($_[0], $ctx) }; +} + +# private subs + +sub emit_html_index { + my ($cb, $ctx) = @_; + my $fh = $cb->([200,['Content-Type'=>'text/html; charset=UTF-8']]); my $max = $ctx->{max} || MAX_PER_PAGE; my $feed_opts = get_feedopts($ctx); @@ -61,11 +69,11 @@ sub generate_html_index { $title = PublicInbox::Hval->new_oneline($title)->as_html; my $atom_url = $feed_opts->{atomurl}; - my $html = "$title" . - "" . - '' . PublicInbox::View::PRE_WRAP . - "$title (Atom feed)\n"; + $fh->write("$title" . + "" . + '' . PublicInbox::View::PRE_WRAP . + "$title (Atom feed)\n"); my $state; my $git = PublicInbox::GitCatFile->new($ctx->{git_dir}); @@ -80,8 +88,7 @@ sub generate_html_index { add_topic($git, $srch, $topics, $path, $ts, $u, $subj); } else { my $mime = do_cat_mail($git, $path) or return 0; - $html .= - PublicInbox::View->index_entry($mime, 0, $state); + PublicInbox::View::index_entry($fh, $mime, 0, $state); 1; } }); @@ -94,12 +101,11 @@ sub generate_html_index { $footer .= "\n" . $list_footer if $list_footer; $footer = "
$footer
"; } - dump_topics(\$html, $topics) if $topics; - $html .= "$footer"; + $fh->write(dump_topics($topics)) if $topics; + $fh->write("$footer"); + $fh->close; } -# private subs - sub nav_footer { my ($cgi, $last, $feed_opts, $state) = @_; $cgi or return ''; @@ -328,27 +334,28 @@ sub add_topic { } sub dump_topics { - my ($dst, $topics) = @_; + my ($topics) = @_; my ($order, $subjs) = @$topics; - $$dst .= "\n[No recent topics]" unless (scalar @$order); + my $dst = ''; + $dst .= "\n[No recent topics]" unless (scalar @$order); while (defined(my $info = shift @$order)) { my ($mid, $ts, $u, $subj) = @$info; my $n = delete $subjs->{$subj}; $mid = PublicInbox::Hval->new($mid)->as_href; $subj = PublicInbox::Hval->new($subj)->as_html; $u = PublicInbox::Hval->new($u)->as_html; - $$dst .= "\n$subj\n- "; + $dst .= "\n$subj\n- "; $ts = POSIX::strftime('%Y-%m-%d %H:%M', gmtime($ts)); if ($n == 1) { - $$dst .= "created by $u @ $ts UTC\n" + $dst .= "created by $u @ $ts UTC\n" } else { # $n isn't the total number of posts on the topic, # just the number of posts in the current "git log" # window, so leave it unlabeled - $$dst .= "updated by $u @ $ts UTC ($n)\n" + $dst .= "updated by $u @ $ts UTC ($n)\n" } } - $$dst .= '' + $dst .= '' } 1; diff --git a/lib/PublicInbox/Mbox.pm b/lib/PublicInbox/Mbox.pm index fcb26693..5f5612a4 100644 --- a/lib/PublicInbox/Mbox.pm +++ b/lib/PublicInbox/Mbox.pm @@ -80,6 +80,7 @@ The administrator needs to install the IO::Compress::Gzip Perl module to support gzipped mboxes. Return to index EOF + $fh->close; } 1; @@ -118,7 +119,7 @@ sub close { my ($self) = @_; $self->{gz}->close; _flush_buf($self); - # do not actually close $fh + $self->{fh}->close; } 1; diff --git a/lib/PublicInbox/View.pm b/lib/PublicInbox/View.pm index 8105affe..ab2720ba 100644 --- a/lib/PublicInbox/View.pm +++ b/lib/PublicInbox/View.pm @@ -48,7 +48,7 @@ sub feed_entry { # this is already inside a 
 # state = [ time, seen = {}, first_commit, page_nr = 0 ]
 sub index_entry {
-	my (undef, $mime, $level, $state) = @_;
+	my ($fh, $mime, $level, $state) = @_;
 	my ($srch, $seen, $first_commit) = @$state;
 	my $midx = $state->[3]++;
 	my ($prev, $next) = ($midx - 1, $midx + 1);
@@ -107,7 +107,7 @@ sub index_entry {
 	if ($prev >= 0) {
 		$rv .= "/prev";
 	}
-	$rv .= "\n\n";
+	$fh->write($rv .= "\n\n");
 
 	my ($fhref, $more_ref);
 	my $mhref = "${path}m/$href.html";
@@ -117,13 +117,12 @@ sub index_entry {
 	}
 	# scan through all parts, looking for displayable text
 	$mime->walk_parts(sub {
-		$rv .= index_walk($_[0], $enc, \$part_nr, $fhref, $more_ref);
+		index_walk($fh, $_[0], $enc, \$part_nr, $fhref, $more_ref);
 	});
 	$mime->body_set('');
 
-	$rv .= "\n$more ";
 	my $txt = "${path}m/$href.txt";
-	$rv .= "raw ";
+	$rv = "\n$more raw ";
 	$rv .= html_footer($mime, 0);
 
 	if (defined $irt) {
@@ -141,23 +140,30 @@ sub index_entry {
 		       "threadlink";
 	}
 
-	$rv .= '
'; + $fh->write($rv .= ''); } sub thread_html { - my (undef, $ctx, $foot, $srch) = @_; + my ($ctx, $foot, $srch) = @_; + sub { emit_thread_html($_[0], $ctx, $foot, $srch) } +} + +# only private functions below. + +sub emit_thread_html { + my ($cb, $ctx, $foot, $srch) = @_; my $mid = mid_compressed($ctx->{mid}); my $res = $srch->get_thread($mid); - my $rv = ''; my $msgs = load_results($res); my $nr = scalar @$msgs; - return $rv if $nr == 0; + return missing_thread($cb) if $nr == 0; + my $fh = $cb->([200,['Content-Type'=>'text/html; charset=UTF-8']]); my $th = thread_results($msgs); my $state = [ $srch, { root_anchor => anchor_for($mid) }, undef, 0 ]; { require PublicInbox::GitCatFile; my $git = PublicInbox::GitCatFile->new($ctx->{git_dir}); - thread_entry(\$rv, $git, $state, $_, 0) for $th->rootset; + thread_entry($fh, $git, $state, $_, 0) for $th->rootset; } my $final_anchor = $state->[3]; my $next = ""; @@ -169,13 +175,13 @@ sub thread_html { } $next .= ", back to index\n"; - $rv .= "
" . PRE_WRAP . $next . $foot . ""; + $fh->write("
" . PRE_WRAP . $next . $foot . + ""); + $fh->close; } -# only private functions below. - sub index_walk { - my ($part, $enc, $part_nr, $fhref, $more) = @_; + my ($fh, $part, $enc, $part_nr, $fhref, $more) = @_; my $s = add_text_body($enc, $part, $part_nr, $fhref); if ($more) { @@ -196,7 +202,7 @@ sub index_walk { $s =~ s/[ \t]+$//sgm; $s .= "\n" unless $s =~ /\n\z/s; } - $s; + $fh->write($s); } sub enc_for { @@ -533,7 +539,7 @@ sub thread_html_head { } sub thread_entry { - my ($dst, $git, $state, $node, $level) = @_; + my ($fh, $git, $state, $node, $level) = @_; return unless $node; # $state = [ $search_res, $seen, undef, 0 (msg_nr) ]; # $seen is overloaded with 3 types of fields: @@ -546,14 +552,14 @@ sub thread_entry { my $path = mid2path(mid_clean($mime->header('Message-ID'))); $mime = eval { Email::MIME->new($git->cat_file("HEAD:$path")) }; if ($mime) { - if (length($$dst) == 0) { - $$dst .= thread_html_head($mime); + if ($state->[3] == 0) { + $fh->write(thread_html_head($mime)); } - $$dst .= index_entry(undef, $mime, $level, $state); + index_entry($fh, $mime, $level, $state); } } - thread_entry($dst, $git, $state, $node->child, $level + 1); - thread_entry($dst, $git, $state, $node->next, $level); + thread_entry($fh, $git, $state, $node->child, $level + 1); + thread_entry($fh, $git, $state, $node->next, $level); } sub load_results { @@ -577,4 +583,13 @@ sub thread_results { $th } +sub missing_thread { + my ($cb) = @_; + my $title = 'Thread does not exist'; + $cb->([404, ['Content-Type' => 'text/html']])->write(<$title
$title
+Return to index
+EOF +} + 1; diff --git a/lib/PublicInbox/WWW.pm b/lib/PublicInbox/WWW.pm index 68839d7c..2de54719 100644 --- a/lib/PublicInbox/WWW.pm +++ b/lib/PublicInbox/WWW.pm @@ -128,8 +128,7 @@ sub get_index { $ctx->{pi_config} = $pi_config; $ctx->{cgi} = $cgi; footer($ctx); - [ 200, [ 'Content-Type' => 'text/html; charset=UTF-8' ], - [ PublicInbox::Feed->generate_html_index($ctx) ] ] + PublicInbox::Feed::generate_html_index($ctx); } # just returns a string ref for the blob in the current ctx @@ -195,10 +194,7 @@ sub get_thread { my $srch = searcher($ctx) or return need_search($ctx); require PublicInbox::View; my $foot = footer($ctx); - my $body = PublicInbox::View->thread_html($ctx, $foot, $srch) or - return r404(); - [ 200, [ 'Content-Type' => 'text/html; charset=UTF-8' ], - [ $body ] ]; + PublicInbox::View::thread_html($ctx, $foot, $srch); } sub self_url { -- cgit v1.2.3-24-ge0c7