www: improve topic view by scanning for ghosts

This should help avoid having too many fake top-level
messages in the topic view since we only have a partial
window for threading results.

3 files changed, 18 insertions, 4 deletions

diff --git a/lib/PublicInbox/Search.pm b/lib/PublicInbox/Search.pm
index 856c8c14..8c0bab18 100644
--- a/lib/PublicInbox/Search.pm
+++ b/lib/PublicInbox/Search.pm
@@ -227,6 +227,12 @@ sub lookup_message {
         $smsg;
 }
 
+sub lookup_mail { # no ghosts!
+        my ($self, $mid) = @_;
+        my $smsg = lookup_message($self, $mid);
+        PublicInbox::SearchMsg->load_doc($smsg->{doc});
+}
+
 sub find_unique_doc_id {
         my ($self, $term, $value) = @_;
 
diff --git a/lib/PublicInbox/SearchMsg.pm b/lib/PublicInbox/SearchMsg.pm
index 28c2037b..d41d2439 100644
--- a/lib/PublicInbox/SearchMsg.pm
+++ b/lib/PublicInbox/SearchMsg.pm
@@ -35,7 +35,7 @@ sub get_val ($$) {
 
 sub load_doc {
         my ($class, $doc) = @_;
-        my $data = $doc->get_data;
+        my $data = $doc->get_data or return;
         my $ts = get_val($doc, &PublicInbox::Search::TS);
         utf8::decode($data);
         my ($subj, $from, $refs, $to, $cc) = split(/\n/, $data);
diff --git a/lib/PublicInbox/View.pm b/lib/PublicInbox/View.pm
index 006da8d0..dfae44f8 100644
--- a/lib/PublicInbox/View.pm
+++ b/lib/PublicInbox/View.pm
@@ -741,19 +741,27 @@ sub sort_ts {
         } @_;
 }
 
+sub _tryload_ghost ($$) {
+        my ($srch, $node) = @_;
+        my $mid = $node->messageid;
+        my $smsg = $srch->lookup_mail($mid) or return;
+        $smsg->mini_mime;
+}
+
 # accumulate recent topics if search is supported
 # returns 1 if done, undef if not
 sub add_topic {
         my ($state, $node, $level) = @_;
         return unless $node;
         my $child_adjust = 1;
-
-        if (my $x = $node->message) {
+        my $srch = $state->{srch};
+        my $x = $node->message || _tryload_ghost($srch, $node);
+        if ($x) {
                 $x = $x->header_obj;
                 my $subj;
 
                 $subj = $x->header('Subject');
-                $subj = $state->{srch}->subject_normalized($subj);
+                $subj = $srch->subject_normalized($subj);
 
                 if (++$state->{subjs}->{$subj} == 1) {
                         push @{$state->{order}}, [ $level, $subj ];