mbox: stream entire thread, regardless of size

Since mbox is usually downloaded, support fetching infinitely large
responses via streaming.

2 files changed, 30 insertions, 18 deletions

diff --git a/lib/PublicInbox/Mbox.pm b/lib/PublicInbox/Mbox.pm
index 2ec50656..fc9df1ad 100644
--- a/lib/PublicInbox/Mbox.pm
+++ b/lib/PublicInbox/Mbox.pm
@@ -9,27 +9,14 @@ use Fcntl qw(SEEK_SET);
 
 sub thread_mbox {
         my ($ctx, $srch) = @_;
-        my $mid = mid_compressed($ctx->{mid});
-        my $res = $srch->get_thread($mid);
-        my $msgs = delete $res->{msgs};
-        require PublicInbox::GitCatFile;
-        require Email::Simple;
-        my $git = PublicInbox::GitCatFile->new($ctx->{git_dir});
-
         sub {
-                my ($res) = @_; # Plack callback
-                my $w = $res->([200, [ 'Content-Type' => 'text/plain' ] ]);
-                while (defined(my $smsg = shift @$msgs)) {
-                        my $msg = eval {
-                                my $path = 'HEAD:' . mid2path($smsg->mid);
-                                Email::Simple->new($git->cat_file($path));
-                        };
-                        emit($w, $msg) if $msg;
-                }
+                my ($response) = @_; # Plack callback
+                my $w = $response->([200, ['Content-Type' => 'text/plain']]);
+                emit_mbox($w, $ctx, $srch);
         }
 }
 
-sub emit {
+sub emit_msg {
         my ($fh, $simple) = @_; # Email::Simple object
 
         # drop potentially confusing headers, ssoma already should've dropped
@@ -52,4 +39,29 @@ sub emit {
         $fh->write($buf);
 }
 
+sub emit_mbox {
+        my ($fh, $ctx, $srch) = @_;
+
+        require PublicInbox::GitCatFile;
+        require Email::Simple;
+        my $mid = mid_compressed($ctx->{mid});
+        my $git = PublicInbox::GitCatFile->new($ctx->{git_dir});
+        my %opts = (offset => 0);
+        my $nr;
+        do {
+                my $res = $srch->get_thread($mid, \%opts);
+                my $msgs = $res->{msgs};
+                $nr = scalar @$msgs;
+                while (defined(my $smsg = shift @$msgs)) {
+                        my $msg = eval {
+                                my $p = 'HEAD:'.mid2path($smsg->mid);
+                                Email::Simple->new($git->cat_file($p));
+                        };
+                        emit_msg($fh, $msg) if $msg;
+                }
+
+                $opts{offset} += $nr;
+        } while ($nr > 0);
+}
+
 1;
diff --git a/lib/PublicInbox/Search.pm b/lib/PublicInbox/Search.pm
index f0040500..f1ff3a4b 100644
--- a/lib/PublicInbox/Search.pm
+++ b/lib/PublicInbox/Search.pm
@@ -247,7 +247,7 @@ sub get_thread {
         my $qtid = $qp->parse_query('thread:'.$smsg->thread_id);
         my $qsub = $qp->parse_query('path:'.mid_compressed($smsg->path));
         my $query = Search::Xapian::Query->new(OP_OR, $qtid, $qsub);
-        $self->do_enquire($query);
+        $self->do_enquire($query, $opts);
 }
 
 # private subs below