* [PATCH] mbox: stream entire thread, regardless of size
@ 2015-08-21 23:31 7% Eric Wong
0 siblings, 0 replies; 1+ results
From: Eric Wong @ 2015-08-21 23:31 UTC (permalink / raw)
To: meta
Since mbox is usually downloaded, support fetching infinitely large
responses via streaming.
---
lib/PublicInbox/Mbox.pm | 46 +++++++++++++++++++++++++++++-----------------
lib/PublicInbox/Search.pm | 2 +-
2 files changed, 30 insertions(+), 18 deletions(-)
diff --git a/lib/PublicInbox/Mbox.pm b/lib/PublicInbox/Mbox.pm
index 2ec5065..fc9df1a 100644
--- a/lib/PublicInbox/Mbox.pm
+++ b/lib/PublicInbox/Mbox.pm
@@ -9,27 +9,14 @@ use Fcntl qw(SEEK_SET);
sub thread_mbox {
my ($ctx, $srch) = @_;
- my $mid = mid_compressed($ctx->{mid});
- my $res = $srch->get_thread($mid);
- my $msgs = delete $res->{msgs};
- require PublicInbox::GitCatFile;
- require Email::Simple;
- my $git = PublicInbox::GitCatFile->new($ctx->{git_dir});
-
sub {
- my ($res) = @_; # Plack callback
- my $w = $res->([200, [ 'Content-Type' => 'text/plain' ] ]);
- while (defined(my $smsg = shift @$msgs)) {
- my $msg = eval {
- my $path = 'HEAD:' . mid2path($smsg->mid);
- Email::Simple->new($git->cat_file($path));
- };
- emit($w, $msg) if $msg;
- }
+ my ($response) = @_; # Plack callback
+ my $w = $response->([200, ['Content-Type' => 'text/plain']]);
+ emit_mbox($w, $ctx, $srch);
}
}
-sub emit {
+sub emit_msg {
my ($fh, $simple) = @_; # Email::Simple object
# drop potentially confusing headers, ssoma already should've dropped
@@ -52,4 +39,29 @@ sub emit {
$fh->write($buf);
}
+sub emit_mbox {
+ my ($fh, $ctx, $srch) = @_;
+
+ require PublicInbox::GitCatFile;
+ require Email::Simple;
+ my $mid = mid_compressed($ctx->{mid});
+ my $git = PublicInbox::GitCatFile->new($ctx->{git_dir});
+ my %opts = (offset => 0);
+ my $nr;
+ do {
+ my $res = $srch->get_thread($mid, \%opts);
+ my $msgs = $res->{msgs};
+ $nr = scalar @$msgs;
+ while (defined(my $smsg = shift @$msgs)) {
+ my $msg = eval {
+ my $p = 'HEAD:'.mid2path($smsg->mid);
+ Email::Simple->new($git->cat_file($p));
+ };
+ emit_msg($fh, $msg) if $msg;
+ }
+
+ $opts{offset} += $nr;
+ } while ($nr > 0);
+}
+
1;
diff --git a/lib/PublicInbox/Search.pm b/lib/PublicInbox/Search.pm
index f004050..f1ff3a4 100644
--- a/lib/PublicInbox/Search.pm
+++ b/lib/PublicInbox/Search.pm
@@ -247,7 +247,7 @@ sub get_thread {
my $qtid = $qp->parse_query('thread:'.$smsg->thread_id);
my $qsub = $qp->parse_query('path:'.mid_compressed($smsg->path));
my $query = Search::Xapian::Query->new(OP_OR, $qtid, $qsub);
- $self->do_enquire($query);
+ $self->do_enquire($query, $opts);
}
# private subs below
--
EW
^ permalink raw reply related [relevance 7%]
Results 1-1 of 1 | reverse | options above
-- pct% links below jump to the message on this page, permalinks otherwise --
2015-08-21 23:31 7% [PATCH] mbox: stream entire thread, regardless of size Eric Wong
Code repositories for project(s) associated with this public inbox
https://80x24.org/public-inbox.git
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).