about summary refs log tree commit homepage
path: root/lib
diff options
context:
space:
mode:
authorEric Wong (Contractor, The Linux Foundation) <e@80x24.org>2018-04-03 11:09:10 +0000
committerEric Wong (Contractor, The Linux Foundation) <e@80x24.org>2018-04-03 12:06:14 +0000
commit0dceebd0a85774c92af247e6da5e2f5a0ee8417c (patch)
treedddcaa1a358a73f0a1da781a9dfd57e2a9ab2ccb /lib
parentb9534449ecce2c59bb4aebad6051f91c3116b187 (diff)
downloadpublic-inbox-0dceebd0a85774c92af247e6da5e2f5a0ee8417c.tar.gz
We can use id_batch in the common case to speed up full mbox
retrievals.  Gigantic msets are still a problem, but will
be fixed in future commits.
Diffstat (limited to 'lib')
-rw-r--r--lib/PublicInbox/Mbox.pm37
-rw-r--r--lib/PublicInbox/Over.pm13
-rw-r--r--lib/PublicInbox/Search.pm4
3 files changed, 39 insertions, 15 deletions
diff --git a/lib/PublicInbox/Mbox.pm b/lib/PublicInbox/Mbox.pm
index 05de6be1..0be19685 100644
--- a/lib/PublicInbox/Mbox.pm
+++ b/lib/PublicInbox/Mbox.pm
@@ -138,8 +138,12 @@ sub thread_mbox {
         my ($ctx, $srch, $sfx) = @_;
         eval { require IO::Compress::Gzip };
         return sub { need_gzip(@_) } if $@;
-
-        my $cb = sub { $srch->get_thread($ctx->{mid}, @_) };
+        my $prev = 0;
+        my $cb = sub {
+                my $msgs = $srch->get_thread($ctx->{mid}, $prev);
+                $prev = $msgs->[-1]->{num} if scalar(@$msgs);
+                $msgs;
+        };
         PublicInbox::MboxGz->response($ctx, $cb);
 }
 
@@ -160,7 +164,25 @@ sub mbox_all {
 
         eval { require IO::Compress::Gzip };
         return sub { need_gzip(@_) } if $@;
-        my $cb = sub { $ctx->{srch}->query($query, @_) };
+        if ($query eq '') {
+                my $prev = 0;
+                my $msgs = [];
+                my $cb = sub {
+                        $ctx->{-inbox}->mm->id_batch($prev, sub {
+                                $msgs = $_[0];
+                        });
+                        $prev = $msgs->[-1] if @$msgs;
+                        $msgs;
+                };
+                return PublicInbox::MboxGz->response($ctx, $cb, 'all');
+        }
+        my $opts = { offset => 0 };
+        my $srch = $ctx->{srch};
+        my $cb = sub { # called by MboxGz->getline
+                my $msgs = $srch->query($query, $opts);
+                $opts->{offset} += scalar @$msgs;
+                $msgs;
+        };
         PublicInbox::MboxGz->response($ctx, $cb, 'results-'.$query);
 }
 
@@ -192,7 +214,6 @@ sub new {
                 cb => $cb,
                 ctx => $ctx,
                 msgs => [],
-                opts => { offset => 0 },
         }, $class;
 }
 
@@ -223,6 +244,10 @@ sub getline {
         do {
                 # work on existing result set
                 while (defined(my $smsg = shift @$msgs)) {
+                        # id_batch may return integers
+                        ref($smsg) or
+                                $smsg = $ctx->{srch}->{over_ro}->get_art($smsg);
+
                         my $msg = eval { $ibx->msg_by_smsg($smsg) } or next;
                         $msg = Email::Simple->new($msg);
                         $gz->write(PublicInbox::Mbox::msg_str($ctx, $msg,
@@ -247,10 +272,10 @@ sub getline {
                 }
 
                 # refill result set
-                $msgs = $self->{msgs} = $self->{cb}->($self->{opts});
-                $self->{opts}->{offset} += scalar @$msgs;
+                $msgs = $self->{msgs} = $self->{cb}->();
         } while (@$msgs);
         $gz->close;
+        # signal that we're done and can return undef next call:
         delete $self->{ctx};
         ${delete $self->{buf}};
 }
diff --git a/lib/PublicInbox/Over.pm b/lib/PublicInbox/Over.pm
index b230d44a..0bd6008b 100644
--- a/lib/PublicInbox/Over.pm
+++ b/lib/PublicInbox/Over.pm
@@ -50,9 +50,7 @@ sub do_get {
         my ($self, $sql, $opts, @args) = @_;
         my $dbh = $self->connect;
         my $lim = (($opts->{limit} || 0) + 0) || 1000;
-        my $off = (($opts->{offset} || 0) + 0) || 0;
         $sql .= "LIMIT $lim";
-        $sql .= " OFFSET $off" if $off > 0;
         my $msgs = $dbh->selectall_arrayref($sql, { Slice => {} }, @args);
         load_from_row($_) for @$msgs;
         $msgs
@@ -77,7 +75,7 @@ ORDER BY num ASC
 sub nothing () { wantarray ? (0, []) : [] };
 
 sub get_thread {
-        my ($self, $mid, $opts) = @_;
+        my ($self, $mid, $prev) = @_;
         my $dbh = $self->connect;
 
         my $id = $dbh->selectrow_array(<<'', undef, $mid);
@@ -96,13 +94,14 @@ SELECT tid,sid FROM over WHERE num = ? LIMIT 1
 
         defined $tid or return nothing; # $sid may be undef
 
-        my $cond = 'FROM over WHERE (tid = ? OR sid = ?) AND num > 0';
-        my $msgs = do_get($self, <<"", $opts, $tid, $sid);
-SELECT * $cond ORDER BY ts ASC
+        $prev ||= 0;
+        my $cond = 'FROM over WHERE (tid = ? OR sid = ?) AND num > ?';
+        my $msgs = do_get($self, <<"", {}, $tid, $sid, $prev);
+SELECT * $cond ORDER BY num ASC
 
         return $msgs unless wantarray;
 
-        my $nr = $dbh->selectrow_array(<<"", undef, $tid, $sid);
+        my $nr = $dbh->selectrow_array(<<"", undef, $tid, $sid, $prev);
 SELECT COUNT(num) $cond
 
         ($nr, $msgs);
diff --git a/lib/PublicInbox/Search.pm b/lib/PublicInbox/Search.pm
index f7fdf854..eca2b0f5 100644
--- a/lib/PublicInbox/Search.pm
+++ b/lib/PublicInbox/Search.pm
@@ -179,8 +179,8 @@ sub query {
 }
 
 sub get_thread {
-        my ($self, $mid, $opts) = @_;
-        $self->{over_ro}->get_thread($mid, $opts);
+        my ($self, $mid, $prev) = @_;
+        $self->{over_ro}->get_thread($mid, $prev);
 }
 
 sub retry_reopen {