From 87dca6d8d5988c5eb54019cca342450b0b7dd6b7 Mon Sep 17 00:00:00 2001 From: "Eric Wong (Contractor, The Linux Foundation)" Date: Mon, 2 Apr 2018 00:04:55 +0000 Subject: www: rework query responses to avoid COUNT in SQLite In many cases, we do not care about the total number of messages. It's a rather expensive operation in SQLite (Xapian only provides an estimate). For LKML, this brings top-level /$INBOX/ loading time from ~375ms to around 60ms on my system. Days ago, this operation was taking 800-900ms(!) for me before introducing the SQLite overview DB. --- lib/PublicInbox/Mbox.pm | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) (limited to 'lib/PublicInbox/Mbox.pm') diff --git a/lib/PublicInbox/Mbox.pm b/lib/PublicInbox/Mbox.pm index 1b68f027..05de6be1 100644 --- a/lib/PublicInbox/Mbox.pm +++ b/lib/PublicInbox/Mbox.pm @@ -217,12 +217,12 @@ sub set_filename ($$) { sub getline { my ($self) = @_; my $ctx = $self->{ctx} or return; - my $res; my $ibx = $ctx->{-inbox}; my $gz = $self->{gz}; + my $msgs = $self->{msgs}; do { # work on existing result set - while (defined(my $smsg = shift @{$self->{msgs}})) { + while (defined(my $smsg = shift @$msgs)) { my $msg = eval { $ibx->msg_by_smsg($smsg) } or next; $msg = Email::Simple->new($msg); $gz->write(PublicInbox::Mbox::msg_str($ctx, $msg, @@ -247,11 +247,9 @@ sub getline { } # refill result set - $res = $self->{cb}->($self->{opts}); - $self->{msgs} = $res->{msgs}; - $res = scalar @{$self->{msgs}}; - $self->{opts}->{offset} += $res; - } while ($res); + $msgs = $self->{msgs} = $self->{cb}->($self->{opts}); + $self->{opts}->{offset} += scalar @$msgs; + } while (@$msgs); $gz->close; delete $self->{ctx}; ${delete $self->{buf}}; -- cgit v1.2.3-24-ge0c7