about summary refs log tree commit homepage
path: root/lib
diff options
context:
space:
mode:
authorEric Wong <e@80x24.org>2015-09-19 02:03:31 +0000
committerEric Wong <e@80x24.org>2015-09-19 04:18:38 +0000
commit7ee29d11b25b09b8919b9a125732e4c2c5784d28 (patch)
treee4b424bb8fa8e430fbf5b3842612087cc680ada7 /lib
parent0d0fde0bff97a283e3b433f2772a99fe9d6d8412 (diff)
downloadpublic-inbox-7ee29d11b25b09b8919b9a125732e4c2c5784d28.tar.gz
LISTGROUP can be expensive for giant groups, too.  Use the
long response API to improve fairness and prevent excessive
buffering.
Diffstat (limited to 'lib')
-rw-r--r--lib/PublicInbox/Msgmap.pm24
-rw-r--r--lib/PublicInbox/NNTP.pm23
2 files changed, 21 insertions, 26 deletions
diff --git a/lib/PublicInbox/Msgmap.pm b/lib/PublicInbox/Msgmap.pm
index c0fc636f..2f64d90c 100644
--- a/lib/PublicInbox/Msgmap.pm
+++ b/lib/PublicInbox/Msgmap.pm
@@ -155,25 +155,17 @@ sub create_tables {
                         'val VARCHAR(255) NOT NULL)');
 }
 
-sub each_id_batch {
-        my ($self, $cb) = @_;
+sub id_batch {
+        my ($self, $num, $cb) = @_;
         my $dbh = $self->{dbh};
-        my $n = 0;
-        my $total = 0;
-        my $nr;
         my $sth = $dbh->prepare('SELECT num FROM msgmap WHERE num > ? '.
                                 'ORDER BY num ASC LIMIT 1000');
-        while (1) {
-                $sth->execute($n);
-                my $ary = $sth->fetchall_arrayref;
-                @$ary = map { $_->[0] } @$ary;
-                $nr = scalar @$ary;
-                last if $nr == 0;
-                $total += $nr;
-                $n = $ary->[-1];
-                $cb->($ary);
-        }
-        $total;
+        $sth->execute($num);
+        my $ary = $sth->fetchall_arrayref;
+        @$ary = map { $_->[0] } @$ary;
+        my $nr = scalar @$ary;
+        $cb->($ary) if $nr;
+        $nr;
 }
 
 1;
diff --git a/lib/PublicInbox/NNTP.pm b/lib/PublicInbox/NNTP.pm
index f86c6335..5d770bdb 100644
--- a/lib/PublicInbox/NNTP.pm
+++ b/lib/PublicInbox/NNTP.pm
@@ -14,6 +14,7 @@ use POSIX qw(strftime);
 use Time::HiRes qw(gettimeofday tv_interval ualarm);
 use constant {
         r501 => '501 command syntax error',
+        long_response_limit => 0xffffffff,
 };
 
 my @OVERVIEW = qw(Subject From Date Message-ID References Bytes Lines);
@@ -142,13 +143,17 @@ sub cmd_listgroup {
                 more($self, $res);
         }
 
-        my $ng = $self->{ng} or return '412 no newsgroup selected';
-        # Ugh this can be silly expensive for big groups
-        $ng->mm->each_id_batch(sub {
-                my ($ary) = @_;
-                more($self, join("\r\n", @$ary));
+        $self->{ng} or return '412 no newsgroup selected';
+        $self->long_response(0, long_response_limit, sub {
+                my ($i) = @_;
+                my $nr = $self->{ng}->mm->id_batch($$i, sub {
+                        my ($ary) = @_;
+                        more($self, join("\r\n", @$ary));
+                });
+
+                # -1 to adjust for implicit increment in long_response
+                $$i = $nr ? $$i + $nr - 1 : long_response_limit;
         });
-        '.'
 }
 
 sub parse_time {
@@ -245,9 +250,7 @@ sub cmd_newnews {
 
         $ts .= '..';
         my $opts = { asc => 1, limit => 1000, offset => 0 };
-
-        my $end = 0xffffffff; # would like to read 4 billion messages?
-        $self->long_response(0, $end, sub {
+        $self->long_response(0, long_response_limit, sub {
                 my ($i) = @_;
                 my $srch = $srch[0];
                 my $res = $srch->query($ts, $opts);
@@ -262,7 +265,7 @@ sub cmd_newnews {
                         if (@srch) { # continue onto next newsgroup
                                 $opts->{offset} = 0;
                         } else { # break out of the long response.
-                                $$i = $end;
+                                $$i = long_response_limit;
                         }
                 }
         });