From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.3.2 (2011-06-06) on dcvr.yhbt.net X-Spam-Level: X-Spam-ASN: X-Spam-Status: No, score=-2.9 required=3.0 tests=ALL_TRUSTED,BAYES_00 shortcircuit=no autolearn=unavailable version=3.3.2 X-Original-To: meta@public-inbox.org Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id 61980633849 for ; Sat, 19 Sep 2015 02:03:46 +0000 (UTC) From: Eric Wong To: meta@public-inbox.org Subject: [PATCH 03/12] nntp: use long response API for LISTGROUP Date: Sat, 19 Sep 2015 02:03:31 +0000 Message-Id: <20150919020340.6484-4-e@80x24.org> In-Reply-To: <20150919020340.6484-1-e@80x24.org> References: <20150919020340.6484-1-e@80x24.org> List-Id: LISTGROUP can be expensive for giant groups, too. Use the long response API to improve fairness and prevent excessive buffering. --- lib/PublicInbox/Msgmap.pm | 24 ++++++++---------------- lib/PublicInbox/NNTP.pm | 23 +++++++++++++---------- 2 files changed, 21 insertions(+), 26 deletions(-) diff --git a/lib/PublicInbox/Msgmap.pm b/lib/PublicInbox/Msgmap.pm index c0fc636..2f64d90 100644 --- a/lib/PublicInbox/Msgmap.pm +++ b/lib/PublicInbox/Msgmap.pm @@ -155,25 +155,17 @@ sub create_tables { 'val VARCHAR(255) NOT NULL)'); } -sub each_id_batch { - my ($self, $cb) = @_; +sub id_batch { + my ($self, $num, $cb) = @_; my $dbh = $self->{dbh}; - my $n = 0; - my $total = 0; - my $nr; my $sth = $dbh->prepare('SELECT num FROM msgmap WHERE num > ? '. 'ORDER BY num ASC LIMIT 1000'); - while (1) { - $sth->execute($n); - my $ary = $sth->fetchall_arrayref; - @$ary = map { $_->[0] } @$ary; - $nr = scalar @$ary; - last if $nr == 0; - $total += $nr; - $n = $ary->[-1]; - $cb->($ary); - } - $total; + $sth->execute($num); + my $ary = $sth->fetchall_arrayref; + @$ary = map { $_->[0] } @$ary; + my $nr = scalar @$ary; + $cb->($ary) if $nr; + $nr; } 1; diff --git a/lib/PublicInbox/NNTP.pm b/lib/PublicInbox/NNTP.pm index f86c633..5d770bd 100644 --- a/lib/PublicInbox/NNTP.pm +++ b/lib/PublicInbox/NNTP.pm @@ -14,6 +14,7 @@ use POSIX qw(strftime); use Time::HiRes qw(gettimeofday tv_interval ualarm); use constant { r501 => '501 command syntax error', + long_response_limit => 0xffffffff, }; my @OVERVIEW = qw(Subject From Date Message-ID References Bytes Lines); @@ -142,13 +143,17 @@ sub cmd_listgroup { more($self, $res); } - my $ng = $self->{ng} or return '412 no newsgroup selected'; - # Ugh this can be silly expensive for big groups - $ng->mm->each_id_batch(sub { - my ($ary) = @_; - more($self, join("\r\n", @$ary)); + $self->{ng} or return '412 no newsgroup selected'; + $self->long_response(0, long_response_limit, sub { + my ($i) = @_; + my $nr = $self->{ng}->mm->id_batch($$i, sub { + my ($ary) = @_; + more($self, join("\r\n", @$ary)); + }); + + # -1 to adjust for implicit increment in long_response + $$i = $nr ? $$i + $nr - 1 : long_response_limit; }); - '.' } sub parse_time { @@ -245,9 +250,7 @@ sub cmd_newnews { $ts .= '..'; my $opts = { asc => 1, limit => 1000, offset => 0 }; - - my $end = 0xffffffff; # would like to read 4 billion messages? - $self->long_response(0, $end, sub { + $self->long_response(0, long_response_limit, sub { my ($i) = @_; my $srch = $srch[0]; my $res = $srch->query($ts, $opts); @@ -262,7 +265,7 @@ sub cmd_newnews { if (@srch) { # continue onto next newsgroup $opts->{offset} = 0; } else { # break out of the long response. - $$i = $end; + $$i = long_response_limit; } } }); -- EW