about summary refs log tree commit homepage
diff options
context:
space:
mode:
authorEric Wong <e@80x24.org>2020-11-28 05:09:12 +0000
committerEric Wong <e@80x24.org>2020-11-29 02:25:47 +0000
commit24e103d37b423c1c718e7f0f6285419005a98be5 (patch)
tree50a191827fc59cb7aa8957fccc638e5eb5336e9e
parent50ac81092ba1034f3055ddabb3d7cc7853edfa41 (diff)
downloadpublic-inbox-24e103d37b423c1c718e7f0f6285419005a98be5.tar.gz
We can amortize the cost of NEWGROUPS time filtering using the
long_response API.  This lets us handle hundreds/thousands of
inboxes without monopolizing the event loop for this command.

Further speedup is possible using MiscSearch, but that requires
not-yet-done indexing changes to MiscIdx.
-rw-r--r--lib/PublicInbox/NNTP.pm21
-rw-r--r--lib/PublicInbox/NNTPD.pm5
-rw-r--r--t/nntp.t1
3 files changed, 16 insertions, 11 deletions
diff --git a/lib/PublicInbox/NNTP.pm b/lib/PublicInbox/NNTP.pm
index 8eec6b91..cc6534b9 100644
--- a/lib/PublicInbox/NNTP.pm
+++ b/lib/PublicInbox/NNTP.pm
@@ -263,6 +263,19 @@ sub group_line ($$) {
         more($self, "$ng->{newsgroup} $max $min n");
 }
 
+sub newgroups_i {
+        my ($self, $ts, $i, $groupnames) = @_;
+        my $end = $$i + 100;
+        my $groups = $self->{nntpd}->{pi_config}->{-by_newsgroup};
+        while ($$i < $end) {
+                my $ngname = $groupnames->[$$i++] // return;
+                my $ibx = $groups->{$ngname} or next; # expired on reload
+                next unless (eval { $ibx->uidvalidity } // 0) > $ts;
+                group_line($self, $ibx);
+        }
+        1;
+}
+
 sub cmd_newgroups ($$$;$$) {
         my ($self, $date, $time, $gmt, $dists) = @_;
         my $ts = eval { parse_time($date, $time, $gmt) };
@@ -270,12 +283,8 @@ sub cmd_newgroups ($$$;$$) {
 
         # TODO dists
         more($self, '231 list of new newsgroups follows');
-        foreach my $ng (@{$self->{nntpd}->{grouplist}}) {
-                my $c = eval { $ng->uidvalidity } // 0;
-                next unless $c > $ts;
-                group_line($self, $ng);
-        }
-        '.'
+        long_response($self, \&newgroups_i, $ts, \(my $i = 0),
+                                $self->{nntpd}->{groupnames});
 }
 
 sub wildmat2re (;$) {
diff --git a/lib/PublicInbox/NNTPD.pm b/lib/PublicInbox/NNTPD.pm
index 4de1944b..33bc5fda 100644
--- a/lib/PublicInbox/NNTPD.pm
+++ b/lib/PublicInbox/NNTPD.pm
@@ -24,7 +24,6 @@ sub new {
                 groups => {},
                 err => \*STDERR,
                 out => \*STDOUT,
-                grouplist => [],
                 pi_config => $pi_config,
                 servername => $name,
                 greet => \"201 $name ready - post via email\r\n",
@@ -60,9 +59,7 @@ sub refresh_groups {
                         delete $groups->{$ngname};
                 }
         });
-        my @names = sort(keys %$groups);
-        $self->{grouplist} = [ map { $groups->{$_} } @names ];
-        $self->{groupnames} = \@names;
+        $self->{groupnames} = [ sort(keys %$groups) ];
         $self->{pi_config} = $pi_config;
         # this will destroy old groups that got deleted
         $self->{groups} = $groups;
diff --git a/t/nntp.t b/t/nntp.t
index 91a2aff7..ea2ef876 100644
--- a/t/nntp.t
+++ b/t/nntp.t
@@ -112,7 +112,6 @@ use PublicInbox::Config;
         my $hdr = $mime->header_obj;
         my $mock_self = {
                 nntpd => {
-                        grouplist => [],
                         servername => 'example.com',
                         pi_config => bless {}, 'PublicInbox::Config',
                 },