From 24e103d37b423c1c718e7f0f6285419005a98be5 Mon Sep 17 00:00:00 2001
From: Eric Wong <e@80x24.org>
Date: Sat, 28 Nov 2020 05:09:12 +0000
Subject: nntp: NEWGROUPS uses long_response

We can amortize the cost of NEWGROUPS time filtering using the
long_response API.  This lets us handle hundreds/thousands of
inboxes without monopolizing the event loop for this command.

Further speedup is possible using MiscSearch, but that requires
not-yet-done indexing changes to MiscIdx.
---
 lib/PublicInbox/NNTP.pm  | 21 +++++++++++++++------
 lib/PublicInbox/NNTPD.pm |  5 +----
 2 files changed, 16 insertions(+), 10 deletions(-)

(limited to 'lib')

diff --git a/lib/PublicInbox/NNTP.pm b/lib/PublicInbox/NNTP.pm
index 8eec6b91..cc6534b9 100644
--- a/lib/PublicInbox/NNTP.pm
+++ b/lib/PublicInbox/NNTP.pm
@@ -263,6 +263,19 @@ sub group_line ($$) {
 	more($self, "$ng->{newsgroup} $max $min n");
 }
 
+sub newgroups_i {
+	my ($self, $ts, $i, $groupnames) = @_;
+	my $end = $$i + 100;
+	my $groups = $self->{nntpd}->{pi_config}->{-by_newsgroup};
+	while ($$i < $end) {
+		my $ngname = $groupnames->[$$i++] // return;
+		my $ibx = $groups->{$ngname} or next; # expired on reload
+		next unless (eval { $ibx->uidvalidity } // 0) > $ts;
+		group_line($self, $ibx);
+	}
+	1;
+}
+
 sub cmd_newgroups ($$$;$$) {
 	my ($self, $date, $time, $gmt, $dists) = @_;
 	my $ts = eval { parse_time($date, $time, $gmt) };
@@ -270,12 +283,8 @@ sub cmd_newgroups ($$$;$$) {
 
 	# TODO dists
 	more($self, '231 list of new newsgroups follows');
-	foreach my $ng (@{$self->{nntpd}->{grouplist}}) {
-		my $c = eval { $ng->uidvalidity } // 0;
-		next unless $c > $ts;
-		group_line($self, $ng);
-	}
-	'.'
+	long_response($self, \&newgroups_i, $ts, \(my $i = 0),
+				$self->{nntpd}->{groupnames});
 }
 
 sub wildmat2re (;$) {
diff --git a/lib/PublicInbox/NNTPD.pm b/lib/PublicInbox/NNTPD.pm
index 4de1944b..33bc5fda 100644
--- a/lib/PublicInbox/NNTPD.pm
+++ b/lib/PublicInbox/NNTPD.pm
@@ -24,7 +24,6 @@ sub new {
 		groups => {},
 		err => \*STDERR,
 		out => \*STDOUT,
-		grouplist => [],
 		pi_config => $pi_config,
 		servername => $name,
 		greet => \"201 $name ready - post via email\r\n",
@@ -60,9 +59,7 @@ sub refresh_groups {
 			delete $groups->{$ngname};
 		}
 	});
-	my @names = sort(keys %$groups);
-	$self->{grouplist} = [ map { $groups->{$_} } @names ];
-	$self->{groupnames} = \@names;
+	$self->{groupnames} = [ sort(keys %$groups) ];
 	$self->{pi_config} = $pi_config;
 	# this will destroy old groups that got deleted
 	$self->{groups} = $groups;
-- 
cgit v1.2.3-24-ge0c7