[PATCH 4/4] imapd: use nntpd_cache to speed up startup/reload time

[Eric Wong <e@80x24.org>]

ConfigIter was still too slow despite being fair.  The addition of
ART_MIN in ALL->misc means it can be used as a startup/reload cache
for -imapd, too.

This results in a ~3x faster startup for -imapd with 50K inboxes.
---
 lib/PublicInbox/IMAP.pm  |  17 +++----
 lib/PublicInbox/IMAPD.pm | 100 +++++++++++++--------------------------
 2 files changed, 41 insertions(+), 76 deletions(-)

diff --git a/lib/PublicInbox/IMAP.pm b/lib/PublicInbox/IMAP.pm
index 19ead70c..9955984b 100644
--- a/lib/PublicInbox/IMAP.pm
+++ b/lib/PublicInbox/IMAP.pm
@@ -350,12 +350,12 @@ sub idle_done ($$) {
 	"$idle_tag OK Idle done\r\n";
 }
 
-sub ensure_slices_exist ($$$) {
-	my ($imapd, $ibx, $max) = @_;
-	defined(my $mb_top = $ibx->{newsgroup}) or return;
+sub ensure_slices_exist ($$) {
+	my ($imapd, $ibx) = @_;
+	my $mb_top = $ibx->{newsgroup} // return;
 	my $mailboxes = $imapd->{mailboxes};
 	my @created;
-	for (my $i = int($max/UID_SLICE); $i >= 0; --$i) {
+	for (my $i = int($ibx->art_max/UID_SLICE); $i >= 0; --$i) {
 		my $sub_mailbox = "$mb_top.$i";
 		last if exists $mailboxes->{$sub_mailbox};
 		$mailboxes->{$sub_mailbox} = $ibx;
@@ -387,7 +387,8 @@ sub inbox_lookup ($$;$) {
 			my $uid_end = $uid_base + UID_SLICE;
 			$exists = $over->imap_exists($uid_base, $uid_end);
 		}
-		ensure_slices_exist($self->{imapd}, $ibx, $over->max);
+		delete $ibx->{-art_max};
+		ensure_slices_exist($self->{imapd}, $ibx);
 	} else {
 		if ($examine) {
 			$self->{uid_base} = $uid_base;
@@ -396,9 +397,9 @@ sub inbox_lookup ($$;$) {
 		}
 		# if "INBOX.foo.bar" is selected and "INBOX.foo.bar.0",
 		# check for new UID ranges (e.g. "INBOX.foo.bar.1")
-		if (my $z = $self->{imapd}->{mailboxes}->{"$mailbox.0"}) {
-			ensure_slices_exist($self->{imapd}, $z,
-						$z->over(1)->max);
+		if (my $ibx = $self->{imapd}->{mailboxes}->{"$mailbox.0"}) {
+			delete $ibx->{-art_max};
+			ensure_slices_exist($self->{imapd}, $ibx);
 		}
 	}
 	($ibx, $exists, $uidmax + 1, $uid_base);
diff --git a/lib/PublicInbox/IMAPD.pm b/lib/PublicInbox/IMAPD.pm
index 6038fd88..5368ff04 100644
--- a/lib/PublicInbox/IMAPD.pm
+++ b/lib/PublicInbox/IMAPD.pm
@@ -6,7 +6,6 @@ package PublicInbox::IMAPD;
 use strict;
 use v5.10.1;
 use PublicInbox::Config;
-use PublicInbox::ConfigIter;
 use PublicInbox::InboxIdle;
 use PublicInbox::IMAP;
 use PublicInbox::DummyInbox;
@@ -15,7 +14,7 @@ my $dummy = bless { uidvalidity => 0 }, 'PublicInbox::DummyInbox';
 sub new {
 	my ($class) = @_;
 	bless {
-		mailboxes => {},
+		# mailboxes => {},
 		err => \*STDERR,
 		out => \*STDOUT,
 		# ssl_ctx_opt => { SSL_cert_file => ..., SSL_key_file => ... }
@@ -25,53 +24,45 @@ sub new {
 }
 
 sub imapd_refresh_ibx { # pi_cfg->each_inbox cb
-	my ($ibx, $imapd) = @_;
-	my $ngname = $ibx->{newsgroup} or return;
+	my ($ibx, $imapd, $cache, $dummies) = @_;
+	my $ngname = $ibx->{newsgroup} // return;
 
 	# We require lower-case since IMAP mailbox names are
 	# case-insensitive (but -nntpd matches INN in being
-	# case-sensitive
+	# case-sensitive)
 	if ($ngname =~ m![^a-z0-9/_\.\-\~\@\+\=:]! ||
 			# don't confuse with 50K slices
 			$ngname =~ /\.[0-9]+\z/) {
 		warn "mailbox name invalid: newsgroup=`$ngname'\n";
 		return;
 	}
-	$ibx->over or return;
-	$ibx->{over} = undef;
-
-	# RFC 3501 2.3.1.1 -  "A good UIDVALIDITY value to use in
-	# this case is a 32-bit representation of the creation
-	# date/time of the mailbox"
-	eval { $ibx->uidvalidity };
-	my $mm = delete($ibx->{mm}) or return;
-	defined($ibx->{uidvalidity}) or return;
-	PublicInbox::IMAP::ensure_slices_exist($imapd, $ibx, $mm->max);
-
-	# preload to avoid fragmentation:
-	$ibx->description;
-	$ibx->base_url;
-
-	# ensure dummies are selectable
-	my $dummies = $imapd->{dummies};
-	do {
-		$dummies->{$ngname} = $dummy;
-	} while ($ngname =~ s/\.[^\.]+\z//);
+	my $ce = $cache->{$ngname};
+	%$ibx = (%$ibx, %$ce) if $ce;
+	# only valid if msgmap and over works:
+	if (defined($ibx->uidvalidity)) {
+		# fill ->{mailboxes}:
+		PublicInbox::IMAP::ensure_slices_exist($imapd, $ibx);
+		# preload to avoid fragmentation:
+		$ibx->description;
+		$ibx->base_url;
+		# ensure dummies are selectable:
+		do {
+			$dummies->{$ngname} = $dummy;
+		} while ($ngname =~ s/\.[^\.]+\z//);
+	}
+	delete @$ibx{qw(mm over)};
 }
 
-sub imapd_refresh_finalize {
-	my ($imapd, $pi_cfg) = @_;
-	my $mailboxes;
-	if (my $next = delete $imapd->{imapd_next}) {
-		$imapd->{mailboxes} = delete $next->{mailboxes};
-		$mailboxes = delete $next->{dummies};
-	} else {
-		$mailboxes = delete $imapd->{dummies};
-	}
-	%$mailboxes = (%$mailboxes, %{$imapd->{mailboxes}});
-	$imapd->{mailboxes} = $mailboxes;
-	$imapd->{mailboxlist} = [
-		map { $_->[2] }
+sub refresh_groups {
+	my ($self, $sig) = @_;
+	my $pi_cfg = PublicInbox::Config->new;
+	my $mailboxes = $self->{mailboxes} = {};
+	my $cache = eval { $pi_cfg->ALL->misc->nntpd_cache_load } // {};
+	my $dummies = {};
+	$pi_cfg->each_inbox(\&imapd_refresh_ibx, $self, $cache, $dummies);
+	%$dummies = (%$dummies, %$mailboxes);
+	$mailboxes = $self->{mailboxes} = $dummies;
+	@{$self->{mailboxlist}} = map { $_->[2] }
 		sort { $a->[0] cmp $b->[0] || $a->[1] <=> $b->[1] }
 		map {
 			my $u = $_; # capitalize "INBOX" for user-familiarity
@@ -85,40 +76,13 @@ sub imapd_refresh_finalize {
 				[ $1, $2 + 0,
 				  qq[* LIST (\\HasNoChildren) "." $u\r\n] ]
 			}
-		} keys %$mailboxes
-	];
-	$imapd->{pi_cfg} = $pi_cfg;
-	if (my $idler = $imapd->{idler}) {
+		} keys %$mailboxes;
+	$self->{pi_cfg} = $pi_cfg;
+	if (my $idler = $self->{idler}) {
 		$idler->refresh($pi_cfg);
 	}
 }
 
-sub imapd_refresh_step { # PublicInbox::ConfigIter cb
-	my ($pi_cfg, $section, $imapd) = @_;
-	if (defined($section)) {
-		return if $section !~ m!\Apublicinbox\.([^/]+)\z!;
-		my $ibx = $pi_cfg->lookup_name($1) or return;
-		imapd_refresh_ibx($ibx, $imapd->{imapd_next});
-	} else { # undef == "EOF"
-		imapd_refresh_finalize($imapd, $pi_cfg);
-	}
-}
-
-sub refresh_groups {
-	my ($self, $sig) = @_;
-	my $pi_cfg = PublicInbox::Config->new;
-	if ($sig) { # SIGHUP is handled through the event loop
-		$self->{imapd_next} = { dummies => {}, mailboxes => {} };
-		my $iter = PublicInbox::ConfigIter->new($pi_cfg,
-						\&imapd_refresh_step, $self);
-		$iter->event_step;
-	} else { # initial start is synchronous
-		$self->{dummies} = {};
-		$pi_cfg->each_inbox(\&imapd_refresh_ibx, $self);
-		imapd_refresh_finalize($self, $pi_cfg);
-	}
-}
-
 sub idler_start {
 	$_[0]->{idler} //= PublicInbox::InboxIdle->new($_[0]->{pi_cfg});
 }