imap: start doing iterative config reloading

This will be used to prevent reloading a giant config with
tens/hundreds of thousands of inboxes from blocking the event
loop.

3 files changed, 71 insertions, 7 deletions

diff --git a/lib/PublicInbox/Config.pm b/lib/PublicInbox/Config.pm
index 458f29b2..c18c9c75 100644
--- a/lib/PublicInbox/Config.pm
+++ b/lib/PublicInbox/Config.pm
@@ -99,6 +99,21 @@ sub each_inbox {
         }
 }
 
+sub iterate_start {
+        my ($self, $cb, $arg) = @_;
+        my $i = 0;
+        $self->{-iter} = [ \$i, $cb, $arg ];
+}
+
+# for PublicInbox::DS::next_tick
+sub event_step {
+        my ($self) = @_;
+        my ($i, $cb, $arg) = @{$self->{-iter}};
+        my $section = $self->{-section_order}->[$$i++];
+        delete($self->{-iter}) unless defined($section);
+        $cb->($self, $section, $arg);
+}
+
 sub lookup_newsgroup {
         my ($self, $ng) = @_;
         _lookup_fill($self, '-by_newsgroup', lc($ng));
diff --git a/lib/PublicInbox/IMAPD.pm b/lib/PublicInbox/IMAPD.pm
index 966879d8..a10fabff 100644
--- a/lib/PublicInbox/IMAPD.pm
+++ b/lib/PublicInbox/IMAPD.pm
@@ -7,6 +7,8 @@ package PublicInbox::IMAPD;
 use strict;
 use parent qw(PublicInbox::NNTPD);
 use PublicInbox::InboxIdle;
+use PublicInbox::IMAP;
+# *UID_BLOCK = \&PublicInbox::IMAP::UID_BLOCK;
 
 sub new {
         my ($class) = @_;
@@ -51,17 +53,64 @@ sub refresh_inboxlist ($) {
         $self->{inboxlist} = \@names;
 }
 
-sub refresh_groups {
-        my ($self) = @_;
-        my $pi_config = $self->{pi_config} = PublicInbox::Config->new;
-        $self->SUPER::refresh_groups($pi_config);
-        refresh_inboxlist($self);
+sub imapd_refresh_ibx { # pi_config->each_inbox cb
+        my ($ibx, $imapd) = @_;
+        my $ngname = $ibx->{newsgroup} or return;
+        if (ref $ngname) {
+                warn 'multiple newsgroups not supported: '.
+                        join(', ', @$ngname). "\n";
+        } elsif ($ngname =~ m![^a-z0-9/_\.\-\~\@\+\=:]! ||
+                 $ngname =~ /\.[0-9]+-[0-9]+\z/) {
+                warn "mailbox name invalid: `$ngname'\n";
+        }
+
+        my $mm = $ibx->mm or return;
+        $ibx->{mm} = undef;
+        defined($ibx->{uidvalidity} = $mm->created_at) or return;
+        $imapd->{tmp_groups}->{$ngname} = $ibx;
 
-        if (my $idler = $self->{idler}) {
+        # preload to avoid fragmentation:
+        $ibx->description;
+        $ibx->base_url;
+        # my $max = $mm->max // 0;
+        # my $uid_min = UID_BLOCK * int($max/UID_BLOCK) + 1;
+}
+
+sub imapd_refresh_finalize {
+        my ($imapd, $pi_config) = @_;
+        $imapd->{groups} = delete $imapd->{tmp_groups};
+        $imapd->{grouplist} = [ values %{$imapd->{groups}} ];
+        refresh_inboxlist($imapd);
+        $imapd->{pi_config} = $pi_config;
+        if (my $idler = $imapd->{idler}) {
                 $idler->refresh($pi_config);
         }
 }
 
+sub imapd_refresh_step { # pi_config->iterate_start cb
+        my ($pi_config, $section, $imapd) = @_;
+        if (defined($section)) {
+                return if $section !~ m!\Apublicinbox\.([^/]+)\z!;
+                my $ibx = $pi_config->lookup_name($1) or return;
+                imapd_refresh_ibx($ibx, $imapd);
+        } else { # "EOF"
+                imapd_refresh_finalize($imapd, $pi_config);
+        }
+}
+
+sub refresh_groups {
+        my ($self, $sig) = @_;
+        my $pi_config = PublicInbox::Config->new;
+        $self->{tmp_groups} = {};
+        if (0 && $sig) { # SIGHUP
+                $pi_config->iterate_start(\&imapd_refresh_step, $self);
+                PublicInbox::DS::requeue($pi_config); # call event_step
+        } else { # initial start
+                $pi_config->each_inbox(\&imapd_refresh_ibx, $self);
+                imapd_refresh_finalize($self, $pi_config);
+        }
+}
+
 sub idler_start {
         $_[0]->{idler} //= PublicInbox::InboxIdle->new($_[0]->{pi_config});
 }
diff --git a/script/public-inbox-imapd b/script/public-inbox-imapd
index 63f865f5..60f2e6d8 100644
--- a/script/public-inbox-imapd
+++ b/script/public-inbox-imapd
@@ -9,6 +9,6 @@ use PublicInbox::IMAPdeflate; # loads PublicInbox::IMAP
 use PublicInbox::IMAPD;
 my $imapd = PublicInbox::IMAPD->new;
 PublicInbox::Daemon::run('0.0.0.0:143',
-        sub { $imapd->refresh_groups }, # refresh
+        sub { $imapd->refresh_groups(@_) }, # refresh
         sub ($$$) { PublicInbox::IMAP->new($_[0], $imapd) }, # post_accept
         $imapd);