public-inbox-netd: a multi-protocol superserver

Since we'll be adding POP3 support as our 4th network protocol;
asking admins to run yet another daemon on top of existing
-httpd, -nntpd, -imapd is a maintenance burden and a waste of
memory.

The goal of public-inbox-netd is to be able to replace all
existing read-only daemons with a single process to save memory
and reduce administrative overhead; hopefully encouraging more
users to self-host their own mirrors.

It's barely-tested at the moment.  Eventually, multiple
PI_CONFIG and HOME directories will be supported, as are
per-listener .psgi config files.

4 files changed, 104 insertions, 32 deletions

diff --git a/lib/PublicInbox/Daemon.pm b/lib/PublicInbox/Daemon.pm
index 50523586..13cce1ec 100644
--- a/lib/PublicInbox/Daemon.pm
+++ b/lib/PublicInbox/Daemon.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2015-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 #
 # Contains common daemon code for the httpd, imapd, and nntpd servers
@@ -75,18 +75,34 @@ sub accept_tls_opt ($) {
         { SSL_server => 1, SSL_startHandshake => 0, SSL_reuse_ctx => $ctx };
 }
 
-sub daemon_prepare ($) {
-        my ($default_listen) = @_;
+sub load_mod ($) {
+        my ($scheme) = @_;
+        my $modc = "PublicInbox::\U$1";
+        my $mod = $modc.'D';
+        eval "require $mod"; # IMAPD|HTTPD|NNTPD|POP3D
+        die $@ if $@;
+        my %xn = map { $_ => $mod->can($_) } qw(refresh post_accept);
+        $xn{tlsd} = $mod->new if $mod->can('refresh_groups'); #!HTTPD
+        my $tlsd = $xn{tlsd};
+        $xn{refresh} //= sub { $tlsd->refresh_groups(@_) };
+        $xn{post_accept} //= sub { $modc->new($_[0], $tlsd) };
+        $xn{af_default} = 'httpready' if $modc eq 'PublicInbox::HTTP';
+        \%xn;
+}
+
+sub daemon_prepare ($$) {
+        my ($default_listen, $xnetd) = @_;
         my $listener_names = {}; # sockname => IO::Handle
         $oldset = PublicInbox::DS::block_signals();
         @CMD = ($0, @ARGV);
         my ($prog) = ($CMD[0] =~ m!([^/]+)\z!g);
+        my $dh = defined($default_listen) ? " (default: $default_listen)" : '';
         my $help = <<EOF;
 usage: $prog [-l ADDRESS] [--cert=FILE] [--key=FILE]
 
 options:
 
-  -l ADDRESS    address to listen on (default: $default_listen)
+  -l ADDRESS    address to listen on$dh
   --cert=FILE   default SSL/TLS certificate
   --key=FILE    default SSL/TLS certificate
   -W WORKERS    number of worker processes to spawn (default: 1)
@@ -121,7 +137,10 @@ EOF
         # ignore daemonize when inheriting
         $daemonize = undef if scalar @listeners;
 
-        push @cfg_listen, $default_listen unless (@listeners || @cfg_listen);
+        unless (@listeners || @cfg_listen) {
+                $default_listen // die "no listeners specified\n";
+                push @cfg_listen, $default_listen
+        }
 
         foreach my $l (@cfg_listen) {
                 my $orig = $l;
@@ -139,7 +158,8 @@ EOF
                 } elsif ($scheme =~ /\A(?:https|imaps|imaps)\z/) {
                         die "$orig specified w/o cert=\n";
                 }
-                # TODO: use scheme to load either NNTP.pm or HTTP.pm
+                $scheme =~ /\A(http|imap|nntp|pop3)/ and
+                        $xnetd->{$l} = load_mod($1);
 
                 next if $listener_names->{$l}; # already inherited
                 my (%o, $sock_pkg);
@@ -585,14 +605,25 @@ sub defer_accept ($$) {
         }
 }
 
-sub daemon_loop ($$$$) {
-        my ($refresh, $post_accept, $tlsd, $af_default) = @_;
+sub daemon_loop ($) {
+        my ($xnetd) = @_;
+        my $refresh = sub {
+                my ($sig) = @_;
+                for my $xn (values %$xnetd) {
+                        eval { $xn->{refresh}->($sig) };
+                        warn "refresh $@\n" if $@;
+                }
+        };
         my %post_accept;
         while (my ($k, $v) = each %tls_opt) {
-                if ($k =~ s!\A(?:https|imaps|nntps)://!!) {
-                        $post_accept{$k} = tls_start_cb($v, $post_accept);
-                } elsif ($tlsd) { # STARTTLS, $k eq '' is OK
-                        $tlsd->{accept_tls} = $v;
+                my $l = $k;
+                $l =~ s!\A([^:]+)://!!;
+                my $scheme = $1;
+                my $xn = $xnetd->{$l} // $xnetd->{''};
+                if ($scheme =~ s!\A(?:https|imaps|nntps)!!) {
+                        $post_accept{$l} = tls_start_cb($v, $xn->{post_accept});
+                } elsif ($xn->{tlsd}) { # STARTTLS, $k eq '' is OK
+                        $xn->{tlsd}->{accept_tls} = $v;
                 }
         }
         my $sig = {
@@ -620,22 +651,29 @@ sub daemon_loop ($$$$) {
         $uid = $gid = undef;
         reopen_logs();
         @listeners = map {;
-                my $tls_cb = $post_accept{sockname($_)};
+                my $l = sockname($_);
+                my $tls_cb = $post_accept{$l};
+                my $xn = $xnetd->{$l} // $xnetd->{''};
 
                 # NNTPS, HTTPS, HTTP, IMAPS and POP3S are client-first traffic
                 # IMAP, NNTP and POP3 are server-first
-                defer_accept($_, $tls_cb ? 'dataready' : $af_default);
+                defer_accept($_, $tls_cb ? 'dataready' : $xn->{af_default});
 
                 # this calls epoll_create:
-                PublicInbox::Listener->new($_, $tls_cb || $post_accept)
+                PublicInbox::Listener->new($_, $tls_cb || $xn->{post_accept})
         } @listeners;
         PublicInbox::DS::event_loop($sig, $oldset);
 }
 
-sub run ($$$;$) {
-        my ($default, $refresh, $post_accept, $tlsd) = @_;
-        daemon_prepare($default);
-        my $af_default = $default =~ /:8080\z/ ? 'httpready' : undef;
+sub run {
+        my ($default_listen) = @_;
+        my $xnetd = {};
+        if ($default_listen) {
+                $default_listen =~ /\A(http|imap|nntp|pop3)/ or
+                        die "BUG: $default_listen";
+                $xnetd->{''} = load_mod($1);
+        }
+        daemon_prepare($default_listen, $xnetd);
         my $for_destroy = daemonize();
 
         # localize GCF2C for tests:
@@ -643,7 +681,7 @@ sub run ($$$;$) {
         local $PublicInbox::Git::async_warn = 1;
         local $SIG{__WARN__} = PublicInbox::Eml::warn_ignore_cb();
 
-        daemon_loop($refresh, $post_accept, $tlsd, $af_default);
+        daemon_loop($xnetd);
         PublicInbox::DS->Reset;
         # ->DESTROY runs when $for_destroy goes out-of-scope
 }
diff --git a/lib/PublicInbox/HTTPD.pm b/lib/PublicInbox/HTTPD.pm
index 02f424c6..715e4538 100644
--- a/lib/PublicInbox/HTTPD.pm
+++ b/lib/PublicInbox/HTTPD.pm
@@ -1,14 +1,15 @@
-# Copyright (C) 2016-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # wraps a listen socket for HTTP and links it to the PSGI app in
 # public-inbox-httpd
 package PublicInbox::HTTPD;
+use v5.10.1;
 use strict;
-use warnings;
-use Plack::Util;
+use Plack::Util ();
+use Plack::Builder;
+use PublicInbox::HTTP;
 use PublicInbox::HTTPD::Async;
-use PublicInbox::Daemon;
 
 sub pi_httpd_async { PublicInbox::HTTPD::Async->new(@_) }
 
@@ -41,10 +42,41 @@ sub new {
                 # detect when to use async paths for slow blobs
                 'pi-httpd.async' => \&pi_httpd_async
         );
-        bless {
-                app => $app,
-                env => \%env
-        }, $class;
+        bless { app => $app, env => \%env }, $class;
+}
+
+my %httpds; # per-listen-FD mapping for HTTPD->{env}->{SERVER_<NAME|PORT>}
+my $default_app; # ugh...
+
+sub refresh {
+        if (@main::ARGV) {
+                eval { $default_app = Plack::Util::load_psgi(@ARGV) };
+                if ($@) {
+                        die $@,
+"$0 runs in /, command-line paths must be absolute\n";
+                }
+        } else {
+                require PublicInbox::WWW;
+                my $www = PublicInbox::WWW->new;
+                $www->preload;
+                $default_app = builder {
+                        eval { enable 'ReverseProxy' };
+                        $@ and warn <<EOM;
+Plack::Middleware::ReverseProxy missing,
+URL generation for redirects may be wrong if behind a reverse proxy
+EOM
+                        enable 'Head';
+                        sub { $www->call(@_) };
+                };
+        }
+        %httpds = (); # invalidate cache
+}
+
+sub post_accept { # Listener->{post_accept}
+        my ($client, $addr, $srv) = @_; # $_[3] - tls_wrap (unused)
+        my $httpd = $httpds{fileno($srv)} //=
+                                __PACKAGE__->new($srv, $default_app, $client);
+        PublicInbox::HTTP->new($client, $addr, $httpd),
 }
 
 1;
diff --git a/lib/PublicInbox/IMAPD.pm b/lib/PublicInbox/IMAPD.pm
index 6aa3d12f..661d6537 100644
--- a/lib/PublicInbox/IMAPD.pm
+++ b/lib/PublicInbox/IMAPD.pm
@@ -1,14 +1,15 @@
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # represents an IMAPD (currently a singleton),
 # see script/public-inbox-imapd for how it is used
 package PublicInbox::IMAPD;
 use strict;
+use v5.10.1;
 use PublicInbox::Config;
 use PublicInbox::ConfigIter;
 use PublicInbox::InboxIdle;
-use PublicInbox::IMAP;
+use PublicInbox::IMAPdeflate; # loads PublicInbox::IMAP
 use PublicInbox::DummyInbox;
 my $dummy = bless { uidvalidity => 0 }, 'PublicInbox::DummyInbox';
 
diff --git a/lib/PublicInbox/NNTPD.pm b/lib/PublicInbox/NNTPD.pm
index 1e4ddd18..0350830b 100644
--- a/lib/PublicInbox/NNTPD.pm
+++ b/lib/PublicInbox/NNTPD.pm
@@ -1,14 +1,15 @@
-# Copyright (C) 2016-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
 # represents an NNTPD (currently a singleton),
 # see script/public-inbox-nntpd for how it is used
 package PublicInbox::NNTPD;
 use strict;
-use warnings;
+use v5.10.1;
 use Sys::Hostname;
 use PublicInbox::Config;
 use PublicInbox::InboxIdle;
+use PublicInbox::NNTPdeflate; # loads PublicInbox::NNTP
 
 sub new {
         my ($class) = @_;