From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.3.2 (2011-06-06) on dcvr.yhbt.net X-Spam-Level: X-Spam-ASN: X-Spam-Status: No, score=-3.0 required=3.0 tests=ALL_TRUSTED,AWL,BAYES_00, URIBL_BLOCKED shortcircuit=no autolearn=unavailable version=3.3.2 X-Original-To: meta@public-inbox.org Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id D3A0F20A21 for ; Mon, 25 Apr 2016 05:27:05 +0000 (UTC) From: Eric Wong To: meta@public-inbox.org Subject: [PATCH] split out NNTPD and HTTPD* modules Date: Mon, 25 Apr 2016 05:27:05 +0000 Message-Id: <20160425052705.32402-1-e@80x24.org> List-Id: Hopefully this modularizes things a little and allows us to work on a combined super server to save RAM. --- .../public-inbox-httpd => lib/PublicInbox/HTTPD.pm | 93 +------------------- .../PublicInbox/HTTPD/Async.pm | 98 ---------------------- .../public-inbox-nntpd => lib/PublicInbox/NNTPD.pm | 22 ++--- script/public-inbox-httpd | 78 +---------------- script/public-inbox-nntpd | 57 +------------ 5 files changed, 11 insertions(+), 337 deletions(-) copy script/public-inbox-httpd => lib/PublicInbox/HTTPD.pm (34%) mode change 100755 => 100644 copy script/public-inbox-httpd => lib/PublicInbox/HTTPD/Async.pm (29%) mode change 100755 => 100644 copy script/public-inbox-nntpd => lib/PublicInbox/NNTPD.pm (72%) mode change 100755 => 100644 diff --git a/script/public-inbox-httpd b/lib/PublicInbox/HTTPD.pm old mode 100755 new mode 100644 similarity index 34% copy from script/public-inbox-httpd copy to lib/PublicInbox/HTTPD.pm index 3ca974c..78efaa5 --- a/script/public-inbox-httpd +++ b/lib/PublicInbox/HTTPD.pm @@ -1,99 +1,12 @@ -#!/usr/bin/perl -w # Copyright (C) 2016 all contributors # License: AGPL-3.0+ -# -# Standalone HTTP server for public-inbox. -use strict; -use warnings; -use Plack::Util; -use PublicInbox::Daemon; -use PublicInbox::HTTP; -use Plack::Request; -use Plack::Builder; -my %httpds; -my $app; -my $refresh = sub { - if (@ARGV) { - eval { $app = Plack::Util::load_psgi(@ARGV) }; - if ($@) { - die $@, -"$0 runs in /, command-line paths must be absolute\n"; - } - } else { - require PublicInbox::WWW; - PublicInbox::WWW->preload; - my $www = PublicInbox::WWW->new; - $app = builder { - enable 'Chunked'; - eval { - enable 'Deflater', - content_type => [ qw( - text/html - text/plain - application/atom+xml - )] - }; - $@ and warn -"Plack::Middleware::Deflater missing, bandwidth will be wasted\n"; - - eval { enable 'ReverseProxy' }; - $@ and warn -"Plack::Middleware::ReverseProxy missing,\n", -"URL generation for redirects may be wrong if behind a reverse proxy\n"; - - enable 'Head'; - sub { $www->call(@_) }; - }; - } -}; - -PublicInbox::Daemon::run('0.0.0.0:8080', $refresh, - sub ($$$) { # post_accept - my ($client, $addr, $srv) = @_; - my $fd = fileno($srv); - my $h = $httpds{$fd} ||= PublicInbox::HTTPD->new($srv, $app); - PublicInbox::HTTP->new($client, $addr, $h), - }); - -1; - -# XXX This is a totally unstable API for public-inbox internal use only -# This is exposed via the 'pi-httpd.async' key in the PSGI env hash. -# The name of this key is not even stable! -# Currently is is intended for use with read-only pipes. -package PublicInbox::HTTPD::Async; -use strict; -use warnings; -use base qw(Danga::Socket); -use fields qw(cb); - -sub new { - my ($class, $io, $cb) = @_; - my $self = fields::new($class); - IO::Handle::blocking($io, 0); - $self->SUPER::new($io); - $self->{cb} = $cb; - $self->watch_read(1); - $self; -} - -sub event_read { $_[0]->{cb}->() } -sub event_hup { $_[0]->{cb}->() } -sub event_err { $_[0]->{cb}->() } -sub sysread { shift->{sock}->sysread(@_) } - -sub close { - my $self = shift; - $self->{cb} = undef; - $self->SUPER::close(@_); -} - -1; package PublicInbox::HTTPD; use strict; use warnings; use Plack::Util; +require PublicInbox::HTTPD::Async; +require PublicInbox::Daemon; sub pi_httpd_async { my ($io, $cb) = @_; @@ -126,7 +39,7 @@ sub new { ); bless { app => $app, - env => \%env, + env => \%env }, $class; } diff --git a/script/public-inbox-httpd b/lib/PublicInbox/HTTPD/Async.pm old mode 100755 new mode 100644 similarity index 29% copy from script/public-inbox-httpd copy to lib/PublicInbox/HTTPD/Async.pm index 3ca974c..6398502 --- a/script/public-inbox-httpd +++ b/lib/PublicInbox/HTTPD/Async.pm @@ -1,62 +1,6 @@ -#!/usr/bin/perl -w # Copyright (C) 2016 all contributors # License: AGPL-3.0+ # -# Standalone HTTP server for public-inbox. -use strict; -use warnings; -use Plack::Util; -use PublicInbox::Daemon; -use PublicInbox::HTTP; -use Plack::Request; -use Plack::Builder; -my %httpds; -my $app; -my $refresh = sub { - if (@ARGV) { - eval { $app = Plack::Util::load_psgi(@ARGV) }; - if ($@) { - die $@, -"$0 runs in /, command-line paths must be absolute\n"; - } - } else { - require PublicInbox::WWW; - PublicInbox::WWW->preload; - my $www = PublicInbox::WWW->new; - $app = builder { - enable 'Chunked'; - eval { - enable 'Deflater', - content_type => [ qw( - text/html - text/plain - application/atom+xml - )] - }; - $@ and warn -"Plack::Middleware::Deflater missing, bandwidth will be wasted\n"; - - eval { enable 'ReverseProxy' }; - $@ and warn -"Plack::Middleware::ReverseProxy missing,\n", -"URL generation for redirects may be wrong if behind a reverse proxy\n"; - - enable 'Head'; - sub { $www->call(@_) }; - }; - } -}; - -PublicInbox::Daemon::run('0.0.0.0:8080', $refresh, - sub ($$$) { # post_accept - my ($client, $addr, $srv) = @_; - my $fd = fileno($srv); - my $h = $httpds{$fd} ||= PublicInbox::HTTPD->new($srv, $app); - PublicInbox::HTTP->new($client, $addr, $h), - }); - -1; - # XXX This is a totally unstable API for public-inbox internal use only # This is exposed via the 'pi-httpd.async' key in the PSGI env hash. # The name of this key is not even stable! @@ -89,45 +33,3 @@ sub close { } 1; - -package PublicInbox::HTTPD; -use strict; -use warnings; -use Plack::Util; - -sub pi_httpd_async { - my ($io, $cb) = @_; - PublicInbox::HTTPD::Async->new($io, $cb); -} - -sub new { - my ($class, $sock, $app) = @_; - my $n = getsockname($sock) or die "not a socket: $sock $!\n"; - my ($host, $port) = PublicInbox::Daemon::host_with_port($n); - - my %env = ( - SERVER_NAME => $host, - SERVER_PORT => $port, - SCRIPT_NAME => '', - 'psgi.version' => [ 1, 1 ], - 'psgi.errors' => \*STDERR, - 'psgi.url_scheme' => 'http', - 'psgi.nonblocking' => Plack::Util::TRUE, - 'psgi.streaming' => Plack::Util::TRUE, - 'psgi.run_once' => Plack::Util::FALSE, - 'psgi.multithread' => Plack::Util::FALSE, - 'psgi.multiprocess' => Plack::Util::TRUE, - 'psgix.harakiri'=> Plack::Util::FALSE, - 'psgix.input.buffered' => Plack::Util::TRUE, - 'pi-httpd.async' => do { - no warnings 'once'; - *pi_httpd_async - }, - ); - bless { - app => $app, - env => \%env, - }, $class; -} - -1; diff --git a/script/public-inbox-nntpd b/lib/PublicInbox/NNTPD.pm old mode 100755 new mode 100644 similarity index 72% copy from script/public-inbox-nntpd copy to lib/PublicInbox/NNTPD.pm index cea8816..85109ea --- a/script/public-inbox-nntpd +++ b/lib/PublicInbox/NNTPD.pm @@ -1,23 +1,13 @@ -#!/usr/bin/perl -w -# Copyright (C) 2015 all contributors -# License: AGPLv3 or later (https://www.gnu.org/licenses/agpl-3.0.txt) -# -# Standalone NNTP server for public-inbox. -use strict; -use warnings; -require PublicInbox::Daemon; -require PublicInbox::NewsGroup; -require PublicInbox::NNTP; -require PublicInbox::Config; -my $nntpd = PublicInbox::NNTPD->new; -PublicInbox::Daemon::run('0.0.0.0:119', - sub { $nntpd->refresh_groups }, # refresh - sub ($$$) { PublicInbox::NNTP->new($_[0], $nntpd) }); # post_accept +# Copyright (C) 2016 all contributors +# License: AGPL-3.0+ -1; +# represents an NNTPD (currently a singleton), +# see script/public-inbox-nntpd for how it is used package PublicInbox::NNTPD; use strict; use warnings; +require PublicInbox::NewsGroup; +require PublicInbox::Config; sub new { my ($class) = @_; diff --git a/script/public-inbox-httpd b/script/public-inbox-httpd index 3ca974c..b29effc 100755 --- a/script/public-inbox-httpd +++ b/script/public-inbox-httpd @@ -8,6 +8,7 @@ use warnings; use Plack::Util; use PublicInbox::Daemon; use PublicInbox::HTTP; +use PublicInbox::HTTPD; use Plack::Request; use Plack::Builder; my %httpds; @@ -54,80 +55,3 @@ PublicInbox::Daemon::run('0.0.0.0:8080', $refresh, my $h = $httpds{$fd} ||= PublicInbox::HTTPD->new($srv, $app); PublicInbox::HTTP->new($client, $addr, $h), }); - -1; - -# XXX This is a totally unstable API for public-inbox internal use only -# This is exposed via the 'pi-httpd.async' key in the PSGI env hash. -# The name of this key is not even stable! -# Currently is is intended for use with read-only pipes. -package PublicInbox::HTTPD::Async; -use strict; -use warnings; -use base qw(Danga::Socket); -use fields qw(cb); - -sub new { - my ($class, $io, $cb) = @_; - my $self = fields::new($class); - IO::Handle::blocking($io, 0); - $self->SUPER::new($io); - $self->{cb} = $cb; - $self->watch_read(1); - $self; -} - -sub event_read { $_[0]->{cb}->() } -sub event_hup { $_[0]->{cb}->() } -sub event_err { $_[0]->{cb}->() } -sub sysread { shift->{sock}->sysread(@_) } - -sub close { - my $self = shift; - $self->{cb} = undef; - $self->SUPER::close(@_); -} - -1; - -package PublicInbox::HTTPD; -use strict; -use warnings; -use Plack::Util; - -sub pi_httpd_async { - my ($io, $cb) = @_; - PublicInbox::HTTPD::Async->new($io, $cb); -} - -sub new { - my ($class, $sock, $app) = @_; - my $n = getsockname($sock) or die "not a socket: $sock $!\n"; - my ($host, $port) = PublicInbox::Daemon::host_with_port($n); - - my %env = ( - SERVER_NAME => $host, - SERVER_PORT => $port, - SCRIPT_NAME => '', - 'psgi.version' => [ 1, 1 ], - 'psgi.errors' => \*STDERR, - 'psgi.url_scheme' => 'http', - 'psgi.nonblocking' => Plack::Util::TRUE, - 'psgi.streaming' => Plack::Util::TRUE, - 'psgi.run_once' => Plack::Util::FALSE, - 'psgi.multithread' => Plack::Util::FALSE, - 'psgi.multiprocess' => Plack::Util::TRUE, - 'psgix.harakiri'=> Plack::Util::FALSE, - 'psgix.input.buffered' => Plack::Util::TRUE, - 'pi-httpd.async' => do { - no warnings 'once'; - *pi_httpd_async - }, - ); - bless { - app => $app, - env => \%env, - }, $class; -} - -1; diff --git a/script/public-inbox-nntpd b/script/public-inbox-nntpd index cea8816..be860a5 100755 --- a/script/public-inbox-nntpd +++ b/script/public-inbox-nntpd @@ -6,64 +6,9 @@ use strict; use warnings; require PublicInbox::Daemon; -require PublicInbox::NewsGroup; require PublicInbox::NNTP; -require PublicInbox::Config; +require PublicInbox::NNTPD; my $nntpd = PublicInbox::NNTPD->new; PublicInbox::Daemon::run('0.0.0.0:119', sub { $nntpd->refresh_groups }, # refresh sub ($$$) { PublicInbox::NNTP->new($_[0], $nntpd) }); # post_accept - -1; -package PublicInbox::NNTPD; -use strict; -use warnings; - -sub new { - my ($class) = @_; - bless { - groups => {}, - err => \*STDERR, - out => \*STDOUT, - grouplist => [], - }, $class; -} - -sub refresh_groups () { - my ($self) = @_; - my $pi_config = PublicInbox::Config->new; - my $new = {}; - my @list; - foreach my $k (keys %$pi_config) { - $k =~ /\Apublicinbox\.([^\.]+)\.mainrepo\z/ or next; - my $g = $1; - my $git_dir = $pi_config->{$k}; - my $addr = $pi_config->{"publicinbox.$g.address"}; - my $ngname = $pi_config->{"publicinbox.$g.newsgroup"}; - if (defined $ngname) { - next if ($ngname eq ''); # disabled - $g = $ngname; - } - my $ng = PublicInbox::NewsGroup->new($g, $git_dir, $addr); - my $old_ng = $self->{groups}->{$g}; - - # Reuse the old one if possible since it can hold - # references to valid mm and gcf objects - if ($old_ng) { - $old_ng->update($ng); - $ng = $old_ng; - } - - # Only valid if msgmap and search works - if ($ng->usable) { - $new->{$g} = $ng; - push @list, $ng; - } - } - @list = sort { $a->{name} cmp $b->{name} } @list; - $self->{grouplist} = \@list; - # this will destroy old groups that got deleted - %{$self->{groups}} = %$new; -} - -1; -- EW