* [PATCH 37/57] nntp: NNTPS and NNTP+STARTTLS working
2019-06-24 2:52 7% [PATCH 00/57] ds: shrink, TLS support, buffer writes to FS Eric Wong
@ 2019-06-24 2:52 3% ` Eric Wong
0 siblings, 0 replies; 2+ results
From: Eric Wong @ 2019-06-24 2:52 UTC (permalink / raw)
To: meta
It kinda, barely works, and I'm most happy I got it working
without any modifications to the main NNTP::event_step callback
thanks to the DS->write(CODE) support we inherited from
Danga::Socket.
---
MANIFEST | 4 +
certs/.gitignore | 4 +
certs/create-certs.perl | 132 ++++++++++++++++++++++++++++++++
lib/PublicInbox/DS.pm | 28 ++++++-
lib/PublicInbox/Daemon.pm | 82 ++++++++++++++++++--
lib/PublicInbox/NNTP.pm | 27 ++++++-
lib/PublicInbox/NNTPD.pm | 1 +
lib/PublicInbox/TLS.pm | 24 ++++++
script/public-inbox-nntpd | 3 +-
t/nntpd-tls.t | 156 ++++++++++++++++++++++++++++++++++++++
t/nntpd.t | 2 +
11 files changed, 450 insertions(+), 13 deletions(-)
create mode 100644 certs/.gitignore
create mode 100755 certs/create-certs.perl
create mode 100644 lib/PublicInbox/TLS.pm
create mode 100644 t/nntpd-tls.t
diff --git a/MANIFEST b/MANIFEST
index c7693976..26ff0d0d 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -31,6 +31,8 @@ MANIFEST
Makefile.PL
README
TODO
+certs/.gitignore
+certs/create-certs.perl
ci/README
ci/deps.perl
ci/profiles.sh
@@ -129,6 +131,7 @@ lib/PublicInbox/Spamcheck/Spamc.pm
lib/PublicInbox/Spawn.pm
lib/PublicInbox/SpawnPP.pm
lib/PublicInbox/Syscall.pm
+lib/PublicInbox/TLS.pm
lib/PublicInbox/Unsubscribe.pm
lib/PublicInbox/UserContent.pm
lib/PublicInbox/V2Writable.pm
@@ -222,6 +225,7 @@ t/msg_iter.t
t/msgmap.t
t/msgtime.t
t/nntp.t
+t/nntpd-tls.t
t/nntpd.t
t/nulsubject.t
t/over.t
diff --git a/certs/.gitignore b/certs/.gitignore
new file mode 100644
index 00000000..0b3a547b
--- /dev/null
+++ b/certs/.gitignore
@@ -0,0 +1,4 @@
+*.pem
+*.der
+*.enc
+*.p12
diff --git a/certs/create-certs.perl b/certs/create-certs.perl
new file mode 100755
index 00000000..bfd8e5f1
--- /dev/null
+++ b/certs/create-certs.perl
@@ -0,0 +1,132 @@
+#!/usr/bin/perl -w
+# License: GPL-1.0+ or Artistic-1.0-Perl
+# from IO::Socket::SSL 2.063 / https://github.com/noxxi/p5-io-socket-ssl
+use strict;
+use warnings;
+use IO::Socket::SSL::Utils;
+use Net::SSLeay;
+
+my $dir = "./";
+my $now = time();
+my $later = $now + 100*365*86400;
+
+Net::SSLeay::SSLeay_add_ssl_algorithms();
+my $sha256 = Net::SSLeay::EVP_get_digestbyname('sha256') or die;
+my $printfp = sub {
+ my ($w,$cert) = @_;
+ print $w.' sha256$'.unpack('H*',Net::SSLeay::X509_digest($cert, $sha256))."\n"
+};
+
+my %time_valid = (not_before => $now, not_after => $later);
+
+my @ca = CERT_create(
+ CA => 1,
+ subject => { CN => 'IO::Socket::SSL Demo CA' },
+ %time_valid,
+);
+save('test-ca.pem',PEM_cert2string($ca[0]));
+
+my @server = CERT_create(
+ CA => 0,
+ subject => { CN => 'server.local' },
+ purpose => 'server',
+ issuer => \@ca,
+ %time_valid,
+);
+save('server-cert.pem',PEM_cert2string($server[0]));
+save('server-key.pem',PEM_key2string($server[1]));
+$printfp->(server => $server[0]);
+
+@server = CERT_create(
+ CA => 0,
+ subject => { CN => 'server2.local' },
+ purpose => 'server',
+ issuer => \@ca,
+ %time_valid,
+);
+save('server2-cert.pem',PEM_cert2string($server[0]));
+save('server2-key.pem',PEM_key2string($server[1]));
+$printfp->(server2 => $server[0]);
+
+@server = CERT_create(
+ CA => 0,
+ subject => { CN => 'server-ecc.local' },
+ purpose => 'server',
+ issuer => \@ca,
+ key => KEY_create_ec(),
+ %time_valid,
+);
+save('server-ecc-cert.pem',PEM_cert2string($server[0]));
+save('server-ecc-key.pem',PEM_key2string($server[1]));
+$printfp->('server-ecc' => $server[0]);
+
+
+my @client = CERT_create(
+ CA => 0,
+ subject => { CN => 'client.local' },
+ purpose => 'client',
+ issuer => \@ca,
+ %time_valid,
+);
+save('client-cert.pem',PEM_cert2string($client[0]));
+save('client-key.pem',PEM_key2string($client[1]));
+$printfp->(client => $client[0]);
+
+my @swc = CERT_create(
+ CA => 0,
+ subject => { CN => 'server.local' },
+ purpose => 'server',
+ issuer => \@ca,
+ subjectAltNames => [
+ [ DNS => '*.server.local' ],
+ [ IP => '127.0.0.1' ],
+ [ DNS => 'www*.other.local' ],
+ [ DNS => 'smtp.mydomain.local' ],
+ [ DNS => 'xn--lwe-sna.idntest.local' ]
+ ],
+ %time_valid,
+);
+save('server-wildcard.pem',PEM_cert2string($swc[0]),PEM_key2string($swc[1]));
+
+
+my @subca = CERT_create(
+ CA => 1,
+ issuer => \@ca,
+ subject => { CN => 'IO::Socket::SSL Demo Sub CA' },
+ %time_valid,
+);
+save('test-subca.pem',PEM_cert2string($subca[0]));
+@server = CERT_create(
+ CA => 0,
+ subject => { CN => 'server.local' },
+ purpose => 'server',
+ issuer => \@subca,
+ %time_valid,
+);
+save('sub-server.pem',PEM_cert2string($server[0]).PEM_key2string($server[1]));
+
+
+
+my @cap = CERT_create(
+ CA => 1,
+ subject => { CN => 'IO::Socket::SSL::Intercept' },
+ %time_valid,
+);
+save('proxyca.pem',PEM_cert2string($cap[0]).PEM_key2string($cap[1]));
+
+sub save {
+ my $file = shift;
+ open(my $fd,'>',$dir.$file) or die $!;
+ print $fd @_;
+}
+
+system(<<CMD);
+cd $dir
+set -x
+openssl x509 -in server-cert.pem -out server-cert.der -outform der
+openssl rsa -in server-key.pem -out server-key.der -outform der
+openssl rsa -in server-key.pem -out server-key.enc -passout pass:bluebell
+openssl rsa -in client-key.pem -out client-key.enc -passout pass:opossum
+openssl pkcs12 -export -in server-cert.pem -inkey server-key.pem -out server.p12 -passout pass:
+openssl pkcs12 -export -in server-cert.pem -inkey server-key.pem -out server_enc.p12 -passout pass:bluebell
+CMD
diff --git a/lib/PublicInbox/DS.pm b/lib/PublicInbox/DS.pm
index 1a1ef7d3..044b991c 100644
--- a/lib/PublicInbox/DS.pm
+++ b/lib/PublicInbox/DS.pm
@@ -465,7 +465,11 @@ next_buf:
}
} else { #($ref eq 'CODE') {
shift @$wbuf;
+ my $before = scalar(@$wbuf);
$bref->($self);
+
+ # bref may be enqueueing more CODE to call (see accept_tls_step)
+ return 0 if (scalar(@$wbuf) > $before);
}
} # while @$wbuf
@@ -479,7 +483,14 @@ sub do_read ($$$$) {
return ($r == 0 ? $self->close : $r) if defined $r;
# common for clients to break connections without warning,
# would be too noisy to log here:
- $! == EAGAIN ? $self->watch_in1 : $self->close;
+ if (ref($self) eq 'IO::Socket::SSL') {
+ my $ev = PublicInbox::TLS::epollbit() or return $self->close;
+ watch($self, $ev | EPOLLONESHOT);
+ } elsif ($! == EAGAIN) {
+ watch($self, EPOLLIN | EPOLLONESHOT);
+ } else {
+ $self->close;
+ }
}
# drop the socket if we hit unrecoverable errors on our system which
@@ -566,7 +577,7 @@ sub msg_more ($$) {
my $self = $_[0];
my $sock = $self->{sock} or return 1;
- if (MSG_MORE && !$self->{wbuf}) {
+ if (MSG_MORE && !$self->{wbuf} && ref($sock) ne 'IO::Socket::SSL') {
my $n = send($sock, $_[1], MSG_MORE);
if (defined $n) {
my $nlen = bytes::length($_[1]) - $n;
@@ -597,6 +608,19 @@ sub watch ($$) {
sub watch_in1 ($) { watch($_[0], EPOLLIN | EPOLLONESHOT) }
+# return true if complete, false if incomplete (or failure)
+sub accept_tls_step ($) {
+ my ($self) = @_;
+ my $sock = $self->{sock} or return;
+ return 1 if $sock->accept_SSL;
+ return $self->close if $! != EAGAIN;
+ if (my $ev = PublicInbox::TLS::epollbit()) {
+ unshift @{$self->{wbuf} ||= []}, \&accept_tls_step;
+ return watch($self, $ev | EPOLLONESHOT);
+ }
+ drop($self, 'BUG? EAGAIN but '.PublicInbox::TLS::err());
+}
+
package PublicInbox::DS::Timer;
# [$abs_float_firetime, $coderef];
sub cancel {
diff --git a/lib/PublicInbox/Daemon.pm b/lib/PublicInbox/Daemon.pm
index b8d6b572..24c13ad2 100644
--- a/lib/PublicInbox/Daemon.pm
+++ b/lib/PublicInbox/Daemon.pm
@@ -22,12 +22,48 @@ my (@cfg_listen, $stdout, $stderr, $group, $user, $pid_file, $daemonize);
my $worker_processes = 1;
my @listeners;
my %pids;
-my %listener_names;
+my %listener_names; # sockname => IO::Handle
+my %tls_opt; # scheme://sockname => args for IO::Socket::SSL->start_SSL
my $reexec_pid;
my $cleanup;
my ($uid, $gid);
+my ($default_cert, $default_key);
END { $cleanup->() if $cleanup };
+sub tls_listen ($$$) {
+ my ($scheme, $sockname, $opt_str) = @_;
+ # opt_str: opt1=val1,opt2=val2 (opt may repeat for multi-value)
+ require PublicInbox::TLS;
+ my $o = {};
+ # allow ',' as delimiter since '&' is shell-unfriendly
+ foreach (split(/[,&]/, $opt_str)) {
+ my ($k, $v) = split(/=/, $_, 2);
+ push @{$o->{$k} ||= []}, $v;
+ }
+
+ # key may be a part of cert. At least
+ # p5-io-socket-ssl/example/ssl_server.pl has this fallback:
+ $o->{cert} //= [ $default_cert ];
+ $o->{key} //= defined($default_key) ? [ $default_key ] : $o->{cert};
+ my %ctx_opt = (SSL_server => 1);
+ # parse out hostname:/path/to/ mappings:
+ foreach my $k (qw(cert key)) {
+ my $x = $ctx_opt{'SSL_'.$k.'_file'} = {};
+ foreach my $path (@{$o->{$k}}) {
+ my $host = '';
+ $path =~ s/\A([^:]+):// and $host = $1;
+ $x->{$host} = $path;
+ }
+ }
+ my $ctx = IO::Socket::SSL::SSL_Context->new(%ctx_opt) or
+ die 'SSL_Context->new: '.PublicInbox::TLS::err();
+ $tls_opt{"$scheme://$sockname"} = {
+ SSL_server => 1,
+ SSL_startHandshake => 0,
+ SSL_reuse_ctx => $ctx
+ };
+}
+
sub daemon_prepare ($) {
my ($default_listen) = @_;
@CMD = ($0, @ARGV);
@@ -42,6 +78,8 @@ sub daemon_prepare ($) {
'u|user=s' => \$user,
'g|group=s' => \$group,
'D|daemonize' => \$daemonize,
+ 'cert=s' => \$default_cert,
+ 'key=s' => \$default_key,
);
GetOptions(%opts) or die "bad command-line args\n";
@@ -55,6 +93,18 @@ sub daemon_prepare ($) {
push @cfg_listen, $default_listen unless (@listeners || @cfg_listen);
foreach my $l (@cfg_listen) {
+ my $orig = $l;
+ my $scheme = '';
+ $l =~ s!\A([^:]+)://!! and $scheme = $1;
+ if ($l =~ s!/?\?(.+)\z!!) {
+ tls_listen($scheme, $l, $1);
+ } elsif (defined($default_cert)) {
+ tls_listen($scheme, $l, '');
+ } elsif ($scheme =~ /\A(?:nntps|https)\z/) {
+ die "$orig specified w/o cert=\n";
+ }
+ # TODO: use scheme to load either NNTP.pm or HTTP.pm
+
next if $listener_names{$l}; # already inherited
my (%o, $sock_pkg);
if (index($l, '/') == 0) {
@@ -461,9 +511,26 @@ sub master_loop {
exit # never gets here, just for documentation
}
-sub daemon_loop ($$) {
- my ($refresh, $post_accept) = @_;
+sub tls_start_cb ($$) {
+ my ($opt, $orig_post_accept) = @_;
+ sub {
+ my ($io, $addr, $srv) = @_;
+ my $ssl = IO::Socket::SSL->start_SSL($io, %$opt);
+ $orig_post_accept->($ssl, $addr, $srv);
+ }
+}
+
+sub daemon_loop ($$$) {
+ my ($refresh, $post_accept, $nntpd) = @_;
PublicInbox::EvCleanup::enable(); # early for $refresh
+ my %post_accept;
+ while (my ($k, $v) = each %tls_opt) {
+ if ($k =~ s!\A(?:nntps|https)://!!) {
+ $post_accept{$k} = tls_start_cb($v, $post_accept);
+ } elsif ($nntpd) { # STARTTLS, $k eq '' is OK
+ $nntpd->{accept_tls} = $v;
+ }
+ }
my $parent_pipe;
if ($worker_processes > 0) {
$refresh->(); # preload by default
@@ -484,18 +551,19 @@ sub daemon_loop ($$) {
$SIG{$_} = 'IGNORE' for qw(USR2 TTIN TTOU WINCH);
# this calls epoll_create:
@listeners = map {
- PublicInbox::Listener->new($_, $post_accept)
+ PublicInbox::Listener->new($_,
+ $post_accept{sockname($_)} || $post_accept)
} @listeners;
PublicInbox::DS->EventLoop;
$parent_pipe = undef;
}
-sub run ($$$) {
- my ($default, $refresh, $post_accept) = @_;
+sub run ($$$;$) {
+ my ($default, $refresh, $post_accept, $nntpd) = @_;
daemon_prepare($default);
daemonize();
- daemon_loop($refresh, $post_accept);
+ daemon_loop($refresh, $post_accept, $nntpd);
}
sub do_chown ($) {
diff --git a/lib/PublicInbox/NNTP.pm b/lib/PublicInbox/NNTP.pm
index a18641d3..659e44d5 100644
--- a/lib/PublicInbox/NNTP.pm
+++ b/lib/PublicInbox/NNTP.pm
@@ -1,4 +1,4 @@
-# Copyright (C) 2015-2018 all contributors <meta@public-inbox.org>
+# Copyright (C) 2015-2019 all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
#
# Each instance of this represents a NNTP client socket
@@ -98,11 +98,19 @@ sub expire_old () {
sub new ($$$) {
my ($class, $sock, $nntpd) = @_;
my $self = fields::new($class);
- $self->SUPER::new($sock, EPOLLOUT | EPOLLONESHOT);
+ my $ev = EPOLLOUT | EPOLLONESHOT;
+ my $wbuf = [];
+ if (ref($sock) eq 'IO::Socket::SSL' && !$sock->accept_SSL) {
+ $ev = PublicInbox::TLS::epollbit() or return $sock->close;
+ $ev |= EPOLLONESHOT;
+ $wbuf->[0] = \&PublicInbox::DS::accept_tls_step;
+ }
+ $self->SUPER::new($sock, $ev);
$self->{nntpd} = $nntpd;
my $greet = "201 $nntpd->{servername} ready - post via email\r\n";
open my $fh, '<:scalar', \$greet or die "open :scalar: $!";
- $self->{wbuf} = [ $fh ];
+ push @$wbuf, $fh;
+ $self->{wbuf} = $wbuf;
$self->{rbuf} = '';
update_idle_time($self);
$expt ||= PublicInbox::EvCleanup::later(*expire_old);
@@ -900,6 +908,19 @@ sub cmd_xover ($;$) {
});
}
+sub cmd_starttls ($) {
+ my ($self) = @_;
+ my $sock = $self->{sock} or return;
+ # RFC 4642 2.2.1
+ (ref($sock) eq 'IO::Socket::SSL') and return '502 Command unavailable';
+ my $opt = $self->{nntpd}->{accept_tls} or
+ return '580 can not initiate TLS negotiation';
+ res($self, '382 Continue with TLS negotiation');
+ $self->{sock} = IO::Socket::SSL->start_SSL($sock, %$opt);
+ requeue($self) if PublicInbox::DS::accept_tls_step($self);
+ undef;
+}
+
sub cmd_xpath ($$) {
my ($self, $mid) = @_;
return r501 unless $mid =~ /\A<(.+)>\z/;
diff --git a/lib/PublicInbox/NNTPD.pm b/lib/PublicInbox/NNTPD.pm
index 32848d7c..6d9ffd5f 100644
--- a/lib/PublicInbox/NNTPD.pm
+++ b/lib/PublicInbox/NNTPD.pm
@@ -25,6 +25,7 @@ sub new {
out => \*STDOUT,
grouplist => [],
servername => $name,
+ # accept_tls => { SSL_server => 1, ..., SSL_reuse_ctx => ... }
}, $class;
}
diff --git a/lib/PublicInbox/TLS.pm b/lib/PublicInbox/TLS.pm
new file mode 100644
index 00000000..576c11d7
--- /dev/null
+++ b/lib/PublicInbox/TLS.pm
@@ -0,0 +1,24 @@
+# Copyright (C) 2019 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# IO::Socket::SSL support code
+package PublicInbox::TLS;
+use strict;
+use IO::Socket::SSL;
+require Carp;
+use Errno qw(EAGAIN);
+use PublicInbox::Syscall qw(EPOLLIN EPOLLOUT);
+
+sub err () { $SSL_ERROR }
+
+# returns the EPOLL event bit which matches the existing SSL error
+sub epollbit () {
+ if ($! == EAGAIN) {
+ return EPOLLIN if $SSL_ERROR == SSL_WANT_READ;
+ return EPOLLOUT if $SSL_ERROR == SSL_WANT_WRITE;
+ die "unexpected SSL error: $SSL_ERROR";
+ }
+ 0;
+}
+
+1;
diff --git a/script/public-inbox-nntpd b/script/public-inbox-nntpd
index 484ce8d6..55bf330e 100755
--- a/script/public-inbox-nntpd
+++ b/script/public-inbox-nntpd
@@ -11,4 +11,5 @@ require PublicInbox::NNTPD;
my $nntpd = PublicInbox::NNTPD->new;
PublicInbox::Daemon::run('0.0.0.0:119',
sub { $nntpd->refresh_groups }, # refresh
- sub ($$$) { PublicInbox::NNTP->new($_[0], $nntpd) }); # post_accept
+ sub ($$$) { PublicInbox::NNTP->new($_[0], $nntpd) }, # post_accept
+ $nntpd);
diff --git a/t/nntpd-tls.t b/t/nntpd-tls.t
new file mode 100644
index 00000000..53890ff2
--- /dev/null
+++ b/t/nntpd-tls.t
@@ -0,0 +1,156 @@
+# Copyright (C) 2019 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+use strict;
+use warnings;
+use Test::More;
+use File::Temp qw(tempdir);
+use Socket qw(SOCK_STREAM);
+foreach my $mod (qw(DBD::SQLite IO::Socket::SSL Net::NNTP)) {
+ eval "require $mod";
+ plan skip_all => "$mod missing for $0" if $@;
+}
+my $cert = 'certs/server-cert.pem';
+my $key = 'certs/server-key.pem';
+unless (-r $key && -r $cert) {
+ plan skip_all =>
+ "certs/ missing for $0, run ./create-certs.perl in certs/";
+}
+
+use_ok 'PublicInbox::TLS';
+use_ok 'IO::Socket::SSL';
+require './t/common.perl';
+require PublicInbox::InboxWritable;
+require PublicInbox::MIME;
+require PublicInbox::SearchIdx;
+my $version = 2; # v2 needs newer git
+require_git('2.6') if $version >= 2;
+my $tmpdir = tempdir('pi-nntpd-tls-XXXXXX', TMPDIR => 1, CLEANUP => 1);
+my $err = "$tmpdir/stderr.log";
+my $out = "$tmpdir/stdout.log";
+my $mainrepo = "$tmpdir";
+my $pi_config = "$tmpdir/pi_config";
+my $group = 'test-nntpd-tls';
+my $addr = $group . '@example.com';
+my $nntpd = 'blib/script/public-inbox-nntpd';
+my %opts = (
+ LocalAddr => '127.0.0.1',
+ ReuseAddr => 1,
+ Proto => 'tcp',
+ Type => SOCK_STREAM,
+ Listen => 1024,
+);
+my $starttls = IO::Socket::INET->new(%opts);
+my $nntps = IO::Socket::INET->new(%opts);
+my ($pid, $tail_pid);
+END {
+ foreach ($pid, $tail_pid) {
+ kill 'TERM', $_ if defined $_;
+ }
+};
+
+my $ibx = PublicInbox::Inbox->new({
+ mainrepo => $mainrepo,
+ name => 'nntpd-tls',
+ version => $version,
+ -primary_address => $addr,
+ indexlevel => 'basic',
+});
+$ibx = PublicInbox::InboxWritable->new($ibx, {nproc=>1});
+$ibx->init_inbox(0);
+{
+ open my $fh, '>', $pi_config or die "open: $!\n";
+ print $fh <<EOF
+[publicinbox "nntpd-tls"]
+ mainrepo = $mainrepo
+ address = $addr
+ indexlevel = basic
+ newsgroup = $group
+EOF
+ ;
+ close $fh or die "close: $!\n";
+}
+
+{
+ my $im = $ibx->importer(0);
+ my $mime = PublicInbox::MIME->new(do {
+ open my $fh, '<', 't/data/0001.patch' or die;
+ local $/;
+ <$fh>
+ });
+ ok($im->add($mime), 'message added');
+ $im->done;
+ if ($version == 1) {
+ my $s = PublicInbox::SearchIdx->new($ibx, 1);
+ $s->index_sync;
+ }
+}
+
+my $nntps_addr = $nntps->sockhost . ':' . $nntps->sockport;
+my $starttls_addr = $starttls->sockhost . ':' . $starttls->sockport;
+my $env = { PI_CONFIG => $pi_config };
+
+for my $args (
+ [ "--cert=$cert", "--key=$key",
+ "-lnntps://$nntps_addr",
+ "-lnntp://$starttls_addr" ],
+) {
+ for ($out, $err) {
+ open my $fh, '>', $_ or die "truncate: $!";
+ }
+ if (my $tail_cmd = $ENV{TAIL}) { # don't assume GNU tail
+ $tail_pid = fork;
+ if (defined $tail_pid && $tail_pid == 0) {
+ exec(split(' ', $tail_cmd), $out, $err);
+ }
+ }
+ my $cmd = [ $nntpd, '-W0', @$args, "--stdout=$out", "--stderr=$err" ];
+ $pid = spawn_listener($env, $cmd, [ $starttls, $nntps ]);
+ my %o = (
+ SSL_hostname => 'server.local',
+ SSL_verifycn_name => 'server.local',
+ SSL => 1,
+ SSL_verify_mode => SSL_VERIFY_PEER(),
+ SSL_ca_file => 'certs/test-ca.pem',
+ );
+ my $expect = { $group => [qw(1 1 n)] };
+
+ # NNTPS
+ my $c = Net::NNTP->new($nntps_addr, %o);
+ my $list = $c->list;
+ is_deeply($list, $expect, 'NNTPS LIST works');
+
+ # STARTTLS
+ delete $o{SSL};
+ $c = Net::NNTP->new($starttls_addr, %o);
+ $list = $c->list;
+ is_deeply($list, $expect, 'plain LIST works');
+ ok($c->starttls, 'STARTTLS succeeds');
+ is($c->code, 382, 'got 382 for STARTTLS');
+ $list = $c->list;
+ is_deeply($list, $expect, 'LIST works after STARTTLS');
+
+ # Net::NNTP won't let us do dumb things, but we need to test
+ # dumb things, so use Net::Cmd directly:
+ my $n = $c->command('STARTTLS')->response();
+ is($n, Net::Cmd::CMD_ERROR(), 'error attempting STARTTLS again');
+ is($c->code, 502, '502 according to RFC 4642 sec#2.2.1');
+
+ $c = undef;
+ kill('TERM', $pid);
+ is($pid, waitpid($pid, 0), 'nntpd exited successfully');
+ is($?, 0, 'no error in exited process');
+ $pid = undef;
+ my $eout = eval {
+ open my $fh, '<', $err or die "open $err failed: $!";
+ local $/;
+ <$fh>;
+ };
+ unlike($eout, qr/wide/i, 'no Wide character warnings');
+ if (defined $tail_pid) {
+ kill 'TERM', $tail_pid;
+ waitpid($tail_pid, 0);
+ $tail_pid = undef;
+ }
+}
+done_testing();
+1;
diff --git a/t/nntpd.t b/t/nntpd.t
index c37880bf..6cba2be4 100644
--- a/t/nntpd.t
+++ b/t/nntpd.t
@@ -106,6 +106,8 @@ EOF
is_deeply($list, { $group => [ qw(1 1 n) ] }, 'LIST works');
is_deeply([$n->group($group)], [ qw(0 1 1), $group ], 'GROUP works');
is_deeply($n->listgroup($group), [1], 'listgroup OK');
+ ok(!$n->starttls, 'STARTTLS fails when unconfigured');
+ is($n->code, 580, 'got 580 code on server w/o TLS');
%opts = (
PeerAddr => $host_port,
--
EW
^ permalink raw reply related [relevance 3%]
* [PATCH 00/57] ds: shrink, TLS support, buffer writes to FS
@ 2019-06-24 2:52 7% Eric Wong
2019-06-24 2:52 3% ` [PATCH 37/57] nntp: NNTPS and NNTP+STARTTLS working Eric Wong
0 siblings, 1 reply; 2+ results
From: Eric Wong @ 2019-06-24 2:52 UTC (permalink / raw)
To: meta
I finally took the step of making changes to DS after
wanting to do something along these lines to Danga::Socket
for the past decade or so And down the rabitt-hole I went.
Write buffering now goes to the filesystem (which is quite fast
on Linux and FreeBSD), so memory usage with giant messages is
slightly reduced compared to before. It could be better if we
replace Email::(Simple|MIME) with something which doesn't
require slurping (but that's a big task).
Fields for read (for NNTP) and all write buffers are lazily
allocated, now, so there's some memory savings with 10K clients
Further memory savings were achieved by passing $self to
DS->write(sub {...}), eliminiating the need for most anonymous
subs.
NNTPS and NNTP+STARTTLS are now supported via public-inbox-nntpd
using the --key and --cert parameters (HTTPS coming). I'm very
happy with how I was able to reuse the write-buffering code for
TLS negotiation and not have to add additional fields or code in
hot paths.
I'm pretty happy with this, so far; but there's still plenty
left to be done. I'm not too impressed with the per-client
memory cost of IO::Socket::SSL, even with
SSL_MODE_RELEASE_BUFFERS, and will need to do further analysis
to see what memory reductions are possible.
Eric Wong (57):
ds: get rid of {closed} field
ds: get rid of more unused debug instance methods
ds: use and export monotonic now()
AddTimer: avoid clock_gettime for the '0' case
ds: get rid of on_incomplete_write wrapper
ds: lazy initialize wbuf_off
ds: split out from ->flush_write and ->write
ds: lazy-initialize wbuf
ds: don't pass `events' arg to EPOLL_CTL_DEL
ds: remove support for DS->write(undef)
http: favor DS->write(strref) when reasonable
ds: share send(..., MSG_MORE) logic
ds: switch write buffering to use a tempfile
ds: get rid of redundant and unnecessary POLL* constants
syscall: get rid of unused EPOLL* constants
syscall: get rid of unnecessary uname local vars
ds: set event flags directly at initialization
ds: import IO::KQueue namespace
ds: share watch_chg between watch_read/watch_write
ds: remove IO::Poll support (for now)
ds: get rid of event_watch field
httpd/async: remove EINTR check
spawn: remove `Blocking' flag handling
qspawn: describe where `$rpipe' come from
http|nntp: favor "$! == EFOO" over $!{EFOO} checks
ds: favor `delete' over assigning fields to `undef'
http: don't pass extra args to PublicInbox::DS::close
ds: pass $self to code references
evcleanup: replace _run_asap with `event_step' callback
ds: remove pointless exit calls
http|nntp: be explicit about bytes::length on rbuf
ds: hoist out do_read from NNTP and HTTP
nntp: simplify re-arming/requeue logic
allow use of PerlIO layers for filesystem writes
ds: deal better with FS-related errors IO buffers
nntp: wait for writability before sending greeting
nntp: NNTPS and NNTP+STARTTLS working
certs/create-certs.perl: fix cert validity on 32-bit
daemon: map inherited sockets to well-known schemes
ds|nntp: use CORE::close on socket
nntp: call SSL_shutdown in normal cases
t/nntpd-tls: slow client connection test
daemon: use SSL_MODE_RELEASE_BUFFERS
ds: allow ->write callbacks to syswrite directly
nntp: reduce allocations for greeting
ds: always use EV_ADD with EV_SET
nntp: simplify long response logic and fix nesting
ds: flush_write runs ->write callbacks even if closed
nntp: lazily allocate and stash rbuf
ci: require IO::KQueue on FreeBSD, for now
nntp: send greeting immediately for plain sockets
daemon: set TCP_DEFER_ACCEPT on everything but NNTP
daemon: use FreeBSD accept filters on non-NNTP
ds: split out IO::KQueue-specific code
ds: reimplement IO::Poll support to look like epoll
Revert "ci: require IO::KQueue on FreeBSD, for now"
ds: reduce overhead of tempfile creation
MANIFEST | 7 +
certs/.gitignore | 4 +
certs/create-certs.perl | 132 +++++++
lib/PublicInbox/DS.pm | 635 ++++++++++++------------------
lib/PublicInbox/DSKQXS.pm | 73 ++++
lib/PublicInbox/DSPoll.pm | 58 +++
lib/PublicInbox/Daemon.pm | 152 ++++++-
lib/PublicInbox/EvCleanup.pm | 20 +-
lib/PublicInbox/GitHTTPBackend.pm | 18 +-
lib/PublicInbox/HTTP.pm | 154 +++-----
lib/PublicInbox/HTTPD/Async.pm | 44 ++-
lib/PublicInbox/Listener.pm | 4 +-
lib/PublicInbox/NNTP.pm | 243 +++++-------
lib/PublicInbox/NNTPD.pm | 2 +
lib/PublicInbox/ParentPipe.pm | 3 +-
lib/PublicInbox/Qspawn.pm | 11 +-
lib/PublicInbox/Spawn.pm | 2 -
lib/PublicInbox/Syscall.pm | 27 +-
lib/PublicInbox/TLS.pm | 24 ++
script/public-inbox-nntpd | 3 +-
t/ds-poll.t | 58 +++
t/httpd-corner.t | 38 +-
t/httpd.t | 18 +
t/nntpd-tls.t | 224 +++++++++++
t/nntpd.t | 2 +
t/spawn.t | 11 -
26 files changed, 1251 insertions(+), 716 deletions(-)
create mode 100644 certs/.gitignore
create mode 100755 certs/create-certs.perl
create mode 100644 lib/PublicInbox/DSKQXS.pm
create mode 100644 lib/PublicInbox/DSPoll.pm
create mode 100644 lib/PublicInbox/TLS.pm
create mode 100644 t/ds-poll.t
create mode 100644 t/nntpd-tls.t
--
EW
^ permalink raw reply [relevance 7%]
Results 1-2 of 2 | reverse | options above
-- pct% links below jump to the message on this page, permalinks otherwise --
2019-06-24 2:52 7% [PATCH 00/57] ds: shrink, TLS support, buffer writes to FS Eric Wong
2019-06-24 2:52 3% ` [PATCH 37/57] nntp: NNTPS and NNTP+STARTTLS working Eric Wong
Code repositories for project(s) associated with this public inbox
https://80x24.org/public-inbox.git
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).