* [PATCH 07/11] http: support HTTPS (kinda)
2019-06-29 19:59 7% [PATCH 00/11] ds: more updates Eric Wong
@ 2019-06-29 19:59 6% ` Eric Wong
0 siblings, 0 replies; 2+ results
From: Eric Wong @ 2019-06-29 19:59 UTC (permalink / raw)
To: meta
It's barely any effort at all to support HTTPS now that we have
NNTPS support and can share all the code for writing daemons.
However, we still depend on Varnish to avoid hug-of-death
situations, so supporting reverse-proxying will be required.
---
MANIFEST | 1 +
lib/PublicInbox/HTTP.pm | 10 +++-
t/httpd-https.t | 141 ++++++++++++++++++++++++++++++++++++++++++++++++
3 files changed, 151 insertions(+), 1 deletion(-)
create mode 100644 t/httpd-https.t
diff --git a/MANIFEST b/MANIFEST
index 29920953..4cb5f38f 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -210,6 +210,7 @@ t/hl_mod.t
t/html_index.t
t/httpd-corner.psgi
t/httpd-corner.t
+t/httpd-https.t
t/httpd-unix.t
t/httpd.t
t/hval.t
diff --git a/lib/PublicInbox/HTTP.pm b/lib/PublicInbox/HTTP.pm
index b8912950..680be72b 100644
--- a/lib/PublicInbox/HTTP.pm
+++ b/lib/PublicInbox/HTTP.pm
@@ -56,8 +56,16 @@ sub http_date () {
sub new ($$$) {
my ($class, $sock, $addr, $httpd) = @_;
my $self = fields::new($class);
- $self->SUPER::new($sock, EPOLLIN | EPOLLONESHOT);
+ my $ev = EPOLLIN;
+ my $wbuf;
+ if (ref($sock) eq 'IO::Socket::SSL' && !$sock->accept_SSL) {
+ return CORE::close($sock) if $! != EAGAIN;
+ $ev = PublicInbox::TLS::epollbit();
+ $wbuf = [ \&PublicInbox::DS::accept_tls_step ];
+ }
+ $self->SUPER::new($sock, $ev | EPOLLONESHOT);
$self->{httpd} = $httpd;
+ $self->{wbuf} = $wbuf if $wbuf;
($self->{remote_addr}, $self->{remote_port}) =
PublicInbox::Daemon::host_with_port($addr);
$self;
diff --git a/t/httpd-https.t b/t/httpd-https.t
new file mode 100644
index 00000000..f6b9806a
--- /dev/null
+++ b/t/httpd-https.t
@@ -0,0 +1,141 @@
+# Copyright (C) 2019 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+use strict;
+use warnings;
+use Test::More;
+use File::Temp qw(tempdir);
+use Socket qw(SOCK_STREAM IPPROTO_TCP SOL_SOCKET);
+# IO::Poll is part of the standard library, but distros may split them off...
+foreach my $mod (qw(IO::Socket::SSL IO::Poll)) {
+ eval "require $mod";
+ plan skip_all => "$mod missing for $0" if $@;
+}
+my $cert = 'certs/server-cert.pem';
+my $key = 'certs/server-key.pem';
+unless (-r $key && -r $cert) {
+ plan skip_all =>
+ "certs/ missing for $0, run ./create-certs.perl in certs/";
+}
+use_ok 'PublicInbox::TLS';
+use_ok 'IO::Socket::SSL';
+require './t/common.perl';
+my $psgi = "./t/httpd-corner.psgi";
+my $tmpdir = tempdir('pi-httpd-https-XXXXXX', TMPDIR => 1, CLEANUP => 1);
+my $err = "$tmpdir/stderr.log";
+my $out = "$tmpdir/stdout.log";
+my $httpd = 'blib/script/public-inbox-httpd';
+my %opts = (
+ LocalAddr => '127.0.0.1',
+ ReuseAddr => 1,
+ Proto => 'tcp',
+ Type => SOCK_STREAM,
+ Listen => 1024,
+);
+my $https = IO::Socket::INET->new(%opts);
+my ($pid, $tail_pid);
+END {
+ foreach ($pid, $tail_pid) {
+ kill 'TERM', $_ if defined $_;
+ }
+};
+my $https_addr = $https->sockhost . ':' . $https->sockport;
+my %opt = ( Proto => 'tcp', PeerAddr => $https_addr, Type => SOCK_STREAM );
+
+for my $args (
+ [ "-lhttps://$https_addr/?key=$key,cert=$cert" ],
+) {
+ for ($out, $err) {
+ open my $fh, '>', $_ or die "truncate: $!";
+ }
+ if (my $tail_cmd = $ENV{TAIL}) { # don't assume GNU tail
+ $tail_pid = fork;
+ if (defined $tail_pid && $tail_pid == 0) {
+ exec(split(' ', $tail_cmd), $out, $err);
+ }
+ }
+ my $cmd = [ $httpd, '-W0', @$args,
+ "--stdout=$out", "--stderr=$err", $psgi ];
+ $pid = spawn_listener(undef, $cmd, [ $https ]);
+ my %o = (
+ SSL_hostname => 'server.local',
+ SSL_verifycn_name => 'server.local',
+ SSL_verify_mode => SSL_VERIFY_PEER(),
+ SSL_ca_file => 'certs/test-ca.pem',
+ );
+ # start negotiating a slow TLS connection
+ my $slow = IO::Socket::INET->new(%opt, Blocking => 0);
+ $slow = IO::Socket::SSL->start_SSL($slow, SSL_startHandshake => 0, %o);
+ my @poll = (fileno($slow));
+ my $slow_done = $slow->connect_SSL;
+ if ($slow_done) {
+ diag('W: connect_SSL early OK, slow client test invalid');
+ push @poll, PublicInbox::Syscall::EPOLLOUT();
+ } else {
+ push @poll, PublicInbox::TLS::epollbit();
+ }
+
+ # normal HTTPS
+ my $c = IO::Socket::INET->new(%opt);
+ IO::Socket::SSL->start_SSL($c, %o);
+ ok($c->print("GET /empty HTTP/1.1\r\n\r\nHost: example.com\r\n\r\n"),
+ 'wrote HTTP request');
+ my $buf = '';
+ sysread($c, $buf, 2007, length($buf)) until $buf =~ /\r\n\r\n/;
+ like($buf, qr!\AHTTP/1\.1 200!, 'read HTTP response');
+
+ # HTTPS with bad hostname
+ $c = IO::Socket::INET->new(%opt);
+ $o{SSL_hostname} = $o{SSL_verifycn_name} = 'server.fail';
+ $c = IO::Socket::SSL->start_SSL($c, %o);
+ is($c, undef, 'HTTPS fails with bad hostname');
+
+ $o{SSL_hostname} = $o{SSL_verifycn_name} = 'server.local';
+ $c = IO::Socket::INET->new(%opt);
+ IO::Socket::SSL->start_SSL($c, %o);
+ ok($c, 'HTTPS succeeds again with valid hostname');
+
+ # slow TLS connection did not block the other fast clients while
+ # connecting, finish it off:
+ until ($slow_done) {
+ IO::Poll::_poll(-1, @poll);
+ $slow_done = $slow->connect_SSL and last;
+ @poll = (fileno($slow), PublicInbox::TLS::epollbit());
+ }
+ $slow->blocking(1);
+ ok($slow->print("GET /empty HTTP/1.1\r\n\r\nHost: example.com\r\n\r\n"),
+ 'wrote HTTP request from slow');
+ $buf = '';
+ sysread($slow, $buf, 666, length($buf)) until $buf =~ /\r\n\r\n/;
+ like($buf, qr!\AHTTP/1\.1 200!, 'read HTTP response from slow');
+ $slow = undef;
+
+ SKIP: {
+ skip 'TCP_DEFER_ACCEPT is Linux-only', 2 if $^O ne 'linux';
+ my $var = Socket::TCP_DEFER_ACCEPT();
+ defined(my $x = getsockopt($https, IPPROTO_TCP, $var)) or die;
+ ok(unpack('i', $x) > 0, 'TCP_DEFER_ACCEPT set on https');
+ };
+ SKIP: {
+ skip 'SO_ACCEPTFILTER is FreeBSD-only', 2 if $^O ne 'freebsd';
+ if (system('kldstat -m accf_data >/dev/null')) {
+ skip 'accf_data not loaded? kldload accf_data', 2;
+ }
+ require PublicInbox::Daemon;
+ my $var = PublicInbox::Daemon::SO_ACCEPTFILTER();
+ my $x = getsockopt($https, SOL_SOCKET, $var);
+ like($x, qr/\Adataready\0+\z/, 'got dataready accf for https');
+ };
+
+ $c = undef;
+ kill('TERM', $pid);
+ is($pid, waitpid($pid, 0), 'httpd exited successfully');
+ is($?, 0, 'no error in exited process');
+ $pid = undef;
+ if (defined $tail_pid) {
+ kill 'TERM', $tail_pid;
+ waitpid($tail_pid, 0);
+ $tail_pid = undef;
+ }
+}
+done_testing();
+1;
--
EW
^ permalink raw reply related [relevance 6%]
* [PATCH 00/11] ds: more updates
@ 2019-06-29 19:59 7% Eric Wong
2019-06-29 19:59 6% ` [PATCH 07/11] http: support HTTPS (kinda) Eric Wong
0 siblings, 1 reply; 2+ results
From: Eric Wong @ 2019-06-29 19:59 UTC (permalink / raw)
To: meta
We can simplify a lot of our async logic now that we don't have
to deal with the buffer-to-heap behavior of Danga::Socket.
The biggest change is now we no longer tie git-http-backend(1)
runtime and memory use to the bandwidth of a slow HTTP client.
This increases buffering on the FS (which may be tmpfs or
a fast SSD); but it's what nginx (and varnish) would be doing,
anyways
We can further remove a lot of the EvCleanup code since that
was to workaround deferred close being deferred for too long
when no I/O events were firing.
HTTPS now works, but more work needs to be done because
Varnish is still a requirement for busy sites.
Eric Wong (11):
ds: share lazy rbuf handling between HTTP and NNTP
ds: move requeue logic over from NNTP
http: use requeue instead of watch_in1
listener: use edge-triggered notifications
ds: handle deferred DS->close after timers
ds: consolidate IO::Socket::SSL checks
http: support HTTPS (kinda)
parentpipe: document and use one-shot wakeups
parentpipe: make the ->close call more obvious
httpd/async: switch to buffering-as-fast-as-possible
http: use bigger, but shorter-lived buffers for pipes
MANIFEST | 1 +
lib/PublicInbox/DS.pm | 85 +++++++++++++++----------
lib/PublicInbox/DSKQXS.pm | 4 +-
lib/PublicInbox/Daemon.pm | 4 +-
lib/PublicInbox/EvCleanup.pm | 80 +++--------------------
lib/PublicInbox/HTTP.pm | 102 +++++++++++++++--------------
lib/PublicInbox/HTTPD/Async.pm | 55 ++++++++--------
lib/PublicInbox/Listener.pm | 7 +-
lib/PublicInbox/NNTP.pm | 47 +++-----------
lib/PublicInbox/ParentPipe.pm | 17 +++--
lib/PublicInbox/Qspawn.pm | 2 +-
lib/PublicInbox/Syscall.pm | 4 +-
lib/PublicInbox/TLS.pm | 9 +--
t/httpd-https.t | 141 +++++++++++++++++++++++++++++++++++++++++
14 files changed, 314 insertions(+), 244 deletions(-)
create mode 100644 t/httpd-https.t
--
EW
^ permalink raw reply [relevance 7%]
Results 1-2 of 2 | reverse | options above
-- pct% links below jump to the message on this page, permalinks otherwise --
2019-06-29 19:59 7% [PATCH 00/11] ds: more updates Eric Wong
2019-06-29 19:59 6% ` [PATCH 07/11] http: support HTTPS (kinda) Eric Wong
Code repositories for project(s) associated with this public inbox
https://80x24.org/public-inbox.git
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).