user/dev discussion of public-inbox itself
 help / color / mirror / code / Atom feed
From: Eric Wong <e@80x24.org>
To: meta@public-inbox.org
Subject: [PATCH 6/6] net_reader: new package split from -watch
Date: Wed, 10 Feb 2021 07:07:49 +0000	[thread overview]
Message-ID: <20210210070749.30391-7-e@80x24.org> (raw)
In-Reply-To: <20210210070749.30391-1-e@80x24.org>

We'll be using some of this for IMAP and NNTP support in lei,
too.  More will need to be done to improve code sharing and
reusability, soon, but this is a start.
---
 MANIFEST                     |   1 +
 lib/PublicInbox/NetReader.pm | 220 +++++++++++++++++++++++++++++++++++
 lib/PublicInbox/Watch.pm     | 204 +-------------------------------
 3 files changed, 222 insertions(+), 203 deletions(-)
 create mode 100644 lib/PublicInbox/NetReader.pm

diff --git a/MANIFEST b/MANIFEST
index f8ee6998..92226d5a 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -209,6 +209,7 @@ lib/PublicInbox/NDC_PP.pm
 lib/PublicInbox/NNTP.pm
 lib/PublicInbox/NNTPD.pm
 lib/PublicInbox/NNTPdeflate.pm
+lib/PublicInbox/NetReader.pm
 lib/PublicInbox/NewsWWW.pm
 lib/PublicInbox/OnDestroy.pm
 lib/PublicInbox/Over.pm
diff --git a/lib/PublicInbox/NetReader.pm b/lib/PublicInbox/NetReader.pm
new file mode 100644
index 00000000..79047fd2
--- /dev/null
+++ b/lib/PublicInbox/NetReader.pm
@@ -0,0 +1,220 @@
+# Copyright (C) 2021 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# common reader code for IMAP and NNTP (and maybe JMAP)
+package PublicInbox::NetReader;
+use strict;
+use v5.10.1;
+use parent qw(Exporter);
+
+# TODO: trim this down, this is huge
+our @EXPORT = qw(uri_new uri_scheme uri_section
+		mic_for nn_new nn_for
+		imap_url nntp_url);
+
+# avoid exposing deprecated "snews" to users.
+my %SCHEME_MAP = ('snews' => 'nntps');
+
+sub uri_scheme ($) {
+	my ($uri) = @_;
+	my $scheme = $uri->scheme;
+	$SCHEME_MAP{$scheme} // $scheme;
+}
+
+# returns the git config section name, e.g [imap "imaps://user@example.com"]
+# without the mailbox, so we can share connections between different inboxes
+sub uri_section ($) {
+	my ($uri) = @_;
+	uri_scheme($uri) . '://' . $uri->authority;
+}
+
+sub auth_anon_cb { '' }; # for Mail::IMAPClient::Authcallback
+
+sub mic_for { # mic = Mail::IMAPClient
+	my ($self, $url, $mic_args) = @_;
+	require PublicInbox::URIimap;
+	my $uri = PublicInbox::URIimap->new($url);
+	require PublicInbox::GitCredential;
+	my $cred = bless {
+		url => $url,
+		protocol => $uri->scheme,
+		host => $uri->host,
+		username => $uri->user,
+		password => $uri->password,
+	}, 'PublicInbox::GitCredential';
+	my $common = $mic_args->{uri_section($uri)} // {};
+	# IMAPClient and Net::Netrc both mishandles `0', so we pass `127.0.0.1'
+	my $host = $cred->{host};
+	$host = '127.0.0.1' if $host eq '0';
+	my $mic_arg = {
+		Port => $uri->port,
+		Server => $host,
+		Ssl => $uri->scheme eq 'imaps',
+		Keepalive => 1, # SO_KEEPALIVE
+		%$common, # may set Starttls, Compress, Debug ....
+	};
+	require PublicInbox::IMAPClient;
+	my $mic = PublicInbox::IMAPClient->new(%$mic_arg) or
+		die "E: <$url> new: $@\n";
+
+	# default to using STARTTLS if it's available, but allow
+	# it to be disabled since I usually connect to localhost
+	if (!$mic_arg->{Ssl} && !defined($mic_arg->{Starttls}) &&
+			$mic->has_capability('STARTTLS') &&
+			$mic->can('starttls')) {
+		$mic->starttls or die "E: <$url> STARTTLS: $@\n";
+	}
+
+	# do we even need credentials?
+	if (!defined($cred->{username}) &&
+			$mic->has_capability('AUTH=ANONYMOUS')) {
+		$cred = undef;
+	}
+	if ($cred) {
+		$cred->check_netrc unless defined $cred->{password};
+		$cred->fill; # may prompt user here
+		$mic->User($mic_arg->{User} = $cred->{username});
+		$mic->Password($mic_arg->{Password} = $cred->{password});
+	} else { # AUTH=ANONYMOUS
+		$mic->Authmechanism($mic_arg->{Authmechanism} = 'ANONYMOUS');
+		$mic->Authcallback($mic_arg->{Authcallback} = \&auth_anon_cb);
+	}
+	if ($mic->login && $mic->IsAuthenticated) {
+		# success! keep IMAPClient->new arg in case we get disconnected
+		$self->{mic_arg}->{uri_section($uri)} = $mic_arg;
+	} else {
+		warn "E: <$url> LOGIN: $@\n";
+		$mic = undef;
+	}
+	$cred->run($mic ? 'approve' : 'reject') if $cred;
+	$mic;
+}
+
+sub uri_new {
+	my ($url) = @_;
+	require URI;
+
+	# URI::snews exists, URI::nntps does not, so use URI::snews
+	$url =~ s!\Anntps://!snews://!i;
+	URI->new($url);
+}
+
+# Net::NNTP doesn't support CAPABILITIES, yet
+sub try_starttls ($) {
+	my ($host) = @_;
+	return if $host =~ /\.onion\z/s;
+	return if $host =~ /\A127\.[0-9]+\.[0-9]+\.[0-9]+\z/s;
+	return if $host eq '::1';
+	1;
+}
+
+sub nn_new ($$$) {
+	my ($nn_arg, $nntp_opt, $url) = @_;
+	my $nn = Net::NNTP->new(%$nn_arg) or die "E: <$url> new: $!\n";
+
+	# default to using STARTTLS if it's available, but allow
+	# it to be disabled for localhost/VPN users
+	if (!$nn_arg->{SSL} && $nn->can('starttls')) {
+		if (!defined($nntp_opt->{starttls}) &&
+				try_starttls($nn_arg->{Host})) {
+			# soft fail by default
+			$nn->starttls or warn <<"";
+W: <$url> STARTTLS tried and failed (not requested)
+
+		} elsif ($nntp_opt->{starttls}) {
+			# hard fail if explicitly configured
+			$nn->starttls or die <<"";
+E: <$url> STARTTLS requested and failed
+
+		}
+	} elsif ($nntp_opt->{starttls}) {
+		$nn->can('starttls') or
+			die "E: <$url> Net::NNTP too old for STARTTLS\n";
+		$nn->starttls or die <<"";
+E: <$url> STARTTLS requested and failed
+
+	}
+	$nn;
+}
+
+sub nn_for ($$$) { # nn = Net::NNTP
+	my ($self, $url, $nn_args) = @_;
+	my $uri = uri_new($url);
+	my $sec = uri_section($uri);
+	my $nntp_opt = $self->{nntp_opt}->{$sec} //= {};
+	my $host = $uri->host;
+	# Net::NNTP and Net::Netrc both mishandle `0', so we pass `127.0.0.1'
+	$host = '127.0.0.1' if $host eq '0';
+	my $cred;
+	my ($u, $p);
+	if (defined(my $ui = $uri->userinfo)) {
+		require PublicInbox::GitCredential;
+		$cred = bless {
+			url => $sec,
+			protocol => uri_scheme($uri),
+			host => $host,
+		}, 'PublicInbox::GitCredential';
+		($u, $p) = split(/:/, $ui, 2);
+		($cred->{username}, $cred->{password}) = ($u, $p);
+		$cred->check_netrc unless defined $p;
+	}
+	my $common = $nn_args->{$sec} // {};
+	my $nn_arg = {
+		Port => $uri->port,
+		Host => $host,
+		SSL => $uri->secure, # snews == nntps
+		%$common, # may Debug ....
+	};
+	my $nn = nn_new($nn_arg, $nntp_opt, $url);
+
+	if ($cred) {
+		$cred->fill; # may prompt user here
+		if ($nn->authinfo($u, $p)) {
+			push @{$nntp_opt->{-postconn}}, [ 'authinfo', $u, $p ];
+		} else {
+			warn "E: <$url> AUTHINFO $u XXXX failed\n";
+			$nn = undef;
+		}
+	}
+
+	if ($nntp_opt->{compress}) {
+		# https://rt.cpan.org/Ticket/Display.html?id=129967
+		if ($nn->can('compress')) {
+			if ($nn->compress) {
+				push @{$nntp_opt->{-postconn}}, [ 'compress' ];
+			} else {
+				warn "W: <$url> COMPRESS failed\n";
+			}
+		} else {
+			delete $nntp_opt->{compress};
+			warn <<"";
+W: <$url> COMPRESS not supported by Net::NNTP
+W: see https://rt.cpan.org/Ticket/Display.html?id=129967 for updates
+
+		}
+	}
+
+	$self->{nn_arg}->{$sec} = $nn_arg;
+	$cred->run($nn ? 'approve' : 'reject') if $cred;
+	$nn;
+}
+
+sub imap_url {
+	my ($url) = @_;
+	require PublicInbox::URIimap;
+	my $uri = PublicInbox::URIimap->new($url);
+	$uri ? $uri->canonical->as_string : undef;
+}
+
+my %IS_NNTP = (news => 1, snews => 1, nntp => 1);
+sub nntp_url {
+	my ($url) = @_;
+	my $uri = uri_new($url);
+	return unless $uri && $IS_NNTP{$uri->scheme} && $uri->group;
+	$url = $uri->canonical->as_string;
+	# nntps is IANA registered, snews is deprecated
+	$url =~ s!\Asnews://!nntps://!;
+	$url;
+}
+
+1;
diff --git a/lib/PublicInbox/Watch.pm b/lib/PublicInbox/Watch.pm
index a4302162..8a457b81 100644
--- a/lib/PublicInbox/Watch.pm
+++ b/lib/PublicInbox/Watch.pm
@@ -9,6 +9,7 @@ use v5.10.1;
 use PublicInbox::Eml;
 use PublicInbox::InboxWritable qw(eml_from_path);
 use PublicInbox::MdirReader;
+use PublicInbox::NetReader;
 use PublicInbox::Filter::Base qw(REJECT);
 use PublicInbox::Spamcheck;
 use PublicInbox::Sigfd;
@@ -279,22 +280,6 @@ sub watch_fs_init ($) {
 	PublicInbox::DirIdle->new([keys %{$self->{mdmap}}], $cb);
 }
 
-# avoid exposing deprecated "snews" to users.
-my %SCHEME_MAP = ('snews' => 'nntps');
-
-sub uri_scheme ($) {
-	my ($uri) = @_;
-	my $scheme = $uri->scheme;
-	$SCHEME_MAP{$scheme} // $scheme;
-}
-
-# returns the git config section name, e.g [imap "imaps://user@example.com"]
-# without the mailbox, so we can share connections between different inboxes
-sub uri_section ($) {
-	my ($uri) = @_;
-	uri_scheme($uri) . '://' . $uri->authority;
-}
-
 sub cfg_intvl ($$$) {
 	my ($cfg, $key, $url) = @_;
 	my $v = $cfg->urlmatch($key, $url) // return;
@@ -344,66 +329,6 @@ sub imap_common_init ($) {
 	$mic_args;
 }
 
-sub auth_anon_cb { '' }; # for Mail::IMAPClient::Authcallback
-
-sub mic_for ($$$) { # mic = Mail::IMAPClient
-	my ($self, $url, $mic_args) = @_;
-	my $uri = PublicInbox::URIimap->new($url);
-	require PublicInbox::GitCredential;
-	my $cred = bless {
-		url => $url,
-		protocol => $uri->scheme,
-		host => $uri->host,
-		username => $uri->user,
-		password => $uri->password,
-	}, 'PublicInbox::GitCredential';
-	my $common = $mic_args->{uri_section($uri)} // {};
-	# IMAPClient and Net::Netrc both mishandles `0', so we pass `127.0.0.1'
-	my $host = $cred->{host};
-	$host = '127.0.0.1' if $host eq '0';
-	my $mic_arg = {
-		Port => $uri->port,
-		Server => $host,
-		Ssl => $uri->scheme eq 'imaps',
-		Keepalive => 1, # SO_KEEPALIVE
-		%$common, # may set Starttls, Compress, Debug ....
-	};
-	my $mic = PublicInbox::IMAPClient->new(%$mic_arg) or
-		die "E: <$url> new: $@\n";
-
-	# default to using STARTTLS if it's available, but allow
-	# it to be disabled since I usually connect to localhost
-	if (!$mic_arg->{Ssl} && !defined($mic_arg->{Starttls}) &&
-			$mic->has_capability('STARTTLS') &&
-			$mic->can('starttls')) {
-		$mic->starttls or die "E: <$url> STARTTLS: $@\n";
-	}
-
-	# do we even need credentials?
-	if (!defined($cred->{username}) &&
-			$mic->has_capability('AUTH=ANONYMOUS')) {
-		$cred = undef;
-	}
-	if ($cred) {
-		$cred->check_netrc unless defined $cred->{password};
-		$cred->fill; # may prompt user here
-		$mic->User($mic_arg->{User} = $cred->{username});
-		$mic->Password($mic_arg->{Password} = $cred->{password});
-	} else { # AUTH=ANONYMOUS
-		$mic->Authmechanism($mic_arg->{Authmechanism} = 'ANONYMOUS');
-		$mic->Authcallback($mic_arg->{Authcallback} = \&auth_anon_cb);
-	}
-	if ($mic->login && $mic->IsAuthenticated) {
-		# success! keep IMAPClient->new arg in case we get disconnected
-		$self->{mic_arg}->{uri_section($uri)} = $mic_arg;
-	} else {
-		warn "E: <$url> LOGIN: $@\n";
-		$mic = undef;
-	}
-	$cred->run($mic ? 'approve' : 'reject') if $cred;
-	$mic;
-}
-
 sub imap_import_msg ($$$$$) {
 	my ($self, $url, $uid, $raw, $flags) = @_;
 	# our target audience expects LF-only, save storage
@@ -805,106 +730,6 @@ sub nntp_common_init ($) {
 	$nn_args;
 }
 
-# Net::NNTP doesn't support CAPABILITIES, yet
-sub try_starttls ($) {
-	my ($host) = @_;
-	return if $host =~ /\.onion\z/s;
-	return if $host =~ /\A127\.[0-9]+\.[0-9]+\.[0-9]+\z/s;
-	return if $host eq '::1';
-	1;
-}
-
-sub nn_new ($$$) {
-	my ($nn_arg, $nntp_opt, $url) = @_;
-	my $nn = Net::NNTP->new(%$nn_arg) or die "E: <$url> new: $!\n";
-
-	# default to using STARTTLS if it's available, but allow
-	# it to be disabled for localhost/VPN users
-	if (!$nn_arg->{SSL} && $nn->can('starttls')) {
-		if (!defined($nntp_opt->{starttls}) &&
-				try_starttls($nn_arg->{Host})) {
-			# soft fail by default
-			$nn->starttls or warn <<"";
-W: <$url> STARTTLS tried and failed (not requested)
-
-		} elsif ($nntp_opt->{starttls}) {
-			# hard fail if explicitly configured
-			$nn->starttls or die <<"";
-E: <$url> STARTTLS requested and failed
-
-		}
-	} elsif ($nntp_opt->{starttls}) {
-		$nn->can('starttls') or
-			die "E: <$url> Net::NNTP too old for STARTTLS\n";
-		$nn->starttls or die <<"";
-E: <$url> STARTTLS requested and failed
-
-	}
-	$nn;
-}
-
-sub nn_for ($$$) { # nn = Net::NNTP
-	my ($self, $url, $nn_args) = @_;
-	my $uri = uri_new($url);
-	my $sec = uri_section($uri);
-	my $nntp_opt = $self->{nntp_opt}->{$sec} //= {};
-	my $host = $uri->host;
-	# Net::NNTP and Net::Netrc both mishandle `0', so we pass `127.0.0.1'
-	$host = '127.0.0.1' if $host eq '0';
-	my $cred;
-	my ($u, $p);
-	if (defined(my $ui = $uri->userinfo)) {
-		require PublicInbox::GitCredential;
-		$cred = bless {
-			url => $sec,
-			protocol => uri_scheme($uri),
-			host => $host,
-		}, 'PublicInbox::GitCredential';
-		($u, $p) = split(/:/, $ui, 2);
-		($cred->{username}, $cred->{password}) = ($u, $p);
-		$cred->check_netrc unless defined $p;
-	}
-	my $common = $nn_args->{$sec} // {};
-	my $nn_arg = {
-		Port => $uri->port,
-		Host => $host,
-		SSL => $uri->secure, # snews == nntps
-		%$common, # may Debug ....
-	};
-	my $nn = nn_new($nn_arg, $nntp_opt, $url);
-
-	if ($cred) {
-		$cred->fill; # may prompt user here
-		if ($nn->authinfo($u, $p)) {
-			push @{$nntp_opt->{-postconn}}, [ 'authinfo', $u, $p ];
-		} else {
-			warn "E: <$url> AUTHINFO $u XXXX failed\n";
-			$nn = undef;
-		}
-	}
-
-	if ($nntp_opt->{compress}) {
-		# https://rt.cpan.org/Ticket/Display.html?id=129967
-		if ($nn->can('compress')) {
-			if ($nn->compress) {
-				push @{$nntp_opt->{-postconn}}, [ 'compress' ];
-			} else {
-				warn "W: <$url> COMPRESS failed\n";
-			}
-		} else {
-			delete $nntp_opt->{compress};
-			warn <<"";
-W: <$url> COMPRESS not supported by Net::NNTP
-W: see https://rt.cpan.org/Ticket/Display.html?id=129967 for updates
-
-		}
-	}
-
-	$self->{nn_arg}->{$sec} = $nn_arg;
-	$cred->run($nn ? 'approve' : 'reject') if $cred;
-	$nn;
-}
-
 sub nntp_fetch_all ($$$) {
 	my ($self, $nn, $url) = @_;
 	my $uri = uri_new($url);
@@ -1132,31 +957,4 @@ EOF
 	undef;
 }
 
-sub uri_new {
-	my ($url) = @_;
-
-	# URI::snews exists, URI::nntps does not, so use URI::snews
-	$url =~ s!\Anntps://!snews://!i;
-	URI->new($url);
-}
-
-sub imap_url {
-	my ($url) = @_;
-	require PublicInbox::URIimap;
-	my $uri = PublicInbox::URIimap->new($url);
-	$uri ? $uri->canonical->as_string : undef;
-}
-
-my %IS_NNTP = (news => 1, snews => 1, nntp => 1);
-sub nntp_url {
-	my ($url) = @_;
-	require URI;
-	my $uri = uri_new($url);
-	return unless $uri && $IS_NNTP{$uri->scheme} && $uri->group;
-	$url = $uri->canonical->as_string;
-	# nntps is IANA registered, snews is deprecated
-	$url =~ s!\Asnews://!nntps://!;
-	$url;
-}
-
 1;

  parent reply	other threads:[~2021-02-10  7:07 UTC|newest]

Thread overview: 8+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2021-02-10  7:07 [PATCH 0/6] more lei stuffs Eric Wong
2021-02-10  7:07 ` [PATCH 1/6] lei *external: glob improvements, ls-external filtering Eric Wong
2021-02-10  7:07 ` [PATCH 2/6] lei_external: remove unnecessary Exporter use Eric Wong
2021-02-10  7:07 ` [PATCH 3/6] test_common: support lei-daemon only testing Eric Wong
2021-02-10  7:07 ` [PATCH 4/6] lei ls-external: support --local and --remote Eric Wong
2021-02-10  7:07 ` [PATCH 5/6] lei: note some TODO items (curl, externals) Eric Wong
2021-02-10  7:07 ` Eric Wong [this message]
2021-02-10  8:38 ` [PATCH 7/6] lei_external: fix+test handling of escaped braces Eric Wong

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

  List information: https://public-inbox.org/README

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20210210070749.30391-7-e@80x24.org \
    --to=e@80x24.org \
    --cc=meta@public-inbox.org \
    --subject='Re: [PATCH 6/6] net_reader: new package split from -watch' \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link

Code repositories for project(s) associated with this inbox:

	https://80x24.org/public-inbox.git

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).