user/dev discussion of public-inbox itself
 help / color / Atom feed
From: Eric Wong <e@yhbt.net>
To: meta@public-inbox.org
Subject: [PATCH 29/34] watch: show user-specified URL consistently.
Date: Sat, 27 Jun 2020 10:03:55 +0000
Message-ID: <20200627100400.9871-30-e@yhbt.net> (raw)
In-Reply-To: <20200627100400.9871-1-e@yhbt.net>

Since we use the non-ref scalar URL in many error messages,
favor keeping the unblessed URL in the long-lived process.

This avoids showing "snews://" to users who've specified
"nntps://" URLs, since "nntps" is IANA-registered nowadays and
what we show in our documentation, while "snews" was just a
draft the URI package picked up decades ago.

---
 lib/PublicInbox/WatchMaildir.pm | 142 ++++++++++++++++++--------------
 t/watch_nntp.t                  |   6 +-
 2 files changed, 84 insertions(+), 64 deletions(-)

diff --git a/lib/PublicInbox/WatchMaildir.pm b/lib/PublicInbox/WatchMaildir.pm
index 616c63a3857..43c8395c79b 100644
--- a/lib/PublicInbox/WatchMaildir.pm
+++ b/lib/PublicInbox/WatchMaildir.pm
@@ -238,11 +238,20 @@ sub watch_fs_init ($) {
 	PublicInbox::DirIdle->new([keys %{$self->{mdmap}}], $cb);
 }
 
+# avoid exposing deprecated "snews" to users.
+my %SCHEME_MAP = ('snews' => 'nntps');
+
+sub uri_scheme ($) {
+	my ($uri) = @_;
+	my $scheme = $uri->scheme;
+	$SCHEME_MAP{$scheme} // $scheme;
+}
+
 # returns the git config section name, e.g [imap "imaps://user@example.com"]
 # without the mailbox, so we can share connections between different inboxes
 sub uri_section ($) {
 	my ($uri) = @_;
-	$uri->scheme . '://' . $uri->authority;
+	uri_scheme($uri) . '://' . $uri->authority;
 }
 
 sub cfg_intvl ($$$) {
@@ -297,8 +306,8 @@ sub imap_common_init ($) {
 sub auth_anon_cb { '' }; # for Mail::IMAPClient::Authcallback
 
 sub mic_for ($$$) { # mic = Mail::IMAPClient
-	my ($self, $uri, $mic_args) = @_;
-	my $url = $uri->as_string;
+	my ($self, $url, $mic_args) = @_;
+	my $uri = PublicInbox::URIimap->new($url);
 	my $cred = {
 		url => $url,
 		protocol => $uri->scheme,
@@ -372,10 +381,10 @@ sub imap_import_msg ($$$$) {
 }
 
 sub imap_fetch_all ($$$) {
-	my ($self, $mic, $uri) = @_;
+	my ($self, $mic, $url) = @_;
+	my $uri = PublicInbox::URIimap->new($url);
 	my $sec = uri_section($uri);
 	my $mbx = $uri->mailbox;
-	my $url = $uri->as_string;
 	$mic->Clear(1); # trim results history
 	$mic->examine($mbx) or return "E: EXAMINE $mbx ($sec) failed: $!";
 	my ($r_uidval, $r_uidnext);
@@ -489,7 +498,8 @@ sub imap_idle_once ($$$$) {
 
 # idles on a single URI
 sub watch_imap_idle_1 ($$$) {
-	my ($self, $uri, $intvl) = @_;
+	my ($self, $url, $intvl) = @_;
+	my $uri = PublicInbox::URIimap->new($url);
 	my $sec = uri_section($uri);
 	my $mic_arg = $self->{mic_arg}->{$sec} or
 			die "BUG: no Mail::IMAPClient->new arg for $sec";
@@ -498,8 +508,8 @@ sub watch_imap_idle_1 ($$$) {
 	until ($self->{quit}) {
 		$mic //= delete($self->{mics}->{$sec}) //
 				PublicInbox::IMAPClient->new(%$mic_arg);
-		my $err = imap_fetch_all($self, $mic, $uri);
-		$err //= imap_idle_once($self, $mic, $intvl, $uri->as_string);
+		my $err = imap_fetch_all($self, $mic, $url);
+		$err //= imap_idle_once($self, $mic, $intvl, $url);
 		if ($err && !$self->{quit}) {
 			warn $err, "\n";
 			$mic = undef;
@@ -526,27 +536,26 @@ sub watch_atfork_parent ($) {
 
 sub imap_idle_reap { # PublicInbox::DS::dwaitpid callback
 	my ($self, $pid) = @_;
-	my $uri_intvl = delete $self->{idle_pids}->{$pid} or
+	my $url_intvl = delete $self->{idle_pids}->{$pid} or
 		die "BUG: PID=$pid (unknown) reaped: \$?=$?\n";
 
-	my ($uri, $intvl) = @$uri_intvl;
-	my $url = $uri->as_string;
+	my ($url, $intvl) = @$url_intvl;
 	return if $self->{quit};
 	warn "W: PID=$pid on $url died: \$?=$?\n" if $?;
-	push @{$self->{idle_todo}}, $uri_intvl;
+	push @{$self->{idle_todo}}, $url_intvl;
 	PubicInbox::DS::requeue($self); # call ->event_step to respawn
 }
 
 sub imap_idle_fork ($$) {
-	my ($self, $uri_intvl) = @_;
-	my ($uri, $intvl) = @$uri_intvl;
+	my ($self, $url_intvl) = @_;
+	my ($url, $intvl) = @$url_intvl;
 	defined(my $pid = fork) or die "fork: $!";
 	if ($pid == 0) {
 		watch_atfork_child($self);
-		watch_imap_idle_1($self, $uri, $intvl);
+		watch_imap_idle_1($self, $url, $intvl);
 		_exit(0);
 	}
-	$self->{idle_pids}->{$pid} = $uri_intvl;
+	$self->{idle_pids}->{$pid} = $url_intvl;
 	PublicInbox::DS::dwaitpid($pid, \&imap_idle_reap, $self);
 }
 
@@ -556,34 +565,35 @@ sub event_step {
 	my $idle_todo = $self->{idle_todo};
 	if ($idle_todo && @$idle_todo) {
 		watch_atfork_parent($self);
-		while (my $uri_intvl = shift(@$idle_todo)) {
-			imap_idle_fork($self, $uri_intvl);
+		while (my $url_intvl = shift(@$idle_todo)) {
+			imap_idle_fork($self, $url_intvl);
 		}
 	}
 	goto(&fs_scan_step) if $self->{mdre};
 }
 
 sub watch_imap_fetch_all ($$) {
-	my ($self, $uris) = @_;
-	for my $uri (@$uris) {
+	my ($self, $urls) = @_;
+	for my $url (@$urls) {
+		my $uri = PublicInbox::URIimap->new($url);
 		my $sec = uri_section($uri);
 		my $mic_arg = $self->{mic_arg}->{$sec} or
 			die "BUG: no Mail::IMAPClient->new arg for $sec";
 		my $mic = PublicInbox::IMAPClient->new(%$mic_arg) or next;
-		my $err = imap_fetch_all($self, $mic, $uri);
+		my $err = imap_fetch_all($self, $mic, $url);
 		last if $self->{quit};
 		warn $err, "\n" if $err;
 	}
 }
 
 sub watch_nntp_fetch_all ($$) {
-	my ($self, $uris) = @_;
-	for my $uri (@$uris) {
+	my ($self, $urls) = @_;
+	for my $url (@$urls) {
+		my $uri = uri_new($url);
 		my $sec = uri_section($uri);
 		my $nn_arg = $self->{nn_arg}->{$sec} or
 			die "BUG: no Net::NNTP->new arg for $sec";
 		my $nntp_opt = $self->{nntp_opt}->{$sec};
-		my $url = $uri->as_string;
 		my $nn = nn_new($nn_arg, $nntp_opt, $url);
 		unless ($nn) {
 			warn "E: $url: \$!=$!\n";
@@ -601,43 +611,42 @@ sub watch_nntp_fetch_all ($$) {
 		}
 		last if $self->{quit};
 		if ($nn) {
-			my $err = nntp_fetch_all($self, $nn, $uri);
+			my $err = nntp_fetch_all($self, $nn, $url);
 			warn $err, "\n" if $err;
 		}
 	}
 }
 
 sub poll_fetch_fork ($) { # DS::add_timer callback
-	my ($self, $intvl, $uris) = @{$_[0]};
+	my ($self, $intvl, $urls) = @{$_[0]};
 	return if $self->{quit};
 	watch_atfork_parent($self);
 	defined(my $pid = fork) or die "fork: $!";
 	if ($pid == 0) {
 		watch_atfork_child($self);
-		if ($uris->[0]->scheme =~ /\Aimaps?\z/) {
-			watch_imap_fetch_all($self, $uris);
+		if ($urls->[0] =~ m!\Aimaps?://!i) {
+			watch_imap_fetch_all($self, $urls);
 		} else {
-			watch_nntp_fetch_all($self, $uris);
+			watch_nntp_fetch_all($self, $urls);
 		}
 		_exit(0);
 	}
-	$self->{poll_pids}->{$pid} = [ $intvl, $uris ];
+	$self->{poll_pids}->{$pid} = [ $intvl, $urls ];
 	PublicInbox::DS::dwaitpid($pid, \&poll_fetch_reap, $self);
 }
 
 sub poll_fetch_reap { # PublicInbox::DS::dwaitpid callback
 	my ($self, $pid) = @_;
-	my $intvl_uris = delete $self->{poll_pids}->{$pid} or
+	my $intvl_urls = delete $self->{poll_pids}->{$pid} or
 		die "BUG: PID=$pid (unknown) reaped: \$?=$?\n";
 	return if $self->{quit};
-	my ($intvl, $uris) = @$intvl_uris;
+	my ($intvl, $urls) = @$intvl_urls;
 	if ($?) {
-		warn "W: PID=$pid died: \$?=$?\n",
-			map { $_->as_string."\n" } @$uris;
+		warn "W: PID=$pid died: \$?=$?\n", map { "$_\n" } @$urls;
 	}
-	warn('I: will check ', $_->as_string, " in ${intvl}s\n") for @$uris;
+	warn("I: will check $_ in ${intvl}s\n") for @$urls;
 	PublicInbox::DS::add_timer($intvl, \&poll_fetch_fork,
-					[$self, $intvl, $uris]);
+					[$self, $intvl, $urls]);
 }
 
 sub watch_imap_init ($) {
@@ -656,11 +665,11 @@ sub watch_imap_init ($) {
 	my $mics = $self->{mics} = {}; # schema://authority => IMAPClient obj
 	for my $url (sort keys %{$self->{imap}}) {
 		my $uri = PublicInbox::URIimap->new($url);
-		$mics->{uri_section($uri)} //= mic_for($self, $uri, $mic_args);
+		$mics->{uri_section($uri)} //= mic_for($self, $url, $mic_args);
 	}
 
-	my $idle = []; # [ [ uri1, intvl1 ], [uri2, intvl2] ]
-	my $poll = {}; # intvl_seconds => [ uri1, uri2 ]
+	my $idle = []; # [ [ url1, intvl1 ], [url2, intvl2] ]
+	my $poll = {}; # intvl_seconds => [ url1, url2 ]
 	for my $url (keys %{$self->{imap}}) {
 		my $uri = PublicInbox::URIimap->new($url);
 		my $sec = uri_section($uri);
@@ -668,9 +677,9 @@ sub watch_imap_init ($) {
 		my $intvl = $self->{imap_opt}->{$sec}->{pollInterval};
 		if ($mic->has_capability('IDLE') && !$intvl) {
 			$intvl = $self->{imap_opt}->{$sec}->{idleInterval};
-			push @$idle, [ $uri, $intvl // () ];
+			push @$idle, [ $url, $intvl // () ];
 		} else {
-			push @{$poll->{$intvl || 120}}, $uri;
+			push @{$poll->{$intvl || 120}}, $url;
 		}
 	}
 	if (scalar @$idle) {
@@ -681,10 +690,10 @@ sub watch_imap_init ($) {
 	return unless scalar keys %$poll;
 	$self->{poll_pids} //= {};
 
-	# poll all URIs for a given interval sequentially
-	while (my ($intvl, $uris) = each %$poll) {
+	# poll all URLs for a given interval sequentially
+	while (my ($intvl, $urls) = each %$poll) {
 		PublicInbox::DS::add_timer(0, \&poll_fetch_fork,
-						[$self, $intvl, $uris]);
+						[$self, $intvl, $urls]);
 	}
 }
 
@@ -694,7 +703,7 @@ sub nntp_common_init ($) {
 	my $cfg = $self->{config};
 	my $nn_args = {}; # scheme://authority => Net::NNTP->new arg
 	for my $url (sort keys %{$self->{nntp}}) {
-		my $sec = uri_section(URI->new($url));
+		my $sec = uri_section(uri_new($url));
 
 		# Debug and Timeout are is passed to Net::NNTP->new
 		my $v = cfg_bool($cfg, 'nntp.Debug', $url);
@@ -756,8 +765,8 @@ E: <$url> STARTTLS requested and failed
 }
 
 sub nn_for ($$$) { # nn = Net::NNTP
-	my ($self, $uri, $nn_args) = @_;
-	my $url = $uri->as_string;
+	my ($self, $url, $nn_args) = @_;
+	my $uri = uri_new($url);
 	my $sec = uri_section($uri);
 	my $nntp_opt = $self->{nntp_opt}->{$sec} //= {};
 	my $cred;
@@ -765,7 +774,7 @@ sub nn_for ($$$) { # nn = Net::NNTP
 	if (defined(my $ui = $uri->userinfo)) {
 		$cred = {
 			url => $sec,
-			protocol => $uri->scheme,
+			protocol => uri_scheme($uri),
 			host => $uri->host,
 		};
 		($u, $p) = split(/:/, $ui, 2);
@@ -814,10 +823,10 @@ W: see https://rt.cpan.org/Ticket/Display.html?id=129967 for updates
 }
 
 sub nntp_fetch_all ($$$) {
-	my ($self, $nn, $uri) = @_;
+	my ($self, $nn, $url) = @_;
+	my $uri = uri_new($url);
 	my ($group, $num_a, $num_b) = $uri->group;
 	my $sec = uri_section($uri);
-	my $url = $uri->as_string;
 	my ($nr, $beg, $end) = $nn->group($group);
 	unless (defined($nr)) {
 		chomp(my $msg = $nn->message);
@@ -897,21 +906,21 @@ sub watch_nntp_init ($) {
 	# make sure we can connect and cache the credentials in memory
 	$self->{nn_arg} = {}; # schema://authority => Net::NNTP->new args
 	for my $url (sort keys %{$self->{nntp}}) {
-		nn_for($self, URI->new($url), $nn_args);
+		nn_for($self, $url, $nn_args);
 	}
-	my $poll = {}; # intvl_seconds => [ uri1, uri2 ]
+	my $poll = {}; # intvl_seconds => [ url1, url2 ]
 	for my $url (keys %{$self->{nntp}}) {
-		my $uri = URI->new($url);
+		my $uri = uri_new($url);
 		my $sec = uri_section($uri);
 		my $intvl = $self->{nntp_opt}->{$sec}->{pollInterval};
-		push @{$poll->{$intvl || 120}}, $uri;
+		push @{$poll->{$intvl || 120}}, $url;
 	}
 	$self->{poll_pids} //= {};
 
-	# poll all URIs for a given interval sequentially
-	while (my ($intvl, $uris) = each %$poll) {
+	# poll all URLs for a given interval sequentially
+	while (my ($intvl, $urls) = each %$poll) {
 		PublicInbox::DS::add_timer(0, \&poll_fetch_fork,
-						[$self, $intvl, $uris]);
+						[$self, $intvl, $urls]);
 	}
 }
 
@@ -1021,6 +1030,14 @@ EOF
 	undef;
 }
 
+sub uri_new {
+	my ($url) = @_;
+
+	# URI::snews exists, URI::nntps does not, so use URI::snews
+	$url =~ s!\Anntps://!snews://!i;
+	URI->new($url);
+}
+
 sub imap_url {
 	my ($url) = @_;
 	require PublicInbox::URIimap;
@@ -1032,11 +1049,12 @@ my %IS_NNTP = (news => 1, snews => 1, nntp => 1);
 sub nntp_url {
 	my ($url) = @_;
 	require URI;
-	# URI::snews exists, URI::nntps does not, so use URI::snews
-	$url =~ s!\Anntps://!snews://!i;
-	my $uri = URI->new($url);
-	return unless $uri && $IS_NNTP{$uri->scheme};
-	$uri->group ? $uri->canonical->as_string : undef;
+	my $uri = uri_new($url);
+	return unless $uri && $IS_NNTP{$uri->scheme} && $uri->group;
+	$url = $uri->canonical->as_string;
+	# nntps is IANA registered, snews is deprecated
+	$url =~ s!\Asnews://!nntps://!;
+	$url;
 }
 
 1;
diff --git a/t/watch_nntp.t b/t/watch_nntp.t
index f919930e7d8..98fb1161d1c 100644
--- a/t/watch_nntp.t
+++ b/t/watch_nntp.t
@@ -9,7 +9,9 @@ use_ok 'PublicInbox::WatchMaildir';
 my $nntp_url = \&PublicInbox::WatchMaildir::nntp_url;
 is('news://example.com/inbox.foo',
 	$nntp_url->('NEWS://examplE.com/inbox.foo'), 'lowercased');
-is('snews://example.com/inbox.foo',
-	$nntp_url->('nntps://example.com/inbox.foo'), 'nntps:// is snews://');
+is('nntps://example.com/inbox.foo',
+	$nntp_url->('nntps://example.com/inbox.foo'), 'nntps:// accepted');
+is('nntps://example.com/inbox.foo',
+	$nntp_url->('SNEWS://example.com/inbox.foo'), 'snews => nntps');
 
 done_testing;

  parent reply index

Thread overview: 45+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2020-06-27 10:03 [PATCH 00/34] watch: add IMAP and NNTP support Eric Wong
2020-06-27 10:03 ` [PATCH 01/34] inboxwritable: ensure ssoma.lock exists on init Eric Wong
2020-06-27 10:03 ` [PATCH 02/34] inbox: warn on ->on_inbox_unlock exception Eric Wong
2020-06-27 10:03 ` [PATCH 03/34] IMAPTracker: Add a helper to track our place in reading imap mailboxes Eric Wong
2020-06-27 10:03 ` [PATCH 04/34] imaptracker: use ~/.local/share/public-inbox/imap.sqlite3 Eric Wong
2020-06-27 10:03 ` [PATCH 05/34] watchmaildir: hoist out compile_watchheaders Eric Wong
2020-06-27 10:03 ` [PATCH 06/34] watchmaildir: fix check for spam vs ham inbox conflicts Eric Wong
2020-06-27 10:03 ` [PATCH 07/34] URI IMAP support Eric Wong
2020-06-27 10:03 ` [PATCH 08/34] watch: preliminary " Eric Wong
2020-06-27 10:03 ` [PATCH 09/34] kqnotify|fake_inotify: detect Maildir write ops Eric Wong
2020-06-27 10:03 ` [PATCH 10/34] watch: remove Filesys::Notify::Simple dependency Eric Wong
2020-06-27 10:03 ` [PATCH 11/34] watch: use signalfd for Maildir watching Eric Wong
2020-06-27 19:05   ` Kyle Meyer
2020-06-27 22:32     ` Eric Wong
2020-06-27 10:03 ` [PATCH 12/34] ds: remove fields.pm usage Eric Wong
2020-06-27 10:03 ` [PATCH 13/34] watch: wire up IMAP IDLE reapers to DS Eric Wong
2020-06-27 10:03 ` [PATCH 14/34] watch: support IMAP polling Eric Wong
2020-06-27 10:03 ` [PATCH 15/34] config: support ->urlmatch method for -watch Eric Wong
2020-06-27 10:03 ` [PATCH 16/34] watch: stop importers before forking Eric Wong
2020-06-27 10:03 ` [PATCH 17/34] watch: use UID SEARCH to avoid empty UID FETCH Eric Wong
2020-06-27 10:03 ` [PATCH 18/34] ds: add_timer: allow passing arg to callback Eric Wong
2020-06-27 10:03 ` [PATCH 19/34] imaptracker: add {url} field to reduce args Eric Wong
2020-06-27 10:03 ` [PATCH 20/34] imaptracker: drop {dbname} field Eric Wong
2020-06-27 10:03 ` [PATCH 21/34] watch: avoid long transaction to IMAPTracker Eric Wong
2020-06-27 10:03 ` [PATCH 22/34] watch: support imap.fetchBatchSize parameter Eric Wong
2020-06-27 10:03 ` [PATCH 23/34] watch: imap: be quiet about disconnecting on quit Eric Wong
2020-06-27 10:03 ` [PATCH 24/34] watch: support multiple watch: directives per-inbox Eric Wong
2020-06-27 10:03 ` [PATCH 25/34] watch: remove {mdir} array Eric Wong
2020-06-27 10:03 ` [PATCH 26/34] watch: just use ->urlmatch Eric Wong
2020-06-27 10:03 ` [PATCH 27/34] testcommon: $ENV{TAIL} supports non-@ARGV redirects Eric Wong
2020-06-27 10:03 ` [PATCH 28/34] watch: add NNTP support Eric Wong
2020-06-27 19:06   ` Kyle Meyer
2020-06-27 10:03 ` Eric Wong [this message]
2020-06-27 10:03 ` [PATCH 30/34] watch: enable autoflush for STDOUT and STDERR Eric Wong
2020-06-27 10:03 ` [PATCH 31/34] watch: use our own "git credential" wrapper Eric Wong
2020-06-27 10:03 ` [PATCH 32/34] watch: support ~/.netrc via Net::Netrc Eric Wong
2020-06-27 10:03 ` [PATCH 33/34] imaptracker: use flock(2) around writes Eric Wong
2020-06-27 10:04 ` [PATCH 34/34] watch: simplify internal structures Eric Wong
2020-06-29 10:34 ` [PATCH 0/5] watch: Maildir fixes Eric Wong
2020-06-29 10:34   ` [PATCH 1/5] watch: check for duplicates in ->over before spamcheck Eric Wong
2020-06-29 10:34   ` [PATCH 2/5] watch: show path for warnings from spam messages Eric Wong
2020-06-29 10:34   ` [PATCH 3/5] watch: ensure SIGCHLD works in forked children Eric Wong
2020-06-29 10:34   ` [PATCH 4/5] spawn: unblock SIGCHLD in subprocess Eric Wong
2020-06-29 10:34   ` [PATCH 5/5] watch: make waitpid() synchronous for Maildir scans Eric Wong
2020-06-29 10:37     ` Eric Wong

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

  List information: http://public-inbox.org/README

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20200627100400.9871-30-e@yhbt.net \
    --to=e@yhbt.net \
    --cc=meta@public-inbox.org \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link

user/dev discussion of public-inbox itself

Archives are clonable:
	git clone --mirror http://public-inbox.org/meta
	git clone --mirror http://czquwvybam4bgbro.onion/meta
	git clone --mirror http://hjrcffqmbrq6wope.onion/meta
	git clone --mirror http://ou63pmih66umazou.onion/meta

Example config snippet for mirrors

Newsgroups are available over NNTP:
	nntp://news.public-inbox.org/inbox.comp.mail.public-inbox.meta
	nntp://ou63pmih66umazou.onion/inbox.comp.mail.public-inbox.meta
	nntp://czquwvybam4bgbro.onion/inbox.comp.mail.public-inbox.meta
	nntp://hjrcffqmbrq6wope.onion/inbox.comp.mail.public-inbox.meta
	nntp://news.gmane.io/gmane.mail.public-inbox.general

 note: .onion URLs require Tor: https://www.torproject.org/

AGPL code for this site: git clone https://public-inbox.org/public-inbox.git