[PATCH 28/34] watch: add NNTP support

[Eric Wong <e@yhbt.net>]

This is similar to IMAP support, but only supports polling.
Automatic altid support is not yet supported, yet; but may
be in the future.
---
 MANIFEST                        |   1 +
 lib/PublicInbox/WatchMaildir.pm | 357 ++++++++++++++++++++++++++++----
 t/nntpd.t                       |  52 +++++
 t/watch_nntp.t                  |  15 ++
 4 files changed, 389 insertions(+), 36 deletions(-)
 create mode 100644 t/watch_nntp.t

diff --git a/MANIFEST b/MANIFEST
index 035c45bf498..f9d1eea5bd9 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -355,6 +355,7 @@ t/watch_imap.t
 t/watch_maildir.t
 t/watch_maildir_v2.t
 t/watch_multiple_headers.t
+t/watch_nntp.t
 t/www_altid.t
 t/www_listing.t
 t/www_static.t
diff --git a/lib/PublicInbox/WatchMaildir.pm b/lib/PublicInbox/WatchMaildir.pm
index 535dadd539c..616c63a3857 100644
--- a/lib/PublicInbox/WatchMaildir.pm
+++ b/lib/PublicInbox/WatchMaildir.pm
@@ -41,7 +41,7 @@ sub compile_watchheaders ($) {
 sub new {
 	my ($class, $config) = @_;
 	my (%mdmap, $spamc);
-	my %imap; # url => [inbox objects] or 'watchspam'
+	my (%imap, %nntp); # url => [inbox objects] or 'watchspam'
 
 	# "publicinboxwatch" is the documented namespace
 	# "publicinboxlearn" is legacy but may be supported
@@ -51,11 +51,14 @@ sub new {
 		defined(my $dirs = $config->{$k}) or next;
 		$dirs = PublicInbox::Config::_array($dirs);
 		for my $dir (@$dirs) {
+			my $url;
 			if (is_maildir($dir)) {
 				# skip "new", no MUA has seen it, yet.
 				$mdmap{"$dir/cur"} = 'watchspam';
-			} elsif (my $url = imap_url($dir)) {
+			} elsif ($url = imap_url($dir)) {
 				$imap{$url} = 'watchspam';
+			} elsif ($url = nntp_url($dir)) {
+				$nntp{$url} = 'watchspam';
 			} else {
 				warn "unsupported $k=$dir\n";
 			}
@@ -74,6 +77,7 @@ sub new {
 		my $watches = $ibx->{watch} or return;
 		$watches = PublicInbox::Config::_array($watches);
 		for my $watch (@$watches) {
+			my $url;
 			if (is_maildir($watch)) {
 				compile_watchheaders($ibx);
 				my ($new, $cur) = ("$watch/new", "$watch/cur");
@@ -81,10 +85,14 @@ sub new {
 				return if is_watchspam($cur, $cur_dst, $ibx);
 				push @{$mdmap{$new} //= []}, $ibx;
 				push @$cur_dst, $ibx;
-			} elsif (my $url = imap_url($watch)) {
+			} elsif ($url = imap_url($watch)) {
 				return if is_watchspam($url, $imap{$url}, $ibx);
 				compile_watchheaders($ibx);
 				push @{$imap{$url} ||= []}, $ibx;
+			} elsif ($url = nntp_url($watch)) {
+				return if is_watchspam($url, $nntp{$url}, $ibx);
+				compile_watchheaders($ibx);
+				push @{$nntp{$url} ||= []}, $ibx;
 			} else {
 				warn "watch unsupported: $k=$watch\n";
 			}
@@ -96,13 +104,15 @@ sub new {
 		$mdre = join('|', map { quotemeta($_) } keys %mdmap);
 		$mdre = qr!\A($mdre)/!;
 	}
-	return unless $mdre || scalar(keys %imap);
+	return unless $mdre || scalar(keys %imap) || scalar(keys %nntp);
+
 	bless {
 		spamcheck => $spamcheck,
 		mdmap => \%mdmap,
 		mdre => $mdre,
 		config => $config,
 		imap => scalar keys %imap ? \%imap : undef,
+		nntp => scalar keys %nntp? \%nntp : undef,
 		importers => {},
 		opendirs => {}, # dirname => dirhandle (in progress scans)
 		ops => [], # 'quit', 'full'
@@ -230,7 +240,7 @@ sub watch_fs_init ($) {
 
 # returns the git config section name, e.g [imap "imaps://user@example.com"]
 # without the mailbox, so we can share connections between different inboxes
-sub imap_section ($) {
+sub uri_section ($) {
 	my ($uri) = @_;
 	$uri->scheme . '://' . $uri->authority;
 }
@@ -247,6 +257,14 @@ sub cfg_intvl ($$$) {
 	}
 }
 
+sub cfg_bool ($$$) {
+	my ($cfg, $key, $url) = @_;
+	my $orig = $cfg->urlmatch($key, $url) // return;
+	my $bool = PublicInbox::Config::_git_config_bool($orig);
+	warn "W: $key=$orig for $url is not boolean\n" unless defined($bool);
+	$bool;
+}
+
 # flesh out common IMAP-specific data structures
 sub imap_common_init ($) {
 	my ($self) = @_;
@@ -254,24 +272,17 @@ sub imap_common_init ($) {
 	my $mic_args = {}; # scheme://authority => Mail:IMAPClient arg
 	for my $url (sort keys %{$self->{imap}}) {
 		my $uri = PublicInbox::URIimap->new($url);
-		my $sec = imap_section($uri);
-		for my $f (qw(Starttls Debug Compress)) {
-			my $k = "imap.$f";
-			my $orig = $cfg->urlmatch($k, $url) // next;
-			my $v = PublicInbox::Config::_git_config_bool($orig);
-			if (defined($v)) {
-				$mic_args->{$sec}->{$f} = $v;
-			} else {
-				warn "W: $k=$orig for $url is not boolean\n";
-			}
+		my $sec = uri_section($uri);
+		for my $k (qw(Starttls Debug Compress)) {
+			my $bool = cfg_bool($cfg, "imap.$k", $url) // next;
+			$mic_args->{$sec}->{$k} = $bool;
 		}
 		my $to = cfg_intvl($cfg, 'imap.timeout', $url);
 		$mic_args->{$sec}->{Timeout} = $to if $to;
-		$to = cfg_intvl($cfg, 'imap.pollInterval', $url);
-		$self->{imap_opt}->{$sec}->{poll_intvl} = $to if $to;
-		$to = cfg_intvl($cfg, 'imap.IdleInterval', $url);
-		$self->{imap_opt}->{$sec}->{idle_intvl} = $to if $to;
-
+		for my $k (qw(pollInterval idleInterval)) {
+			$to = cfg_intvl($cfg, "imap.$k", $url) // next;
+			$self->{imap_opt}->{$sec}->{$k} = $to;
+		}
 		my $k = 'imap.fetchBatchSize';
 		my $bs = $cfg->urlmatch($k, $url) // next;
 		if ($bs =~ /\A([0-9]+)\z/) {
@@ -295,7 +306,7 @@ sub mic_for ($$$) { # mic = Mail::IMAPClient
 		username => $uri->user,
 		password => $uri->password,
 	};
-	my $common = $mic_args->{imap_section($uri)} // {};
+	my $common = $mic_args->{uri_section($uri)} // {};
 	my $host = $cred->{host};
 	my $mic_arg = {
 		Port => $uri->port,
@@ -331,7 +342,7 @@ sub mic_for ($$$) { # mic = Mail::IMAPClient
 	}
 	if ($mic->login && $mic->IsAuthenticated) {
 		# success! keep IMAPClient->new arg in case we get disconnected
-		$self->{mic_arg}->{imap_section($uri)} = $mic_arg;
+		$self->{mic_arg}->{uri_section($uri)} = $mic_arg;
 	} else {
 		warn "E: <$url> LOGIN: $@\n";
 		$mic = undef;
@@ -362,7 +373,7 @@ sub imap_import_msg ($$$$) {
 
 sub imap_fetch_all ($$$) {
 	my ($self, $mic, $uri) = @_;
-	my $sec = imap_section($uri);
+	my $sec = uri_section($uri);
 	my $mbx = $uri->mailbox;
 	my $url = $uri->as_string;
 	$mic->Clear(1); # trim results history
@@ -479,7 +490,7 @@ sub imap_idle_once ($$$$) {
 # idles on a single URI
 sub watch_imap_idle_1 ($$$) {
 	my ($self, $uri, $intvl) = @_;
-	my $sec = imap_section($uri);
+	my $sec = uri_section($uri);
 	my $mic_arg = $self->{mic_arg}->{$sec} or
 			die "BUG: no Mail::IMAPClient->new arg for $sec";
 	my $mic;
@@ -555,7 +566,7 @@ sub event_step {
 sub watch_imap_fetch_all ($$) {
 	my ($self, $uris) = @_;
 	for my $uri (@$uris) {
-		my $sec = imap_section($uri);
+		my $sec = uri_section($uri);
 		my $mic_arg = $self->{mic_arg}->{$sec} or
 			die "BUG: no Mail::IMAPClient->new arg for $sec";
 		my $mic = PublicInbox::IMAPClient->new(%$mic_arg) or next;
@@ -565,21 +576,56 @@ sub watch_imap_fetch_all ($$) {
 	}
 }
 
-sub imap_fetch_fork ($) { # DS::add_timer callback
+sub watch_nntp_fetch_all ($$) {
+	my ($self, $uris) = @_;
+	for my $uri (@$uris) {
+		my $sec = uri_section($uri);
+		my $nn_arg = $self->{nn_arg}->{$sec} or
+			die "BUG: no Net::NNTP->new arg for $sec";
+		my $nntp_opt = $self->{nntp_opt}->{$sec};
+		my $url = $uri->as_string;
+		my $nn = nn_new($nn_arg, $nntp_opt, $url);
+		unless ($nn) {
+			warn "E: $url: \$!=$!\n";
+			next;
+		}
+		last if $self->{quit};
+		if (my $postconn = $nntp_opt->{-postconn}) {
+			for my $m_arg (@$postconn) {
+				my ($method, @args) = @$m_arg;
+				$nn->$method(@args) and next;
+				warn "E: <$url> $method failed\n";
+				$nn = undef;
+				last;
+			}
+		}
+		last if $self->{quit};
+		if ($nn) {
+			my $err = nntp_fetch_all($self, $nn, $uri);
+			warn $err, "\n" if $err;
+		}
+	}
+}
+
+sub poll_fetch_fork ($) { # DS::add_timer callback
 	my ($self, $intvl, $uris) = @{$_[0]};
 	return if $self->{quit};
 	watch_atfork_parent($self);
 	defined(my $pid = fork) or die "fork: $!";
 	if ($pid == 0) {
 		watch_atfork_child($self);
-		watch_imap_fetch_all($self, $uris);
+		if ($uris->[0]->scheme =~ /\Aimaps?\z/) {
+			watch_imap_fetch_all($self, $uris);
+		} else {
+			watch_nntp_fetch_all($self, $uris);
+		}
 		_exit(0);
 	}
 	$self->{poll_pids}->{$pid} = [ $intvl, $uris ];
-	PublicInbox::DS::dwaitpid($pid, \&imap_fetch_reap, $self);
+	PublicInbox::DS::dwaitpid($pid, \&poll_fetch_reap, $self);
 }
 
-sub imap_fetch_reap { # PublicInbox::DS::dwaitpid callback
+sub poll_fetch_reap { # PublicInbox::DS::dwaitpid callback
 	my ($self, $pid) = @_;
 	my $intvl_uris = delete $self->{poll_pids}->{$pid} or
 		die "BUG: PID=$pid (unknown) reaped: \$?=$?\n";
@@ -590,7 +636,7 @@ sub imap_fetch_reap { # PublicInbox::DS::dwaitpid callback
 			map { $_->as_string."\n" } @$uris;
 	}
 	warn('I: will check ', $_->as_string, " in ${intvl}s\n") for @$uris;
-	PublicInbox::DS::add_timer($intvl, \&imap_fetch_fork,
+	PublicInbox::DS::add_timer($intvl, \&poll_fetch_fork,
 					[$self, $intvl, $uris]);
 }
 
@@ -610,18 +656,18 @@ sub watch_imap_init ($) {
 	my $mics = $self->{mics} = {}; # schema://authority => IMAPClient obj
 	for my $url (sort keys %{$self->{imap}}) {
 		my $uri = PublicInbox::URIimap->new($url);
-		$mics->{imap_section($uri)} //= mic_for($self, $uri, $mic_args);
+		$mics->{uri_section($uri)} //= mic_for($self, $uri, $mic_args);
 	}
 
 	my $idle = []; # [ [ uri1, intvl1 ], [uri2, intvl2] ]
 	my $poll = {}; # intvl_seconds => [ uri1, uri2 ]
 	for my $url (keys %{$self->{imap}}) {
 		my $uri = PublicInbox::URIimap->new($url);
-		my $sec = imap_section($uri);
+		my $sec = uri_section($uri);
 		my $mic = $mics->{$sec};
-		my $intvl = $self->{imap_opt}->{$sec}->{poll_intvl};
+		my $intvl = $self->{imap_opt}->{$sec}->{pollInterval};
 		if ($mic->has_capability('IDLE') && !$intvl) {
-			$intvl = $self->{imap_opt}->{$sec}->{idle_intvl};
+			$intvl = $self->{imap_opt}->{$sec}->{idleInterval};
 			push @$idle, [ $uri, $intvl // () ];
 		} else {
 			push @{$poll->{$intvl || 120}}, $uri;
@@ -633,11 +679,238 @@ sub watch_imap_init ($) {
 		PublicInbox::DS::requeue($self); # ->event_step to fork
 	}
 	return unless scalar keys %$poll;
-	$self->{poll_pids} = {};
+	$self->{poll_pids} //= {};
+
+	# poll all URIs for a given interval sequentially
+	while (my ($intvl, $uris) = each %$poll) {
+		PublicInbox::DS::add_timer(0, \&poll_fetch_fork,
+						[$self, $intvl, $uris]);
+	}
+}
+
+# flesh out common NNTP-specific data structures
+sub nntp_common_init ($) {
+	my ($self) = @_;
+	my $cfg = $self->{config};
+	my $nn_args = {}; # scheme://authority => Net::NNTP->new arg
+	for my $url (sort keys %{$self->{nntp}}) {
+		my $sec = uri_section(URI->new($url));
+
+		# Debug and Timeout are is passed to Net::NNTP->new
+		my $v = cfg_bool($cfg, 'nntp.Debug', $url);
+		$nn_args->{$sec}->{Debug} = $v if defined $v;
+		my $to = cfg_intvl($cfg, 'nntp.Timeout', $url);
+		$nn_args->{$sec}->{Timeout} = $to if $to;
+
+		# Net::NNTP post-connect commands
+		for my $k (qw(starttls compress)) {
+			$v = cfg_bool($cfg, "nntp.$k", $url) // next;
+			$self->{nntp_opt}->{$sec}->{$k} = $v;
+		}
+
+		# internal option
+		for my $k (qw(pollInterval)) {
+			$to = cfg_intvl($cfg, "nntp.$k", $url) // next;
+			$self->{nntp_opt}->{$sec}->{$k} = $to;
+		}
+	}
+	$nn_args;
+}
+
+# Net::NNTP doesn't support CAPABILITIES, yet
+sub try_starttls ($) {
+	my ($host) = @_;
+	return if $host =~ /\.onion\z/s;
+	return if $host =~ /\A127\.[0-9]+\.[0-9]+\.[0-9]+\z/s;
+	return if $host eq '::1';
+	1;
+}
+
+sub nn_new ($$$) {
+	my ($nn_arg, $nntp_opt, $url) = @_;
+	my $nn = Net::NNTP->new(%$nn_arg) or die "E: <$url> new: $!\n";
+
+	# default to using STARTTLS if it's available, but allow
+	# it to be disabled for localhost/VPN users
+	if (!$nn_arg->{SSL} && $nn->can('starttls')) {
+		if (!defined($nntp_opt->{starttls}) &&
+				try_starttls($nn_arg->{Host})) {
+			# soft fail by default
+			$nn->starttls or warn <<"";
+W: <$url> STARTTLS tried and failed (not requested)
+
+		} elsif ($nntp_opt->{starttls}) {
+			# hard fail if explicitly configured
+			$nn->starttls or die <<"";
+E: <$url> STARTTLS requested and failed
+
+		}
+	} elsif ($nntp_opt->{starttls}) {
+		$nn->can('starttls') or
+			die "E: <$url> Net::NNTP too old for STARTTLS\n";
+		$nn->starttls or die <<"";
+E: <$url> STARTTLS requested and failed
+
+	}
+	$nn;
+}
+
+sub nn_for ($$$) { # nn = Net::NNTP
+	my ($self, $uri, $nn_args) = @_;
+	my $url = $uri->as_string;
+	my $sec = uri_section($uri);
+	my $nntp_opt = $self->{nntp_opt}->{$sec} //= {};
+	my $cred;
+	my ($u, $p);
+	if (defined(my $ui = $uri->userinfo)) {
+		$cred = {
+			url => $sec,
+			protocol => $uri->scheme,
+			host => $uri->host,
+		};
+		($u, $p) = split(/:/, $ui, 2);
+		($cred->{username}, $cred->{password}) = ($u, $p);
+	}
+	my $common = $nn_args->{$sec} // {};
+	my $nn_arg = {
+		Port => $uri->port,
+		# Net::NNTP mishandles `0', so we pass `127.0.0.1'
+		Host => $uri->host eq '0' ? '127.0.0.1' : $uri->host,
+		SSL => $uri->secure, # snews == nntps
+		%$common, # may Debug ....
+	};
+	my $nn = nn_new($nn_arg, $nntp_opt, $url);
+
+	if ($cred) {
+		Git::credential($cred, 'fill'); # may prompt user here
+		if ($nn->authinfo($u, $p)) {
+			push @{$nntp_opt->{-postconn}}, [ 'authinfo', $u, $p ];
+		} else {
+			warn "E: <$url> AUTHINFO $u XXXX failed\n";
+			$nn = undef;
+		}
+	}
+
+	if ($nntp_opt->{compress}) {
+		# https://rt.cpan.org/Ticket/Display.html?id=129967
+		if ($nn->can('compress')) {
+			if ($nn->compress) {
+				push @{$nntp_opt->{-postconn}}, [ 'compress' ];
+			} else {
+				warn "W: <$url> COMPRESS failed\n";
+			}
+		} else {
+			delete $nntp_opt->{compress};
+			warn <<"";
+W: <$url> COMPRESS not supported by Net::NNTP
+W: see https://rt.cpan.org/Ticket/Display.html?id=129967 for updates
+
+		}
+	}
+
+	$self->{nn_arg}->{$sec} = $nn_arg;
+	Git::credential($cred, $nn ? 'approve' : 'reject') if $cred;
+	$nn;
+}
+
+sub nntp_fetch_all ($$$) {
+	my ($self, $nn, $uri) = @_;
+	my ($group, $num_a, $num_b) = $uri->group;
+	my $sec = uri_section($uri);
+	my $url = $uri->as_string;
+	my ($nr, $beg, $end) = $nn->group($group);
+	unless (defined($nr)) {
+		chomp(my $msg = $nn->message);
+		return "E: GROUP $group <$sec> $msg";
+	}
+
+	# IMAPTracker is also used for tracking NNTP, UID == article number
+	# LIST.ACTIVE can get the equivalent of UIDVALIDITY, but that's
+	# expensive.  So we assume newsgroups don't change:
+	my $itrk = PublicInbox::IMAPTracker->new($url);
+	my (undef, $l_art) = $itrk->get_last;
+	$l_art //= $beg; # initial import
+
+	# allow users to specify articles to refetch
+	# cf. https://tools.ietf.org/id/draft-gilman-news-url-01.txt
+	# nntp://example.com/inbox.foo/$num_a-$num_b
+	$l_art = $num_a if defined($num_a) && $num_a < $l_art;
+	$end = $num_b if defined($num_b) && $num_b < $end;
+
+	return if $l_art >= $end; # nothing to do
+	$beg = $l_art + 1;
+
+	warn "I: $url fetching ARTICLE $beg..$end\n";
+	my $warn_cb = $SIG{__WARN__} || sub { print STDERR @_ };
+	my ($err, $art);
+	local $SIG{__WARN__} = sub {
+		$warn_cb->("$url ", $art ? ("ARTICLE $art") : (), "\n", @_);
+	};
+	my $inboxes = $self->{nntp}->{$url};
+	my $last_art;
+	for ($beg..$end) {
+		last if $self->{quit};
+		$art = $_;
+		my $raw = $nn->article($art);
+		unless (defined($raw)) {
+			my $msg = $nn->message;
+			if ($nn->code == 421) { # pseudo response from Net::Cmd
+				$err = "E: $msg";
+				last;
+			} else { # probably just a deleted message (spam)
+				warn "W: $msg";
+				next;
+			}
+		}
+		s/\r\n/\n/ for @$raw;
+		$raw = join('', @$raw);
+		if (ref($inboxes)) {
+			for my $ibx (@$inboxes) {
+				my $eml = PublicInbox::Eml->new($raw);
+				import_eml($self, $ibx, $eml);
+			}
+		} elsif ($inboxes eq 'watchspam') {
+			my $eml = PublicInbox::Eml->new(\$raw);
+			my $arg = [ $self, $eml, "$url ARTICLE $art" ];
+			$self->{config}->each_inbox(\&remove_eml_i, $arg);
+		} else {
+			die "BUG: destination unknown $inboxes";
+		}
+		$last_art = $art;
+	}
+	$itrk->update_last(0, $last_art) if defined $last_art;
+	_done_for_now($self);
+	$err;
+}
+
+sub watch_nntp_init ($) {
+	my ($self) = @_;
+	eval { require Net::NNTP } or
+		die "Net::NNTP is required for NNTP:\n$@\n";
+	eval { require Git } or
+		die "Git (Perl module) is required for NNTP:\n$@\n";
+	eval { require PublicInbox::IMAPTracker } or
+		die "DBD::SQLite is required for NNTP\n:$@\n";
+
+	my $nn_args = nntp_common_init($self); # read args from config
+
+	# make sure we can connect and cache the credentials in memory
+	$self->{nn_arg} = {}; # schema://authority => Net::NNTP->new args
+	for my $url (sort keys %{$self->{nntp}}) {
+		nn_for($self, URI->new($url), $nn_args);
+	}
+	my $poll = {}; # intvl_seconds => [ uri1, uri2 ]
+	for my $url (keys %{$self->{nntp}}) {
+		my $uri = URI->new($url);
+		my $sec = uri_section($uri);
+		my $intvl = $self->{nntp_opt}->{$sec}->{pollInterval};
+		push @{$poll->{$intvl || 120}}, $uri;
+	}
+	$self->{poll_pids} //= {};
 
 	# poll all URIs for a given interval sequentially
 	while (my ($intvl, $uris) = each %$poll) {
-		PublicInbox::DS::add_timer(0, \&imap_fetch_fork,
+		PublicInbox::DS::add_timer(0, \&poll_fetch_fork,
 						[$self, $intvl, $uris]);
 	}
 }
@@ -647,6 +920,7 @@ sub watch {
 	$self->{oldset} = $oldset;
 	$self->{sig} = $sig;
 	watch_imap_init($self) if $self->{imap};
+	watch_nntp_init($self) if $self->{nntp};
 	watch_fs_init($self) if $self->{mdre};
 	PublicInbox::DS->SetPostLoopCallback(sub {});
 	PublicInbox::DS->EventLoop until $self->{quit};
@@ -754,4 +1028,15 @@ sub imap_url {
 	$uri ? $uri->canonical->as_string : undef;
 }
 
+my %IS_NNTP = (news => 1, snews => 1, nntp => 1);
+sub nntp_url {
+	my ($url) = @_;
+	require URI;
+	# URI::snews exists, URI::nntps does not, so use URI::snews
+	$url =~ s!\Anntps://!snews://!i;
+	my $uri = URI->new($url);
+	return unless $uri && $IS_NNTP{$uri->scheme};
+	$uri->group ? $uri->canonical->as_string : undef;
+}
+
 1;
diff --git a/t/nntpd.t b/t/nntpd.t
index 9d0ee2baa6d..d72d6a1ce7e 100644
--- a/t/nntpd.t
+++ b/t/nntpd.t
@@ -352,6 +352,7 @@ Date: Fri, 02 Oct 1993 00:00:00 +0000
 		my @of = xqx([$lsof, '-p', $td->{pid}], undef, $noerr);
 		is(scalar(grep(/\(deleted\)/, @of)), 0, 'no deleted files');
 	};
+	SKIP: { test_watch($tmpdir, $sock, $group) };
 	{
 		setsockopt($s, IPPROTO_TCP, TCP_NODELAY, 1);
 		syswrite($s, 'HDR List-id 1-');
@@ -391,4 +392,55 @@ sub read_til_dot {
 	$buf;
 }
 
+sub test_watch {
+	my ($tmpdir, $sock, $group) = @_;
+	use_ok 'PublicInbox::WatchMaildir';
+	use_ok 'PublicInbox::InboxIdle';
+	require_git('1.8.5', 1) or skip('git 1.8.5+ needed for --urlmatch', 4);
+	my $old_env = { HOME => $ENV{HOME} };
+	my $home = "$tmpdir/watch_home";
+	mkdir $home or BAIL_OUT $!;
+	mkdir "$home/.public-inbox" or BAIL_OUT $!;
+	local $ENV{HOME} = $home;
+	my $name = 'watchnntp';
+	my $addr = "i1\@example.com";
+	my $url = "http://example.com/i1";
+	my $inboxdir = "$tmpdir/watchnntp";
+	my $cmd = ['-init', '-V1', '-Lbasic', $name, $inboxdir, $url, $addr];
+	my ($ihost, $iport) = ($sock->sockhost, $sock->sockport);
+	my $nntpurl = "nntp://$ihost:$iport/$group";
+	run_script($cmd) or BAIL_OUT("init $name");
+	xsys(qw(git config), "--file=$home/.public-inbox/config",
+			"publicinbox.$name.watch",
+			$nntpurl) == 0 or BAIL_OUT "git config $?";
+	# try again with polling
+	xsys(qw(git config), "--file=$home/.public-inbox/config",
+		'nntp.PollInterval', 0.11) == 0
+		or BAIL_OUT "git config $?";
+	my $cfg = PublicInbox::Config->new;
+	PublicInbox::DS->Reset;
+	my $ii = PublicInbox::InboxIdle->new($cfg);
+	my $cb = sub { PublicInbox::DS->SetPostLoopCallback(sub {}) };
+	my $obj = bless \$cb, 'PublicInbox::TestCommon::InboxWakeup';
+	$cfg->each_inbox(sub { $_[0]->subscribe_unlock('ident', $obj) });
+	my $watcherr = "$tmpdir/watcherr";
+	open my $err_wr, '>', $watcherr or BAIL_OUT $!;
+	open my $err, '<', $watcherr or BAIL_OUT $!;
+	my $w = start_script(['-watch'], undef, { 2 => $err_wr });
+
+	diag 'waiting for initial fetch...';
+	PublicInbox::DS->EventLoop;
+	diag 'inbox unlocked on initial fetch';
+	$w->kill;
+	$w->join;
+	is($?, 0, 'no error in exited -watch process');
+	$cfg->each_inbox(sub { shift->unsubscribe_unlock('ident') });
+	$ii->close;
+	PublicInbox::DS->Reset;
+	my @err = grep(!/^I:/, <$err>);
+	is(@err, 0, 'no warnings/errors from -watch'.join(' ', @err));
+	my @ls = xqx(['git', "--git-dir=$inboxdir", qw(ls-tree -r HEAD)]);
+	isnt(scalar(@ls), 0, 'imported something');
+}
+
 1;
diff --git a/t/watch_nntp.t b/t/watch_nntp.t
new file mode 100644
index 00000000000..f919930e7d8
--- /dev/null
+++ b/t/watch_nntp.t
@@ -0,0 +1,15 @@
+# Copyright (C) 2020 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+use strict;
+use Test::More;
+use PublicInbox::Config;
+# see t/nntpd*.t for tests against a live NNTP server
+
+use_ok 'PublicInbox::WatchMaildir';
+my $nntp_url = \&PublicInbox::WatchMaildir::nntp_url;
+is('news://example.com/inbox.foo',
+	$nntp_url->('NEWS://examplE.com/inbox.foo'), 'lowercased');
+is('snews://example.com/inbox.foo',
+	$nntp_url->('nntps://example.com/inbox.foo'), 'nntps:// is snews://');
+
+done_testing;