[PATCH 0/8] nntp: eliminate response-duration cyclic refs

[PATCH 0/8] nntp: eliminate response-duration cyclic refs

[Eric Wong]

While there are no known memory leaks in our code, the short-lived
reference cycle introduced by the long_response API is a potential
source of bugs.

Change the long_response API and update callers to pass
parameter explicitly to named subroutines instead of anonymous
ones.

Since creating new anonymous subs (aka "closures") take a fair
amount of memory in Perl, this ought to reduce memory pressure,
as well, since we're now only creating lightweight references to
long-lived subs.  The tradeoff is we make a few more hash
lookups since we no longer cache some hash lookups into local
variables onto the stack.

Eric Wong (8):
  nntp: get_range: return scalarref for $beg
  nntp: remove cyclic refs from long_response
  nntp: hdr_searchmsg: use named sub for numeric range response
  nntp: cmd_xrover: use named sub for long_response
  nntp: cmd_listgroup: use named subs for long_response
  nntp: cmd_newnews: use named sub for long_response
  nntp: hdr_msg_id: use named sub for long_response
  nntp: cmd_xover: use named sub for long_response

 lib/PublicInbox/NNTP.pm | 293 +++++++++++++++++++++-------------------
 1 file changed, 152 insertions(+), 141 deletions(-)

[PATCH 1/8] nntp: get_range: return scalarref for $beg

[Eric Wong]

...Instead of just returning a plain scalar inside an arrayref.

This is because we usually pass the result of NNTP::get_range to
Msgmap::msg_range.  Upcoming changes will move us away from
anonymous subroutines, so this change will make followup commits
easier-to-digest as modifications to the underlying scalar can
be more easily propagated between non-anonymous-subs.
---
 lib/PublicInbox/NNTP.pm | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/lib/PublicInbox/NNTP.pm b/lib/PublicInbox/NNTP.pm
index 6cd2b84c..cd5d7bba 100644
--- a/lib/PublicInbox/NNTP.pm
+++ b/lib/PublicInbox/NNTP.pm
@@ -204,7 +204,7 @@ sub cmd_listgroup ($;$$) {
 		return $r unless ref $r;
 		my ($beg, $end) = @$r;
 		long_response($self, sub {
-			$r = $mm->msg_range(\$beg, $end, 'num');
+			$r = $mm->msg_range($beg, $end, 'num');
 			scalar(@$r) or return;
 			more($self, join("\r\n", map { $_->[0] } @$r));
 			1;
@@ -593,7 +593,7 @@ sub get_range ($$) {
 	$beg = $min if ($beg < $min);
 	$end = $max if ($end > $max);
 	return '420 No article(s) selected' if ($beg > $end);
-	[ $beg, $end ];
+	[ \$beg, $end ];
 }
 
 sub long_response ($$) {
@@ -661,7 +661,7 @@ sub hdr_message_id ($$$) { # optimize XHDR Message-ID [range] for slrnpull.
 		my ($beg, $end) = @$r;
 		more($self, $xhdr ? r221 : r225);
 		long_response($self, sub {
-			my $r = $mm->msg_range(\$beg, $end);
+			my $r = $mm->msg_range($beg, $end);
 			@$r or return;
 			more($self, join("\r\n", map {
 				"$_->[0] <$_->[1]>"
@@ -704,7 +704,7 @@ sub hdr_xref ($$$) { # optimize XHDR Xref [range] for rtin
 		my ($beg, $end) = @$r;
 		more($self, $xhdr ? r221 : r225);
 		long_response($self, sub {
-			my $r = $mm->msg_range(\$beg, $end);
+			my $r = $mm->msg_range($beg, $end);
 			@$r or return;
 			more($self, join("\r\n", map {
 				my $num = $_->[0];
@@ -737,7 +737,7 @@ sub hdr_searchmsg ($$$$) {
 		return $r unless ref $r;
 		my ($beg, $end) = @$r;
 		more($self, $xhdr ? r221 : r225);
-		my $cur = $beg;
+		my $cur = $$beg;
 		long_response($self, sub {
 			my $msgs = $over->query_xover($cur, $end);
 			my $nr = scalar @$msgs or return;
@@ -823,9 +823,9 @@ sub cmd_xrover ($;$) {
 	more($self, '224 Overview information follows');
 
 	long_response($self, sub {
-		my $h = over_header_for($over, $beg, 'references');
-		more($self, "$beg $h") if defined($h);
-		$beg++ < $end;
+		my $h = over_header_for($over, $$beg, 'references');
+		more($self, "$$beg $h") if defined($h);
+		$$beg++ < $end;
 	});
 }
 
@@ -870,9 +870,9 @@ sub cmd_xover ($;$) {
 	my $r = get_range($self, $range);
 	return $r unless ref $r;
 	my ($beg, $end) = @$r;
-	more($self, "224 Overview information follows for $beg to $end");
+	more($self, "224 Overview information follows for $$beg to $end");
 	my $over = $self->{ng}->over;
-	my $cur = $beg;
+	my $cur = $$beg;
 	long_response($self, sub {
 		my $msgs = $over->query_xover($cur, $end);
 		my $nr = scalar @$msgs or return;

[PATCH 2/8] nntp: remove cyclic refs from long_response

[Eric Wong]

Leftover cyclic references are a source of memory leaks.  While
our code is AFAIK unaffected by such leaks at the moment,
eliminating a potential source of bugs will make maintenance
easier.

We make the long_response API cycle-free by stashing the
callback into the NNTP object.  However, callers will need
to be updated to get rid of the circular reference to $self.
We do that be replacing anonymous subs with name subroutine
references, such as xref_range_i replacing the formerly
anonymous sub inside hdr_xref.
---
 lib/PublicInbox/NNTP.pm | 117 +++++++++++++++++++++-------------------
 1 file changed, 61 insertions(+), 56 deletions(-)

diff --git a/lib/PublicInbox/NNTP.pm b/lib/PublicInbox/NNTP.pm
index cd5d7bba..33c557f5 100644
--- a/lib/PublicInbox/NNTP.pm
+++ b/lib/PublicInbox/NNTP.pm
@@ -6,7 +6,7 @@ package PublicInbox::NNTP;
 use strict;
 use warnings;
 use base qw(PublicInbox::DS);
-use fields qw(nntpd article ng);
+use fields qw(nntpd article ng long_cb);
 use PublicInbox::Search;
 use PublicInbox::Msgmap;
 use PublicInbox::MID qw(mid_escape);
@@ -596,53 +596,57 @@ sub get_range ($$) {
 	[ \$beg, $end ];
 }
 
-sub long_response ($$) {
-	my ($self, $cb) = @_; # cb returns true if more, false if done
+sub long_step {
+	my ($self) = @_;
+	# wbuf is unset or empty, here; {long} may add to it
+	my ($cb, $t0, @args) = @{$self->{long_cb}};
+	my $more = eval { $cb->($self, @args) };
+	if ($@ || !$self->{sock}) { # something bad happened...
+		delete $self->{long_cb};
+		my $elapsed = now() - $t0;
+		my $fd = fileno($self->{sock});
+		if ($@) {
+			err($self,
+			    "%s during long response[$fd] - %0.6f",
+			    $@, $elapsed);
+		}
+		out($self, " deferred[$fd] aborted - %0.6f", $elapsed);
+		$self->close;
+	} elsif ($more) { # $self->{wbuf}:
+		$self->update_idle_time;
+
+		# COMPRESS users all share the same DEFLATE context.
+		# Flush it here to ensure clients don't see
+		# each other's data
+		$self->zflush;
+
+		# no recursion, schedule another call ASAP
+		# but only after all pending writes are done
+		my $wbuf = $self->{wbuf} ||= [];
+		push @$wbuf, \&long_step;
+
+		# wbuf may be populated by $cb, no need to rearm if so:
+		$self->requeue if scalar(@$wbuf) == 1;
+	} else { # all done!
+		delete $self->{long_cb};
+		res($self, '.');
+		my $elapsed = now() - $t0;
+		my $fd = fileno($self->{sock});
+		out($self, " deferred[$fd] done - %0.6f", $elapsed);
+		my $wbuf = $self->{wbuf};
+		$self->requeue unless $wbuf && @$wbuf;
+	}
+}
 
-	my $fd = fileno($self->{sock});
-	defined $fd or return;
+sub long_response ($$;@) {
+	my ($self, $cb, @args) = @_; # cb returns true if more, false if done
+
+	$self->{sock} or return;
 	# make sure we disable reading during a long response,
 	# clients should not be sending us stuff and making us do more
 	# work while we are stream a response to them
-	my $t0 = now();
-	my $long_cb; # DANGER: self-referential
-	$long_cb = sub {
-		# wbuf is unset or empty, here; $cb may add to it
-		my $more = eval { $cb->() };
-		if ($@ || !$self->{sock}) { # something bad happened...
-			$long_cb = undef;
-			my $diff = now() - $t0;
-			if ($@) {
-				err($self,
-				    "%s during long response[$fd] - %0.6f",
-				    $@, $diff);
-			}
-			out($self, " deferred[$fd] aborted - %0.6f", $diff);
-			$self->close;
-		} elsif ($more) { # $self->{wbuf}:
-			$self->update_idle_time;
-
-			# COMPRESS users all share the same DEFLATE context.
-			# Flush it here to ensure clients don't see
-			# each other's data
-			$self->zflush;
-
-			# no recursion, schedule another call ASAP
-			# but only after all pending writes are done
-			my $wbuf = $self->{wbuf} ||= [];
-			push @$wbuf, $long_cb;
-
-			# wbuf may be populated by $cb, no need to rearm if so:
-			$self->requeue if scalar(@$wbuf) == 1;
-		} else { # all done!
-			$long_cb = undef;
-			res($self, '.');
-			out($self, " deferred[$fd] done - %0.6f", now() - $t0);
-			my $wbuf = $self->{wbuf};
-			$self->requeue unless $wbuf && @$wbuf;
-		}
-	};
-	$self->write($long_cb); # kick off!
+	$self->{long_cb} = [ $cb, now(), @args ];
+	long_step($self); # kick off!
 	undef;
 }
 
@@ -686,6 +690,18 @@ sub mid_lookup ($$) {
 	(undef, undef);
 }
 
+sub xref_range_i {
+	my ($self, $beg, $end) = @_;
+	my $ng = $self->{ng};
+	my $r = $ng->mm->msg_range($beg, $end);
+	@$r or return;
+	more($self, join("\r\n", map {
+		my $num = $_->[0];
+		"$num ".xref($self, $ng, $num, $_->[1]);
+	} @$r));
+	1;
+}
+
 sub hdr_xref ($$$) { # optimize XHDR Xref [range] for rtin
 	my ($self, $xhdr, $range) = @_;
 
@@ -699,19 +715,8 @@ sub hdr_xref ($$$) { # optimize XHDR Xref [range] for rtin
 		$range = $self->{article} unless defined $range;
 		my $r = get_range($self, $range);
 		return $r unless ref $r;
-		my $ng = $self->{ng};
-		my $mm = $ng->mm;
-		my ($beg, $end) = @$r;
 		more($self, $xhdr ? r221 : r225);
-		long_response($self, sub {
-			my $r = $mm->msg_range($beg, $end);
-			@$r or return;
-			more($self, join("\r\n", map {
-				my $num = $_->[0];
-				"$num ".xref($self, $ng, $num, $_->[1]);
-			} @$r));
-			1;
-		});
+		long_response($self, \&xref_range_i, @$r);
 	}
 }
 

[PATCH 3/8] nntp: hdr_searchmsg: use named sub for numeric range response

[Eric Wong]

Introduce searchmsg_range_i, which does the same thing as
the anonymous sub it replaces.
---
 lib/PublicInbox/NNTP.pm | 30 +++++++++++++++---------------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/lib/PublicInbox/NNTP.pm b/lib/PublicInbox/NNTP.pm
index 33c557f5..30862dca 100644
--- a/lib/PublicInbox/NNTP.pm
+++ b/lib/PublicInbox/NNTP.pm
@@ -727,6 +727,20 @@ sub over_header_for {
 	$smsg->{$field};
 }
 
+sub searchmsg_range_i {
+	my ($self, $beg, $end, $field) = @_;
+	my $over = $self->{ng}->over;
+	my $msgs = $over->query_xover($$beg, $end);
+	scalar(@$msgs) or return;
+	my $tmp = '';
+	foreach my $s (@$msgs) {
+		$tmp .= $s->{num} . ' ' . $s->$field . "\r\n";
+	}
+	utf8::encode($tmp);
+	$self->msg_more($tmp);
+	$$beg = $msgs->[-1]->{num} + 1;
+}
+
 sub hdr_searchmsg ($$$$) {
 	my ($self, $xhdr, $field, $range) = @_;
 	if (defined $range && $range =~ /\A<(.+)>\z/) { # Message-ID
@@ -736,24 +750,10 @@ sub hdr_searchmsg ($$$$) {
 		hdr_mid_response($self, $xhdr, $ng, $n, $range, $v);
 	} else { # numeric range
 		$range = $self->{article} unless defined $range;
-		my $over = $self->{ng}->over;
-		my $mm = $self->{ng}->mm;
 		my $r = get_range($self, $range);
 		return $r unless ref $r;
-		my ($beg, $end) = @$r;
 		more($self, $xhdr ? r221 : r225);
-		my $cur = $$beg;
-		long_response($self, sub {
-			my $msgs = $over->query_xover($cur, $end);
-			my $nr = scalar @$msgs or return;
-			my $tmp = '';
-			foreach my $s (@$msgs) {
-				$tmp .= $s->{num} . ' ' . $s->$field . "\r\n";
-			}
-			utf8::encode($tmp);
-			$self->msg_more($tmp);
-			$cur = $msgs->[-1]->{num} + 1;
-		});
+		long_response($self, \&searchmsg_range_i, @$r, $field);
 	}
 }
 

[PATCH 4/8] nntp: cmd_xrover: use named sub for long_response

[Eric Wong]

Introduce xrover_i which does the same thing as the anonymous
sub it replaces.
---
 lib/PublicInbox/NNTP.pm | 17 ++++++++---------
 1 file changed, 8 insertions(+), 9 deletions(-)

diff --git a/lib/PublicInbox/NNTP.pm b/lib/PublicInbox/NNTP.pm
index 30862dca..dff1bcd2 100644
--- a/lib/PublicInbox/NNTP.pm
+++ b/lib/PublicInbox/NNTP.pm
@@ -813,6 +813,13 @@ sub hdr_mid_response ($$$$$$) {
 	undef;
 }
 
+sub xrover_i {
+	my ($self, $beg, $end) = @_;
+	my $h = over_header_for($self->{ng}->over, $$beg, 'references');
+	more($self, "$$beg $h") if defined($h);
+	$$beg++ < $end;
+}
+
 sub cmd_xrover ($;$) {
 	my ($self, $range) = @_;
 	my $ng = $self->{ng} or return '412 no newsgroup selected';
@@ -822,16 +829,8 @@ sub cmd_xrover ($;$) {
 	$range = $self->{article} unless defined $range;
 	my $r = get_range($self, $range);
 	return $r unless ref $r;
-	my ($beg, $end) = @$r;
-	my $mm = $ng->mm;
-	my $over = $ng->over;
 	more($self, '224 Overview information follows');
-
-	long_response($self, sub {
-		my $h = over_header_for($over, $$beg, 'references');
-		more($self, "$$beg $h") if defined($h);
-		$$beg++ < $end;
-	});
+	long_response($self, \&xrover_i, @$r);
 }
 
 sub over_line ($$$$) {

[PATCH 5/8] nntp: cmd_listgroup: use named subs for long_response

[Eric Wong]

Introduce listgroup_range_i and listgroup_all_i subs which
do the same things as the anonymous subs they replace.
---
 lib/PublicInbox/NNTP.pm | 35 +++++++++++++++++++----------------
 1 file changed, 19 insertions(+), 16 deletions(-)

diff --git a/lib/PublicInbox/NNTP.pm b/lib/PublicInbox/NNTP.pm
index dff1bcd2..c288d06b 100644
--- a/lib/PublicInbox/NNTP.pm
+++ b/lib/PublicInbox/NNTP.pm
@@ -190,6 +190,22 @@ sub cmd_list ($;$$) {
 	'.'
 }
 
+sub listgroup_range_i {
+	my ($self, $beg, $end) = @_;
+	my $r = $self->{ng}->mm->msg_range($beg, $end, 'num');
+	scalar(@$r) or return;
+	more($self, join("\r\n", map { $_->[0] } @$r));
+	1;
+}
+
+sub listgroup_all_i {
+	my ($self, $num) = @_;
+	my $ary = $self->{ng}->mm->ids_after($num);
+	scalar(@$ary) or return;
+	more($self, join("\r\n", @$ary));
+	1;
+}
+
 sub cmd_listgroup ($;$$) {
 	my ($self, $group, $range) = @_;
 	if (defined $group) {
@@ -197,26 +213,13 @@ sub cmd_listgroup ($;$$) {
 		return $res if ($res !~ /\A211 /);
 		more($self, $res);
 	}
-	my $ng = $self->{ng} or return '412 no newsgroup selected';
-	my $mm = $ng->mm;
+	$self->{ng} or return '412 no newsgroup selected';
 	if (defined $range) {
 		my $r = get_range($self, $range);
 		return $r unless ref $r;
-		my ($beg, $end) = @$r;
-		long_response($self, sub {
-			$r = $mm->msg_range($beg, $end, 'num');
-			scalar(@$r) or return;
-			more($self, join("\r\n", map { $_->[0] } @$r));
-			1;
-		});
+		long_response($self, \&listgroup_range_i, @$r);
 	} else { # grab every article number
-		my $n = 0;
-		long_response($self, sub {
-			my $ary = $mm->ids_after(\$n);
-			scalar(@$ary) or return;
-			more($self, join("\r\n", @$ary));
-			1;
-		});
+		long_response($self, \&listgroup_all_i, \(my $num = 0));
 	}
 }
 

[PATCH 6/8] nntp: cmd_newnews: use named sub for long_response

[Eric Wong]

Introduce newnews_i, which does the same thing as the anonymous
sub it replaces.
---
 lib/PublicInbox/NNTP.pm | 45 ++++++++++++++++++++++-------------------
 1 file changed, 24 insertions(+), 21 deletions(-)

diff --git a/lib/PublicInbox/NNTP.pm b/lib/PublicInbox/NNTP.pm
index c288d06b..ec9ac287 100644
--- a/lib/PublicInbox/NNTP.pm
+++ b/lib/PublicInbox/NNTP.pm
@@ -300,6 +300,26 @@ sub ngpat2re (;$) {
 	$_[0] = qr/\A(?:$_[0])\z/;
 }
 
+sub newnews_i {
+	my ($self, $overs, $ts, $prev) = @_;
+	my $over = $overs->[0];
+	my $msgs = $over->query_ts($ts, $$prev);
+	if (scalar @$msgs) {
+		more($self, '<' .
+			join(">\r\n<", map { $_->mid } @$msgs ).
+			'>');
+		$$prev = $msgs->[-1]->{num};
+	} else {
+		shift @$overs;
+		if (@$overs) { # continue onto next newsgroup
+			$$prev = 0;
+			return 1;
+		} else { # break out of the long response.
+			return;
+		}
+	}
+}
+
 sub cmd_newnews ($$$$;$$) {
 	my ($self, $newsgroups, $date, $time, $gmt, $dists) = @_;
 	my $ts = eval { parse_time($date, $time, $gmt) };
@@ -308,34 +328,17 @@ sub cmd_newnews ($$$$;$$) {
 	my ($keep, $skip) = split('!', $newsgroups, 2);
 	ngpat2re($keep);
 	ngpat2re($skip);
-	my @over;
+	my @overs;
 	foreach my $ng (@{$self->{nntpd}->{grouplist}}) {
 		$ng->{newsgroup} =~ $keep or next;
 		$ng->{newsgroup} =~ $skip and next;
 		my $over = $ng->over or next;
-		push @over, $over;
+		push @overs, $over;
 	};
-	return '.' unless @over;
+	return '.' unless @overs;
 
 	my $prev = 0;
-	long_response($self, sub {
-		my $over = $over[0];
-		my $msgs = $over->query_ts($ts, $prev);
-		if (scalar @$msgs) {
-			more($self, '<' .
-				join(">\r\n<", map { $_->mid } @$msgs ).
-				'>');
-			$prev = $msgs->[-1]->{num};
-		} else {
-			shift @over;
-			if (@over) { # continue onto next newsgroup
-				$prev = 0;
-				return 1;
-			} else { # break out of the long response.
-				return;
-			}
-		}
-	});
+	long_response($self, \&newnews_i, \@overs, $ts, \$prev);
 }
 
 sub cmd_group ($$) {

[PATCH 7/8] nntp: hdr_msg_id: use named sub for long_response

[Eric Wong]

Introduce hdr_msgid_range_i, which does the same thing as the
anonymous sub it replaces.
---
 lib/PublicInbox/NNTP.pm | 19 +++++++++----------
 1 file changed, 9 insertions(+), 10 deletions(-)

diff --git a/lib/PublicInbox/NNTP.pm b/lib/PublicInbox/NNTP.pm
index ec9ac287..b2927b79 100644
--- a/lib/PublicInbox/NNTP.pm
+++ b/lib/PublicInbox/NNTP.pm
@@ -656,6 +656,14 @@ sub long_response ($$;@) {
 	undef;
 }
 
+sub hdr_msgid_range_i {
+	my ($self, $beg, $end) = @_;
+	my $r = $self->{ng}->mm->msg_range($beg, $end);
+	@$r or return;
+	more($self, join("\r\n", map { "$_->[0] <$_->[1]>" } @$r));
+	1;
+}
+
 sub hdr_message_id ($$$) { # optimize XHDR Message-ID [range] for slrnpull.
 	my ($self, $xhdr, $range) = @_;
 
@@ -667,17 +675,8 @@ sub hdr_message_id ($$$) { # optimize XHDR Message-ID [range] for slrnpull.
 		$range = $self->{article} unless defined $range;
 		my $r = get_range($self, $range);
 		return $r unless ref $r;
-		my $mm = $self->{ng}->mm;
-		my ($beg, $end) = @$r;
 		more($self, $xhdr ? r221 : r225);
-		long_response($self, sub {
-			my $r = $mm->msg_range($beg, $end);
-			@$r or return;
-			more($self, join("\r\n", map {
-				"$_->[0] <$_->[1]>"
-			} @$r));
-			1;
-		});
+		long_response($self, \&hdr_msgid_range_i, @$r);
 	}
 }
 

[PATCH 8/8] nntp: cmd_xover: use named sub for long_response

[Eric Wong]

Introduce xover_i, which does the same thing as the anonymous
sub it replaces.
---
 lib/PublicInbox/NNTP.pm | 26 ++++++++++++++------------
 1 file changed, 14 insertions(+), 12 deletions(-)

diff --git a/lib/PublicInbox/NNTP.pm b/lib/PublicInbox/NNTP.pm
index b2927b79..3d3ad539 100644
--- a/lib/PublicInbox/NNTP.pm
+++ b/lib/PublicInbox/NNTP.pm
@@ -873,6 +873,19 @@ sub cmd_over ($;$) {
 	}
 }
 
+sub xover_i {
+	my ($self, $beg, $end) = @_;
+	my $ng = $self->{ng};
+	my $msgs = $ng->over->query_xover($$beg, $end);
+	my $nr = scalar @$msgs or return;
+
+	# OVERVIEW.FMT
+	more($self, join("\r\n", map {
+		over_line($self, $ng, $_->{num}, $_);
+		} @$msgs));
+	$$beg = $msgs->[-1]->{num} + 1;
+}
+
 sub cmd_xover ($;$) {
 	my ($self, $range) = @_;
 	$range = $self->{article} unless defined $range;
@@ -880,18 +893,7 @@ sub cmd_xover ($;$) {
 	return $r unless ref $r;
 	my ($beg, $end) = @$r;
 	more($self, "224 Overview information follows for $$beg to $end");
-	my $over = $self->{ng}->over;
-	my $cur = $$beg;
-	long_response($self, sub {
-		my $msgs = $over->query_xover($cur, $end);
-		my $nr = scalar @$msgs or return;
-
-		# OVERVIEW.FMT
-		more($self, join("\r\n", map {
-			over_line($self, $self->{ng}, $_->{num}, $_);
-			} @$msgs));
-		$cur = $msgs->[-1]->{num} + 1;
-	});
+	long_response($self, \&xover_i, @$r);
 }
 
 sub compressed { undef }