* [PATCH 2/3] nntp: event_step: prepare for async git reads
2020-06-16 22:31 6% [PATCH 0/3] nntpd: updates from imapd Eric Wong
@ 2020-06-16 22:31 7% ` Eric Wong
0 siblings, 0 replies; 2+ results
From: Eric Wong @ 2020-06-16 22:31 UTC (permalink / raw)
To: meta
This matches PublicInbox::IMAP::event_step and will allow us to
handle blob retrievals from git asynchronously without falling
over on pipelined requests.
---
lib/PublicInbox/NNTP.pm | 39 ++++++++++++++++------------------
lib/PublicInbox/NNTPdeflate.pm | 10 +++++++++
2 files changed, 28 insertions(+), 21 deletions(-)
diff --git a/lib/PublicInbox/NNTP.pm b/lib/PublicInbox/NNTP.pm
index be3bddc3f5d..80dd8614fe8 100644
--- a/lib/PublicInbox/NNTP.pm
+++ b/lib/PublicInbox/NNTP.pm
@@ -956,38 +956,35 @@ sub out ($$;@) {
sub event_step {
my ($self) = @_;
- return unless $self->flush_write && $self->{sock};
+ return unless $self->flush_write && $self->{sock} && !$self->{long_cb};
$self->update_idle_time;
# only read more requests if we've drained the write buffer,
# otherwise we can be buffering infinitely w/o backpressure
- my $rbuf = $self->{rbuf} // (\(my $x = ''));
- my $r = 1;
-
- if (index($$rbuf, "\n") < 0) {
- my $off = bytes::length($$rbuf);
- $r = $self->do_read($rbuf, LINE_MAX, $off) or return;
- }
- while ($r > 0 && $$rbuf =~ s/\A[ \t]*([^\n]*?)\r?\n//) {
- my $line = $1;
- return $self->close if $line =~ /[[:cntrl:]]/s;
- my $t0 = now();
- my $fd = fileno($self->{sock});
- $r = eval { process_line($self, $line) };
- my $pending = $self->{wbuf} ? ' pending' : '';
- out($self, "[$fd] %s - %0.6f$pending", $line, now() - $t0);
- }
-
+ my $rbuf = $self->{rbuf} // \(my $x = '');
+ my $line = index($$rbuf, "\n");
+ while ($line < 0) {
+ return $self->close if length($$rbuf) >= LINE_MAX;
+ $self->do_read($rbuf, LINE_MAX, length($$rbuf)) or return;
+ $line = index($$rbuf, "\n");
+ }
+ $line = substr($$rbuf, 0, $line + 1, '');
+ $line =~ s/\r?\n\z//s;
+ return $self->close if $line =~ /[[:cntrl:]]/s;
+
+ my $t0 = now();
+ my $fd = fileno($self->{sock});
+ my $r = eval { process_line($self, $line) };
+ my $pending = $self->{wbuf} ? ' pending' : '';
+ out($self, "[$fd] %s - %0.6f$pending", $line, now() - $t0);
return $self->close if $r < 0;
- my $len = bytes::length($$rbuf);
- return $self->close if ($len >= LINE_MAX);
$self->rbuf_idle($rbuf);
$self->update_idle_time;
# maybe there's more pipelined data, or we'll have
# to register it for socket-readiness notifications
- $self->requeue unless $self->{wbuf};
+ $self->requeue unless $pending;
}
# for graceful shutdown in PublicInbox::Daemon:
diff --git a/lib/PublicInbox/NNTPdeflate.pm b/lib/PublicInbox/NNTPdeflate.pm
index eb400c9c220..dec88aba3a5 100644
--- a/lib/PublicInbox/NNTPdeflate.pm
+++ b/lib/PublicInbox/NNTPdeflate.pm
@@ -71,6 +71,16 @@ sub do_read ($$$$) {
$doff = length($dbuf);
my $r = PublicInbox::DS::do_read($self, \$dbuf, $len, $doff) or return;
+ # Workaround inflate bug appending to OOK scalars:
+ # <https://rt.cpan.org/Ticket/Display.html?id=132734>
+ # We only have $off if the client is pipelining, and pipelining
+ # is where our substr() OOK optimization in event_step makes sense.
+ if ($off) {
+ my $copy = $$rbuf;
+ undef $$rbuf;
+ $$rbuf = $copy;
+ }
+
# assert(length($$rbuf) == $off) as far as NNTP.pm is concerned
# -ConsumeInput is true, so $dbuf is automatically emptied
my $err = $zin->inflate($dbuf, $rbuf);
^ permalink raw reply related [relevance 7%]
* [PATCH 0/3] nntpd: updates from imapd
@ 2020-06-16 22:31 6% Eric Wong
2020-06-16 22:31 7% ` [PATCH 2/3] nntp: event_step: prepare for async git reads Eric Wong
0 siblings, 1 reply; 2+ results
From: Eric Wong @ 2020-06-16 22:31 UTC (permalink / raw)
To: meta
Since our IMAP code was forked from NNTP and a proving ground
for some new ideas, start bringing the NNTP up-to-date with slow
git storage support and make it less dependent on
IO::Socket::SSL for TLS.
Eric Wong (3):
daemon: use ->can to check for IO::Socket::SSL
nntp: event_step: prepare for async git reads
nntp: support slow blob retrievals
lib/PublicInbox/DS.pm | 6 +-
lib/PublicInbox/HTTP.pm | 2 +-
lib/PublicInbox/NNTP.pm | 150 +++++++++++++++++++--------------
lib/PublicInbox/NNTPdeflate.pm | 10 +++
t/nntp.t | 12 ++-
t/nntpd.t | 20 ++++-
6 files changed, 129 insertions(+), 71 deletions(-)
^ permalink raw reply [relevance 6%]
Results 1-2 of 2 | reverse | options above
-- pct% links below jump to the message on this page, permalinks otherwise --
2020-06-16 22:31 6% [PATCH 0/3] nntpd: updates from imapd Eric Wong
2020-06-16 22:31 7% ` [PATCH 2/3] nntp: event_step: prepare for async git reads Eric Wong
Code repositories for project(s) associated with this public inbox
https://80x24.org/public-inbox.git
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).