user/dev discussion of public-inbox itself
 help / color / mirror / code / Atom feed
From: "Eric Wong (Contractor, The Linux Foundation)" <e@80x24.org>
To: meta@public-inbox.org
Subject: [PATCH 6/7] msgmap: replace id_batch with ids_after
Date: Tue,  3 Apr 2018 11:09:11 +0000	[thread overview]
Message-ID: <20180403110912.24231-7-e@80x24.org> (raw)
In-Reply-To: <20180403110912.24231-1-e@80x24.org>

id_batch had a an overly complicated interface, replace it
with id_batch which is simpler and takes advantage of
selectcol_arrayref in DBI.  This allows simplification of
callers and the diffstat agrees with me.
---
 lib/PublicInbox/Mbox.pm   | 11 ++---------
 lib/PublicInbox/Msgmap.pm | 19 ++++++++-----------
 lib/PublicInbox/NNTP.pm   | 12 ++++--------
 t/nntpd.t                 |  1 +
 t/v2writable.t            |  7 +++++++
 5 files changed, 22 insertions(+), 28 deletions(-)

diff --git a/lib/PublicInbox/Mbox.pm b/lib/PublicInbox/Mbox.pm
index 0be1968..c66ccaa 100644
--- a/lib/PublicInbox/Mbox.pm
+++ b/lib/PublicInbox/Mbox.pm
@@ -166,14 +166,7 @@ sub mbox_all {
 	return sub { need_gzip(@_) } if $@;
 	if ($query eq '') {
 		my $prev = 0;
-		my $msgs = [];
-		my $cb = sub {
-			$ctx->{-inbox}->mm->id_batch($prev, sub {
-				$msgs = $_[0];
-			});
-			$prev = $msgs->[-1] if @$msgs;
-			$msgs;
-		};
+		my $cb = sub { $ctx->{-inbox}->mm->ids_after(\$prev) };
 		return PublicInbox::MboxGz->response($ctx, $cb, 'all');
 	}
 	my $opts = { offset => 0 };
@@ -244,7 +237,7 @@ sub getline {
 	do {
 		# work on existing result set
 		while (defined(my $smsg = shift @$msgs)) {
-			# id_batch may return integers
+			# ids_after may return integers
 			ref($smsg) or
 				$smsg = $ctx->{srch}->{over_ro}->get_art($smsg);
 
diff --git a/lib/PublicInbox/Msgmap.pm b/lib/PublicInbox/Msgmap.pm
index dea9573..26565d4 100644
--- a/lib/PublicInbox/Msgmap.pm
+++ b/lib/PublicInbox/Msgmap.pm
@@ -186,17 +186,14 @@ sub create_tables {
 }
 
 # used by NNTP.pm
-sub id_batch {
-	my ($self, $num, $cb) = @_;
-	my $dbh = $self->{dbh};
-	my $sth = $dbh->prepare('SELECT num FROM msgmap WHERE num > ? '.
-				'ORDER BY num ASC LIMIT 1000');
-	$sth->execute($num);
-	my $ary = $sth->fetchall_arrayref;
-	@$ary = map { $_->[0] } @$ary;
-	my $nr = scalar @$ary;
-	$cb->($ary) if $nr;
-	$nr;
+sub ids_after {
+	my ($self, $num) = @_;
+	my $ids = $self->{dbh}->selectcol_arrayref(<<'', undef, $$num);
+SELECT num FROM msgmap WHERE num > ?
+ORDER BY num ASC LIMIT 1000
+
+	$$num = $ids->[-1] if @$ids;
+	$ids;
 }
 
 # only used for mapping external serial numbers (e.g. articles from gmane)
diff --git a/lib/PublicInbox/NNTP.pm b/lib/PublicInbox/NNTP.pm
index ff6d895..b91cda1 100644
--- a/lib/PublicInbox/NNTP.pm
+++ b/lib/PublicInbox/NNTP.pm
@@ -34,7 +34,6 @@ my $LIST_HEADERS = join("\r\n", @OVERVIEW,
 			qw(:bytes :lines Xref To Cc)) . "\r\n";
 
 # disable commands with easy DoS potential:
-# LISTGROUP could get pretty bad, too...
 my %DISABLED; # = map { $_ => 1 } qw(xover list_overview_fmt newnews xhdr);
 
 my $EXPMAP; # fd -> [ idle_time, $self ]
@@ -225,15 +224,12 @@ sub cmd_listgroup ($;$) {
 	}
 
 	$self->{ng} or return '412 no newsgroup selected';
+	my $n = 0;
 	long_response($self, 0, long_response_limit, sub {
 		my ($i) = @_;
-		my $nr = $self->{ng}->mm->id_batch($$i, sub {
-			my ($ary) = @_;
-			more($self, join("\r\n", @$ary));
-		});
-
-		# -1 to adjust for implicit increment in long_response
-		$$i = $nr ? $$i + $nr - 1 : long_response_limit;
+		my $ary = $self->{ng}->mm->ids_after(\$n);
+		scalar @$ary or return ($$i = long_response_limit);
+		more($self, join("\r\n", @$ary));
 	});
 }
 
diff --git a/t/nntpd.t b/t/nntpd.t
index de781d7..c6e34ed 100644
--- a/t/nntpd.t
+++ b/t/nntpd.t
@@ -123,6 +123,7 @@ EOF
 	my $list = $n->list;
 	is_deeply($list, { $group => [ qw(1 1 n) ] }, 'LIST works');
 	is_deeply([$n->group($group)], [ qw(0 1 1), $group ], 'GROUP works');
+	is_deeply($n->listgroup($group), [1], 'listgroup OK');
 
 	%opts = (
 		PeerAddr => $host_port,
diff --git a/t/v2writable.t b/t/v2writable.t
index 1e8e404..6294735 100644
--- a/t/v2writable.t
+++ b/t/v2writable.t
@@ -189,6 +189,13 @@ EOF
 		is($nn{$mid}++, 0, "MID is unique in NEWNEWS");
 	}
 	is_deeply([sort keys %nn], [sort keys %uniq]);
+
+	my %lg;
+	foreach my $num (@{$n->listgroup($group)}) {
+		is($lg{$num}++, 0, "num is unique in LISTGROUP");
+	}
+	is_deeply([sort keys %lg], [sort keys %$x],
+		'XOVER and LISTGROUPS return the same article numbers');
 };
 {
 	local $ENV{NPROC} = 2;
-- 
EW


  parent reply	other threads:[~2018-04-03 11:09 UTC|newest]

Thread overview: 8+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2018-04-03 11:09 [PATCH 0/7] optimize V2 Eric Wong (Contractor, The Linux Foundation)
2018-04-03 11:09 ` [PATCH 1/7] t/thread-all.t: modernize test to support modern inboxes Eric Wong (Contractor, The Linux Foundation)
2018-04-03 11:09 ` [PATCH 2/7] rename+rewrite test using Benchmark module Eric Wong (Contractor, The Linux Foundation)
2018-04-03 11:09 ` [PATCH 3/7] nntp: make XOVER, XHDR, OVER, HDR and NEWNEWS faster Eric Wong (Contractor, The Linux Foundation)
2018-04-03 11:09 ` [PATCH 4/7] view: avoid offset during pagination Eric Wong (Contractor, The Linux Foundation)
2018-04-03 11:09 ` [PATCH 5/7] mbox: remove remaining OFFSET usage in SQLite Eric Wong (Contractor, The Linux Foundation)
2018-04-03 11:09 ` Eric Wong (Contractor, The Linux Foundation) [this message]
2018-04-03 11:09 ` [PATCH 7/7] nntp: simplify the long_response API Eric Wong (Contractor, The Linux Foundation)

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

  List information: https://public-inbox.org/README

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20180403110912.24231-7-e@80x24.org \
    --to=e@80x24.org \
    --cc=meta@public-inbox.org \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
Code repositories for project(s) associated with this public inbox

	https://80x24.org/public-inbox.git

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).