From: "Eric Wong (Contractor, The Linux Foundation)" <e@80x24.org>
To: meta@public-inbox.org
Subject: [PATCH 6/7] msgmap: replace id_batch with ids_after
Date: Tue, 3 Apr 2018 11:09:11 +0000 [thread overview]
Message-ID: <20180403110912.24231-7-e@80x24.org> (raw)
In-Reply-To: <20180403110912.24231-1-e@80x24.org>
id_batch had a an overly complicated interface, replace it
with id_batch which is simpler and takes advantage of
selectcol_arrayref in DBI. This allows simplification of
callers and the diffstat agrees with me.
---
lib/PublicInbox/Mbox.pm | 11 ++---------
lib/PublicInbox/Msgmap.pm | 19 ++++++++-----------
lib/PublicInbox/NNTP.pm | 12 ++++--------
t/nntpd.t | 1 +
t/v2writable.t | 7 +++++++
5 files changed, 22 insertions(+), 28 deletions(-)
diff --git a/lib/PublicInbox/Mbox.pm b/lib/PublicInbox/Mbox.pm
index 0be1968..c66ccaa 100644
--- a/lib/PublicInbox/Mbox.pm
+++ b/lib/PublicInbox/Mbox.pm
@@ -166,14 +166,7 @@ sub mbox_all {
return sub { need_gzip(@_) } if $@;
if ($query eq '') {
my $prev = 0;
- my $msgs = [];
- my $cb = sub {
- $ctx->{-inbox}->mm->id_batch($prev, sub {
- $msgs = $_[0];
- });
- $prev = $msgs->[-1] if @$msgs;
- $msgs;
- };
+ my $cb = sub { $ctx->{-inbox}->mm->ids_after(\$prev) };
return PublicInbox::MboxGz->response($ctx, $cb, 'all');
}
my $opts = { offset => 0 };
@@ -244,7 +237,7 @@ sub getline {
do {
# work on existing result set
while (defined(my $smsg = shift @$msgs)) {
- # id_batch may return integers
+ # ids_after may return integers
ref($smsg) or
$smsg = $ctx->{srch}->{over_ro}->get_art($smsg);
diff --git a/lib/PublicInbox/Msgmap.pm b/lib/PublicInbox/Msgmap.pm
index dea9573..26565d4 100644
--- a/lib/PublicInbox/Msgmap.pm
+++ b/lib/PublicInbox/Msgmap.pm
@@ -186,17 +186,14 @@ sub create_tables {
}
# used by NNTP.pm
-sub id_batch {
- my ($self, $num, $cb) = @_;
- my $dbh = $self->{dbh};
- my $sth = $dbh->prepare('SELECT num FROM msgmap WHERE num > ? '.
- 'ORDER BY num ASC LIMIT 1000');
- $sth->execute($num);
- my $ary = $sth->fetchall_arrayref;
- @$ary = map { $_->[0] } @$ary;
- my $nr = scalar @$ary;
- $cb->($ary) if $nr;
- $nr;
+sub ids_after {
+ my ($self, $num) = @_;
+ my $ids = $self->{dbh}->selectcol_arrayref(<<'', undef, $$num);
+SELECT num FROM msgmap WHERE num > ?
+ORDER BY num ASC LIMIT 1000
+
+ $$num = $ids->[-1] if @$ids;
+ $ids;
}
# only used for mapping external serial numbers (e.g. articles from gmane)
diff --git a/lib/PublicInbox/NNTP.pm b/lib/PublicInbox/NNTP.pm
index ff6d895..b91cda1 100644
--- a/lib/PublicInbox/NNTP.pm
+++ b/lib/PublicInbox/NNTP.pm
@@ -34,7 +34,6 @@ my $LIST_HEADERS = join("\r\n", @OVERVIEW,
qw(:bytes :lines Xref To Cc)) . "\r\n";
# disable commands with easy DoS potential:
-# LISTGROUP could get pretty bad, too...
my %DISABLED; # = map { $_ => 1 } qw(xover list_overview_fmt newnews xhdr);
my $EXPMAP; # fd -> [ idle_time, $self ]
@@ -225,15 +224,12 @@ sub cmd_listgroup ($;$) {
}
$self->{ng} or return '412 no newsgroup selected';
+ my $n = 0;
long_response($self, 0, long_response_limit, sub {
my ($i) = @_;
- my $nr = $self->{ng}->mm->id_batch($$i, sub {
- my ($ary) = @_;
- more($self, join("\r\n", @$ary));
- });
-
- # -1 to adjust for implicit increment in long_response
- $$i = $nr ? $$i + $nr - 1 : long_response_limit;
+ my $ary = $self->{ng}->mm->ids_after(\$n);
+ scalar @$ary or return ($$i = long_response_limit);
+ more($self, join("\r\n", @$ary));
});
}
diff --git a/t/nntpd.t b/t/nntpd.t
index de781d7..c6e34ed 100644
--- a/t/nntpd.t
+++ b/t/nntpd.t
@@ -123,6 +123,7 @@ EOF
my $list = $n->list;
is_deeply($list, { $group => [ qw(1 1 n) ] }, 'LIST works');
is_deeply([$n->group($group)], [ qw(0 1 1), $group ], 'GROUP works');
+ is_deeply($n->listgroup($group), [1], 'listgroup OK');
%opts = (
PeerAddr => $host_port,
diff --git a/t/v2writable.t b/t/v2writable.t
index 1e8e404..6294735 100644
--- a/t/v2writable.t
+++ b/t/v2writable.t
@@ -189,6 +189,13 @@ EOF
is($nn{$mid}++, 0, "MID is unique in NEWNEWS");
}
is_deeply([sort keys %nn], [sort keys %uniq]);
+
+ my %lg;
+ foreach my $num (@{$n->listgroup($group)}) {
+ is($lg{$num}++, 0, "num is unique in LISTGROUP");
+ }
+ is_deeply([sort keys %lg], [sort keys %$x],
+ 'XOVER and LISTGROUPS return the same article numbers');
};
{
local $ENV{NPROC} = 2;
--
EW
next prev parent reply other threads:[~2018-04-03 11:09 UTC|newest]
Thread overview: 8+ messages / expand[flat|nested] mbox.gz Atom feed top
2018-04-03 11:09 [PATCH 0/7] optimize V2 Eric Wong (Contractor, The Linux Foundation)
2018-04-03 11:09 ` [PATCH 1/7] t/thread-all.t: modernize test to support modern inboxes Eric Wong (Contractor, The Linux Foundation)
2018-04-03 11:09 ` [PATCH 2/7] rename+rewrite test using Benchmark module Eric Wong (Contractor, The Linux Foundation)
2018-04-03 11:09 ` [PATCH 3/7] nntp: make XOVER, XHDR, OVER, HDR and NEWNEWS faster Eric Wong (Contractor, The Linux Foundation)
2018-04-03 11:09 ` [PATCH 4/7] view: avoid offset during pagination Eric Wong (Contractor, The Linux Foundation)
2018-04-03 11:09 ` [PATCH 5/7] mbox: remove remaining OFFSET usage in SQLite Eric Wong (Contractor, The Linux Foundation)
2018-04-03 11:09 ` Eric Wong (Contractor, The Linux Foundation) [this message]
2018-04-03 11:09 ` [PATCH 7/7] nntp: simplify the long_response API Eric Wong (Contractor, The Linux Foundation)
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
List information: https://public-inbox.org/README
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20180403110912.24231-7-e@80x24.org \
--to=e@80x24.org \
--cc=meta@public-inbox.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
Code repositories for project(s) associated with this public inbox
https://80x24.org/public-inbox.git
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).