[PATCH 0/4] a few more odds and ends for v2

[PATCH 0/4] a few more odds and ends for v2

[Eric Wong (Contractor, The Linux Foundation)]

Eric Wong (Contractor, The Linux Foundation) (4):
  altid: fix miscopied field name
  nntp: set Xref across multiple inboxes
  www: favor reading more from SQLite, and less from Xapian
  ensure Xapian and SQLite are still optional for v1 tests

 lib/PublicInbox/AltId.pm      |  2 +-
 lib/PublicInbox/Inbox.pm      |  2 +-
 lib/PublicInbox/Mbox.pm       | 39 +++++++++++++-------------------
 lib/PublicInbox/NNTP.pm       | 43 ++++++++++++++++++++++-------------
 lib/PublicInbox/Over.pm       | 29 ++++++++++++++++++++++++
 lib/PublicInbox/Search.pm     | 52 ++++---------------------------------------
 lib/PublicInbox/SearchIdx.pm  |  3 ++-
 lib/PublicInbox/V2Writable.pm | 34 ++++++++++++----------------
 lib/PublicInbox/View.pm       | 35 ++++++++++-------------------
 script/public-inbox-compact   |  2 +-
 scripts/import_vger_from_mbox |  2 +-
 t/convert-compact.t           |  2 +-
 t/nntp.t                      |  6 +++--
 t/psgi_v2.t                   |  1 +
 t/search.t                    | 16 +++++--------
 t/v2mirror.t                  |  2 +-
 t/v2writable.t                | 18 +++++++--------
 17 files changed, 129 insertions(+), 159 deletions(-)

-- 
EW

[PATCH 1/4] altid: fix miscopied field name

[Eric Wong (Contractor, The Linux Foundation)]

Oops :x
---
 lib/PublicInbox/AltId.pm | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/PublicInbox/AltId.pm b/lib/PublicInbox/AltId.pm
index f8aa4cb..4a6ff97 100644
--- a/lib/PublicInbox/AltId.pm
+++ b/lib/PublicInbox/AltId.pm
@@ -39,7 +39,7 @@ sub mm_alt {
 	my ($self) = @_;
 	$self->{mm_alt} ||= eval {
 		my $f = $self->{filename};
-		my $writable = $self->{filename};
+		my $writable = $self->{writable};
 		PublicInbox::Msgmap->new_file($f, $writable);
 	};
 }
-- 
EW

[PATCH 2/4] nntp: set Xref across multiple inboxes

[Eric Wong (Contractor, The Linux Foundation)]

Noted by Jonathan Corbet in https://lwn.net/Articles/748184/
---
 lib/PublicInbox/NNTP.pm | 43 ++++++++++++++++++++++++++++---------------
 t/nntp.t                |  6 ++++--
 2 files changed, 32 insertions(+), 17 deletions(-)

diff --git a/lib/PublicInbox/NNTP.pm b/lib/PublicInbox/NNTP.pm
index e517935..fa890cb 100644
--- a/lib/PublicInbox/NNTP.pm
+++ b/lib/PublicInbox/NNTP.pm
@@ -407,12 +407,30 @@ sub header_append ($$$) {
 	$hdr->header_set($k, @v, $v);
 }
 
-sub set_nntp_headers {
-	my ($hdr, $ng, $n, $mid) = @_;
+sub xref ($$$$) {
+	my ($self, $ng, $n, $mid) = @_;
+	my $ret = "$ng->{domain} $ng->{newsgroup}:$n";
+
+	# num_for is pretty cheap and sometimes we'll lookup the existence
+	# of an article without getting even the OVER info.  In other words,
+	# I'm not sure if its worth optimizing by scanning To:/Cc: and
+	# PublicInbox::ExtMsg on the PSGI end is just as expensive
+	foreach my $other (@{$self->{nntpd}->{grouplist}}) {
+		next if $ng eq $other;
+		my $num = eval { $other->mm->num_for($mid) } or next;
+		$ret .= " $other->{newsgroup}:$num";
+	}
+	$ret;
+}
+
+sub set_nntp_headers ($$$$$) {
+	my ($self, $hdr, $ng, $n, $mid) = @_;
 
 	# clobber some
-	$hdr->header_set('Newsgroups', $ng->{newsgroup});
-	$hdr->header_set('Xref', xref($ng, $n));
+	my $xref = xref($self, $ng, $n, $mid);
+	$hdr->header_set('Xref', $xref);
+	$xref =~ s/:\d+//g;
+	$hdr->header_set('Newsgroups', (split(/ /, $xref, 2))[1]);
 	header_append($hdr, 'List-Post', "<mailto:$ng->{-primary_address}>");
 	if (my $url = $ng->base_url) {
 		$mid = mid_escape($mid);
@@ -461,7 +479,7 @@ found:
 	my $msg = $ng->msg_by_smsg($smsg) or return $err;
 	my $s = Email::Simple->new($msg);
 	if ($set_headers) {
-		set_nntp_headers($s->header_obj, $ng, $n, $mid);
+		set_nntp_headers($self, $s->header_obj, $ng, $n, $mid);
 
 		# must be last
 		$s->body_set('') if ($set_headers == 2);
@@ -635,11 +653,6 @@ sub hdr_message_id ($$$) { # optimize XHDR Message-ID [range] for slrnpull.
 	}
 }
 
-sub xref ($$) {
-	my ($ng, $n) = @_;
-	"$ng->{domain} $ng->{newsgroup}:$n"
-}
-
 sub mid_lookup ($$) {
 	my ($self, $mid) = @_;
 	my $self_ng = $self->{ng};
@@ -659,9 +672,11 @@ sub hdr_xref ($$$) { # optimize XHDR Xref [range] for rtin
 	my ($self, $xhdr, $range) = @_;
 
 	if (defined $range && $range =~ /\A<(.+)>\z/) { # Message-ID
-		my ($ng, $n) = mid_lookup($self, $1);
+		my $mid = $1;
+		my ($ng, $n) = mid_lookup($self, $mid);
 		return r430 unless $n;
-		hdr_mid_response($self, $xhdr, $ng, $n, $range, xref($ng, $n));
+		hdr_mid_response($self, $xhdr, $ng, $n, $range,
+				xref($self, $ng, $n, $mid));
 	} else { # numeric range
 		$range = $self->{article} unless defined $range;
 		my $r = get_range($self, $range);
@@ -674,10 +689,8 @@ sub hdr_xref ($$$) { # optimize XHDR Xref [range] for rtin
 			my $r = $mm->msg_range(\$beg, $end);
 			@$r or return;
 			more($self, join("\r\n", map {
-				# TODO: use $_->[1] (mid) to fill
-				# Xref: from other inboxes
 				my $num = $_->[0];
-				"$num ".xref($ng, $num);
+				"$num ".xref($self, $ng, $num, $_->[1]);
 			} @$r));
 			1;
 		});
diff --git a/t/nntp.t b/t/nntp.t
index 03c7f08..57fef48 100644
--- a/t/nntp.t
+++ b/t/nntp.t
@@ -109,7 +109,9 @@ use_ok 'PublicInbox::Inbox';
 	is($ng->base_url, $u, 'URL expanded');
 	my $mid = 'a@b';
 	my $mime = Email::MIME->new("Message-ID: <$mid>\r\n\r\n");
-	PublicInbox::NNTP::set_nntp_headers($mime->header_obj, $ng, 1, $mid);
+	my $hdr = $mime->header_obj;
+	my $mock_self = { nntpd => { grouplist => [] } };
+	PublicInbox::NNTP::set_nntp_headers($mock_self, $hdr, $ng, 1, $mid);
 	is_deeply([ $mime->header('Message-ID') ], [ "<$mid>" ],
 		'Message-ID unchanged');
 	is_deeply([ $mime->header('Archived-At') ], [ "<${u}a\@b/>" ],
@@ -124,7 +126,7 @@ use_ok 'PublicInbox::Inbox';
 		'Xref: set');
 
 	$ng->{-base_url} = 'http://mirror.example.com/m/';
-	PublicInbox::NNTP::set_nntp_headers($mime->header_obj, $ng, 2, $mid);
+	PublicInbox::NNTP::set_nntp_headers($mock_self, $hdr, $ng, 2, $mid);
 	is_deeply([ $mime->header('Message-ID') ], [ "<$mid>" ],
 		'Message-ID unchanged');
 	is_deeply([ $mime->header('Archived-At') ],
-- 
EW

[PATCH 3/4] www: favor reading more from SQLite, and less from Xapian

[Eric Wong (Contractor, The Linux Foundation)]

Favor simpler internal APIs this time around, this cuts
a fair amount of code out and takes another step towards
removing Xapian as a dependency for v2 repos.
---
 lib/PublicInbox/Inbox.pm      |  2 +-
 lib/PublicInbox/Mbox.pm       | 39 +++++++++++++-------------------
 lib/PublicInbox/Over.pm       | 29 ++++++++++++++++++++++++
 lib/PublicInbox/Search.pm     | 52 ++++---------------------------------------
 lib/PublicInbox/SearchIdx.pm  |  3 ++-
 lib/PublicInbox/V2Writable.pm | 34 ++++++++++++----------------
 lib/PublicInbox/View.pm       | 35 ++++++++++-------------------
 t/psgi_v2.t                   |  1 +
 t/search.t                    | 16 +++++--------
 t/v2writable.t                | 18 +++++++--------
 10 files changed, 92 insertions(+), 137 deletions(-)

diff --git a/lib/PublicInbox/Inbox.pm b/lib/PublicInbox/Inbox.pm
index 0ea18b4..f71493a 100644
--- a/lib/PublicInbox/Inbox.pm
+++ b/lib/PublicInbox/Inbox.pm
@@ -273,7 +273,7 @@ sub msg_by_smsg ($$;$) {
 	my ($self, $smsg, $ref) = @_;
 
 	# ghosts may have undef smsg (from SearchThread.node) or
-	# no {blob} field (from each_smsg_by_mid)
+	# no {blob} field
 	return unless defined $smsg;
 	defined(my $blob = $smsg->{blob}) or return;
 
diff --git a/lib/PublicInbox/Mbox.pm b/lib/PublicInbox/Mbox.pm
index c5e1cb9..4427ae5 100644
--- a/lib/PublicInbox/Mbox.pm
+++ b/lib/PublicInbox/Mbox.pm
@@ -34,19 +34,17 @@ sub mb_stream {
 # called by PSGI server as body response
 sub getline {
 	my ($more) = @_; # self
-	my ($ctx, $head, $tail, $db, $cur) = @$more;
-	if ($cur) {
+	my ($ctx, $id, $prev, $next, $cur) = @$more;
+	if ($cur) { # first
 		pop @$more;
 		return msg_str($ctx, $cur);
 	}
-	for (; !defined($cur) && $head != $tail; $head++) {
-		my $smsg = PublicInbox::SearchMsg->get($head, $db, $ctx->{mid});
-		my $mref = $ctx->{-inbox}->msg_by_smsg($smsg) or next;
-		$cur = Email::Simple->new($mref);
-		$cur = msg_str($ctx, $cur);
-	}
-	$more->[1] = $head;
-	$cur;
+	$cur = $next or return;
+	my $ibx = $ctx->{-inbox};
+	$next = $ibx->search->next_by_mid($ctx->{mid}, \$id, \$prev);
+	@$more = ($ctx, $id, $prev, $next); # $next may be undef, here
+	my $mref = $ibx->msg_by_smsg($cur) or return;
+	msg_str($ctx, Email::Simple->new($mref));
 }
 
 sub close {} # noop
@@ -57,21 +55,14 @@ sub emit_raw {
 	my $ibx = $ctx->{-inbox};
 	my $first;
 	my $more;
-	my ($head, $tail, $db);
-	my %seen;
 	if (my $srch = $ibx->search) {
-		$srch->retry_reopen(sub {
-			($head, $tail, $db) = $srch->each_smsg_by_mid($mid);
-			for (; !defined($first) && $head != $tail; $head++) {
-				my @args = ($head, $db, $mid);
-				my $smsg = PublicInbox::SearchMsg->get(@args);
-				my $mref = $ibx->msg_by_smsg($smsg) or next;
-				$first = Email::Simple->new($mref);
-			}
-			if ($head != $tail) {
-				$more = [ $ctx, $head, $tail, $db, $first ];
-			}
-		});
+		my ($id, $prev);
+		my $smsg = $srch->next_by_mid($mid, \$id, \$prev) or return;
+		my $mref = $ibx->msg_by_smsg($smsg) or return;
+		$first = Email::Simple->new($mref);
+		my $next = $srch->next_by_mid($mid, \$id, \$prev);
+		# $more is for ->getline
+		$more = [ $ctx, $id, $prev, $next, $first ] if $next;
 	} else {
 		my $mref = $ibx->msg_by_mid($mid) or return;
 		$first = Email::Simple->new($mref);
diff --git a/lib/PublicInbox/Over.pm b/lib/PublicInbox/Over.pm
index 0bd6008..309e044 100644
--- a/lib/PublicInbox/Over.pm
+++ b/lib/PublicInbox/Over.pm
@@ -147,4 +147,33 @@ SELECT * from OVER where num = ? LIMIT 1
 	undef;
 }
 
+sub next_by_mid {
+	my ($self, $mid, $id, $prev) = @_;
+	my $dbh = $self->connect;
+
+	unless (defined $$id) {
+		my $sth = $dbh->prepare_cached(<<'', undef, 1);
+	SELECT id FROM msgid WHERE mid = ? LIMIT 1
+
+		$sth->execute($mid);
+		$$id = $sth->fetchrow_array;
+		defined $$id or return;
+	}
+	my $sth = $dbh->prepare_cached(<<"", undef, 1);
+SELECT num FROM id2num WHERE id = ? AND num > ?
+ORDER BY num ASC LIMIT 1
+
+	$$prev ||= 0;
+	$sth->execute($$id, $$prev);
+	my $num = $sth->fetchrow_array or return;
+	$$prev = $num;
+
+	$sth = $dbh->prepare_cached(<<"", undef, 1);
+SELECT num,ts,ds,ddd FROM over WHERE num = ? LIMIT 1
+
+	$sth->execute($num);
+	my $smsg = $sth->fetchrow_hashref or return;
+	load_from_row($smsg);
+}
+
 1;
diff --git a/lib/PublicInbox/Search.pm b/lib/PublicInbox/Search.pm
index 34ebd1a..7175ddc 100644
--- a/lib/PublicInbox/Search.pm
+++ b/lib/PublicInbox/Search.pm
@@ -304,58 +304,14 @@ sub query_ts {
 	$self->{over_ro}->query_ts($ts, $prev);
 }
 
-sub first_smsg_by_mid {
-	my ($self, $mid) = @_;
-	my $smsg;
-	retry_reopen($self, sub {
-		each_smsg_by_mid($self, $mid, sub { $smsg = $_[0]; undef });
-	});
-	$smsg;
-}
-
 sub lookup_article {
 	my ($self, $num) = @_;
-	my $term = 'XNUM'.$num;
-	my $db = $self->{xdb};
-	retry_reopen($self, sub {
-		my $head = $db->postlist_begin($term);
-		my $tail = $db->postlist_end($term);
-		return if $head->equal($tail);
-		my $doc_id = $head->get_docid;
-		return unless defined $doc_id;
-		$head->inc;
-		if ($head->nequal($tail)) {
-			warn "article #$num is not unique\n";
-		}
-		# raises on error:
-		my $doc = $db->get_document($doc_id);
-		my $smsg = PublicInbox::SearchMsg->wrap($doc);
-		$smsg->{doc_id} = $doc_id;
-		$smsg->load_expand;
-	});
+	$self->{over_ro}->get_art($num);
 }
 
-sub each_smsg_by_mid {
-	my ($self, $mid, $cb) = @_;
-	# XXX retry_reopen isn't necessary for V2Writable, but the PSGI
-	# interface will need it...
-	my $db = $self->{xdb};
-	my $term = 'Q' . $mid;
-	my $head = $db->postlist_begin($term);
-	my $tail = $db->postlist_end($term);
-	if ($head == $tail) {
-		$db->reopen;
-		$head = $db->postlist_begin($term);
-		$tail = $db->postlist_end($term);
-	}
-	return ($head, $tail, $db) if wantarray;
-	for (; $head->nequal($tail); $head->inc) {
-		my $doc_id = $head->get_docid;
-		my $doc = $db->get_document($doc_id);
-		my $smsg = PublicInbox::SearchMsg->wrap($doc, $mid);
-		$smsg->{doc_id} = $doc_id;
-		$cb->($smsg) or return;
-	}
+sub next_by_mid {
+	my ($self, $mid, $id, $prev) = @_;
+	$self->{over_ro}->next_by_mid($mid, $id, $prev);
 }
 
 # normalize subjects so they are suitable as pathnames for URLs
diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm
index 4256263..3596972 100644
--- a/lib/PublicInbox/SearchIdx.pm
+++ b/lib/PublicInbox/SearchIdx.pm
@@ -76,7 +76,8 @@ sub new {
 	if ($version == 1) {
 		$self->{lock_path} = "$mainrepo/ssoma.lock";
 		my $dir = $self->xdir;
-		$self->{over} = PublicInbox::OverIdx->new("$dir/over.sqlite3");
+		$self->{over_ro} = $self->{over} =
+				PublicInbox::OverIdx->new("$dir/over.sqlite3");
 	} elsif ($version == 2) {
 		defined $part or die "partition is required for v2\n";
 		# partition is a number
diff --git a/lib/PublicInbox/V2Writable.pm b/lib/PublicInbox/V2Writable.pm
index a8117aa..877a459 100644
--- a/lib/PublicInbox/V2Writable.pm
+++ b/lib/PublicInbox/V2Writable.pm
@@ -271,22 +271,20 @@ sub remove_internal {
 
 	foreach my $mid (@$mids) {
 		my %gone;
-		$srch->reopen->each_smsg_by_mid($mid, sub {
-			my ($smsg) = @_;
-			$smsg->load_expand;
+		my ($id, $prev);
+		while (my $smsg = $srch->next_by_mid($mid, \$id, \$prev)) {
 			my $msg = $ibx->msg_by_smsg($smsg);
 			if (!defined($msg)) {
 				warn "broken smsg for $mid\n";
-				return 1; # continue
+				next; # continue
 			}
 			my $orig = $$msg;
 			my $cur = PublicInbox::MIME->new($msg);
 			if (content_id($cur) eq $cid) {
 				$smsg->{mime} = $cur;
-				$gone{$smsg->num} = [ $smsg, \$orig ];
+				$gone{$smsg->{num}} = [ $smsg, \$orig ];
 			}
-			1; # continue
-		});
+		}
 		my $n = scalar keys %gone;
 		next unless $n;
 		if ($n > 1) {
@@ -552,26 +550,23 @@ sub lookup_content {
 	my $srch = $ibx->search->reopen;
 	my $cid = content_id($mime);
 	my $found;
-	$srch->each_smsg_by_mid($mid, sub {
-		my ($smsg) = @_;
-		$smsg->load_expand;
+	my ($id, $prev);
+	while (my $smsg = $srch->next_by_mid($mid, \$id, \$prev)) {
 		my $msg = $ibx->msg_by_smsg($smsg);
 		if (!defined($msg)) {
 			warn "broken smsg for $mid\n";
-			return 1; # continue
+			next;
 		}
 		my $cur = PublicInbox::MIME->new($msg);
 		if (content_id($cur) eq $cid) {
 			$smsg->{mime} = $cur;
 			$found = $smsg;
-			return 0; # break out of loop
+			last;
 		}
 
 		# XXX DEBUG_DIFF is experimental and may be removed
 		diff($mid, $cur, $mime) if $ENV{DEBUG_DIFF};
-
-		1; # continue
-	});
+	}
 	$found;
 }
 
@@ -770,15 +765,14 @@ sub unindex_oid {
 	my $mime = PublicInbox::MIME->new($msgref);
 	my $mids = mids($mime->header_obj);
 	$mime = $msgref = undef;
-
+	my $srch = $self->{-inbox}->search;
 	foreach my $mid (@$mids) {
 		my %gone;
-		$self->{-inbox}->search->reopen->each_smsg_by_mid($mid, sub {
-			my ($smsg) = @_;
-			$smsg->load_expand;
+		my ($id, $prev);
+		while (my $smsg = $srch->next_by_mid($mid, \$id, \$prev)) {
 			$gone{$smsg->num} = 1 if $oid eq $smsg->{blob};
 			1; # continue
-		});
+		}
 		my $n = scalar keys %gone;
 		next unless $n;
 		if ($n > 1) {
diff --git a/lib/PublicInbox/View.pm b/lib/PublicInbox/View.pm
index cbed916..94058ed 100644
--- a/lib/PublicInbox/View.pm
+++ b/lib/PublicInbox/View.pm
@@ -58,20 +58,16 @@ sub msg_page {
 	my ($ctx) = @_;
 	my $mid = $ctx->{mid};
 	my $ibx = $ctx->{-inbox};
-	my ($first, $more, $head, $tail, $db);
+	my ($first, $more);
 	my $smsg;
 	if (my $srch = $ibx->search) {
-		$srch->retry_reopen(sub {
-			($head, $tail, $db) = $srch->each_smsg_by_mid($mid);
-			for (; !defined($first) && $head != $tail; $head++) {
-				my @args = ($head, $db, $mid);
-				$smsg = PublicInbox::SearchMsg->get(@args);
-				$first = $ibx->msg_by_smsg($smsg);
-			}
-			if ($head != $tail) {
-				$more = [ $head, $tail, $db ];
-			}
-		});
+		my ($id, $prev);
+		$smsg = $srch->next_by_mid($mid, \$id, \$prev);
+		$first = $ibx->msg_by_smsg($smsg) if $smsg;
+		if ($first) {
+			my $next = $srch->next_by_mid($mid, \$id, \$prev);
+			$more = [ $id, $prev, $next ] if $next;
+		}
 		return unless $first;
 	} else {
 		$first = $ibx->msg_by_mid($mid) or return;
@@ -82,18 +78,11 @@ sub msg_page {
 sub msg_html_more {
 	my ($ctx, $more, $nr) = @_;
 	my $str = eval {
-		my $smsg;
-		my ($head, $tail, $db) = @$more;
+		my ($id, $prev, $smsg) = @$more;
 		my $mid = $ctx->{mid};
-		for (; !defined($smsg) && $head != $tail; $head++) {
-			my $m = PublicInbox::SearchMsg->get($head, $db, $mid);
-			$smsg = $ctx->{-inbox}->smsg_mime($m);
-		}
-		if ($head == $tail) { # done
-			@$more = ();
-		} else {
-			$more->[0] = $head;
-		}
+		$smsg = $ctx->{-inbox}->smsg_mime($smsg);
+		my $next = $ctx->{srch}->next_by_mid($mid, \$id, \$prev);
+		@$more = $next ? ($id, $prev, $next) : ();
 		if ($smsg) {
 			my $mime = $smsg->{mime};
 			my $upfx = '../' . mid_escape($smsg->mid) . '/';
diff --git a/t/psgi_v2.t b/t/psgi_v2.t
index bdf23de..faa139f 100644
--- a/t/psgi_v2.t
+++ b/t/psgi_v2.t
@@ -129,6 +129,7 @@ test_psgi(sub { $www->call(@_) }, sub {
 		is(scalar(@from_), 3, 'three From_ lines in t.mbox.gz');
 
 		# search interface
+		$config->each_inbox(sub { $_[0]->search->reopen });
 		$res = $cb->(POST('/v2test/?q=m:a-mid@b&x=m'));
 		$in = $res->content;
 		$status = IO::Uncompress::Gunzip::gunzip(\$in => \$out);
diff --git a/t/search.t b/t/search.t
index 2f7b795..fda32d3 100644
--- a/t/search.t
+++ b/t/search.t
@@ -89,10 +89,9 @@ sub filter_mids {
 {
 	$rw_commit->();
 	$ro->reopen;
-	my $found = $ro->first_smsg_by_mid('root@s');
-	ok($found, "message found");
-	is($root_id, $found->{doc_id}, 'doc_id set correctly');
-	is($found->mid, 'root@s', 'mid set correctly');
+	my $found = $ro->query('m:root@s');
+	is(scalar(@$found), 1, "message found");
+	is($found->[0]->mid, 'root@s', 'mid set correctly');
 
 	my ($res, @res);
 	my @exp = sort qw(root@s last@s);
@@ -276,10 +275,9 @@ sub filter_mids {
 		],
 		body => "LOOP!\n"));
 	ok($doc_id > 0, "doc_id defined with circular reference");
-	my $smsg = $rw->first_smsg_by_mid('circle@a');
+	my $smsg = $rw->query('m:circle@a', {limit=>1})->[0];
 	is($smsg->references, '', "no references created");
-	my $msg = PublicInbox::SearchMsg->load_doc($smsg->{doc});
-	is($s, $msg->subject, 'long subject not rewritten');
+	is($s, $smsg->subject, 'long subject not rewritten');
 }
 
 {
@@ -293,9 +291,7 @@ sub filter_mids {
 	my $mime = Email::MIME->new($str);
 	my $doc_id = $rw->add_message($mime);
 	ok($doc_id > 0, 'message indexed doc_id with UTF-8');
-	my $smsg = $rw->first_smsg_by_mid('testmessage@example.com');
-	my $msg = PublicInbox::SearchMsg->load_doc($smsg->{doc});
-
+	my $msg = $rw->query('m:testmessage@example.com', {limit => 1})->[0];
 	is($mime->header('Subject'), $msg->subject, 'UTF-8 subject preserved');
 }
 
diff --git a/t/v2writable.t b/t/v2writable.t
index ab85e9a..4a42c01 100644
--- a/t/v2writable.t
+++ b/t/v2writable.t
@@ -108,14 +108,13 @@ if ('ensure git configs are correct') {
 	$mime->header_set('References', '<zz-mid@b>');
 	ok($im->add($mime), 'message with multiple Message-ID');
 	$im->done;
-	my @found;
 	my $srch = $ibx->search;
-	$srch->reopen->each_smsg_by_mid('abcde@1', sub { push @found, @_; 1 });
-	is(scalar(@found), 1, 'message found by first MID');
-	$srch->reopen->each_smsg_by_mid('abcde@2', sub { push @found, @_; 1 });
-	is(scalar(@found), 2, 'message found by second MID');
-	is($found[0]->{doc_id}, $found[1]->{doc_id}, 'same document');
-	ok($found[1]->{doc_id} > 0, 'doc_id is positive');
+	my $mset1 = $srch->reopen->query('m:abcde@1', { mset => 1 });
+	is($mset1->size, 1, 'message found by first MID');
+	my $mset2 = $srch->reopen->query('m:abcde@2', { mset => 1 });
+	is($mset2->size, 1, 'message found by second MID');
+	is((($mset1->items)[0])->get_docid, (($mset2->items)[0])->get_docid,
+		'same document');
 }
 
 SKIP: {
@@ -224,9 +223,8 @@ EOF
 	like($smsg->num, qr/\A\d+\z/, 'numeric number in return message');
 	is($ibx->mm->mid_for($smsg->num), undef, 'no longer in Msgmap by num');
 	my $srch = $ibx->search->reopen;
-	my @found = ();
-	$srch->each_smsg_by_mid($smsg->mid, sub { push @found, @_; 1 });
-	is(scalar(@found), 0, 'no longer found in Xapian');
+	my $mset = $srch->query('m:'.$smsg->mid, { mset => 1});
+	is($mset->size, 0, 'no longer found in Xapian');
 	my @log1 = qw(log -1 --pretty=raw --raw -r --no-abbrev --no-renames);
 	is($srch->{over_ro}->get_art($smsg->num), undef,
 		'removal propagated to Over DB');
-- 
EW

[PATCH 4/4] ensure Xapian and SQLite are still optional for v1 tests

[Eric Wong (Contractor, The Linux Foundation)]

Xapian is size-intensive and SQLite is not strictly necessary for v1.
---
 script/public-inbox-compact   | 2 +-
 scripts/import_vger_from_mbox | 2 +-
 t/convert-compact.t           | 2 +-
 t/v2mirror.t                  | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/script/public-inbox-compact b/script/public-inbox-compact
index 43e9460..d855b9e 100755
--- a/script/public-inbox-compact
+++ b/script/public-inbox-compact
@@ -4,9 +4,9 @@
 use strict;
 use warnings;
 use Getopt::Long qw(:config gnu_getopt no_ignore_case auto_abbrev);
-use PublicInbox::V2Writable;
 use PublicInbox::Search;
 use PublicInbox::Config;
+use PublicInbox::InboxWritable;
 use Cwd 'abs_path';
 use File::Temp qw(tempdir);
 use File::Path qw(remove_tree);
diff --git a/scripts/import_vger_from_mbox b/scripts/import_vger_from_mbox
index 191f75d..ca5a408 100644
--- a/scripts/import_vger_from_mbox
+++ b/scripts/import_vger_from_mbox
@@ -6,7 +6,6 @@ use warnings;
 use Getopt::Long qw/:config gnu_getopt no_ignore_case auto_abbrev/;
 use PublicInbox::MIME;
 use PublicInbox::InboxWritable;
-use PublicInbox::V2Writable;
 use PublicInbox::Import;
 use PublicInbox::MDA;
 my $usage = "usage: $0 NAME EMAIL DIR <MBOX\n";
@@ -35,6 +34,7 @@ my $ibx = {
 $ibx = PublicInbox::Inbox->new($ibx);
 unless ($dry_run) {
 	if ($version >= 2) {
+		require PublicInbox::V2Writable;
 		PublicInbox::V2Writable->new($ibx, 1)->init_inbox(0);
 	} else {
 		system(qw(git init --bare -q), $mainrepo) == 0 or die;
diff --git a/t/convert-compact.t b/t/convert-compact.t
index e2ba40a..5caa0ac 100644
--- a/t/convert-compact.t
+++ b/t/convert-compact.t
@@ -10,7 +10,7 @@ foreach my $mod (@mods) {
 	eval "require $mod";
 	plan skip_all => "$mod missing for convert-compact.t" if $@;
 }
-use PublicInbox::V2Writable;
+use_ok 'PublicInbox::V2Writable';
 use PublicInbox::Import;
 my $tmpdir = tempdir('convert-compact-XXXXXX', TMPDIR => 1, CLEANUP => 1);
 my $ibx = {
diff --git a/t/v2mirror.t b/t/v2mirror.t
index 0c66aef..9e0c9e1 100644
--- a/t/v2mirror.t
+++ b/t/v2mirror.t
@@ -13,7 +13,7 @@ foreach my $mod (qw(Plack::Util Plack::Builder Danga::Socket
 use File::Temp qw/tempdir/;
 use IO::Socket;
 use POSIX qw(dup2);
-use PublicInbox::V2Writable;
+use_ok 'PublicInbox::V2Writable';
 use PublicInbox::MIME;
 use PublicInbox::Config;
 use Fcntl qw(FD_CLOEXEC F_SETFD F_GETFD);
-- 
EW