* [PATCH 1/4] testcommon: add require_mods method and use it
2019-12-22 22:17 [PATCH 0/4] support SWIG-generated Xapian binding Eric Wong
@ 2019-12-22 22:17 ` Eric Wong
2019-12-22 22:17 ` [PATCH 2/4] searchidx: call "++" on PostingIterator instead of "->inc" Eric Wong
` (2 subsequent siblings)
3 siblings, 0 replies; 6+ messages in thread
From: Eric Wong @ 2019-12-22 22:17 UTC (permalink / raw)
To: meta
This cuts down on lines of code in individual test cases and
fixes some misnamed error messages by using "$0" consistently.
This will also provide us with a method of swapping out
dependencies which provide equivalent functionality (e.g
"Xapian" SWIG can replace "Search::Xapian" XS bindings).
---
lib/PublicInbox/TestCommon.pm | 16 +++++++++++++++-
t/altid.t | 6 +-----
t/altid_v2.t | 6 +-----
t/convert-compact.t | 6 +-----
t/edit.t | 7 +------
t/filter_rubylang.t | 3 +--
t/httpd-corner.t | 9 ++-------
t/httpd-https.t | 7 ++-----
t/httpd-unix.t | 10 ++--------
t/httpd.t | 8 ++------
t/indexlevels-mirror.t | 6 +-----
t/init.t | 5 +----
t/mda_filter_rubylang.t | 7 +------
t/msgmap.t | 7 +------
t/nntp.t | 8 ++------
t/nntpd-tls.t | 7 ++-----
t/nntpd.t | 13 ++++++-------
t/over.t | 6 +-----
t/plack.t | 5 +----
t/psgi_attach.t | 5 +----
t/psgi_bad_mids.t | 5 +----
t/psgi_mount.t | 10 ++--------
t/psgi_multipart_not.t | 5 +----
t/psgi_scan_all.t | 5 +----
t/psgi_search.t | 6 +-----
t/psgi_text.t | 5 +----
t/psgi_v2.t | 5 +----
t/purge.t | 6 +-----
t/replace.t | 6 +-----
t/search-thr-index.t | 8 ++------
t/search.t | 8 ++------
t/solver_git.t | 13 ++-----------
t/v1-add-remove-add.t | 6 +-----
t/v1reindex.t | 6 +-----
t/v2-add-remove-add.t | 6 +-----
t/v2mda.t | 5 +----
t/v2mirror.t | 7 ++-----
t/v2reindex.t | 6 +-----
t/v2writable.t | 5 +----
t/watch_filter_rubylang.t | 7 +------
t/watch_maildir.t | 7 +------
t/watch_maildir_v2.t | 6 +-----
t/www_listing.t | 8 ++------
t/xcpdb-reshard.t | 6 +-----
xt/git-http-backend.t | 8 ++------
xt/msgtime_cmp.t | 2 +-
xt/nntpd-validate.t | 8 ++------
xt/perf-nntpd.t | 1 -
48 files changed, 80 insertions(+), 243 deletions(-)
diff --git a/lib/PublicInbox/TestCommon.pm b/lib/PublicInbox/TestCommon.pm
index 372cafa6..9680b94c 100644
--- a/lib/PublicInbox/TestCommon.pm
+++ b/lib/PublicInbox/TestCommon.pm
@@ -8,7 +8,7 @@ use parent qw(Exporter);
use Fcntl qw(FD_CLOEXEC F_SETFD F_GETFD :seek);
use POSIX qw(dup2);
use IO::Socket::INET;
-our @EXPORT = qw(tmpdir tcp_server tcp_connect require_git
+our @EXPORT = qw(tmpdir tcp_server tcp_connect require_git require_mods
run_script start_script key2sub);
sub tmpdir (;$) {
@@ -59,6 +59,20 @@ sub require_git ($;$) {
1;
}
+sub require_mods {
+ my @mods = @_;
+ my $maybe = pop @mods if $mods[-1] =~ /\A[0-9]+\z/;
+ my @need;
+ for my $mod (@mods) {
+ eval "require $mod";
+ push @need, $mod if $@;
+ }
+ return unless @need;
+ my $m = join(', ', @need)." missing for $0";
+ Test::More::skip($m, $maybe) if $maybe;
+ Test::More::plan(skip_all => $m)
+}
+
sub key2script ($) {
my ($key) = @_;
return $key if (index($key, '/') >= 0);
diff --git a/t/altid.t b/t/altid.t
index fc5bcbc2..a3f71044 100644
--- a/t/altid.t
+++ b/t/altid.t
@@ -4,11 +4,7 @@ use strict;
use warnings;
use Test::More;
use PublicInbox::TestCommon;
-foreach my $mod (qw(DBD::SQLite Search::Xapian)) {
- eval "require $mod";
- plan skip_all => "$mod missing for altid.t" if $@;
-}
-
+require_mods(qw(DBD::SQLite Search::Xapian));
use_ok 'PublicInbox::Msgmap';
use_ok 'PublicInbox::SearchIdx';
use_ok 'PublicInbox::Import';
diff --git a/t/altid_v2.t b/t/altid_v2.t
index 5bd62e13..2d2379ec 100644
--- a/t/altid_v2.t
+++ b/t/altid_v2.t
@@ -5,11 +5,7 @@ use warnings;
use Test::More;
use PublicInbox::TestCommon;
require_git(2.6);
-foreach my $mod (qw(DBD::SQLite Search::Xapian)) {
- eval "require $mod";
- plan skip_all => "$mod missing for altid_v2.t" if $@;
-}
-
+require_mods(qw(DBD::SQLite Search::Xapian));
use_ok 'PublicInbox::V2Writable';
use_ok 'PublicInbox::Inbox';
my ($tmpdir, $for_destroy) = tmpdir();
diff --git a/t/convert-compact.t b/t/convert-compact.t
index c6da64ea..6e4a8268 100644
--- a/t/convert-compact.t
+++ b/t/convert-compact.t
@@ -7,11 +7,7 @@ use PublicInbox::MIME;
use PublicInbox::Spawn qw(which);
use PublicInbox::TestCommon;
require_git(2.6);
-my @mods = qw(DBD::SQLite Search::Xapian);
-foreach my $mod (@mods) {
- eval "require $mod";
- plan skip_all => "$mod missing for convert-compact.t" if $@;
-}
+require_mods(qw(DBD::SQLite Search::Xapian));
which('xapian-compact') or
plan skip_all => 'xapian-compact missing for '.__FILE__;
diff --git a/t/edit.t b/t/edit.t
index 9a78b45d..89283291 100644
--- a/t/edit.t
+++ b/t/edit.t
@@ -11,12 +11,7 @@ require PublicInbox::InboxWritable;
require PublicInbox::Config;
use PublicInbox::MID qw(mid_clean);
-my @mods = qw(DBI DBD::SQLite);
-foreach my $mod (@mods) {
- eval "require $mod";
- plan skip_all => "missing $mod for $0" if $@;
-};
-
+require_mods('DBD::SQLite');
my ($tmpdir, $for_destroy) = tmpdir();
my $inboxdir = "$tmpdir/v2";
my $ibx = PublicInbox::Inbox->new({
diff --git a/t/filter_rubylang.t b/t/filter_rubylang.t
index 576cbdeb..1288b72f 100644
--- a/t/filter_rubylang.t
+++ b/t/filter_rubylang.t
@@ -23,8 +23,7 @@ is($ret, $mime, "delivery successful");
is($mime->body, "keep this\n", 'normal message filtered OK');
SKIP: {
- eval 'require DBD::SQLite';
- skip 'DBD::SQLite missing for altid mapping', 4 if $@;
+ require_mods('DBD::SQLite', 4);
use_ok 'PublicInbox::Inbox';
my ($git_dir, $for_destroy) = tmpdir();
is(mkdir("$git_dir/public-inbox"), 1, "created public-inbox dir");
diff --git a/t/httpd-corner.t b/t/httpd-corner.t
index fb3ffcf1..4ef1618a 100644
--- a/t/httpd-corner.t
+++ b/t/httpd-corner.t
@@ -7,19 +7,14 @@ use warnings;
use Test::More;
use Time::HiRes qw(gettimeofday tv_interval);
use PublicInbox::Spawn qw(which spawn);
-
-foreach my $mod (qw(Plack::Util Plack::Builder HTTP::Date HTTP::Status)) {
- eval "require $mod";
- plan skip_all => "$mod missing for httpd-corner.t" if $@;
-}
-
+use PublicInbox::TestCommon;
+require_mods(qw(Plack::Util Plack::Builder HTTP::Date HTTP::Status));
use Digest::SHA qw(sha1_hex);
use IO::Socket;
use IO::Socket::UNIX;
use Fcntl qw(:seek);
use Socket qw(IPPROTO_TCP TCP_NODELAY SOL_SOCKET);
use POSIX qw(mkfifo);
-use PublicInbox::TestCommon;
my ($tmpdir, $for_destroy) = tmpdir();
my $fifo = "$tmpdir/fifo";
ok(defined mkfifo($fifo, 0777), 'created FIFO');
diff --git a/t/httpd-https.t b/t/httpd-https.t
index 1d2e4d5c..265febe5 100644
--- a/t/httpd-https.t
+++ b/t/httpd-https.t
@@ -4,11 +4,9 @@ use strict;
use warnings;
use Test::More;
use Socket qw(SOCK_STREAM IPPROTO_TCP SOL_SOCKET);
+use PublicInbox::TestCommon;
# IO::Poll is part of the standard library, but distros may split them off...
-foreach my $mod (qw(IO::Socket::SSL IO::Poll)) {
- eval "require $mod";
- plan skip_all => "$mod missing for $0" if $@;
-}
+require_mods(qw(IO::Socket::SSL IO::Poll));
my $cert = 'certs/server-cert.pem';
my $key = 'certs/server-key.pem';
unless (-r $key && -r $cert) {
@@ -17,7 +15,6 @@ unless (-r $key && -r $cert) {
}
use_ok 'PublicInbox::TLS';
use_ok 'IO::Socket::SSL';
-use PublicInbox::TestCommon;
my $psgi = "./t/httpd-corner.psgi";
my ($tmpdir, $for_destroy) = tmpdir();
my $err = "$tmpdir/stderr.log";
diff --git a/t/httpd-unix.t b/t/httpd-unix.t
index bd4ee12e..6da2ac6b 100644
--- a/t/httpd-unix.t
+++ b/t/httpd-unix.t
@@ -6,12 +6,7 @@ use warnings;
use Test::More;
use PublicInbox::TestCommon;
use Errno qw(EADDRINUSE);
-
-foreach my $mod (qw(Plack::Util Plack::Builder HTTP::Date HTTP::Status)) {
- eval "require $mod";
- plan skip_all => "$mod missing for httpd-unix.t" if $@;
-}
-
+require_mods(qw(Plack::Util Plack::Builder HTTP::Date HTTP::Status));
use IO::Socket::UNIX;
my ($tmpdir, $for_destroy) = tmpdir();
my $unix = "$tmpdir/unix.sock";
@@ -85,8 +80,7 @@ check_sock($unix);
}
SKIP: {
- eval 'require Net::Server::Daemonize';
- skip('Net::Server missing for pid-file/daemonization test', 20) if $@;
+ require_mods('Net::Server::Daemonize', 20);
my $pid_file = "$tmpdir/pid";
for my $w (qw(-W0 -W1)) {
# wait for daemonization
diff --git a/t/httpd.t b/t/httpd.t
index 517329fb..880c69e6 100644
--- a/t/httpd.t
+++ b/t/httpd.t
@@ -3,13 +3,9 @@
use strict;
use warnings;
use Test::More;
-
-foreach my $mod (qw(Plack::Util Plack::Builder HTTP::Date HTTP::Status)) {
- eval "require $mod";
- plan skip_all => "$mod missing for httpd.t" if $@;
-}
-use Socket qw(IPPROTO_TCP SOL_SOCKET);
use PublicInbox::TestCommon;
+use Socket qw(IPPROTO_TCP SOL_SOCKET);
+require_mods(qw(Plack::Util Plack::Builder HTTP::Date HTTP::Status));
# FIXME: too much setup
my ($tmpdir, $for_destroy) = tmpdir();
diff --git a/t/indexlevels-mirror.t b/t/indexlevels-mirror.t
index 876abd2c..e4313faa 100644
--- a/t/indexlevels-mirror.t
+++ b/t/indexlevels-mirror.t
@@ -10,11 +10,7 @@ require PublicInbox::Admin;
use PublicInbox::TestCommon;
my $PI_TEST_VERSION = $ENV{PI_TEST_VERSION} || 2;
require_git('2.6') if $PI_TEST_VERSION == 2;
-
-foreach my $mod (qw(DBD::SQLite)) {
- eval "require $mod";
- plan skip_all => "$mod missing for $0" if $@;
-}
+require_mods(qw(DBD::SQLite));
my $mime = PublicInbox::MIME->create(
header => [
diff --git a/t/init.t b/t/init.t
index f70d1c20..786add13 100644
--- a/t/init.t
+++ b/t/init.t
@@ -50,10 +50,7 @@ sub quiet_fail {
}
SKIP: {
- foreach my $mod (qw(DBD::SQLite Search::Xapian::WritableDatabase)) {
- eval "require $mod";
- skip "$mod missing for v2", 2 if $@;
- }
+ require_mods(qw(DBD::SQLite Search::Xapian::WritableDatabase), 2);
require_git(2.6, 1) or skip "git 2.6+ required", 2;
local $ENV{PI_DIR} = "$tmpdir/.public-inbox/";
my $cfgfile = "$ENV{PI_DIR}/config";
diff --git a/t/mda_filter_rubylang.t b/t/mda_filter_rubylang.t
index a5b6af3c..fec61b68 100644
--- a/t/mda_filter_rubylang.t
+++ b/t/mda_filter_rubylang.t
@@ -7,12 +7,7 @@ use PublicInbox::MIME;
use PublicInbox::Config;
use PublicInbox::TestCommon;
require_git(2.6);
-my @mods = qw(DBD::SQLite Search::Xapian);
-foreach my $mod (@mods) {
- eval "require $mod";
- plan skip_all => "$mod missing for mda_filter_rubylang.t" if $@;
-}
-
+require_mods(qw(DBD::SQLite Search::Xapian));
use_ok 'PublicInbox::V2Writable';
my ($tmpdir, $for_destroy) = tmpdir();
my $pi_config = "$tmpdir/pi_config";
diff --git a/t/msgmap.t b/t/msgmap.t
index 6edeed56..197b0e55 100644
--- a/t/msgmap.t
+++ b/t/msgmap.t
@@ -4,12 +4,7 @@ use strict;
use warnings;
use Test::More;
use PublicInbox::TestCommon;
-
-foreach my $mod (qw(DBD::SQLite)) {
- eval "require $mod";
- plan skip_all => "$mod missing for nntpd.t" if $@;
-}
-
+require_mods('DBD::SQLite');
use_ok 'PublicInbox::Msgmap';
my ($tmpdir, $for_destroy) = tmpdir();
my $d = PublicInbox::Msgmap->new($tmpdir, 1);
diff --git a/t/nntp.t b/t/nntp.t
index aabfd4ff..11a175bb 100644
--- a/t/nntp.t
+++ b/t/nntp.t
@@ -3,12 +3,8 @@
use strict;
use warnings;
use Test::More;
-
-foreach my $mod (qw(DBD::SQLite Data::Dumper)) {
- eval "require $mod";
- plan skip_all => "$mod missing for nntp.t" if $@;
-}
-
+use PublicInbox::TestCommon;
+require_mods(qw(DBD::SQLite Data::Dumper));
use_ok 'PublicInbox::NNTP';
use_ok 'PublicInbox::Inbox';
diff --git a/t/nntpd-tls.t b/t/nntpd-tls.t
index 25b7bd1c..edc1fab4 100644
--- a/t/nntpd-tls.t
+++ b/t/nntpd-tls.t
@@ -4,12 +4,10 @@ use strict;
use warnings;
use Test::More;
use Socket qw(SOCK_STREAM IPPROTO_TCP SOL_SOCKET);
+use PublicInbox::TestCommon;
# IO::Poll and Net::NNTP are part of the standard library, but
# distros may split them off...
-foreach my $mod (qw(DBD::SQLite IO::Socket::SSL Net::NNTP IO::Poll)) {
- eval "require $mod";
- plan skip_all => "$mod missing for $0" if $@;
-}
+require_mods(qw(DBD::SQLite IO::Socket::SSL Net::NNTP IO::Poll));
Net::NNTP->can('starttls') or
plan skip_all => 'Net::NNTP does not support TLS';
IO::Socket::SSL->VERSION(2.007) or
@@ -24,7 +22,6 @@ unless (-r $key && -r $cert) {
use_ok 'PublicInbox::TLS';
use_ok 'IO::Socket::SSL';
-use PublicInbox::TestCommon;
require PublicInbox::InboxWritable;
require PublicInbox::MIME;
require PublicInbox::SearchIdx;
diff --git a/t/nntpd.t b/t/nntpd.t
index c3712b67..243b8e56 100644
--- a/t/nntpd.t
+++ b/t/nntpd.t
@@ -3,10 +3,8 @@
use strict;
use warnings;
use Test::More;
-foreach my $mod (qw(DBD::SQLite)) {
- eval "require $mod";
- plan skip_all => "$mod missing for nntpd.t" if $@;
-}
+use PublicInbox::TestCommon;
+require_mods(qw(DBD::SQLite));
require PublicInbox::SearchIdx;
require PublicInbox::Msgmap;
require PublicInbox::InboxWritable;
@@ -15,7 +13,6 @@ use IO::Socket;
use Socket qw(IPPROTO_TCP TCP_NODELAY);
use Net::NNTP;
use Sys::Hostname;
-use PublicInbox::TestCommon;
# FIXME: make easier to test both versions
my $version = $ENV{PI_TEST_VERSION} || 2;
@@ -106,10 +103,12 @@ EOF
SKIP: {
$n->can('starttls') or
skip('Net::NNTP too old to support STARTTLS', 2);
+ require_mods('IO::Socket::SSL', 2);
eval {
- require IO::Socket::SSL;
IO::Socket::SSL->VERSION(2.007);
- } or skip('IO::Socket::SSL <2.007 not supported by Net::NNTP');
+ } or skip(<<EOF, 2);
+IO::Socket::SSL <2.007 not supported by Net::NNTP
+EOF
ok(!$n->starttls, 'STARTTLS fails when unconfigured');
is($n->code, 580, 'got 580 code on server w/o TLS');
};
diff --git a/t/over.t b/t/over.t
index 5e33a0b0..4e630bcd 100644
--- a/t/over.t
+++ b/t/over.t
@@ -5,11 +5,7 @@ use warnings;
use Test::More;
use Compress::Zlib qw(compress);
use PublicInbox::TestCommon;
-foreach my $mod (qw(DBD::SQLite)) {
- eval "require $mod";
- plan skip_all => "$mod missing for over.t" if $@;
-}
-
+require_mods('DBD::SQLite');
use_ok 'PublicInbox::OverIdx';
my ($tmpdir, $for_destroy) = tmpdir();
my $over = PublicInbox::OverIdx->new("$tmpdir/over.sqlite3");
diff --git a/t/plack.t b/t/plack.t
index c9a2cf79..f0dc4bbb 100644
--- a/t/plack.t
+++ b/t/plack.t
@@ -12,10 +12,7 @@ my $maindir = "$tmpdir/main.git";
my $addr = 'test-public@example.com';
my $cfgpfx = "publicinbox.test";
my @mods = qw(HTTP::Request::Common Plack::Test URI::Escape);
-foreach my $mod (@mods) {
- eval "require $mod";
- plan skip_all => "$mod missing for plack.t" if $@;
-}
+require_mods(@mods);
use_ok 'PublicInbox::Import';
use_ok 'PublicInbox::Git';
my @ls;
diff --git a/t/psgi_attach.t b/t/psgi_attach.t
index 5a7129f9..0c3174bc 100644
--- a/t/psgi_attach.t
+++ b/t/psgi_attach.t
@@ -10,10 +10,7 @@ my $maindir = "$tmpdir/main.git";
my $addr = 'test-public@example.com';
my $cfgpfx = "publicinbox.test";
my @mods = qw(HTTP::Request::Common Plack::Test URI::Escape);
-foreach my $mod (@mods) {
- eval "require $mod";
- plan skip_all => "$mod missing for plack.t" if $@;
-}
+require_mods(@mods);
use_ok $_ foreach @mods;
use PublicInbox::Import;
use PublicInbox::Git;
diff --git a/t/psgi_bad_mids.t b/t/psgi_bad_mids.t
index b22f3652..5cdd249e 100644
--- a/t/psgi_bad_mids.t
+++ b/t/psgi_bad_mids.t
@@ -9,10 +9,7 @@ use PublicInbox::WWW;
use PublicInbox::TestCommon;
my @mods = qw(DBD::SQLite HTTP::Request::Common Plack::Test
URI::Escape Plack::Builder);
-foreach my $mod (@mods) {
- eval "require $mod";
- plan skip_all => "$mod missing for psgi_bad_mids.t" if $@;
-}
+require_mods(@mods);
use_ok($_) for @mods;
use_ok 'PublicInbox::V2Writable';
my ($inboxdir, $for_destroy) = tmpdir();
diff --git a/t/psgi_mount.t b/t/psgi_mount.t
index a2621a2a..751c13b7 100644
--- a/t/psgi_mount.t
+++ b/t/psgi_mount.t
@@ -10,10 +10,7 @@ my $maindir = "$tmpdir/main.git";
my $addr = 'test-public@example.com';
my $cfgpfx = "publicinbox.test";
my @mods = qw(HTTP::Request::Common Plack::Test URI::Escape);
-foreach my $mod (@mods) {
- eval "require $mod";
- plan skip_all => "$mod missing for plack.t" if $@;
-}
+require_mods(@mods);
use_ok $_ foreach @mods;
use PublicInbox::Import;
use PublicInbox::Git;
@@ -88,10 +85,7 @@ test_psgi($app, sub {
});
SKIP: {
- my @mods = qw(DBI DBD::SQLite Search::Xapian IO::Uncompress::Gunzip);
- foreach my $mod (@mods) {
- eval "require $mod" or skip "$mod not available: $@", 3;
- }
+ require_mods(qw(DBD::SQLite Search::Xapian IO::Uncompress::Gunzip), 3);
my $ibx = $config->lookup_name('test');
require_ok 'PublicInbox::SearchIdx';
PublicInbox::SearchIdx->new($ibx, 1)->index_sync;
diff --git a/t/psgi_multipart_not.t b/t/psgi_multipart_not.t
index 9384d14a..fbc09f55 100644
--- a/t/psgi_multipart_not.t
+++ b/t/psgi_multipart_not.t
@@ -9,10 +9,7 @@ use PublicInbox::WWW;
use PublicInbox::TestCommon;
my @mods = qw(DBD::SQLite Search::Xapian HTTP::Request::Common
Plack::Test URI::Escape Plack::Builder Plack::Test);
-foreach my $mod (@mods) {
- eval "require $mod";
- plan skip_all => "$mod missing for psgi_multipart_not.t" if $@;
-}
+require_mods(@mods);
use_ok($_) for @mods;
use_ok 'PublicInbox::V2Writable';
my ($repo, $for_destroy) = tmpdir();
diff --git a/t/psgi_scan_all.t b/t/psgi_scan_all.t
index fa636d32..09ed96e0 100644
--- a/t/psgi_scan_all.t
+++ b/t/psgi_scan_all.t
@@ -7,10 +7,7 @@ use Email::MIME;
use PublicInbox::Config;
use PublicInbox::TestCommon;
my @mods = qw(HTTP::Request::Common Plack::Test URI::Escape DBD::SQLite);
-foreach my $mod (@mods) {
- eval "require $mod";
- plan skip_all => "$mod missing for psgi_scan_all.t" if $@;
-}
+require_mods(@mods);
use_ok 'PublicInbox::V2Writable';
foreach my $mod (@mods) { use_ok $mod; }
my ($tmp, $for_destroy) = tmpdir();
diff --git a/t/psgi_search.t b/t/psgi_search.t
index d1050437..95749c66 100644
--- a/t/psgi_search.t
+++ b/t/psgi_search.t
@@ -12,11 +12,7 @@ use bytes (); # only for bytes::length
use PublicInbox::TestCommon;
my @mods = qw(DBD::SQLite Search::Xapian HTTP::Request::Common Plack::Test
URI::Escape Plack::Builder);
-foreach my $mod (@mods) {
- eval "require $mod";
- plan skip_all => "$mod missing for psgi_search.t" if $@;
-}
-
+require_mods(@mods);
use_ok $_ foreach (@mods, qw(PublicInbox::SearchIdx));
my ($tmpdir, $for_destroy) = tmpdir();
diff --git a/t/psgi_text.t b/t/psgi_text.t
index ee4d9f14..757a1294 100644
--- a/t/psgi_text.t
+++ b/t/psgi_text.t
@@ -10,10 +10,7 @@ my $maindir = "$tmpdir/main.git";
my $addr = 'test-public@example.com';
my $cfgpfx = "publicinbox.test";
my @mods = qw(HTTP::Request::Common Plack::Test URI::Escape);
-foreach my $mod (@mods) {
- eval "require $mod";
- plan skip_all => "$mod missing for psgi_text.t" if $@;
-}
+require_mods(@mods);
use_ok $_ foreach @mods;
use PublicInbox::Import;
use PublicInbox::Git;
diff --git a/t/psgi_v2.t b/t/psgi_v2.t
index 10176516..9a81ee9a 100644
--- a/t/psgi_v2.t
+++ b/t/psgi_v2.t
@@ -11,10 +11,7 @@ use PublicInbox::WWW;
use PublicInbox::MID qw(mids);
my @mods = qw(DBD::SQLite Search::Xapian HTTP::Request::Common Plack::Test
URI::Escape Plack::Builder);
-foreach my $mod (@mods) {
- eval "require $mod";
- plan skip_all => "$mod missing for psgi_v2_dupes.t" if $@;
-}
+require_mods(@mods);
use_ok($_) for @mods;
use_ok 'PublicInbox::V2Writable';
my ($inboxdir, $for_destroy) = tmpdir();
diff --git a/t/purge.t b/t/purge.t
index 0262f791..7f49bc60 100644
--- a/t/purge.t
+++ b/t/purge.t
@@ -5,11 +5,7 @@ use warnings;
use Test::More;
use PublicInbox::TestCommon;
require_git(2.6);
-my @mods = qw(DBI DBD::SQLite);
-foreach my $mod (@mods) {
- eval "require $mod";
- plan skip_all => "missing $mod for t/purge.t" if $@;
-};
+require_mods(qw(DBD::SQLite));
use Cwd qw(abs_path); # we need this since we chdir below
local $ENV{HOME} = abs_path('t');
my $purge = abs_path('blib/script/public-inbox-purge');
diff --git a/t/replace.t b/t/replace.t
index 2346c3a1..085b49ee 100644
--- a/t/replace.t
+++ b/t/replace.t
@@ -8,11 +8,7 @@ use PublicInbox::InboxWritable;
use PublicInbox::TestCommon;
use Cwd qw(abs_path);
require_git(2.6); # replace is v2 only, for now...
-foreach my $mod (qw(DBD::SQLite)) {
- eval "require $mod";
- plan skip_all => "$mod missing for $0" if $@;
-}
-
+require_mods(qw(DBD::SQLite));
local $ENV{HOME} = abs_path('t');
sub test_replace ($$$) {
diff --git a/t/search-thr-index.t b/t/search-thr-index.t
index 47454fe7..4750503c 100644
--- a/t/search-thr-index.t
+++ b/t/search-thr-index.t
@@ -4,16 +4,12 @@ use strict;
use warnings;
use bytes (); # only for bytes::length
use Test::More;
+use PublicInbox::TestCommon;
use PublicInbox::MID qw(mids);
use Email::MIME;
-my @mods = qw(DBI DBD::SQLite Search::Xapian);
-foreach my $mod (@mods) {
- eval "require $mod";
- plan skip_all => "missing $mod for $0" if $@;
-}
+require_mods(qw(DBD::SQLite Search::Xapian));
require PublicInbox::SearchIdx;
require PublicInbox::Inbox;
-use PublicInbox::TestCommon;
my ($tmpdir, $for_destroy) = tmpdir();
my $git_dir = "$tmpdir/a.git";
diff --git a/t/search.t b/t/search.t
index 33a7596e..c40e1363 100644
--- a/t/search.t
+++ b/t/search.t
@@ -3,14 +3,10 @@
use strict;
use warnings;
use Test::More;
-my @mods = qw(DBI DBD::SQLite Search::Xapian);
-foreach my $mod (@mods) {
- eval "require $mod";
- plan skip_all => "missing $mod for $0" if $@;
-};
+use PublicInbox::TestCommon;
+require_mods(qw(DBD::SQLite Search::Xapian));
require PublicInbox::SearchIdx;
require PublicInbox::Inbox;
-use PublicInbox::TestCommon;
use Email::MIME;
my ($tmpdir, $for_destroy) = tmpdir();
my $git_dir = "$tmpdir/a.git";
diff --git a/t/solver_git.t b/t/solver_git.t
index 99ee9b5e..4cadd06a 100644
--- a/t/solver_git.t
+++ b/t/solver_git.t
@@ -7,12 +7,7 @@ use Cwd qw(abs_path);
use PublicInbox::TestCommon;
require_git(2.6);
use PublicInbox::Spawn qw(spawn);
-
-my @mods = qw(DBD::SQLite Search::Xapian);
-foreach my $mod (@mods) {
- eval "require $mod";
- plan skip_all => "$mod missing for $0" if $@;
-}
+require_mods(qw(DBD::SQLite Search::Xapian));
chomp(my $git_dir = `git rev-parse --git-dir 2>/dev/null`);
plan skip_all => "$0 must be run from a git working tree" if $?;
@@ -116,11 +111,7 @@ is_deeply($res, $hinted, 'hints work (or did not hurt :P');
my @psgi = qw(HTTP::Request::Common Plack::Test URI::Escape Plack::Builder);
SKIP: {
- my @missing;
- for my $mod (@psgi) {
- eval("require $mod") or push(@missing, $mod);
- }
- skip("missing: ".join(', ', @missing), 7 + scalar(@psgi)) if @missing;
+ require_mods(@psgi, 7 + scalar(@psgi));
use_ok($_) for @psgi;
my $binfoo = "$inboxdir/binfoo.git";
system(qw(git init --bare -q), $binfoo) == 0 or die "git init: $?";
diff --git a/t/v1-add-remove-add.t b/t/v1-add-remove-add.t
index aabb2fa5..6fac5a02 100644
--- a/t/v1-add-remove-add.t
+++ b/t/v1-add-remove-add.t
@@ -6,11 +6,7 @@ use Test::More;
use PublicInbox::MIME;
use PublicInbox::Import;
use PublicInbox::TestCommon;
-
-foreach my $mod (qw(DBD::SQLite Search::Xapian)) {
- eval "require $mod";
- plan skip_all => "$mod missing for v1-add-remove-add.t" if $@;
-}
+require_mods(qw(DBD::SQLite Search::Xapian));
require PublicInbox::SearchIdx;
my ($inboxdir, $for_destroy) = tmpdir();
is(system(qw(git init --bare -q), $inboxdir), 0);
diff --git a/t/v1reindex.t b/t/v1reindex.t
index 321b3b21..bf1d45f0 100644
--- a/t/v1reindex.t
+++ b/t/v1reindex.t
@@ -8,11 +8,7 @@ use PublicInbox::ContentId qw(content_digest);
use File::Path qw(remove_tree);
use PublicInbox::TestCommon;
require_git(2.6);
-
-foreach my $mod (qw(DBD::SQLite Search::Xapian)) {
- eval "require $mod";
- plan skip_all => "$mod missing for v1reindex.t" if $@;
-}
+require_mods(qw(DBD::SQLite Search::Xapian));
use_ok 'PublicInbox::SearchIdx';
use_ok 'PublicInbox::Import';
my ($inboxdir, $for_destroy) = tmpdir();
diff --git a/t/v2-add-remove-add.t b/t/v2-add-remove-add.t
index 7cce2493..8be55123 100644
--- a/t/v2-add-remove-add.t
+++ b/t/v2-add-remove-add.t
@@ -6,11 +6,7 @@ use Test::More;
use PublicInbox::MIME;
use PublicInbox::TestCommon;
require_git(2.6);
-
-foreach my $mod (qw(DBD::SQLite Search::Xapian)) {
- eval "require $mod";
- plan skip_all => "$mod missing for v2-add-remove-add.t" if $@;
-}
+require_mods(qw(DBD::SQLite Search::Xapian));
use_ok 'PublicInbox::V2Writable';
my ($inboxdir, $for_destroy) = tmpdir();
my $ibx = {
diff --git a/t/v2mda.t b/t/v2mda.t
index 4b1249ea..bd49a38a 100644
--- a/t/v2mda.t
+++ b/t/v2mda.t
@@ -10,10 +10,7 @@ use PublicInbox::TestCommon;
require_git(2.6);
my $V = 2;
-foreach my $mod (qw(DBD::SQLite Search::Xapian)) {
- eval "require $mod";
- plan skip_all => "$mod missing for v2mda.t" if $@;
-}
+require_mods(qw(DBD::SQLite Search::Xapian));
use_ok 'PublicInbox::V2Writable';
my ($tmpdir, $for_destroy) = tmpdir();
my $ibx = {
diff --git a/t/v2mirror.t b/t/v2mirror.t
index e3c384fa..64b2f636 100644
--- a/t/v2mirror.t
+++ b/t/v2mirror.t
@@ -9,11 +9,8 @@ require_git(2.6);
local $ENV{HOME} = abs_path('t');
# Integration tests for HTTP cloning + mirroring
-foreach my $mod (qw(Plack::Util Plack::Builder
- HTTP::Date HTTP::Status Search::Xapian DBD::SQLite)) {
- eval "require $mod";
- plan skip_all => "$mod missing for v2mirror.t" if $@;
-}
+require_mods(qw(Plack::Util Plack::Builder
+ HTTP::Date HTTP::Status Search::Xapian DBD::SQLite));
use IO::Socket;
use POSIX qw(dup2);
use_ok 'PublicInbox::V2Writable';
diff --git a/t/v2reindex.t b/t/v2reindex.t
index a14bf798..2c6bdd57 100644
--- a/t/v2reindex.t
+++ b/t/v2reindex.t
@@ -8,11 +8,7 @@ use PublicInbox::ContentId qw(content_digest);
use File::Path qw(remove_tree);
use PublicInbox::TestCommon;
require_git(2.6);
-
-foreach my $mod (qw(DBD::SQLite Search::Xapian)) {
- eval "require $mod";
- plan skip_all => "$mod missing for v2reindex.t" if $@;
-}
+require_mods(qw(DBD::SQLite Search::Xapian));
use_ok 'PublicInbox::V2Writable';
my ($inboxdir, $for_destroy) = tmpdir();
my $ibx_config = {
diff --git a/t/v2writable.t b/t/v2writable.t
index bf2064d0..63f27f96 100644
--- a/t/v2writable.t
+++ b/t/v2writable.t
@@ -8,10 +8,7 @@ use PublicInbox::ContentId qw(content_digest);
use PublicInbox::TestCommon;
use Cwd qw(abs_path);
require_git(2.6);
-foreach my $mod (qw(DBD::SQLite Search::Xapian)) {
- eval "require $mod";
- plan skip_all => "$mod missing for nntpd.t" if $@;
-}
+require_mods(qw(DBD::SQLite Search::Xapian));
local $ENV{HOME} = abs_path('t');
use_ok 'PublicInbox::V2Writable';
umask 007;
diff --git a/t/watch_filter_rubylang.t b/t/watch_filter_rubylang.t
index 0ea680ee..72895d85 100644
--- a/t/watch_filter_rubylang.t
+++ b/t/watch_filter_rubylang.t
@@ -6,12 +6,7 @@ use PublicInbox::TestCommon;
use Test::More;
use PublicInbox::MIME;
use PublicInbox::Config;
-my @mods = qw(Filesys::Notify::Simple DBD::SQLite Search::Xapian);
-foreach my $mod (@mods) {
- eval "require $mod";
- plan skip_all => "$mod missing for watch_filter_rubylang_v2.t" if $@;
-}
-
+require_mods(qw(Filesys::Notify::Simple DBD::SQLite Search::Xapian));
use_ok 'PublicInbox::WatchMaildir';
use_ok 'PublicInbox::Emergency';
my ($tmpdir, $for_destroy) = tmpdir();
diff --git a/t/watch_maildir.t b/t/watch_maildir.t
index 38297043..a5fb52b2 100644
--- a/t/watch_maildir.t
+++ b/t/watch_maildir.t
@@ -6,12 +6,7 @@ use Email::MIME;
use Cwd;
use PublicInbox::Config;
use PublicInbox::TestCommon;
-my @mods = qw(Filesys::Notify::Simple);
-foreach my $mod (@mods) {
- eval "require $mod";
- plan skip_all => "$mod missing for watch_maildir.t" if $@;
-}
-
+require_mods(qw(Filesys::Notify::Simple));
my ($tmpdir, $for_destroy) = tmpdir();
my $git_dir = "$tmpdir/test.git";
my $maildir = "$tmpdir/md";
diff --git a/t/watch_maildir_v2.t b/t/watch_maildir_v2.t
index 77e4b981..db407f78 100644
--- a/t/watch_maildir_v2.t
+++ b/t/watch_maildir_v2.t
@@ -7,11 +7,7 @@ use Cwd;
use PublicInbox::Config;
use PublicInbox::TestCommon;
require_git(2.6);
-my @mods = qw(Search::Xapian DBD::SQLite Filesys::Notify::Simple);
-foreach my $mod (@mods) {
- eval "require $mod";
- plan skip_all => "$mod missing for watch_maildir_v2.t" if $@;
-}
+require_mods(qw(Search::Xapian DBD::SQLite Filesys::Notify::Simple));
require PublicInbox::V2Writable;
my ($tmpdir, $for_destroy) = tmpdir();
my $inboxdir = "$tmpdir/v2";
diff --git a/t/www_listing.t b/t/www_listing.t
index 75698ee5..4fcb43ea 100644
--- a/t/www_listing.t
+++ b/t/www_listing.t
@@ -6,12 +6,8 @@ use warnings;
use Test::More;
use PublicInbox::Spawn qw(which);
use PublicInbox::TestCommon;
-my @mods = qw(URI::Escape Plack::Builder Digest::SHA
- IO::Compress::Gzip IO::Uncompress::Gunzip HTTP::Tiny);
-foreach my $mod (@mods) {
- eval("require $mod") or plan skip_all => "$mod missing for $0";
-}
-
+require_mods(qw(URI::Escape Plack::Builder Digest::SHA
+ IO::Compress::Gzip IO::Uncompress::Gunzip HTTP::Tiny));
require PublicInbox::WwwListing;
my $json = eval { PublicInbox::WwwListing::_json() };
plan skip_all => "JSON module missing: $@" if $@;
diff --git a/t/xcpdb-reshard.t b/t/xcpdb-reshard.t
index 0d78d2b2..1eae234d 100644
--- a/t/xcpdb-reshard.t
+++ b/t/xcpdb-reshard.t
@@ -3,12 +3,8 @@
use strict;
use warnings;
use Test::More;
-my @mods = qw(DBI DBD::SQLite Search::Xapian);
-foreach my $mod (@mods) {
- eval "require $mod";
- plan skip_all => "missing $mod for $0" if $@;
-};
use PublicInbox::TestCommon;
+require_mods(qw(DBD::SQLite Search::Xapian));
require_git('2.6');
use PublicInbox::MIME;
use PublicInbox::InboxWritable;
diff --git a/xt/git-http-backend.t b/xt/git-http-backend.t
index 5f96369f..421c6316 100644
--- a/xt/git-http-backend.t
+++ b/xt/git-http-backend.t
@@ -11,12 +11,8 @@ use PublicInbox::TestCommon;
my $git_dir = $ENV{GIANT_GIT_DIR};
plan 'skip_all' => 'GIANT_GIT_DIR not defined' unless $git_dir;
-foreach my $mod (qw(BSD::Resource
- Plack::Util Plack::Builder
- HTTP::Date HTTP::Status Net::HTTP)) {
- eval "require $mod";
- plan skip_all => "$mod missing for git-http-backend.t" if $@;
-}
+require_mods(qw(BSD::Resource Plack::Util Plack::Builder
+ HTTP::Date HTTP::Status Net::HTTP));
my $psgi = "./t/git-http-backend.psgi";
my ($tmpdir, $for_destroy) = tmpdir();
my $err = "$tmpdir/stderr.log";
diff --git a/xt/msgtime_cmp.t b/xt/msgtime_cmp.t
index 469756c6..4f798006 100644
--- a/xt/msgtime_cmp.t
+++ b/xt/msgtime_cmp.t
@@ -9,7 +9,7 @@ use PublicInbox::Inbox;
use PublicInbox::Git;
use PublicInbox::MsgTime qw(msg_timestamp msg_datestamp);
use POSIX qw(strftime);
-eval { require Date::Parse } or plan skip_all => "Date::Parse missing for $0";
+require_mods('Date::Parse');
my $git;
my ($inboxdir, $git_dir) = @ENV{qw(GIANT_INBOX_DIR GIANT_GIT_DIR)};
if (defined $inboxdir) {
diff --git a/xt/nntpd-validate.t b/xt/nntpd-validate.t
index c82d46e2..bf5c1ba8 100644
--- a/xt/nntpd-validate.t
+++ b/xt/nntpd-validate.t
@@ -8,16 +8,13 @@ use Test::More;
use Symbol qw(gensym);
use Time::HiRes qw(clock_gettime CLOCK_MONOTONIC);
use POSIX qw(_exit);
+use PublicInbox::TestCommon;
my $inbox_dir = $ENV{GIANT_INBOX_DIR};
plan skip_all => "GIANT_INBOX_DIR not defined for $0" unless $inbox_dir;
my $mid = $ENV{TEST_MID};
# Net::NNTP is part of the standard library, but distros may split it off...
-foreach my $mod (qw(DBD::SQLite Net::NNTP Compress::Raw::Zlib)) {
- eval "require $mod";
- plan skip_all => "$mod missing for $0" if $@;
-}
-
+require_mods(qw(DBD::SQLite Net::NNTP Compress::Raw::Zlib));
my $test_compress = Net::NNTP->can('compress');
if (!$test_compress) {
diag 'Your Net::NNTP does not yet support compression';
@@ -29,7 +26,6 @@ my $key = 'certs/server-key.pem';
if ($test_tls && !-r $key || !-r $cert) {
plan skip_all => "certs/ missing for $0, run $^X ./certs/create-certs.perl";
}
-use PublicInbox::TestCommon;
my ($tmpdir, $ftd) = tmpdir();
$File::Temp::KEEP_ALL = !!$ENV{TEST_KEEP_TMP};
my (%OPT, $td, $host_port, $group);
diff --git a/xt/perf-nntpd.t b/xt/perf-nntpd.t
index df5ecb41..e4fd7355 100644
--- a/xt/perf-nntpd.t
+++ b/xt/perf-nntpd.t
@@ -8,7 +8,6 @@ use PublicInbox::Inbox;
use Net::NNTP;
my $inboxdir = $ENV{GIANT_INBOX_DIR} // $ENV{GIANT_PI_DIR};
plan skip_all => "GIANT_INBOX_DIR not defined for $0" unless defined($inboxdir);
-eval { require PublicInbox::Search };
my ($host_port, $group, %opts, $s, $td, $tmp_obj);
use PublicInbox::TestCommon;
^ permalink raw reply related [flat|nested] 6+ messages in thread* [PATCH 3/4] search: support SWIG-generated Xapian.pm
2019-12-22 22:17 [PATCH 0/4] support SWIG-generated Xapian binding Eric Wong
2019-12-22 22:17 ` [PATCH 1/4] testcommon: add require_mods method and use it Eric Wong
2019-12-22 22:17 ` [PATCH 2/4] searchidx: call "++" on PostingIterator instead of "->inc" Eric Wong
@ 2019-12-22 22:17 ` Eric Wong
2019-12-23 18:07 ` Eric Wong
2019-12-22 22:17 ` [PATCH 4/4] t/search.t: update permissions check for OpenBSD Eric Wong
3 siblings, 1 reply; 6+ messages in thread
From: Eric Wong @ 2019-12-22 22:17 UTC (permalink / raw)
To: meta
Xapian upstream is slowly phasing out the XS-based Search::Xapian
in favor of the SWIG-generated "Xapian" package. While Debian and
both FreeBSD have Search::Xapian, OpenBSD only includes the "Xapian"
binding.
More information about the status of the "Xapian" Perl module here:
https://trac.xapian.org/ticket/523
---
lib/PublicInbox/Admin.pm | 4 +++
lib/PublicInbox/Search.pm | 57 ++++++++++++++++++++++++-----------
lib/PublicInbox/SearchIdx.pm | 32 ++++++++++++++------
lib/PublicInbox/SearchMsg.pm | 3 +-
lib/PublicInbox/TestCommon.pm | 10 +++++-
lib/PublicInbox/V2Writable.pm | 5 ++-
lib/PublicInbox/Xapcmd.pm | 25 +++++++++------
t/indexlevels-mirror.t | 3 +-
t/psgi_multipart_not.t | 2 +-
t/psgi_search.t | 2 +-
t/psgi_v2.t | 7 ++---
t/xcpdb-reshard.t | 4 ++-
12 files changed, 105 insertions(+), 49 deletions(-)
diff --git a/lib/PublicInbox/Admin.pm b/lib/PublicInbox/Admin.pm
index 3d0d80b9..32a9f65e 100644
--- a/lib/PublicInbox/Admin.pm
+++ b/lib/PublicInbox/Admin.pm
@@ -159,6 +159,10 @@ sub check_require {
while (my $mod = shift @mods) {
if (my $groups = $mod_groups{$mod}) {
push @mods, @$groups;
+ } elsif ($mod eq 'Search::Xapian') {
+ require PublicInbox::Search;
+ PublicInbox::Search::load_xapian() or
+ $err->{'Search::Xapian || Xapian'} = $@;
} else {
eval "require $mod";
$err->{$mod} = $@ if $@;
diff --git a/lib/PublicInbox/Search.pm b/lib/PublicInbox/Search.pm
index 88c8dc70..7d83419d 100644
--- a/lib/PublicInbox/Search.pm
+++ b/lib/PublicInbox/Search.pm
@@ -17,20 +17,42 @@ use PublicInbox::MIME;
use PublicInbox::MID qw/id_compress/;
use PublicInbox::Over;
my $QP_FLAGS;
+our %X = map { $_ => 0 } qw(BoolWeight Database Enquire
+ NumberValueRangeProcessor QueryParser Stem);
+our $Xap; # 'Search::Xapian' or 'Xapian'
+my $ENQ_ASCENDING;
+
sub load_xapian () {
- $QP_FLAGS ||= eval {
- require Search::Xapian;
- Search::Xapian->import(qw(:standard));
+ return 1 if defined $Xap;
+ for my $x (qw(Xapian Search::Xapian)) {
+ eval "require $x";
+ next if $@;
+
+ $x->import(qw(:standard));
+ $Xap = $x;
+ $X{$_} = $Xap.'::'.$_ for (keys %X);
+ # ENQ_ASCENDING doesn't seem exported by SWIG Xapian.pm,
+ # so lets hope this part of the ABI is stable because it's
+ # just an integer:
+ $ENQ_ASCENDING = $x eq 'Xapian' ?
+ 1 : Search::Xapian::ENQ_ASCENDING();
+
+ # for SearchMsg:
+ *PublicInbox::SearchMsg::sortable_unserialise =
+ $Xap.'::sortable_unserialise';
# n.b. FLAG_PURE_NOT is expensive not suitable for a public
# website as it could become a denial-of-service vector
# FLAG_PHRASE also seems to cause performance problems chert
# (and probably earlier Xapian DBs). glass seems fine...
# TODO: make this an option, maybe?
# or make indexlevel=medium as default
- FLAG_PHRASE()|FLAG_BOOLEAN()|FLAG_LOVEHATE()|FLAG_WILDCARD();
- };
-};
+ $QP_FLAGS = FLAG_PHRASE() | FLAG_BOOLEAN() | FLAG_LOVEHATE() |
+ FLAG_WILDCARD();
+ last;
+ }
+ undef;
+}
# This is English-only, everything else is non-standard and may be confused as
# a prefix common in patch emails
@@ -148,7 +170,7 @@ sub _xdb ($) {
if ($self->{version} >= 2) {
foreach my $shard (<$dir/*>) {
-d $shard && $shard =~ m!/[0-9]+\z! or next;
- my $sub = Search::Xapian::Database->new($shard);
+ my $sub = $X{Database}->new($shard);
if ($xdb) {
$xdb->add_database($sub);
} else {
@@ -158,7 +180,7 @@ sub _xdb ($) {
}
} else {
$slow_phrase = -f "$dir/iamchert";
- $xdb = Search::Xapian::Database->new($dir);
+ $xdb = $X{Database}->new($dir);
}
$$qpf |= FLAG_PHRASE() unless $slow_phrase;
$xdb;
@@ -222,7 +244,7 @@ sub retry_reopen {
}
# Exception: The revision being read has been discarded -
# you should call Xapian::Database::reopen()
- if (ref($@) eq 'Search::Xapian::DatabaseModifiedError') {
+ if (ref($@) =~ /\bDatabaseModifiedError\b/) {
warn "reopen try #$i on $@\n";
reopen($self);
} else {
@@ -243,13 +265,13 @@ sub _do_enquire {
sub _enquire_once {
my ($self, $query, $opts) = @_;
my $xdb = xdb($self);
- my $enquire = Search::Xapian::Enquire->new($xdb);
+ my $enquire = $X{Enquire}->new($xdb);
$enquire->set_query($query);
$opts ||= {};
my $desc = !$opts->{asc};
if (($opts->{mset} || 0) == 2) {
- $enquire->set_docid_order(Search::Xapian::ENQ_ASCENDING());
- $enquire->set_weighting_scheme(Search::Xapian::BoolWeight->new);
+ $enquire->set_docid_order($ENQ_ASCENDING);
+ $enquire->set_weighting_scheme($X{BoolWeight}->new);
} elsif ($opts->{relevance}) {
$enquire->set_sort_by_relevance_then_value(TS, $desc);
} else {
@@ -268,7 +290,7 @@ sub _enquire_once {
}
# read-write
-sub stemmer { Search::Xapian::Stem->new($LANG) }
+sub stemmer { $X{Stem}->new($LANG) }
# read-only
sub qp {
@@ -278,16 +300,15 @@ sub qp {
return $qp if $qp;
my $xdb = xdb($self);
# new parser
- $qp = Search::Xapian::QueryParser->new;
+ $qp = $X{QueryParser}->new;
$qp->set_default_op(OP_AND());
$qp->set_database($xdb);
$qp->set_stemmer($self->stemmer);
$qp->set_stemming_strategy(STEM_SOME());
$qp->set_max_wildcard_expansion(100);
- $qp->add_valuerangeprocessor(
- Search::Xapian::NumberValueRangeProcessor->new(YYYYMMDD, 'd:'));
- $qp->add_valuerangeprocessor(
- Search::Xapian::NumberValueRangeProcessor->new(DT, 'dt:'));
+ my $nvrp = $X{NumberValueRangeProcessor};
+ $qp->add_valuerangeprocessor($nvrp->new(YYYYMMDD, 'd:'));
+ $qp->add_valuerangeprocessor($nvrp->new(DT, 'dt:'));
while (my ($name, $prefix) = each %bool_pfx_external) {
$qp->add_boolean_prefix($name, $_) foreach split(/ /, $prefix);
diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm
index b56fd0ee..21ab8119 100644
--- a/lib/PublicInbox/SearchIdx.pm
+++ b/lib/PublicInbox/SearchIdx.pm
@@ -19,7 +19,8 @@ use POSIX qw(strftime);
use PublicInbox::OverIdx;
use PublicInbox::Spawn qw(spawn);
use PublicInbox::Git qw(git_unquote);
-
+my $X = \%PublicInbox::Search::X;
+my ($DB_CREATE_OR_OPEN, $DB_OPEN);
use constant {
BATCH_BYTES => defined($ENV{XAPIAN_FLUSH_THRESHOLD}) ?
0x7fffffff : 1_000_000,
@@ -85,17 +86,28 @@ sub _xdb_release {
undef;
}
+sub load_xapian_writable () {
+ return 1 if $X->{WritableDatabase};
+ PublicInbox::Search::load_xapian() or return;
+ my $xap = $PublicInbox::Search::Xap;
+ for (qw(Document TermGenerator WritableDatabase)) {
+ $X->{$_} = $xap.'::'.$_;
+ }
+ eval 'require '.$X->{WritableDatabase} or die;
+ *sortable_serialise = $xap.'::sortable_serialise';
+ $DB_CREATE_OR_OPEN = eval($xap.'::DB_CREATE_OR_OPEN()');
+ $DB_OPEN = eval($xap.'::DB_OPEN()');
+ 1;
+}
+
sub _xdb_acquire {
my ($self) = @_;
my $flag;
my $dir = $self->xdir;
if (need_xapian($self)) {
croak 'already acquired' if $self->{xdb};
- PublicInbox::Search::load_xapian();
- require Search::Xapian::WritableDatabase;
- $flag = $self->{creat} ?
- Search::Xapian::DB_CREATE_OR_OPEN() :
- Search::Xapian::DB_OPEN();
+ load_xapian_writable();
+ $flag = $self->{creat} ? $DB_CREATE_OR_OPEN : $DB_OPEN;
}
if ($self->{creat}) {
require File::Path;
@@ -108,7 +120,7 @@ sub _xdb_acquire {
}
}
return unless defined $flag;
- my $xdb = eval { Search::Xapian::WritableDatabase->new($dir, $flag) };
+ my $xdb = eval { ($X->{WritableDatabase})->new($dir, $flag) };
if ($@) {
die "Failed opening $dir: ", $@;
}
@@ -117,7 +129,7 @@ sub _xdb_acquire {
sub add_val ($$$) {
my ($doc, $col, $num) = @_;
- $num = Search::Xapian::sortable_serialise($num);
+ $num = sortable_serialise($num);
$doc->add_value($col, $num);
}
@@ -274,7 +286,7 @@ sub index_body ($$$) {
sub add_xapian ($$$$$) {
my ($self, $mime, $num, $oid, $mids, $mid0) = @_;
my $smsg = PublicInbox::SearchMsg->new($mime);
- my $doc = Search::Xapian::Document->new;
+ my $doc = $X->{Document}->new;
my $subj = $smsg->subject;
add_val($doc, PublicInbox::Search::TS(), $smsg->ts);
my @ds = gmtime($smsg->ds);
@@ -458,7 +470,7 @@ sub term_generator { # write-only
my $tg = $self->{term_generator};
return $tg if $tg;
- $tg = Search::Xapian::TermGenerator->new;
+ $tg = $X->{TermGenerator}->new;
$tg->set_stemmer($self->stemmer);
$self->{term_generator} = $tg;
diff --git a/lib/PublicInbox/SearchMsg.pm b/lib/PublicInbox/SearchMsg.pm
index 7561e7f2..53882f73 100644
--- a/lib/PublicInbox/SearchMsg.pm
+++ b/lib/PublicInbox/SearchMsg.pm
@@ -27,7 +27,8 @@ sub wrap {
sub get_val ($$) {
my ($doc, $col) = @_;
- Search::Xapian::sortable_unserialise($doc->get_value($col));
+ # sortable_unserialise is defined by PublicInbox::Search::load_xapian()
+ sortable_unserialise($doc->get_value($col));
}
sub to_doc_data {
diff --git a/lib/PublicInbox/TestCommon.pm b/lib/PublicInbox/TestCommon.pm
index 9680b94c..b0b1f4d9 100644
--- a/lib/PublicInbox/TestCommon.pm
+++ b/lib/PublicInbox/TestCommon.pm
@@ -64,7 +64,15 @@ sub require_mods {
my $maybe = pop @mods if $mods[-1] =~ /\A[0-9]+\z/;
my @need;
for my $mod (@mods) {
- eval "require $mod";
+ if ($mod eq 'Search::Xapian') {
+ require PublicInbox::Search;
+ PublicInbox::Search::load_xapian() and next;
+ } elsif ($mod eq 'Search::Xapian::WritableDatabase') {
+ require PublicInbox::SearchIdx;
+ PublicInbox::SearchIdx::load_xapian_writable() and next;
+ } else {
+ eval "require $mod";
+ }
push @need, $mod if $@;
}
return unless @need;
diff --git a/lib/PublicInbox/V2Writable.pm b/lib/PublicInbox/V2Writable.pm
index ab80941a..77b3bde4 100644
--- a/lib/PublicInbox/V2Writable.pm
+++ b/lib/PublicInbox/V2Writable.pm
@@ -62,10 +62,13 @@ sub count_shards ($) {
# Also, shard count may change while -watch is running
# due to "xcpdb --reshard"
if (-d $xpfx) {
+ require PublicInbox::Search;
+ PublicInbox::Search::load_xapian();
+ my $XapianDatabase = $PublicInbox::Search::X{Database};
foreach my $shard (<$xpfx/*>) {
-d $shard && $shard =~ m!/[0-9]+\z! or next;
eval {
- Search::Xapian::Database->new($shard)->close;
+ $XapianDatabase->new($shard)->close;
$n++;
};
}
diff --git a/lib/PublicInbox/Xapcmd.pm b/lib/PublicInbox/Xapcmd.pm
index 78b3a9ac..9f897dad 100644
--- a/lib/PublicInbox/Xapcmd.pm
+++ b/lib/PublicInbox/Xapcmd.pm
@@ -5,7 +5,7 @@ use strict;
use warnings;
use PublicInbox::Spawn qw(which spawn);
use PublicInbox::Over;
-use PublicInbox::Search;
+use PublicInbox::SearchIdx;
use File::Temp ();
use File::Path qw(remove_tree);
use File::Basename qw(dirname);
@@ -99,7 +99,7 @@ sub prepare_reindex ($$$) {
my ($ibx, $im, $reindex) = @_;
if ($ibx->{version} == 1) {
my $dir = $ibx->search->xdir(1);
- my $xdb = Search::Xapian::Database->new($dir);
+ my $xdb = $PublicInbox::Search::X{Database}->new($dir);
if (my $lc = $xdb->get_metadata('last_commit')) {
$reindex->{from} = $lc;
}
@@ -164,7 +164,8 @@ sub run {
if (!$opt->{-coarse_lock}) {
$reindex = $opt->{reindex} = {};
$from = $reindex->{from} = [];
- require Search::Xapian::WritableDatabase;
+ require PublicInbox::SearchIdx;
+ PublicInbox::SearchIdx::load_xapian_writable();
}
$ibx->umask_prepare;
@@ -251,7 +252,7 @@ sub run {
sub cpdb_retryable ($$) {
my ($src, $pfx) = @_;
- if (ref($@) eq 'Search::Xapian::DatabaseModifiedError') {
+ if (ref($@) =~ /\bDatabaseModifiedError\b/) {
warn "$pfx Xapian DB modified, reopening and retrying\n";
$src->reopen;
return 1;
@@ -361,6 +362,8 @@ sub cpdb ($$) {
my $new = $newdir->dirname;
my ($src, $cur_shard);
my $reshard;
+ PublicInbox::SearchIdx::load_xapian_writable() or die;
+ my $XapianDatabase = $PublicInbox::Search::X{Database};
if (ref($old) eq 'ARRAY') {
($cur_shard) = ($new =~ m!xap[0-9]+/([0-9]+)\b!);
defined $cur_shard or
@@ -371,14 +374,14 @@ sub cpdb ($$) {
# resharding, M:N copy means have full read access
foreach (@$old) {
if ($src) {
- my $sub = Search::Xapian::Database->new($_);
+ my $sub = $XapianDatabase->new($_);
$src->add_database($sub);
} else {
- $src = Search::Xapian::Database->new($_);
+ $src = $XapianDatabase->new($_);
}
}
} else {
- $src = Search::Xapian::Database->new($old);
+ $src = $XapianDatabase->new($old);
}
my ($tmp, $ft);
@@ -395,8 +398,10 @@ sub cpdb ($$) {
# like copydatabase(1), be sure we don't overwrite anything in case
# of other bugs:
- my $creat = Search::Xapian::DB_CREATE();
- my $dst = Search::Xapian::WritableDatabase->new($tmp, $creat);
+ my $creat = eval($PublicInbox::Search::Xap.'::DB_CREATE()');
+ die if $@;
+ my $XapianWritableDatabase = $PublicInbox::Search::X{WritableDatabase};
+ my $dst = $XapianWritableDatabase->new($tmp, $creat);
my $pr = $opt->{-progress};
my $pfx = $opt->{-progress_pfx} = progress_pfx($new);
my $pr_data = { pr => $pr, pfx => $pfx, nr => 0 } if $pr;
@@ -439,7 +444,7 @@ sub cpdb ($$) {
# individually.
$src = undef;
foreach (@$old) {
- my $old = Search::Xapian::Database->new($_);
+ my $old = $XapianDatabase->new($_);
cpdb_loop($old, $dst, $pr_data, $cur_shard, $reshard);
}
} else {
diff --git a/t/indexlevels-mirror.t b/t/indexlevels-mirror.t
index e4313faa..2244870b 100644
--- a/t/indexlevels-mirror.t
+++ b/t/indexlevels-mirror.t
@@ -165,7 +165,8 @@ import_index_incremental($PI_TEST_VERSION, 'basic', $mime);
SKIP: {
require PublicInbox::Search;
- PublicInbox::Search::load_xapian() or skip 'Search::Xapian missing', 2;
+ PublicInbox::Search::load_xapian() or
+ skip('Xapian perl binding missing', 2);
foreach my $l (qw(medium full)) {
import_index_incremental($PI_TEST_VERSION, $l, $mime);
}
diff --git a/t/psgi_multipart_not.t b/t/psgi_multipart_not.t
index fbc09f55..606151c4 100644
--- a/t/psgi_multipart_not.t
+++ b/t/psgi_multipart_not.t
@@ -10,7 +10,7 @@ use PublicInbox::TestCommon;
my @mods = qw(DBD::SQLite Search::Xapian HTTP::Request::Common
Plack::Test URI::Escape Plack::Builder Plack::Test);
require_mods(@mods);
-use_ok($_) for @mods;
+use_ok($_) for (qw(HTTP::Request::Common Plack::Test));
use_ok 'PublicInbox::V2Writable';
my ($repo, $for_destroy) = tmpdir();
my $ibx = PublicInbox::Inbox->new({
diff --git a/t/psgi_search.t b/t/psgi_search.t
index 95749c66..534063f8 100644
--- a/t/psgi_search.t
+++ b/t/psgi_search.t
@@ -13,7 +13,7 @@ use PublicInbox::TestCommon;
my @mods = qw(DBD::SQLite Search::Xapian HTTP::Request::Common Plack::Test
URI::Escape Plack::Builder);
require_mods(@mods);
-use_ok $_ foreach (@mods, qw(PublicInbox::SearchIdx));
+use_ok($_) for (qw(HTTP::Request::Common Plack::Test));
my ($tmpdir, $for_destroy) = tmpdir();
my $ibx = PublicInbox::Inbox->new({
diff --git a/t/psgi_v2.t b/t/psgi_v2.t
index 9a81ee9a..a02b90fb 100644
--- a/t/psgi_v2.t
+++ b/t/psgi_v2.t
@@ -9,10 +9,9 @@ use PublicInbox::MIME;
use PublicInbox::Config;
use PublicInbox::WWW;
use PublicInbox::MID qw(mids);
-my @mods = qw(DBD::SQLite Search::Xapian HTTP::Request::Common Plack::Test
- URI::Escape Plack::Builder);
-require_mods(@mods);
-use_ok($_) for @mods;
+require_mods(qw(DBD::SQLite Search::Xapian HTTP::Request::Common Plack::Test
+ URI::Escape Plack::Builder));
+use_ok($_) for (qw(HTTP::Request::Common Plack::Test));
use_ok 'PublicInbox::V2Writable';
my ($inboxdir, $for_destroy) = tmpdir();
my $ibx = {
diff --git a/t/xcpdb-reshard.t b/t/xcpdb-reshard.t
index 1eae234d..2a0aeb45 100644
--- a/t/xcpdb-reshard.t
+++ b/t/xcpdb-reshard.t
@@ -8,6 +8,7 @@ require_mods(qw(DBD::SQLite Search::Xapian));
require_git('2.6');
use PublicInbox::MIME;
use PublicInbox::InboxWritable;
+use PublicInbox::Search;
my $mime = PublicInbox::MIME->create(
header => [
@@ -61,8 +62,9 @@ for my $R (qw(2 4 1 3 3)) {
# ensure docids in Xapian match NNTP article numbers
my $tot = 0;
my %tmp = %nums;
+ my $XapianDatabase = $PublicInbox::Search::X{Database};
foreach my $d (@new_shards) {
- my $xdb = Search::Xapian::Database->new($d);
+ my $xdb = $XapianDatabase->new($d);
$tot += $xdb->get_doccount;
my $it = $xdb->postlist_begin('');
my $end = $xdb->postlist_end('');
^ permalink raw reply related [flat|nested] 6+ messages in thread