From 0b1de991a099b5e8b9a9e3e85b5eaaacc9362dbb Mon Sep 17 00:00:00 2001 From: Eric Wong Date: Wed, 15 May 2019 01:18:08 +0000 Subject: lazy load Xapian and make it optional for v2 More tests work without Search::Xapian, now. Usability issues still need to be fixed --- t/admin.t | 2 +- t/cgi.t | 16 ++++++++++++---- t/indexlevels-mirror.t | 17 +++++++++++------ t/nntp.t | 2 +- t/nntpd.t | 15 +++++++++++---- t/over.t | 3 +-- t/psgi_bad_mids.t | 3 ++- t/psgi_scan_all.t | 4 ++-- t/psgi_search.t | 4 ++-- t/purge.t | 2 +- t/search-thr-index.t | 5 +++-- t/search.t | 7 ++++--- t/v1reindex.t | 3 +-- t/v2reindex.t | 5 +++-- t/watch_maildir_v2.t | 4 ++-- 15 files changed, 57 insertions(+), 35 deletions(-) (limited to 't') diff --git a/t/admin.t b/t/admin.t index b5a4383e..3790c9e2 100644 --- a/t/admin.t +++ b/t/admin.t @@ -50,7 +50,7 @@ SKIP: { # v2 SKIP: { - for my $m (qw(DBD::SQLite Search::Xapian)) { + for my $m (qw(DBD::SQLite)) { skip "$m missing", 5 unless eval "require $m"; } use_ok 'PublicInbox::V2Writable'; diff --git a/t/cgi.t b/t/cgi.t index b24bbc4a..d3172bf7 100644 --- a/t/cgi.t +++ b/t/cgi.t @@ -30,6 +30,7 @@ my $cfgpfx = "publicinbox.test"; my %cfg = ( "$cfgpfx.address" => $addr, "$cfgpfx.mainrepo" => $maindir, + "$cfgpfx.indexlevel" => 'basic', ); while (my ($k,$v) = each %cfg) { is(0, system(qw(git config --file), $pi_config, $k, $v), @@ -39,9 +40,12 @@ my $cfgpfx = "publicinbox.test"; use_ok 'PublicInbox::Git'; use_ok 'PublicInbox::Import'; -use_ok 'Email::MIME'; -my $git = PublicInbox::Git->new($maindir); -my $im = PublicInbox::Import->new($git, 'test', $addr); +use_ok 'PublicInbox::Inbox'; +use_ok 'PublicInbox::V1Writable'; +use_ok 'PublicInbox::Config'; +my $cfg = PublicInbox::Config->new($pi_config); +my $ibx = $cfg->lookup_name('test'); +my $im = PublicInbox::V1Writable->new($ibx); { local $ENV{HOME} = $home; @@ -103,8 +107,9 @@ EOF like($res->{head}, qr/^Status: 501 /, "search not-yet-enabled"); my $indexed; eval { + require DBD::SQLite; require PublicInbox::SearchIdx; - my $s = PublicInbox::SearchIdx->new($maindir, 1); + my $s = PublicInbox::SearchIdx->new($ibx, 1); $s->index_sync; $indexed = 1; }; @@ -120,6 +125,7 @@ EOF }; } else { like($res->{head}, qr/^Status: 501 /, "search not available"); + SKIP: { skip 'DBD::SQLite not available', 2 }; } my $have_xml_feed = eval { require XML::Feed; 1 } if $indexed; @@ -132,6 +138,8 @@ EOF my $p = XML::Feed->parse(\($res->{body})); is($p->format, "Atom", "parsed atom feed"); is(scalar $p->entries, 3, "parsed three entries"); + } else { + SKIP: { skip 'DBD::SQLite or XML::Feed missing', 2 }; } } diff --git a/t/indexlevels-mirror.t b/t/indexlevels-mirror.t index e25b827f..cac7050b 100644 --- a/t/indexlevels-mirror.t +++ b/t/indexlevels-mirror.t @@ -10,8 +10,7 @@ require './t/common.perl'; require_git(2.6); my $this = (split('/', __FILE__))[-1]; -# TODO: remove Search::Xapian as a requirement for basic -foreach my $mod (qw(DBD::SQLite Search::Xapian)) { +foreach my $mod (qw(DBD::SQLite)) { eval "require $mod"; plan skip_all => "$mod missing for $this" if $@; } @@ -47,8 +46,11 @@ sub import_index_incremental { $im->done; # index master (required for v1) - is(system($index, $ibx->{mainrepo}), 0, 'index master OK'); - my $ro_master = PublicInbox::Inbox->new({mainrepo => $ibx->{mainrepo}}); + is(system($index, $ibx->{mainrepo}, "-L$level"), 0, 'index master OK'); + my $ro_master = PublicInbox::Inbox->new({ + mainrepo => $ibx->{mainrepo}, + indexlevel => $level + }); my ($nr, $msgs) = $ro_master->recent; is($nr, 1, 'only one message in master, so far'); is($msgs->[0]->{mid}, 'm@1', 'first message in master indexed'); @@ -75,7 +77,10 @@ sub import_index_incremental { is(system($index, $mirror), 0, "v$v index mirror OK"); # read-only access - my $ro_mirror = PublicInbox::Inbox->new({mainrepo => $mirror}); + my $ro_mirror = PublicInbox::Inbox->new({ + mainrepo => $mirror, + indexlevel => 'basic' + }); ($nr, $msgs) = $ro_mirror->recent; is($nr, 1, 'only one message, so far'); is($msgs->[0]->{mid}, 'm@1', 'read first message'); @@ -94,7 +99,7 @@ sub import_index_incremental { ['m@1','m@2'], 'got both messages in mirror'); # incremental index master (required for v1) - is(system($index, $ibx->{mainrepo}), 0, 'index master OK'); + is(system($index, $ibx->{mainrepo}, "-L$level"), 0, 'index master OK'); ($nr, $msgs) = $ro_master->recent; is($nr, 2, '2nd message seen in master'); is_deeply([sort { $a cmp $b } map { $_->{mid} } @$msgs], diff --git a/t/nntp.t b/t/nntp.t index c4b9737c..7ab801c1 100644 --- a/t/nntp.t +++ b/t/nntp.t @@ -4,7 +4,7 @@ use strict; use warnings; use Test::More; -foreach my $mod (qw(DBD::SQLite Search::Xapian Data::Dumper)) { +foreach my $mod (qw(DBD::SQLite Data::Dumper)) { eval "require $mod"; plan skip_all => "$mod missing for nntp.t" if $@; } diff --git a/t/nntpd.t b/t/nntpd.t index b8a84a0c..c7ea3197 100644 --- a/t/nntpd.t +++ b/t/nntpd.t @@ -3,7 +3,7 @@ use strict; use warnings; use Test::More; -foreach my $mod (qw(DBD::SQLite Search::Xapian)) { +foreach my $mod (qw(DBD::SQLite)) { eval "require $mod"; plan skip_all => "$mod missing for nntpd.t" if $@; } @@ -55,12 +55,13 @@ my $ibx = { name => $group, version => $version, -primary_address => $addr, + indexlevel => 'basic', }; $ibx = PublicInbox::Inbox->new($ibx); { local $ENV{HOME} = $home; my @cmd = ($init, $group, $mainrepo, 'http://example.com/', $addr); - push @cmd, "-V$version"; + push @cmd, "-V$version", '-Lbasic'; is(system(@cmd), 0, 'init OK'); is(system(qw(git config), "--file=$home/.public-inbox/config", "publicinbox.$group.newsgroup", $group), @@ -71,8 +72,8 @@ $ibx = PublicInbox::Inbox->new($ibx); if ($version == 2) { $im = PublicInbox::V2Writable->new($ibx); } elsif ($version == 1) { - my $git = PublicInbox::Git->new($mainrepo); - $im = PublicInbox::Import->new($git, 'test', $addr); + use_ok 'PublicInbox::V1Writable'; + $im = PublicInbox::V1Writable->new($ibx); } else { die "unsupported version: $version"; } @@ -261,6 +262,12 @@ EOF is($rdr, waitpid($rdr, 0), 'reader done'); is($? >> 8, 0, 'no errors'); } + SKIP: { + my @of = `lsof -p $pid 2>/dev/null`; + skip('lsof broken', 1) if (!scalar(@of) || $?); + my @xap = grep m!Search/Xapian!, @of; + is_deeply(\@xap, [], 'Xapian not loaded in nntpd'); + } { setsockopt($s, IPPROTO_TCP, TCP_NODELAY, 1); syswrite($s, 'HDR List-id 1-'); diff --git a/t/over.t b/t/over.t index 7a3c9721..c0d9d5e5 100644 --- a/t/over.t +++ b/t/over.t @@ -5,8 +5,7 @@ use warnings; use Test::More; use File::Temp qw/tempdir/; use Compress::Zlib qw(compress); -# FIXME: allow using Over w/o Xapian -foreach my $mod (qw(DBD::SQLite Search::Xapian)) { +foreach my $mod (qw(DBD::SQLite)) { eval "require $mod"; plan skip_all => "$mod missing for over.t" if $@; } diff --git a/t/psgi_bad_mids.t b/t/psgi_bad_mids.t index 5008f5be..71eefe59 100644 --- a/t/psgi_bad_mids.t +++ b/t/psgi_bad_mids.t @@ -7,7 +7,7 @@ use File::Temp qw/tempdir/; use PublicInbox::MIME; use PublicInbox::Config; use PublicInbox::WWW; -my @mods = qw(DBD::SQLite Search::Xapian HTTP::Request::Common Plack::Test +my @mods = qw(DBD::SQLite HTTP::Request::Common Plack::Test URI::Escape Plack::Builder); foreach my $mod (@mods) { eval "require $mod"; @@ -22,6 +22,7 @@ my $ibx = { name => 'bad-mids', version => 2, -primary_address => 'test@example.com', + indexlevel => 'basic', }; $ibx = PublicInbox::Inbox->new($ibx); my $im = PublicInbox::V2Writable->new($ibx, 1); diff --git a/t/psgi_scan_all.t b/t/psgi_scan_all.t index e9c439ec..2f54c820 100644 --- a/t/psgi_scan_all.t +++ b/t/psgi_scan_all.t @@ -6,8 +6,7 @@ use Test::More; use Email::MIME; use File::Temp qw/tempdir/; use PublicInbox::Config; -my @mods = qw(HTTP::Request::Common Plack::Test URI::Escape Search::Xapian - DBD::SQLite); +my @mods = qw(HTTP::Request::Common Plack::Test URI::Escape DBD::SQLite); foreach my $mod (@mods) { eval "require $mod"; plan skip_all => "$mod missing for psgi_scan_all.t" if $@; @@ -26,6 +25,7 @@ foreach my $i (1..2) { mainrepo => $mainrepo, name => "test-$i", version => 2, + indexlevel => 'basic', -primary_address => $addr, }; my $ibx = PublicInbox::Inbox->new($opt); diff --git a/t/psgi_search.t b/t/psgi_search.t index da6cc682..1adc1d6e 100644 --- a/t/psgi_search.t +++ b/t/psgi_search.t @@ -8,13 +8,13 @@ use Email::MIME; use PublicInbox::Config; use PublicInbox::WWW; use bytes (); # only for bytes::length -my @mods = qw(PublicInbox::SearchIdx HTTP::Request::Common Plack::Test +my @mods = qw(Search::Xapian HTTP::Request::Common Plack::Test URI::Escape Plack::Builder); foreach my $mod (@mods) { eval "require $mod"; plan skip_all => "$mod missing for psgi_search.t" if $@; } -use_ok $_ foreach @mods; +use_ok $_ foreach (@mods, qw(PublicInbox::SearchIdx)); my $tmpdir = tempdir('pi-psgi-search.XXXXXX', TMPDIR => 1, CLEANUP => 1); my $git_dir = "$tmpdir/a.git"; diff --git a/t/purge.t b/t/purge.t index 574935e0..c1e0e9a0 100644 --- a/t/purge.t +++ b/t/purge.t @@ -6,7 +6,7 @@ use Test::More; use File::Temp qw/tempdir/; require './t/common.perl'; require_git(2.6); -my @mods = qw(IPC::Run DBI DBD::SQLite Search::Xapian); +my @mods = qw(IPC::Run DBI DBD::SQLite); foreach my $mod (@mods) { eval "require $mod"; plan skip_all => "missing $mod for t/purge.t" if $@; diff --git a/t/search-thr-index.t b/t/search-thr-index.t index ed3e4e76..848dc564 100644 --- a/t/search-thr-index.t +++ b/t/search-thr-index.t @@ -7,8 +7,9 @@ use Test::More; use File::Temp qw/tempdir/; use PublicInbox::MID qw(mids); use Email::MIME; -eval { require PublicInbox::SearchIdx; }; -plan skip_all => "Xapian missing for search" if $@; +eval { require Search::Xapian }; +plan skip_all => "Search::Xapian missing for search" if $@; +require PublicInbox::SearchIdx; my $tmpdir = tempdir('pi-search-thr-index.XXXXXX', TMPDIR => 1, CLEANUP => 1); my $git_dir = "$tmpdir/a.git"; diff --git a/t/search.t b/t/search.t index 493e00de..a9d0c920 100644 --- a/t/search.t +++ b/t/search.t @@ -3,8 +3,9 @@ use strict; use warnings; use Test::More; -eval { require PublicInbox::SearchIdx; }; -plan skip_all => "Xapian missing for search" if $@; +eval { require Search::Xapian }; +plan skip_all => "Search::Xapian missing for search" if $@; +require PublicInbox::SearchIdx; use File::Temp qw/tempdir/; use Email::MIME; my $tmpdir = tempdir('pi-search-XXXXXX', TMPDIR => 1, CLEANUP => 1); @@ -12,7 +13,7 @@ my $git_dir = "$tmpdir/a.git"; my ($root_id, $last_id); is(0, system(qw(git init --shared -q --bare), $git_dir), "git init (main)"); -eval { PublicInbox::Search->new($git_dir) }; +eval { PublicInbox::Search->new($git_dir)->xdb }; ok($@, "exception raised on non-existent DB"); my $rw = PublicInbox::SearchIdx->new($git_dir, 1); diff --git a/t/v1reindex.t b/t/v1reindex.t index 33a36fad..402ecd78 100644 --- a/t/v1reindex.t +++ b/t/v1reindex.t @@ -209,8 +209,7 @@ ok(!-d $xap, 'Xapian directories removed again'); delete $ibx->{mm}; is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged'); is($ibx->mm->num_highwater, 10, 'num_highwater as expected'); - my $mset = $ibx->search->reopen->query('hello world', {mset=>1}); - is($mset->size, 0, "no Xapian search results"); + isnt($ibx->search, 'no search for basic'); my ($min, $max) = $ibx->mm->minmax; is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged'); diff --git a/t/v2reindex.t b/t/v2reindex.t index c416629c..2a6fc555 100644 --- a/t/v2reindex.t +++ b/t/v2reindex.t @@ -225,8 +225,9 @@ ok(!-d $xap, 'Xapian directories removed again'); delete $ibx->{mm}; is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged'); is($ibx->mm->num_highwater, 10, 'num_highwater as expected'); - my $mset = $ibx->search->query('freedom', {mset=>1}); - is($mset->size, 0, "search fails on indexlevel='basic'"); + + isnt($ibx->search, 'no search for basic'); + for (<"$xap/*/*">) { $sizes{$ibx->{indexlevel}} += -s _ if -f $_ } ok($sizes{medium} > $sizes{basic}, 'basic is smaller than medium'); diff --git a/t/watch_maildir_v2.t b/t/watch_maildir_v2.t index 5f968919..b2cff4ba 100644 --- a/t/watch_maildir_v2.t +++ b/t/watch_maildir_v2.t @@ -7,12 +7,12 @@ use Cwd; use PublicInbox::Config; require './t/common.perl'; require_git(2.6); -my @mods = qw(Filesys::Notify::Simple PublicInbox::V2Writable); +my @mods = qw(Search::Xapian DBD::SQLite Filesys::Notify::Simple); foreach my $mod (@mods) { eval "require $mod"; plan skip_all => "$mod missing for watch_maildir_v2.t" if $@; } - +require PublicInbox::V2Writable; my $tmpdir = tempdir('watch_maildir-v2-XXXXXX', TMPDIR => 1, CLEANUP => 1); my $mainrepo = "$tmpdir/v2"; my $maildir = "$tmpdir/md"; -- cgit v1.2.3-24-ge0c7