diff options
| -rw-r--r-- | INSTALL | 2 | ||||
| -rw-r--r-- | TODO | 2 | ||||
| -rwxr-xr-x | ci/deps.perl | 7 | ||||
| -rwxr-xr-x | ci/profiles.sh | 3 | ||||
| -rw-r--r-- | lib/PublicInbox/Inbox.pm | 19 | ||||
| -rw-r--r-- | lib/PublicInbox/Search.pm | 84 | ||||
| -rw-r--r-- | lib/PublicInbox/SearchIdx.pm | 50 | ||||
| -rw-r--r-- | lib/PublicInbox/WWW.pm | 7 | ||||
| -rwxr-xr-x | script/public-inbox-index | 25 | ||||
| -rwxr-xr-x | script/public-inbox-init | 1 | ||||
| -rwxr-xr-x | script/public-inbox-purge | 5 | ||||
| -rw-r--r-- | t/admin.t | 2 | ||||
| -rw-r--r-- | t/cgi.t | 16 | ||||
| -rw-r--r-- | t/indexlevels-mirror.t | 17 | ||||
| -rw-r--r-- | t/nntp.t | 2 | ||||
| -rw-r--r-- | t/nntpd.t | 15 | ||||
| -rw-r--r-- | t/over.t | 3 | ||||
| -rw-r--r-- | t/psgi_bad_mids.t | 3 | ||||
| -rw-r--r-- | t/psgi_scan_all.t | 4 | ||||
| -rw-r--r-- | t/psgi_search.t | 4 | ||||
| -rw-r--r-- | t/purge.t | 2 | ||||
| -rw-r--r-- | t/search-thr-index.t | 5 | ||||
| -rw-r--r-- | t/search.t | 7 | ||||
| -rw-r--r-- | t/v1reindex.t | 3 | ||||
| -rw-r--r-- | t/v2reindex.t | 5 | ||||
| -rw-r--r-- | t/watch_maildir_v2.t | 4 |
26 files changed, 184 insertions, 113 deletions
@@ -69,7 +69,7 @@ Numerous optional modules are likely to be useful as well: - Search::Xapian deb: libsearch-xapian-perl pkg: p5-Search-Xapian rpm: perl-Search-Xapian - (for v2, HTTP search) + (HTTP search) - Net::Server deb: libnet-server-perl pkg: pkg-Net-Server @@ -69,8 +69,6 @@ all need to be considered for everything we introduce) * large mbox/Maildir/MH/NNTP spool import (see PublicInbox::Import) -* Allow NNTP and more of PSGI code to work without Xapian - * Read-only WebDAV interface to the git repo so it can be mounted via davfs2 or fusedav to avoid full clones. davfs2 needs Range: request support for this to be feasible: diff --git a/ci/deps.perl b/ci/deps.perl index e0fda011..32b0226e 100755 --- a/ci/deps.perl +++ b/ci/deps.perl @@ -70,12 +70,7 @@ if (@precious) { # bare minimum for v2 -$profiles->{v2essential} = [ @{$profiles->{essential}}, qw( - DBD::SQLite - DBI - Search::Xapian - xapian-compact -) ]; +$profiles->{v2essential} = [ @{$profiles->{essential}}, qw(DBD::SQLite DBI) ]; # package names which can't be mapped automatically: my $non_auto = { diff --git a/ci/profiles.sh b/ci/profiles.sh index 31140f35..5d7c2cf6 100755 --- a/ci/profiles.sh +++ b/ci/profiles.sh @@ -62,8 +62,9 @@ essential devtest- EOF ;; debian-sid|debian-9|debian-10) sed "s/^/$PKG_FMT /" <<EOF -all devtest- all devtest +all devtest Search::Xapian- +all devtest- v2essential essential essential devtest- diff --git a/lib/PublicInbox/Inbox.pm b/lib/PublicInbox/Inbox.pm index dc186b73..813ed997 100644 --- a/lib/PublicInbox/Inbox.pm +++ b/lib/PublicInbox/Inbox.pm @@ -25,7 +25,7 @@ sub cleanup_task () { my $next = {}; for my $ibx (values %$CLEANUP) { my $again; - foreach my $f (qw(mm search)) { + foreach my $f (qw(mm search over)) { delete $ibx->{$f} if SvREFCNT($ibx->{$f}) == 1; } my $expire = time - 60; @@ -37,7 +37,7 @@ sub cleanup_task () { $again = 1 if $git->cleanup($expire); } } - $again ||= !!($ibx->{mm} || $ibx->{search}); + $again ||= !!($ibx->{over} || $ibx->{mm} || $ibx->{search}); $next->{"$ibx"} = $ibx if $again; } $CLEANUP = $next; @@ -175,14 +175,17 @@ sub search ($;$) { require PublicInbox::Search; PublicInbox::Search->new($self, $self->{altid}); }; - # TODO: lazily load Xapian - # return $srch if $over_only || eval { $srch->xdb }; - # undef; + ($over_only || eval { $srch->xdb }) ? $srch : undef; } sub over ($) { - my $srch = search($_[0], 1) or return; - $srch->{over_ro}; + my ($self) = @_; + my $srch = search($self, 1) or return; + $self->{over} ||= eval { + my $over = $srch->{over_ro}; + $over->dbh_new; # may fail + $over; + } } sub try_cat { @@ -290,7 +293,7 @@ sub nntp_url { sub nntp_usable { my ($self) = @_; my $ret = mm($self) && over($self); - $self->{mm} = $self->{search} = undef; + $self->{mm} = $self->{over} = $self->{search} = undef; $ret; } diff --git a/lib/PublicInbox/Search.pm b/lib/PublicInbox/Search.pm index e79ec0f8..b1e62f4c 100644 --- a/lib/PublicInbox/Search.pm +++ b/lib/PublicInbox/Search.pm @@ -12,11 +12,21 @@ use constant TS => 0; # Received: header in Unix time use constant YYYYMMDD => 1; # Date: header for searching in the WWW UI use constant DT => 2; # Date: YYYYMMDDHHMMSS -use Search::Xapian qw/:standard/; use PublicInbox::SearchMsg; use PublicInbox::MIME; use PublicInbox::MID qw/id_compress/; use PublicInbox::Over; +my $QP_FLAGS; +sub load_xapian () { + $QP_FLAGS ||= eval { + require Search::Xapian; + Search::Xapian->import(qw(:standard)); + + # n.b. FLAG_PURE_NOT is expensive not suitable for a public + # website as it could become a denial-of-service vector + FLAG_PHRASE()|FLAG_BOOLEAN()|FLAG_LOVEHATE()|FLAG_WILDCARD(); + }; +}; # This is English-only, everything else is non-standard and may be confused as # a prefix common in patch emails @@ -41,10 +51,6 @@ use constant { # (commit 83425ef12e4b65cdcecd11ddcb38175d4a91d5a0) # 14 - fix ghost root vivification SCHEMA_VERSION => 15, - - # n.b. FLAG_PURE_NOT is expensive not suitable for a public website - # as it could become a denial-of-service vector - QP_FLAGS => FLAG_PHRASE|FLAG_BOOLEAN|FLAG_LOVEHATE|FLAG_WILDCARD, }; my %bool_pfx_external = ( @@ -113,18 +119,43 @@ EOF ); chomp @HELP; -sub xdir { - my ($self) = @_; +sub xdir ($;$) { + my ($self, $rdonly) = @_; if ($self->{version} == 1) { "$self->{mainrepo}/public-inbox/xapian" . SCHEMA_VERSION; } else { my $dir = "$self->{mainrepo}/xap" . SCHEMA_VERSION; + return $dir if $rdonly; + my $part = $self->{partition}; defined $part or die "partition not given"; $dir .= "/$part"; } } +sub xdb ($) { + my ($self) = @_; + $self->{xdb} ||= do { + load_xapian(); + my $dir = xdir($self, 1); + if ($self->{version} >= 2) { + my $xdb; + foreach my $part (<$dir/*>) { + -d $part && $part =~ m!/\d+\z! or next; + my $sub = Search::Xapian::Database->new($part); + if ($xdb) { + $xdb->add_database($sub); + } else { + $xdb = $sub; + } + } + $xdb; + } else { + Search::Xapian::Database->new($dir); + } + }; +} + sub new { my ($class, $mainrepo, $altid) = @_; my $version = 1; @@ -138,33 +169,16 @@ sub new { altid => $altid, version => $version, }, $class; - my $dir; - if ($version >= 2) { - $dir = "$self->{mainrepo}/xap" . SCHEMA_VERSION; - my $xdb; - my $parts = 0; - foreach my $part (<$dir/*>) { - -d $part && $part =~ m!/\d+\z! or next; - $parts++; - my $sub = Search::Xapian::Database->new($part); - if ($xdb) { - $xdb->add_database($sub); - } else { - $xdb = $sub; - } - } - $self->{xdb} = $xdb; - } else { - $dir = $self->xdir; - $self->{xdb} = Search::Xapian::Database->new($dir); - } + my $dir = xdir($self, 1); $self->{over_ro} = PublicInbox::Over->new("$dir/over.sqlite3"); $self; } sub reopen { my ($self) = @_; - $self->{xdb}->reopen; + if (my $xdb = $self->{xdb}) { + $xdb->reopen; + } $self; # make chaining easier } @@ -175,7 +189,8 @@ sub query { if ($query_string eq '' && !$opts->{mset}) { $self->{over_ro}->recent($opts); } else { - my $query = $self->qp->parse_query($query_string, QP_FLAGS); + my $qp = qp($self); + my $query = $qp->parse_query($query_string, $QP_FLAGS); $opts->{relevance} = 1 unless exists $opts->{relevance}; _do_enquire($self, $query, $opts); } @@ -213,7 +228,8 @@ sub _do_enquire { sub _enquire_once { my ($self, $query, $opts) = @_; - my $enquire = Search::Xapian::Enquire->new($self->{xdb}); + my $xdb = xdb($self); + my $enquire = Search::Xapian::Enquire->new($xdb); $enquire->set_query($query); $opts ||= {}; my $desc = !$opts->{asc}; @@ -246,13 +262,13 @@ sub qp { my $qp = $self->{query_parser}; return $qp if $qp; - + my $xdb = xdb($self); # new parser $qp = Search::Xapian::QueryParser->new; - $qp->set_default_op(OP_AND); - $qp->set_database($self->{xdb}); + $qp->set_default_op(OP_AND()); + $qp->set_database($xdb); $qp->set_stemmer($self->stemmer); - $qp->set_stemming_strategy(STEM_SOME); + $qp->set_stemming_strategy(STEM_SOME()); $qp->set_max_wildcard_expansion(100); $qp->add_valuerangeprocessor( Search::Xapian::NumberValueRangeProcessor->new(YYYYMMDD, 'd:')); diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm index 1b86f727..135b5eb9 100644 --- a/lib/PublicInbox/SearchIdx.pm +++ b/lib/PublicInbox/SearchIdx.pm @@ -54,11 +54,10 @@ sub new { die("Invalid indexlevel $ibx->{indexlevel}\n"); } } - } else { # v1 + } else { # FIXME: old tests: old tests $ibx = { mainrepo => $git_dir, version => 1 }; } $ibx = PublicInbox::InboxWritable->new($ibx); - require Search::Xapian::WritableDatabase; my $self = bless { mainrepo => $mainrepo, -inbox => $ibx, @@ -84,25 +83,36 @@ sub new { $self; } +sub need_xapian ($) { $_[0]->{indexlevel} =~ $xapianlevels } + sub _xdb_release { my ($self) = @_; - my $xdb = delete $self->{xdb} or croak 'not acquired'; - $xdb->close; + if (need_xapian($self)) { + my $xdb = delete $self->{xdb} or croak 'not acquired'; + $xdb->close; + } $self->lock_release if $self->{creat}; undef; } sub _xdb_acquire { my ($self) = @_; - croak 'already acquired' if $self->{xdb}; + my $flag; my $dir = $self->xdir; - my $flag = Search::Xapian::DB_OPEN; + if (need_xapian($self)) { + croak 'already acquired' if $self->{xdb}; + PublicInbox::Search::load_xapian(); + require Search::Xapian::WritableDatabase; + $flag = $self->{creat} ? + Search::Xapian::DB_CREATE_OR_OPEN() : + Search::Xapian::DB_OPEN(); + } if ($self->{creat}) { require File::Path; $self->lock_acquire; File::Path::mkpath($dir); - $flag = Search::Xapian::DB_CREATE_OR_OPEN; } + return unless defined $flag; $self->{xdb} = Search::Xapian::WritableDatabase->new($dir, $flag); } @@ -342,7 +352,7 @@ sub add_message { $num = index_mm($self, $mime); } eval { - if ($self->{indexlevel} =~ $xapianlevels) { + if (need_xapian($self)) { $self->add_xapian($mime, $num, $oid, $mids, $mid0) } if (my $over = $self->{over}) { @@ -383,7 +393,6 @@ sub batch_do { # v1 only, where $mid is unique sub remove_message { my ($self, $mid) = @_; - my $db = $self->{xdb}; $mid = mid_clean($mid); if (my $over = $self->{over}) { @@ -394,7 +403,8 @@ sub remove_message { warn "<$mid> missing for removal from overview\n"; } } - return if $self->{indexlevel} !~ $xapianlevels; + return unless need_xapian($self); + my $db = $self->{xdb}; my $nr = 0; eval { batch_do($self, 'Q' . $mid, sub { @@ -413,10 +423,12 @@ sub remove_message { # MID is a hint in V2 sub remove_by_oid { my ($self, $oid, $mid) = @_; - my $db = $self->{xdb}; $self->{over}->remove_oid($oid, $mid) if $self->{over}; + return unless need_xapian($self); + my $db = $self->{xdb}; + # XXX careful, we cannot use batch_do here since we conditionally # delete documents based on other factors, so we cannot call # find_doc_ids twice. @@ -664,7 +676,7 @@ sub _last_x_commit { my ($self, $mm) = @_; my $lm = $mm->last_commit || ''; my $lx = ''; - if ($self->{indexlevel} =~ $xapianlevels) { + if (need_xapian($self)) { $lx = $self->{xdb}->get_metadata('last_commit') || ''; } else { $lx = $lm; @@ -695,7 +707,7 @@ sub _index_sync { $self->{over}->disconnect; $git->cleanup; delete $self->{txn}; - $xdb->cancel_transaction; + $xdb->cancel_transaction if $xdb; $xdb = _xdb_release($self); # ensure we leak no FDs to "git log" with Xapian <= 1.2 @@ -717,7 +729,7 @@ sub _index_sync { } $dbh->commit; } - if ($newest && $self->{indexlevel} =~ $xapianlevels) { + if ($newest && need_xapian($self)) { my $cur = $xdb->get_metadata('last_commit'); if (need_update($self, $cur, $newest)) { $xdb->set_metadata('last_commit', $newest); @@ -785,7 +797,7 @@ sub begin_txn_lazy { $self->{-inbox}->with_umask(sub { my $xdb = $self->{xdb} || $self->_xdb_acquire; $self->{over}->begin_lazy if $self->{over}; - $xdb->begin_transaction; + $xdb->begin_transaction if $xdb; $self->{txn} = 1; $xdb; }); @@ -795,14 +807,18 @@ sub commit_txn_lazy { my ($self) = @_; delete $self->{txn} or return; $self->{-inbox}->with_umask(sub { - $self->{xdb}->commit_transaction; + if (my $xdb = $self->{xdb}) { + $xdb->commit_transaction; + } $self->{over}->commit_lazy if $self->{over}; }); } sub worker_done { my ($self) = @_; - die "$$ $0 xdb not released\n" if $self->{xdb}; + if (need_xapian($self)) { + die "$$ $0 xdb not released\n" if $self->{xdb}; + } die "$$ $0 still in transaction\n" if $self->{txn}; } diff --git a/lib/PublicInbox/WWW.pm b/lib/PublicInbox/WWW.pm index 0f963dcb..8e1b1afe 100644 --- a/lib/PublicInbox/WWW.pm +++ b/lib/PublicInbox/WWW.pm @@ -149,8 +149,11 @@ sub preload { require PublicInbox::MIME; require Digest::SHA; require POSIX; - - foreach (qw(PublicInbox::Search PublicInbox::SearchView + eval { + require PublicInbox::Search; + PublicInbox::Search::load_xapian(); + }; + foreach (qw(PublicInbox::SearchView PublicInbox::Mbox IO::Compress::Gzip PublicInbox::NewsWWW)) { eval "require $_;"; diff --git a/script/public-inbox-index b/script/public-inbox-index index b353093e..53def9a7 100755 --- a/script/public-inbox-index +++ b/script/public-inbox-index @@ -26,10 +26,12 @@ if ($@) { my $reindex; my $prune; my $jobs = undef; +my $indexlevel; my %opts = ( '--reindex' => \$reindex, '--jobs|j=i' => \$jobs, '--prune' => \$prune, + 'L|indexlevel=s' => \$indexlevel, ); GetOptions(%opts) or die "bad command-line args\n$usage"; die "--jobs must be positive\n" if defined $jobs && $jobs < 0; @@ -55,18 +57,27 @@ defined($config) and $config->each_inbox(sub { }); foreach my $dir (@dirs) { - if (!ref($dir) && -f "$dir/inbox.lock") { # v2 - my $ibx = { mainrepo => $dir, name => 'unnamed' }; + if (!ref($dir)) { + unless (-d $dir) { + die "$dir does not appear to be an inbox repository\n"; + } + my $ibx = { + mainrepo => $dir, + name => 'unnamed', + indexlevel => $indexlevel, + version => -f "$dir/inbox.lock" ? 2 : 1, + }; $dir = PublicInbox::Inbox->new($ibx); + } elsif (defined $indexlevel && !defined($dir->{indexlevel})) { + # XXX: users can shoot themselves in the foot, with this... + $dir->{indexlevel} = $indexlevel; } - index_dir($dir); + + index_inbox($dir); } -sub index_dir { +sub index_inbox { my ($repo) = @_; - if (!ref $repo && ! -d $repo) { - die "$repo does not appear to be an inbox repository\n"; - } if (ref($repo) && ($repo->{version} || 1) == 2) { eval { require PublicInbox::V2Writable }; die "v2 requirements not met: $@\n" if $@; diff --git a/script/public-inbox-init b/script/public-inbox-init index 8bb78451..9f0bd1bd 100755 --- a/script/public-inbox-init +++ b/script/public-inbox-init @@ -109,6 +109,7 @@ my $ibx = PublicInbox::Inbox->new({ name => $name, version => $version, -primary_address => $address[0], + indexlevel => $indexlevel, }); if ($version >= 2) { diff --git a/script/public-inbox-purge b/script/public-inbox-purge index 688dd950..264bcdef 100755 --- a/script/public-inbox-purge +++ b/script/public-inbox-purge @@ -63,6 +63,11 @@ if ($all) { mainrepo => $dir, }); }; + + # somebody could "rm -r" all the Xapian directories; + # let them purge the overview, at least + $ibx->{indexlevel} ||= 'basic' unless $ibx->search; + push @inboxes, $ibx; } @@ -50,7 +50,7 @@ SKIP: { # v2 SKIP: { - for my $m (qw(DBD::SQLite Search::Xapian)) { + for my $m (qw(DBD::SQLite)) { skip "$m missing", 5 unless eval "require $m"; } use_ok 'PublicInbox::V2Writable'; @@ -30,6 +30,7 @@ my $cfgpfx = "publicinbox.test"; my %cfg = ( "$cfgpfx.address" => $addr, "$cfgpfx.mainrepo" => $maindir, + "$cfgpfx.indexlevel" => 'basic', ); while (my ($k,$v) = each %cfg) { is(0, system(qw(git config --file), $pi_config, $k, $v), @@ -39,9 +40,12 @@ my $cfgpfx = "publicinbox.test"; use_ok 'PublicInbox::Git'; use_ok 'PublicInbox::Import'; -use_ok 'Email::MIME'; -my $git = PublicInbox::Git->new($maindir); -my $im = PublicInbox::Import->new($git, 'test', $addr); +use_ok 'PublicInbox::Inbox'; +use_ok 'PublicInbox::V1Writable'; +use_ok 'PublicInbox::Config'; +my $cfg = PublicInbox::Config->new($pi_config); +my $ibx = $cfg->lookup_name('test'); +my $im = PublicInbox::V1Writable->new($ibx); { local $ENV{HOME} = $home; @@ -103,8 +107,9 @@ EOF like($res->{head}, qr/^Status: 501 /, "search not-yet-enabled"); my $indexed; eval { + require DBD::SQLite; require PublicInbox::SearchIdx; - my $s = PublicInbox::SearchIdx->new($maindir, 1); + my $s = PublicInbox::SearchIdx->new($ibx, 1); $s->index_sync; $indexed = 1; }; @@ -120,6 +125,7 @@ EOF }; } else { like($res->{head}, qr/^Status: 501 /, "search not available"); + SKIP: { skip 'DBD::SQLite not available', 2 }; } my $have_xml_feed = eval { require XML::Feed; 1 } if $indexed; @@ -132,6 +138,8 @@ EOF my $p = XML::Feed->parse(\($res->{body})); is($p->format, "Atom", "parsed atom feed"); is(scalar $p->entries, 3, "parsed three entries"); + } else { + SKIP: { skip 'DBD::SQLite or XML::Feed missing', 2 }; } } diff --git a/t/indexlevels-mirror.t b/t/indexlevels-mirror.t index e25b827f..cac7050b 100644 --- a/t/indexlevels-mirror.t +++ b/t/indexlevels-mirror.t @@ -10,8 +10,7 @@ require './t/common.perl'; require_git(2.6); my $this = (split('/', __FILE__))[-1]; -# TODO: remove Search::Xapian as a requirement for basic -foreach my $mod (qw(DBD::SQLite Search::Xapian)) { +foreach my $mod (qw(DBD::SQLite)) { eval "require $mod"; plan skip_all => "$mod missing for $this" if $@; } @@ -47,8 +46,11 @@ sub import_index_incremental { $im->done; # index master (required for v1) - is(system($index, $ibx->{mainrepo}), 0, 'index master OK'); - my $ro_master = PublicInbox::Inbox->new({mainrepo => $ibx->{mainrepo}}); + is(system($index, $ibx->{mainrepo}, "-L$level"), 0, 'index master OK'); + my $ro_master = PublicInbox::Inbox->new({ + mainrepo => $ibx->{mainrepo}, + indexlevel => $level + }); my ($nr, $msgs) = $ro_master->recent; is($nr, 1, 'only one message in master, so far'); is($msgs->[0]->{mid}, 'm@1', 'first message in master indexed'); @@ -75,7 +77,10 @@ sub import_index_incremental { is(system($index, $mirror), 0, "v$v index mirror OK"); # read-only access - my $ro_mirror = PublicInbox::Inbox->new({mainrepo => $mirror}); + my $ro_mirror = PublicInbox::Inbox->new({ + mainrepo => $mirror, + indexlevel => 'basic' + }); ($nr, $msgs) = $ro_mirror->recent; is($nr, 1, 'only one message, so far'); is($msgs->[0]->{mid}, 'm@1', 'read first message'); @@ -94,7 +99,7 @@ sub import_index_incremental { ['m@1','m@2'], 'got both messages in mirror'); # incremental index master (required for v1) - is(system($index, $ibx->{mainrepo}), 0, 'index master OK'); + is(system($index, $ibx->{mainrepo}, "-L$level"), 0, 'index master OK'); ($nr, $msgs) = $ro_master->recent; is($nr, 2, '2nd message seen in master'); is_deeply([sort { $a cmp $b } map { $_->{mid} } @$msgs], @@ -4,7 +4,7 @@ use strict; use warnings; use Test::More; -foreach my $mod (qw(DBD::SQLite Search::Xapian Data::Dumper)) { +foreach my $mod (qw(DBD::SQLite Data::Dumper)) { eval "require $mod"; plan skip_all => "$mod missing for nntp.t" if $@; } @@ -3,7 +3,7 @@ use strict; use warnings; use Test::More; -foreach my $mod (qw(DBD::SQLite Search::Xapian)) { +foreach my $mod (qw(DBD::SQLite)) { eval "require $mod"; plan skip_all => "$mod missing for nntpd.t" if $@; } @@ -55,12 +55,13 @@ my $ibx = { name => $group, version => $version, -primary_address => $addr, + indexlevel => 'basic', }; $ibx = PublicInbox::Inbox->new($ibx); { local $ENV{HOME} = $home; my @cmd = ($init, $group, $mainrepo, 'http://example.com/', $addr); - push @cmd, "-V$version"; + push @cmd, "-V$version", '-Lbasic'; is(system(@cmd), 0, 'init OK'); is(system(qw(git config), "--file=$home/.public-inbox/config", "publicinbox.$group.newsgroup", $group), @@ -71,8 +72,8 @@ $ibx = PublicInbox::Inbox->new($ibx); if ($version == 2) { $im = PublicInbox::V2Writable->new($ibx); } elsif ($version == 1) { - my $git = PublicInbox::Git->new($mainrepo); - $im = PublicInbox::Import->new($git, 'test', $addr); + use_ok 'PublicInbox::V1Writable'; + $im = PublicInbox::V1Writable->new($ibx); } else { die "unsupported version: $version"; } @@ -261,6 +262,12 @@ EOF is($rdr, waitpid($rdr, 0), 'reader done'); is($? >> 8, 0, 'no errors'); } + SKIP: { + my @of = `lsof -p $pid 2>/dev/null`; + skip('lsof broken', 1) if (!scalar(@of) || $?); + my @xap = grep m!Search/Xapian!, @of; + is_deeply(\@xap, [], 'Xapian not loaded in nntpd'); + } { setsockopt($s, IPPROTO_TCP, TCP_NODELAY, 1); syswrite($s, 'HDR List-id 1-'); @@ -5,8 +5,7 @@ use warnings; use Test::More; use File::Temp qw/tempdir/; use Compress::Zlib qw(compress); -# FIXME: allow using Over w/o Xapian -foreach my $mod (qw(DBD::SQLite Search::Xapian)) { +foreach my $mod (qw(DBD::SQLite)) { eval "require $mod"; plan skip_all => "$mod missing for over.t" if $@; } diff --git a/t/psgi_bad_mids.t b/t/psgi_bad_mids.t index 5008f5be..71eefe59 100644 --- a/t/psgi_bad_mids.t +++ b/t/psgi_bad_mids.t @@ -7,7 +7,7 @@ use File::Temp qw/tempdir/; use PublicInbox::MIME; use PublicInbox::Config; use PublicInbox::WWW; -my @mods = qw(DBD::SQLite Search::Xapian HTTP::Request::Common Plack::Test +my @mods = qw(DBD::SQLite HTTP::Request::Common Plack::Test URI::Escape Plack::Builder); foreach my $mod (@mods) { eval "require $mod"; @@ -22,6 +22,7 @@ my $ibx = { name => 'bad-mids', version => 2, -primary_address => 'test@example.com', + indexlevel => 'basic', }; $ibx = PublicInbox::Inbox->new($ibx); my $im = PublicInbox::V2Writable->new($ibx, 1); diff --git a/t/psgi_scan_all.t b/t/psgi_scan_all.t index e9c439ec..2f54c820 100644 --- a/t/psgi_scan_all.t +++ b/t/psgi_scan_all.t @@ -6,8 +6,7 @@ use Test::More; use Email::MIME; use File::Temp qw/tempdir/; use PublicInbox::Config; -my @mods = qw(HTTP::Request::Common Plack::Test URI::Escape Search::Xapian - DBD::SQLite); +my @mods = qw(HTTP::Request::Common Plack::Test URI::Escape DBD::SQLite); foreach my $mod (@mods) { eval "require $mod"; plan skip_all => "$mod missing for psgi_scan_all.t" if $@; @@ -26,6 +25,7 @@ foreach my $i (1..2) { mainrepo => $mainrepo, name => "test-$i", version => 2, + indexlevel => 'basic', -primary_address => $addr, }; my $ibx = PublicInbox::Inbox->new($opt); diff --git a/t/psgi_search.t b/t/psgi_search.t index da6cc682..1adc1d6e 100644 --- a/t/psgi_search.t +++ b/t/psgi_search.t @@ -8,13 +8,13 @@ use Email::MIME; use PublicInbox::Config; use PublicInbox::WWW; use bytes (); # only for bytes::length -my @mods = qw(PublicInbox::SearchIdx HTTP::Request::Common Plack::Test +my @mods = qw(Search::Xapian HTTP::Request::Common Plack::Test URI::Escape Plack::Builder); foreach my $mod (@mods) { eval "require $mod"; plan skip_all => "$mod missing for psgi_search.t" if $@; } -use_ok $_ foreach @mods; +use_ok $_ foreach (@mods, qw(PublicInbox::SearchIdx)); my $tmpdir = tempdir('pi-psgi-search.XXXXXX', TMPDIR => 1, CLEANUP => 1); my $git_dir = "$tmpdir/a.git"; @@ -6,7 +6,7 @@ use Test::More; use File::Temp qw/tempdir/; require './t/common.perl'; require_git(2.6); -my @mods = qw(IPC::Run DBI DBD::SQLite Search::Xapian); +my @mods = qw(IPC::Run DBI DBD::SQLite); foreach my $mod (@mods) { eval "require $mod"; plan skip_all => "missing $mod for t/purge.t" if $@; diff --git a/t/search-thr-index.t b/t/search-thr-index.t index ed3e4e76..848dc564 100644 --- a/t/search-thr-index.t +++ b/t/search-thr-index.t @@ -7,8 +7,9 @@ use Test::More; use File::Temp qw/tempdir/; use PublicInbox::MID qw(mids); use Email::MIME; -eval { require PublicInbox::SearchIdx; }; -plan skip_all => "Xapian missing for search" if $@; +eval { require Search::Xapian }; +plan skip_all => "Search::Xapian missing for search" if $@; +require PublicInbox::SearchIdx; my $tmpdir = tempdir('pi-search-thr-index.XXXXXX', TMPDIR => 1, CLEANUP => 1); my $git_dir = "$tmpdir/a.git"; @@ -3,8 +3,9 @@ use strict; use warnings; use Test::More; -eval { require PublicInbox::SearchIdx; }; -plan skip_all => "Xapian missing for search" if $@; +eval { require Search::Xapian }; +plan skip_all => "Search::Xapian missing for search" if $@; +require PublicInbox::SearchIdx; use File::Temp qw/tempdir/; use Email::MIME; my $tmpdir = tempdir('pi-search-XXXXXX', TMPDIR => 1, CLEANUP => 1); @@ -12,7 +13,7 @@ my $git_dir = "$tmpdir/a.git"; my ($root_id, $last_id); is(0, system(qw(git init --shared -q --bare), $git_dir), "git init (main)"); -eval { PublicInbox::Search->new($git_dir) }; +eval { PublicInbox::Search->new($git_dir)->xdb }; ok($@, "exception raised on non-existent DB"); my $rw = PublicInbox::SearchIdx->new($git_dir, 1); diff --git a/t/v1reindex.t b/t/v1reindex.t index 33a36fad..402ecd78 100644 --- a/t/v1reindex.t +++ b/t/v1reindex.t @@ -209,8 +209,7 @@ ok(!-d $xap, 'Xapian directories removed again'); delete $ibx->{mm}; is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged'); is($ibx->mm->num_highwater, 10, 'num_highwater as expected'); - my $mset = $ibx->search->reopen->query('hello world', {mset=>1}); - is($mset->size, 0, "no Xapian search results"); + isnt($ibx->search, 'no search for basic'); my ($min, $max) = $ibx->mm->minmax; is_deeply($ibx->mm->msg_range(\$min, $max), $msgmap, 'msgmap unchanged'); diff --git a/t/v2reindex.t b/t/v2reindex.t index c416629c..2a6fc555 100644 --- a/t/v2reindex.t +++ b/t/v2reindex.t @@ -225,8 +225,9 @@ ok(!-d $xap, 'Xapian directories removed again'); delete $ibx->{mm}; is_deeply([ $ibx->mm->minmax ], $minmax, 'minmax unchanged'); is($ibx->mm->num_highwater, 10, 'num_highwater as expected'); - my $mset = $ibx->search->query('freedom', {mset=>1}); - is($mset->size, 0, "search fails on indexlevel='basic'"); + + isnt($ibx->search, 'no search for basic'); + for (<"$xap/*/*">) { $sizes{$ibx->{indexlevel}} += -s _ if -f $_ } ok($sizes{medium} > $sizes{basic}, 'basic is smaller than medium'); diff --git a/t/watch_maildir_v2.t b/t/watch_maildir_v2.t index 5f968919..b2cff4ba 100644 --- a/t/watch_maildir_v2.t +++ b/t/watch_maildir_v2.t @@ -7,12 +7,12 @@ use Cwd; use PublicInbox::Config; require './t/common.perl'; require_git(2.6); -my @mods = qw(Filesys::Notify::Simple PublicInbox::V2Writable); +my @mods = qw(Search::Xapian DBD::SQLite Filesys::Notify::Simple); foreach my $mod (@mods) { eval "require $mod"; plan skip_all => "$mod missing for watch_maildir_v2.t" if $@; } - +require PublicInbox::V2Writable; my $tmpdir = tempdir('watch_maildir-v2-XXXXXX', TMPDIR => 1, CLEANUP => 1); my $mainrepo = "$tmpdir/v2"; my $maildir = "$tmpdir/md"; |