about summary refs log tree commit homepage
path: root/t/extsearch.t
diff options
context:
space:
mode:
Diffstat (limited to 't/extsearch.t')
-rw-r--r--t/extsearch.t118
1 files changed, 74 insertions, 44 deletions
diff --git a/t/extsearch.t b/t/extsearch.t
index 2d7375d6..090f6db5 100644
--- a/t/extsearch.t
+++ b/t/extsearch.t
@@ -1,30 +1,25 @@
 #!perl -w
 # Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
-use strict;
-use Test::More;
+use v5.12;
 use PublicInbox::TestCommon;
 use PublicInbox::Config;
 use PublicInbox::InboxWritable;
-use Fcntl qw(:seek);
 require_git(2.6);
-require_mods(qw(json DBD::SQLite Search::Xapian));
+require_mods(qw(json DBD::SQLite Xapian));
+use autodie qw(open rename truncate unlink);
 require PublicInbox::Search;
 use_ok 'PublicInbox::ExtSearch';
 use_ok 'PublicInbox::ExtSearchIdx';
 use_ok 'PublicInbox::OverIdx';
-my $sock = tcp_server();
-my $host_port = tcp_host_port($sock);
 my ($home, $for_destroy) = tmpdir();
 local $ENV{HOME} = $home;
 mkdir "$home/.public-inbox" or BAIL_OUT $!;
 my $cfg_path = "$home/.public-inbox/config";
-open my $fh, '>', $cfg_path or BAIL_OUT $!;
-print $fh <<EOF or BAIL_OUT $!;
+PublicInbox::IO::write_file '>', $cfg_path, <<EOF;
 [publicinboxMda]
         spamcheck = none
 EOF
-close $fh or BAIL_OUT $!;
 my $v2addr = 'v2test@example.com';
 my $v1addr = 'v1test@example.com';
 ok(run_script([qw(-init -Lbasic -V2 v2test --newsgroup v2.example),
@@ -33,24 +28,18 @@ my $env = { ORIGINAL_RECIPIENT => $v2addr };
 my $eml = eml_load('t/utf8.eml');
 
 $eml->header_set('List-Id', '<v2.example.com>');
-open($fh, '+>', undef) or BAIL_OUT $!;
-$fh->autoflush(1);
-print $fh $eml->as_string or BAIL_OUT $!;
-seek($fh, 0, SEEK_SET) or BAIL_OUT $!;
 
-run_script(['-mda', '--no-precheck'], $env, { 0 => $fh }) or BAIL_OUT '-mda';
+my $in = \($eml->as_string);
+run_script(['-mda', '--no-precheck'], $env, { 0 => $in }) or BAIL_OUT '-mda';
 
 ok(run_script([qw(-init -V1 v1test --newsgroup v1.example), "$home/v1test",
         'http://example.com/v1test', $v1addr ]), 'v1test init');
 
 $eml->header_set('List-Id', '<v1.example.com>');
-seek($fh, 0, SEEK_SET) or BAIL_OUT $!;
-truncate($fh, 0) or BAIL_OUT $!;
-print $fh $eml->as_string or BAIL_OUT $!;
-seek($fh, 0, SEEK_SET) or BAIL_OUT $!;
+$in = \$eml->as_string;
 
 $env = { ORIGINAL_RECIPIENT => $v1addr };
-run_script(['-mda', '--no-precheck'], $env, { 0 => $fh }) or BAIL_OUT '-mda';
+run_script(['-mda', '--no-precheck'], $env, { 0 => $in }) or BAIL_OUT '-mda';
 
 run_script([qw(-index -Lbasic), "$home/v1test"]) or BAIL_OUT "index $?";
 
@@ -106,14 +95,11 @@ if ('with boost') {
 }
 
 { # TODO: -extindex should write this to config
-        open $fh, '>>', $cfg_path or BAIL_OUT $!;
-        print $fh <<EOF or BAIL_OUT $!;
+        PublicInbox::IO::write_file '>>', $cfg_path, <<EOF;
 ; for ->ALL
 [extindex "all"]
         topdir = $home/extindex
 EOF
-        close $fh or BAIL_OUT $!;
-
         my $pi_cfg = PublicInbox::Config->new;
         $pi_cfg->fill_all;
         ok($pi_cfg->ALL, '->ALL');
@@ -125,6 +111,8 @@ EOF
 
 SKIP: {
         require_mods(qw(Net::NNTP), 1);
+        my $sock = tcp_server();
+        my $host_port = tcp_host_port($sock);
         my ($out, $err) = ("$home/nntpd.out.log", "$home/nntpd.err.log");
         my $cmd = [ '-nntpd', '-W0', "--stdout=$out", "--stderr=$err" ];
         my $td = start_script($cmd, undef, { 3 => $sock });
@@ -152,7 +140,7 @@ if ('inbox edited') {
         my ($in, $out, $err);
         $in = $out = $err = '';
         my $opt = { 0 => \$in, 1 => \$out, 2 => \$err };
-        my $env = { MAIL_EDITOR => "$^X -i -p -e 's/test message/BEST MSG/'" };
+        my $env = { MAIL_EDITOR => "$^X -w -i -p -e 's/test message/BEST MSG/'" };
         my $cmd = [ qw(-edit -Ft/utf8.eml), "$home/v2test" ];
         ok(run_script($cmd, $env, $opt), '-edit');
         ok(run_script([qw(-extindex --all), "$home/extindex"], undef, $opt),
@@ -203,11 +191,7 @@ if ('inbox edited') {
         is_deeply($res, $exp, 'isearch limited results');
         $pi_cfg = $res = $exp = undef;
 
-        open my $rmfh, '+>', undef or BAIL_OUT $!;
-        $rmfh->autoflush(1);
-        print $rmfh $eml2->as_string or BAIL_OUT $!;
-        seek($rmfh, 0, SEEK_SET) or BAIL_OUT $!;
-        $opt->{0} = $rmfh;
+        $opt->{0} = \($eml2->as_string);
         ok(run_script([qw(-learn rm --all)], undef, $opt), '-learn rm');
 
         ok(run_script([qw(-extindex --all), "$home/extindex"], undef, undef),
@@ -246,13 +230,11 @@ if ('inject w/o indexing') {
         isnt($tip, $cmt, '0.git v2 updated');
 
         # inject a message w/o updating index
-        rename("$home/v1test/public-inbox", "$home/v1test/skip-index") or
-                BAIL_OUT $!;
-        open(my $eh, '<', 't/iso-2202-jp.eml') or BAIL_OUT $!;
+        rename("$home/v1test/public-inbox", "$home/v1test/skip-index");
+        open(my $eh, '<', 't/iso-2202-jp.eml');
         run_script(['-mda', '--no-precheck'], $env, { 0 => $eh}) or
                 BAIL_OUT '-mda';
-        rename("$home/v1test/skip-index", "$home/v1test/public-inbox") or
-                BAIL_OUT $!;
+        rename("$home/v1test/skip-index", "$home/v1test/public-inbox");
 
         my ($in, $out, $err);
         $in = $out = $err = '';
@@ -309,7 +291,7 @@ if ('reindex catches missed messages') {
         is($oidx->eidx_meta($lc_key), $cmt_b, 'lc-v2 stays unchanged');
         my @err = split(/^/, $err);
         is(scalar(@err), 1, 'only one warning') or diag "err=$err";
-        like($err[0], qr/I: reindex_unseen/, 'got reindex_unseen message');
+        like($err[0], qr/# reindex_unseen/, 'got reindex_unseen message');
         my $new = $oidx->get_art($max + 1);
         is($new->{subject}, $eml->header('Subject'), 'new message added');
 
@@ -415,8 +397,8 @@ if ('remove v1test and test gc') {
         my $opt = { 2 => \(my $err = '') };
         ok(run_script([qw(-extindex --gc), "$home/extindex"], undef, $opt),
                 'extindex --gc');
-        like($err, qr/^I: remove #1 v1\.example /ms, 'removed v1 message');
-        is(scalar(grep(!/^I:/, split(/^/m, $err))), 0,
+        like($err, qr/^# remove #1 v1\.example /ms, 'removed v1 message');
+        is(scalar(grep(!/^#/, split(/^/m, $err))), 0,
                 'no non-informational messages');
         $misc->{xdb}->reopen;
         @it = $misc->mset('')->items;
@@ -481,7 +463,7 @@ SKIP: {
         for my $i (2..3) {
                 is(grep(m!/ei[0-9]+/$i\z!, @dirs), 0, "no shard [$i]");
         }
-        skip 'xapian-compact missing', 4 unless have_xapian_compact;
+        have_xapian_compact 1;
         ok(run_script([qw(-compact), $d], undef, $o), 'compact');
         # n.b. stderr contains xapian-compact output
 
@@ -501,10 +483,8 @@ SKIP: {
                 "$home/v2tmp", 'http://example.com/v2tmp', $tmp_addr ])
                 or xbail '-init';
         $env = { ORIGINAL_RECIPIENT => $tmp_addr };
-        open $fh, '+>', undef or xbail "open $!";
-        $fh->autoflush(1);
         my $mid = 'tmpmsg@example.com';
-        print $fh <<EOM or xbail "print $!";
+        my $in = \<<EOM;
 From: b\@z
 To: b\@r
 Message-Id: <$mid>
@@ -512,8 +492,7 @@ Subject: tmpmsg
 Date: Tue, 19 Jan 2038 03:14:07 +0000
 
 EOM
-        seek $fh, 0, SEEK_SET or xbail "seek $!";
-        run_script([qw(-mda --no-precheck)], $env, {0 => $fh}) or xbail '-mda';
+        run_script([qw(-mda --no-precheck)], $env, {0 => $in}) or xbail '-mda';
         ok(run_script([qw(-extindex --all), "$home/extindex"]), 'update');
         my $nr;
         {
@@ -526,7 +505,7 @@ EOM
                 $mset = $es->search->mset('z:0..');
                 $nr = $mset->size;
         }
-        truncate($cfg_path, $old_size) or xbail "truncate $!";
+        truncate($cfg_path, $old_size);
         my $rdr = { 2 => \(my $err) };
         ok(run_script([qw(-extindex --gc), "$home/extindex"], undef, $rdr),
                 'gc to get rid of removed inbox');
@@ -554,4 +533,55 @@ EOM
         is_deeply($x, $o, 'xref3 and over docids match');
 }
 
+{
+        my $d = "$home/eidx-med";
+        ok(run_script([qw(-extindex --dangerous --all -L medium -j3), $d]),
+                'extindex medium init');
+        my $es = PublicInbox::ExtSearch->new($d);
+        is($es->xdb->get_metadata('indexlevel'), 'medium',
+                'es indexlevel before');
+        my @xdb = $es->xdb_shards_flat;
+        is($xdb[0]->get_metadata('indexlevel'), 'medium',
+                '0 indexlevel before');
+        shift @xdb;
+        for (@xdb) {
+                ok(!$_->get_metadata('indexlevel'), 'no indexlevel in >0 shard')
+        }
+        is($es->xdb->get_metadata('indexlevel'), 'medium', 'indexlevel before');
+        ok(run_script([qw(-xcpdb -R5), $d]), 'xcpdb R5');
+        $es = PublicInbox::ExtSearch->new($d);
+        is($es->xdb->get_metadata('indexlevel'), 'medium',
+                '0 indexlevel after');
+        @xdb = $es->xdb_shards_flat;
+        is(scalar(@xdb), 5, 'got 5 shards');
+        is($xdb[0]->get_metadata('indexlevel'), 'medium', '0 indexlevel after');
+        shift @xdb;
+        for (@xdb) {
+                ok(!$_->get_metadata('indexlevel'), 'no indexlevel in >0 shard')
+        }
+}
+
+test_lei(sub {
+        my $d = "$home/extindex";
+        lei_ok('convert', '-o', "$home/md1", $d);
+        lei_ok('convert', '-o', "$home/md2", "extindex:$d");
+        my $dst = [];
+        my $cb = sub { push @$dst, $_[2]->as_string };
+        require PublicInbox::MdirReader;
+        PublicInbox::MdirReader->new->maildir_each_eml("$home/md1", $cb);
+        my @md1 = sort { $a cmp $b } @$dst;
+        ok(scalar(@md1), 'dumped messages to md1');
+        $dst = [];
+        PublicInbox::MdirReader->new->maildir_each_eml("$home/md2", $cb);
+        @$dst = sort { $a cmp $b } @$dst;
+        is_deeply($dst, \@md1,
+                "convert from extindex w/ or w/o `extindex' prefix");
+
+        my @o = glob "$home/extindex/ei*/over.sqlite*";
+        unlink(@o);
+        ok(!lei('convert', '-o', "$home/fail", "extindex:$d"));
+        like($lei_err, qr/unindexed .*?not supported/,
+                'noted unindexed extindex is unsupported');
+});
+
 done_testing;