From: Eric Wong <e@yhbt.net>
To: meta@public-inbox.org
Subject: [PATCH 1/7] xapcmd: quietly no-op on indexlevel=basic
Date: Fri, 7 Aug 2020 01:14:00 +0000 [thread overview]
Message-ID: <20200807011406.12285-2-e@yhbt.net> (raw)
In-Reply-To: <20200807011406.12285-1-e@yhbt.net>
I find myself mindlessly adding "-c" to public-inbox-index,
and other users may do the same. Instead of erroring out,
we'll just silently ignore it, for now and allow
public-inbox-compact to work on SQLite-only inboxes.
We'll only check for xapian-compact if search exists, since
it won't be needed in case we support SQLite VACUUM.
---
lib/PublicInbox/Xapcmd.pm | 14 ++++++++------
t/indexlevels-mirror.t | 24 +++++++++++++++++-------
2 files changed, 25 insertions(+), 13 deletions(-)
diff --git a/lib/PublicInbox/Xapcmd.pm b/lib/PublicInbox/Xapcmd.pm
index f1c80831..8d09ed27 100644
--- a/lib/PublicInbox/Xapcmd.pm
+++ b/lib/PublicInbox/Xapcmd.pm
@@ -167,9 +167,11 @@ sub prepare_run {
my ($ibx, $opt) = @_;
my $tmp = {}; # old shard dir => File::Temp->newdir object or undef
my @queue; # ([old//src,newdir]) - list of args for cpdb() or compact()
-
- my $old = $ibx->search->xdir(1);
- -d $old or die "$old does not exist\n";
+ my $old;
+ if (my $srch = $ibx->search) {
+ $old = $srch->xdir(1);
+ -d $old or die "$old does not exist\n";
+ }
my $reshard = $opt->{reshard};
if (defined $reshard && $reshard <= 0) {
die "--reshard must be a positive number\n";
@@ -177,7 +179,7 @@ sub prepare_run {
# we want temporary directories to be as deep as possible,
# so v2 shards can keep "xap$SCHEMA_VERSION" on a separate FS.
- if ($ibx->version == 1) {
+ if ($old && $ibx->version == 1) {
if (defined $reshard) {
warn
"--reshard=$reshard ignored for v1 $ibx->{inboxdir}\n";
@@ -189,7 +191,7 @@ sub prepare_run {
$tmp->{$old} = $wip;
nodatacow_dir($wip->dirname);
push @queue, [ $old, $wip ];
- } else {
+ } elsif ($old) {
opendir my $dh, $old or die "Failed to opendir $old: $!\n";
my @old_shards;
while (defined(my $dn = readdir($dh))) {
@@ -255,7 +257,7 @@ sub run {
PublicInbox::Admin::progress_prepare($opt ||= {});
defined(my $dir = $ibx->{inboxdir}) or die "no inboxdir defined\n";
-d $dir or die "inboxdir=$dir does not exist\n";
- check_compact() if $opt->{compact};
+ check_compact() if $opt->{compact} && $ibx->search;
my $reindex; # v1:{ from => $x40 }, v2:{ from => [ $x40, $x40, .. ] } }
if (!$opt->{-coarse_lock}) {
diff --git a/t/indexlevels-mirror.t b/t/indexlevels-mirror.t
index 44313e40..859c2c17 100644
--- a/t/indexlevels-mirror.t
+++ b/t/indexlevels-mirror.t
@@ -6,11 +6,13 @@ use Test::More;
use PublicInbox::Eml;
use PublicInbox::Inbox;
use PublicInbox::InboxWritable;
+use PublicInbox::Spawn qw(which);
require PublicInbox::Admin;
use PublicInbox::TestCommon;
my $PI_TEST_VERSION = $ENV{PI_TEST_VERSION} || 2;
require_git('2.6') if $PI_TEST_VERSION == 2;
require_mods(qw(DBD::SQLite));
+my $have_xapian_compact = which($ENV{XAPIAN_COMPACT} || 'xapian-compact');
my $mime = PublicInbox::Eml->new(<<'EOF');
From: a@example.com
@@ -21,8 +23,9 @@ Date: Fri, 02 Oct 1993 00:00:00 +0000
hello world
EOF
-sub import_index_incremental {
+my $import_index_incremental = sub {
my ($v, $level, $mime) = @_;
+ my $err = '';
my $this = "pi-$v-$level-indexlevels";
my ($tmpdir, $for_destroy) = tmpdir();
local $ENV{PI_CONFIG} = "$tmpdir/config";
@@ -39,8 +42,9 @@ sub import_index_incremental {
$im->done;
# index master (required for v1)
- ok(run_script([qw(-index -j0), $ibx->{inboxdir}, "-L$level"]),
- 'index master OK');
+ my @cmd = (qw(-index -j0), $ibx->{inboxdir}, "-L$level");
+ push @cmd, '-c' if $have_xapian_compact;
+ ok(run_script(\@cmd, undef, { 2 => \$err }), 'index master');
my $ro_master = PublicInbox::Inbox->new({
inboxdir => $ibx->{inboxdir},
indexlevel => $level
@@ -50,7 +54,7 @@ sub import_index_incremental {
is($msgs->[0]->{mid}, 'm@1', 'first message in master indexed');
# clone
- my @cmd = (qw(git clone --mirror -q));
+ @cmd = (qw(git clone --mirror -q));
my $mirror = "$tmpdir/mirror-$v";
if ($v == 1) {
push @cmd, $ibx->{inboxdir}, $mirror;
@@ -157,17 +161,23 @@ sub import_index_incremental {
is(PublicInbox::Admin::detect_indexlevel($ro_mirror), $level,
'indexlevel detectable by Admin '.$v.$level);
-}
+
+ SKIP: {
+ skip 'xapian-compact missing', 1 if !$have_xapian_compact;
+ my $cmd = [ qw(-compact), $mirror ];
+ ok(run_script($cmd, undef, { 2 => \$err}), "compact $level");
+ }
+};
# we can probably cull some other tests
-import_index_incremental($PI_TEST_VERSION, 'basic', $mime);
+$import_index_incremental->($PI_TEST_VERSION, 'basic', $mime);
SKIP: {
require PublicInbox::Search;
PublicInbox::Search::load_xapian() or
skip('Xapian perl binding missing', 2);
foreach my $l (qw(medium full)) {
- import_index_incremental($PI_TEST_VERSION, $l, $mime);
+ $import_index_incremental->($PI_TEST_VERSION, $l, $mime);
}
}
next prev parent reply other threads:[~2020-08-07 1:14 UTC|newest]
Thread overview: 8+ messages / expand[flat|nested] mbox.gz Atom feed top
2020-08-07 1:13 [PATCH 0/7] index: --sequential-shard and other stuff Eric Wong
2020-08-07 1:14 ` Eric Wong [this message]
2020-08-07 1:14 ` [PATCH 2/7] xapcmd: remove redundant searchidx require Eric Wong
2020-08-07 1:14 ` [PATCH 3/7] xapcmd: drop outdated comment Eric Wong
2020-08-07 1:14 ` [PATCH 4/7] v2writable: fix rethread cleanup Eric Wong
2020-08-07 1:14 ` [PATCH 5/7] index: v2: --sequential-shard option Eric Wong
2020-08-07 1:14 ` [PATCH 6/7] index: support --xapian-only switch Eric Wong
2020-08-07 1:14 ` [PATCH 7/7] index+xcpdb: rename `--no-sync' to `--no-fsync' Eric Wong
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
List information: https://public-inbox.org/README
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20200807011406.12285-2-e@yhbt.net \
--to=e@yhbt.net \
--cc=meta@public-inbox.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
Code repositories for project(s) associated with this public inbox
https://80x24.org/public-inbox.git
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).