user/dev discussion of public-inbox itself
 help / color / mirror / code / Atom feed
From: Eric Wong <e@yhbt.net>
To: meta@public-inbox.org
Subject: [PATCH 1/7] xapcmd: quietly no-op on indexlevel=basic
Date: Fri,  7 Aug 2020 01:14:00 +0000	[thread overview]
Message-ID: <20200807011406.12285-2-e@yhbt.net> (raw)
In-Reply-To: <20200807011406.12285-1-e@yhbt.net>

I find myself mindlessly adding "-c" to public-inbox-index,
and other users may do the same.  Instead of erroring out,
we'll just silently ignore it, for now and allow
public-inbox-compact to work on SQLite-only inboxes.

We'll only check for xapian-compact if search exists, since
it won't be needed in case we support SQLite VACUUM.
---
 lib/PublicInbox/Xapcmd.pm | 14 ++++++++------
 t/indexlevels-mirror.t    | 24 +++++++++++++++++-------
 2 files changed, 25 insertions(+), 13 deletions(-)

diff --git a/lib/PublicInbox/Xapcmd.pm b/lib/PublicInbox/Xapcmd.pm
index f1c80831..8d09ed27 100644
--- a/lib/PublicInbox/Xapcmd.pm
+++ b/lib/PublicInbox/Xapcmd.pm
@@ -167,9 +167,11 @@ sub prepare_run {
 	my ($ibx, $opt) = @_;
 	my $tmp = {}; # old shard dir => File::Temp->newdir object or undef
 	my @queue; # ([old//src,newdir]) - list of args for cpdb() or compact()
-
-	my $old = $ibx->search->xdir(1);
-	-d $old or die "$old does not exist\n";
+	my $old;
+	if (my $srch = $ibx->search) {
+		$old = $srch->xdir(1);
+		-d $old or die "$old does not exist\n";
+	}
 	my $reshard = $opt->{reshard};
 	if (defined $reshard && $reshard <= 0) {
 		die "--reshard must be a positive number\n";
@@ -177,7 +179,7 @@ sub prepare_run {
 
 	# we want temporary directories to be as deep as possible,
 	# so v2 shards can keep "xap$SCHEMA_VERSION" on a separate FS.
-	if ($ibx->version == 1) {
+	if ($old && $ibx->version == 1) {
 		if (defined $reshard) {
 			warn
 "--reshard=$reshard ignored for v1 $ibx->{inboxdir}\n";
@@ -189,7 +191,7 @@ sub prepare_run {
 		$tmp->{$old} = $wip;
 		nodatacow_dir($wip->dirname);
 		push @queue, [ $old, $wip ];
-	} else {
+	} elsif ($old) {
 		opendir my $dh, $old or die "Failed to opendir $old: $!\n";
 		my @old_shards;
 		while (defined(my $dn = readdir($dh))) {
@@ -255,7 +257,7 @@ sub run {
 	PublicInbox::Admin::progress_prepare($opt ||= {});
 	defined(my $dir = $ibx->{inboxdir}) or die "no inboxdir defined\n";
 	-d $dir or die "inboxdir=$dir does not exist\n";
-	check_compact() if $opt->{compact};
+	check_compact() if $opt->{compact} && $ibx->search;
 	my $reindex; # v1:{ from => $x40 }, v2:{ from => [ $x40, $x40, .. ] } }
 
 	if (!$opt->{-coarse_lock}) {
diff --git a/t/indexlevels-mirror.t b/t/indexlevels-mirror.t
index 44313e40..859c2c17 100644
--- a/t/indexlevels-mirror.t
+++ b/t/indexlevels-mirror.t
@@ -6,11 +6,13 @@ use Test::More;
 use PublicInbox::Eml;
 use PublicInbox::Inbox;
 use PublicInbox::InboxWritable;
+use PublicInbox::Spawn qw(which);
 require PublicInbox::Admin;
 use PublicInbox::TestCommon;
 my $PI_TEST_VERSION = $ENV{PI_TEST_VERSION} || 2;
 require_git('2.6') if $PI_TEST_VERSION == 2;
 require_mods(qw(DBD::SQLite));
+my $have_xapian_compact = which($ENV{XAPIAN_COMPACT} || 'xapian-compact');
 
 my $mime = PublicInbox::Eml->new(<<'EOF');
 From: a@example.com
@@ -21,8 +23,9 @@ Date: Fri, 02 Oct 1993 00:00:00 +0000
 hello world
 EOF
 
-sub import_index_incremental {
+my $import_index_incremental = sub {
 	my ($v, $level, $mime) = @_;
+	my $err = '';
 	my $this = "pi-$v-$level-indexlevels";
 	my ($tmpdir, $for_destroy) = tmpdir();
 	local $ENV{PI_CONFIG} = "$tmpdir/config";
@@ -39,8 +42,9 @@ sub import_index_incremental {
 	$im->done;
 
 	# index master (required for v1)
-	ok(run_script([qw(-index -j0), $ibx->{inboxdir}, "-L$level"]),
-		'index master OK');
+	my @cmd = (qw(-index -j0), $ibx->{inboxdir}, "-L$level");
+	push @cmd, '-c' if $have_xapian_compact;
+	ok(run_script(\@cmd, undef, { 2 => \$err }), 'index master');
 	my $ro_master = PublicInbox::Inbox->new({
 		inboxdir => $ibx->{inboxdir},
 		indexlevel => $level
@@ -50,7 +54,7 @@ sub import_index_incremental {
 	is($msgs->[0]->{mid}, 'm@1', 'first message in master indexed');
 
 	# clone
-	my @cmd = (qw(git clone --mirror -q));
+	@cmd = (qw(git clone --mirror -q));
 	my $mirror = "$tmpdir/mirror-$v";
 	if ($v == 1) {
 		push @cmd, $ibx->{inboxdir}, $mirror;
@@ -157,17 +161,23 @@ sub import_index_incremental {
 
 	is(PublicInbox::Admin::detect_indexlevel($ro_mirror), $level,
 	   'indexlevel detectable by Admin '.$v.$level);
-}
+
+	SKIP: {
+		skip 'xapian-compact missing', 1 if !$have_xapian_compact;
+		my $cmd = [ qw(-compact), $mirror ];
+		ok(run_script($cmd, undef, { 2 => \$err}), "compact $level");
+	}
+};
 
 # we can probably cull some other tests
-import_index_incremental($PI_TEST_VERSION, 'basic', $mime);
+$import_index_incremental->($PI_TEST_VERSION, 'basic', $mime);
 
 SKIP: {
 	require PublicInbox::Search;
 	PublicInbox::Search::load_xapian() or
 		skip('Xapian perl binding missing', 2);
 	foreach my $l (qw(medium full)) {
-		import_index_incremental($PI_TEST_VERSION, $l, $mime);
+		$import_index_incremental->($PI_TEST_VERSION, $l, $mime);
 	}
 }
 

  reply	other threads:[~2020-08-07  1:14 UTC|newest]

Thread overview: 8+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2020-08-07  1:13 [PATCH 0/7] index: --sequential-shard and other stuff Eric Wong
2020-08-07  1:14 ` Eric Wong [this message]
2020-08-07  1:14 ` [PATCH 2/7] xapcmd: remove redundant searchidx require Eric Wong
2020-08-07  1:14 ` [PATCH 3/7] xapcmd: drop outdated comment Eric Wong
2020-08-07  1:14 ` [PATCH 4/7] v2writable: fix rethread cleanup Eric Wong
2020-08-07  1:14 ` [PATCH 5/7] index: v2: --sequential-shard option Eric Wong
2020-08-07  1:14 ` [PATCH 6/7] index: support --xapian-only switch Eric Wong
2020-08-07  1:14 ` [PATCH 7/7] index+xcpdb: rename `--no-sync' to `--no-fsync' Eric Wong

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

  List information: https://public-inbox.org/README

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20200807011406.12285-2-e@yhbt.net \
    --to=e@yhbt.net \
    --cc=meta@public-inbox.org \
    --subject='Re: [PATCH 1/7] xapcmd: quietly no-op on indexlevel=basic' \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link

user/dev discussion of public-inbox itself

This inbox may be cloned and mirrored by anyone:

	git clone --mirror https://public-inbox.org/meta
	git clone --mirror http://czquwvybam4bgbro.onion/meta
	git clone --mirror http://hjrcffqmbrq6wope.onion/meta
	git clone --mirror http://ou63pmih66umazou.onion/meta

	# If you have public-inbox 1.1+ installed, you may
	# initialize and index your mirror using the following commands:
	public-inbox-init -V1 meta meta/ https://public-inbox.org/meta \
		meta@public-inbox.org
	public-inbox-index meta

Example config snippet for mirrors.
Newsgroups are available over NNTP:
	nntp://news.public-inbox.org/inbox.comp.mail.public-inbox.meta
	nntp://7fh6tueqddpjyxjmgtdiueylzoqt6pt7hec3pukyptlmohoowvhde4yd.onion/inbox.comp.mail.public-inbox.meta
	nntp://ie5yzdi7fg72h7s4sdcztq5evakq23rdt33mfyfcddc5u3ndnw24ogqd.onion/inbox.comp.mail.public-inbox.meta
	nntp://4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/inbox.comp.mail.public-inbox.meta
	nntp://news.gmane.io/gmane.mail.public-inbox.general
 note: .onion URLs require Tor: https://www.torproject.org/

code repositories for project(s) associated with this inbox:

	https://80x24.org/public-inbox.git

AGPL code for this site: git clone https://public-inbox.org/public-inbox.git