[PATCH 0/2] improve support for unconfigured inboxes

[PATCH 0/2] improve support for unconfigured inboxes

[Eric Wong]

Asking users to configure inboxes every time -index runs is
annoying, and there are valid reasons explained in [PATCH 1/2]
to leave inboxes unconfigured.

Eric Wong (2):
  index: remove warning on unconfigured inboxes
  searchidx: store indexlevel=medium as metadata

 lib/PublicInbox/Admin.pm     | 25 +++++++++++++++++++++++++
 lib/PublicInbox/SearchIdx.pm |  8 ++++++++
 lib/PublicInbox/Xapcmd.pm    |  8 ++++++++
 script/public-inbox-index    | 11 ++++-------
 t/indexlevels-mirror.t       | 14 +++++++++++---
 5 files changed, 56 insertions(+), 10 deletions(-)

-- 
EW

[PATCH 1/2] index: remove warning on unconfigured inboxes

[Eric Wong]

It's annoying for people using "git fetch && public-inbox-index"
as one user; and running -httpd/-nntpd as a different user
(where users see different config files).
---
 script/public-inbox-index | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/script/public-inbox-index b/script/public-inbox-index
index c0d637b..e4a7be1 100755
--- a/script/public-inbox-index
+++ b/script/public-inbox-index
@@ -20,9 +20,7 @@ die "--jobs must be positive\n" if defined $opt->{jobs} && $opt->{jobs} <= 0;
 
 sub usage { print STDERR "Usage: $usage\n"; exit 1 }
 
-# do we really care about this message?  It's annoying...
-my $warn = 'public-inbox unconfigured for serving, indexing anyways...';
-my @ibxs = PublicInbox::Admin::resolve_inboxes(\@ARGV, $warn);
+my @ibxs = PublicInbox::Admin::resolve_inboxes(\@ARGV);
 PublicInbox::Admin::require_or_die('-index');
 usage() unless @ibxs;
 my $mods = {};
-- 
EW

[PATCH 2/2] searchidx: store indexlevel=medium as metadata

[Eric Wong]

And use it from Admin.

It's easy to tell what indexlevel=basic is from unconfigured
inboxes, but distinguishing between 'medium' and 'full' would
require stat()-ing position.* files which is fragile and
Xapian-implementation-dependent.

So use the metadata facility of Xapian and store it in the main
partition so Admin tools can deal better with unconfigured
inboxes copied using generic tools like cp(1) or rsync(1).
---
 lib/PublicInbox/Admin.pm     | 25 +++++++++++++++++++++++++
 lib/PublicInbox/SearchIdx.pm |  8 ++++++++
 lib/PublicInbox/Xapcmd.pm    |  8 ++++++++
 script/public-inbox-index    |  7 +++----
 t/indexlevels-mirror.t       | 14 +++++++++++---
 5 files changed, 55 insertions(+), 7 deletions(-)

diff --git a/lib/PublicInbox/Admin.pm b/lib/PublicInbox/Admin.pm
index 07d8b57..4a862c6 100644
--- a/lib/PublicInbox/Admin.pm
+++ b/lib/PublicInbox/Admin.pm
@@ -41,6 +41,31 @@ sub resolve_repo_dir {
 	}
 }
 
+# for unconfigured inboxes
+sub detect_indexlevel ($) {
+	my ($ibx) = @_;
+
+	# brand new or never before indexed inboxes default to full
+	return 'full' unless $ibx->over;
+	delete $ibx->{over}; # don't leave open FD lying around
+
+	my $l = 'basic';
+	my $srch = $ibx->search or return $l;
+	delete $ibx->{search}; # don't leave open FD lying around
+	if (my $xdb = $srch->xdb) {
+		$l = 'full';
+		my $m = $xdb->get_metadata('indexlevel');
+		if ($m eq 'medium') {
+			$l = $m;
+		} elsif ($m ne '') {
+			warn <<"";
+$ibx->{mainrepo} has unexpected indexlevel in Xapian: $m
+
+		}
+	}
+	$l;
+}
+
 sub resolve_inboxes {
 	my ($argv, $warn_on_unconfigured) = @_;
 	require PublicInbox::Config;
diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm
index b963805..9985628 100644
--- a/lib/PublicInbox/SearchIdx.pm
+++ b/lib/PublicInbox/SearchIdx.pm
@@ -828,6 +828,14 @@ sub commit_txn_lazy {
 	delete $self->{txn} or return;
 	$self->{-inbox}->with_umask(sub {
 		if (my $xdb = $self->{xdb}) {
+
+			# store 'indexlevel=medium' in v2 part=0 and v1 (only part)
+			# This metadata is read by Admin::detect_indexlevel:
+			if (!$self->{partition} # undef or 0, not >0
+			    && $self->{indexlevel} eq 'medium') {
+				$xdb->set_metadata('indexlevel', 'medium');
+			}
+
 			$xdb->commit_transaction;
 		}
 		$self->{over}->commit_lazy if $self->{over};
diff --git a/lib/PublicInbox/Xapcmd.pm b/lib/PublicInbox/Xapcmd.pm
index 7e3d47f..9067231 100644
--- a/lib/PublicInbox/Xapcmd.pm
+++ b/lib/PublicInbox/Xapcmd.pm
@@ -275,6 +275,14 @@ sub cpdb ($$) {
 			my $lc = $src->get_metadata('last_commit');
 			$dst->set_metadata('last_commit', $lc) if $lc;
 
+			# only the first xapian partition (0) gets 'indexlevel'
+			if ($old =~ m!(?:xapian\d+|xap\d+/0)\z!) {
+				my $l = $src->get_metadata('indexlevel');
+				if ($l eq 'medium') {
+					$dst->set_metadata('indexlevel', $l);
+				}
+			}
+
 			$it = $src->postlist_begin('');
 			$end = $src->postlist_end('');
 			if ($pr) {
diff --git a/script/public-inbox-index b/script/public-inbox-index
index e4a7be1..439da15 100755
--- a/script/public-inbox-index
+++ b/script/public-inbox-index
@@ -25,10 +25,9 @@ PublicInbox::Admin::require_or_die('-index');
 usage() unless @ibxs;
 my $mods = {};
 foreach my $ibx (@ibxs) {
-	if (defined $opt->{indexlevel} && !defined($ibx->{indexlevel})) {
-		# XXX: users can shoot themselves in the foot, with this...
-		$ibx->{indexlevel} = $opt->{indexlevel};
-	}
+	# XXX: users can shoot themselves in the foot, with opt->{indexlevel}
+	$ibx->{indexlevel} //= $opt->{indexlevel} //
+			PublicInbox::Admin::detect_indexlevel($ibx);
 	PublicInbox::Admin::scan_ibx_modules($mods, $ibx);
 }
 
diff --git a/t/indexlevels-mirror.t b/t/indexlevels-mirror.t
index 1251136..bf0f801 100644
--- a/t/indexlevels-mirror.t
+++ b/t/indexlevels-mirror.t
@@ -7,6 +7,7 @@ use PublicInbox::MIME;
 use PublicInbox::Inbox;
 use PublicInbox::InboxWritable;
 use File::Temp qw/tempdir/;
+require PublicInbox::Admin;
 require './t/common.perl';
 require_git(2.6);
 my $this = (split('/', __FILE__))[-1];
@@ -119,6 +120,8 @@ sub import_index_incremental {
 
 	if ($level ne 'basic') {
 		is(system(@xcpdb, $mirror), 0, "v$v xcpdb OK");
+		is(PublicInbox::Admin::detect_indexlevel($ro_mirror), $level,
+		   'indexlevel detectable by Admin after xcpdb v' .$v.$level);
 		delete $ro_mirror->{$_} for (qw(over search));
 		($nr, $msgs) = $ro_mirror->search->query('m:m@2');
 		is($nr, 1, "v$v found m\@2 via Xapian on $level");
@@ -157,6 +160,9 @@ sub import_index_incremental {
 	@rw_nums = map { $_->{num} } @{$ibx->over->query_ts(0, 0)};
 	is_deeply(\@rw_nums, \@expect, "v$v master has expected NNTP articles");
 	is_deeply(\@ro_nums, \@expect, "v$v mirror matches master articles");
+
+	is(PublicInbox::Admin::detect_indexlevel($ro_mirror), $level,
+	   'indexlevel detectable by Admin '.$v.$level);
 }
 
 # we can probably cull some other tests and put full/medium tests, here
@@ -172,9 +178,11 @@ SKIP: {
 	require PublicInbox::Search;
 	PublicInbox::Search::load_xapian() or skip 'Search::Xapian missing', 2;
 	for my $v (1..2) {
-		subtest("v$v indexlevel=medium" => sub {
-			import_index_incremental($v, 'medium');
-		})
+		foreach my $l (qw(medium full)) {
+			subtest("v$v indexlevel=$l" => sub {
+				import_index_incremental($v, $l);
+			});
+		}
 	}
 }
 
-- 
EW