Merge remote-tracking branch 'origin/xap-optional' into master

* origin/xap-optional:
  admin: improve warnings and errors for missing modules
  searchidx: do not create empty Xapian partitions for basic
  lazy load Xapian and make it optional for v2
  www: use Inbox->over where appropriate
  nntp: use Inbox->over directly
  inbox: add ->over method to ease access

3 files changed, 76 insertions, 23 deletions

diff --git a/script/public-inbox-index b/script/public-inbox-index
index b353093e..cf001cc1 100755
--- a/script/public-inbox-index
+++ b/script/public-inbox-index
@@ -10,26 +10,24 @@ use strict;
 use warnings;
 use Getopt::Long qw(:config gnu_getopt no_ignore_case auto_abbrev);
 my $usage = "public-inbox-index REPO_DIR";
-use PublicInbox::Config;
 use PublicInbox::Admin qw(resolve_repo_dir);
+PublicInbox::Admin::require_or_die('-index');
+require PublicInbox::Config;
 
 my $config = eval { PublicInbox::Config->new } || eval {
         warn "public-inbox unconfigured for serving, indexing anyways...\n";
         undef;
 };
-eval { require PublicInbox::SearchIdx };
-if ($@) {
-        print STDERR "Search::Xapian required for $0\n";
-        exit 1;
-}
 
 my $reindex;
 my $prune;
 my $jobs = undef;
+my $indexlevel;
 my %opts = (
         '--reindex' => \$reindex,
         '--jobs|j=i' => \$jobs,
         '--prune' => \$prune,
+        'L|indexlevel=s' => \$indexlevel,
 );
 GetOptions(%opts) or die "bad command-line args\n$usage";
 die "--jobs must be positive\n" if defined $jobs && $jobs < 0;
@@ -54,19 +52,36 @@ defined($config) and $config->each_inbox(sub {
         }
 });
 
+my @inboxes;
+my $mods = {};
+
 foreach my $dir (@dirs) {
-        if (!ref($dir) && -f "$dir/inbox.lock") { # v2
-                my $ibx = { mainrepo => $dir, name => 'unnamed' };
-                $dir = PublicInbox::Inbox->new($ibx);
+        my $ibx = $dir;
+        if (!ref($ibx)) {
+                unless (-d $dir) {
+                        die "$dir does not appear to be an inbox repository\n";
+                }
+                $ibx = PublicInbox::Inbox->new({
+                        mainrepo => $dir,
+                        name => 'unnamed',
+                        indexlevel => $indexlevel,
+                        version => -f "$dir/inbox.lock" ? 2 : 1,
+                });
+        } elsif (defined $indexlevel && !defined($ibx->{indexlevel})) {
+                # XXX: users can shoot themselves in the foot, with this...
+                $ibx->{indexlevel} = $indexlevel;
         }
-        index_dir($dir);
+        push @inboxes, $ibx;
+        PublicInbox::Admin::scan_ibx_modules($mods, $ibx);
 }
 
-sub index_dir {
+PublicInbox::Admin::require_or_die(keys %$mods);
+
+require PublicInbox::SearchIdx;
+index_inbox($_) for @inboxes;
+
+sub index_inbox {
         my ($repo) = @_;
-        if (!ref $repo && ! -d $repo) {
-                die "$repo does not appear to be an inbox repository\n";
-        }
         if (ref($repo) && ($repo->{version} || 1) == 2) {
                 eval { require PublicInbox::V2Writable };
                 die "v2 requirements not met: $@\n" if $@;
diff --git a/script/public-inbox-init b/script/public-inbox-init
index 8bb78451..5516e798 100755
--- a/script/public-inbox-init
+++ b/script/public-inbox-init
@@ -7,8 +7,10 @@ use strict;
 use warnings;
 my $usage = "public-inbox-init NAME REPO_DIR HTTP_URL ADDRESS [ADDRESS..]";
 use Getopt::Long qw/:config gnu_getopt no_ignore_case auto_abbrev/;
-use PublicInbox::Config;
-use PublicInbox::Inbox;
+use PublicInbox::Admin;
+PublicInbox::Admin::require_or_die('-base');
+require PublicInbox::Config;
+require PublicInbox::Inbox;
 use File::Temp qw/tempfile/;
 use File::Basename qw/dirname/;
 use File::Path qw/mkpath/;
@@ -24,6 +26,7 @@ my %opts = ( 'V|version=i' => \$version,
              'S|skip=i' => \$skip,
 );
 GetOptions(%opts) or usage();
+PublicInbox::Admin::indexlevel_ok_or_die($indexlevel) if defined $indexlevel;
 my $name = shift @ARGV or usage();
 my $mainrepo = shift @ARGV or usage();
 my $http_url = shift @ARGV or usage();
@@ -109,6 +112,7 @@ my $ibx = PublicInbox::Inbox->new({
         name => $name,
         version => $version,
         -primary_address => $address[0],
+        indexlevel => $indexlevel,
 });
 
 if ($version >= 2) {
diff --git a/script/public-inbox-purge b/script/public-inbox-purge
index 688dd950..381826dc 100755
--- a/script/public-inbox-purge
+++ b/script/public-inbox-purge
@@ -7,16 +7,16 @@
 use strict;
 use warnings;
 use Getopt::Long qw(:config gnu_getopt no_ignore_case auto_abbrev);
-use PublicInbox::Config;
-use PublicInbox::MIME;
 use PublicInbox::Admin qw(resolve_repo_dir);
-use PublicInbox::Filter::Base;
-*REJECT = *PublicInbox::Filter::Base::REJECT;
+PublicInbox::Admin::check_require('-index');
+require PublicInbox::Filter::Base;
+require PublicInbox::Config;
+require PublicInbox::MIME;
+require PublicInbox::V2Writable;
 
-my $usage = "$0 [--all] [INBOX_DIRS] </path/to/message";
+{ no warnings 'once'; *REJECT = *PublicInbox::Filter::Base::REJECT }
 
-eval { require PublicInbox::V2Writable } or die
-        "DBI, DBD::SQLite and Search::Xapian required for purge\n";
+my $usage = "$0 [--all] [INBOX_DIRS] </path/to/message";
 my $config = eval { PublicInbox::Config->new };
 my $cfgfile = PublicInbox::Config::default_file();
 my ($all, $force);
@@ -72,6 +72,40 @@ if ($all) {
         }
 }
 
+foreach my $ibx (@inboxes) {
+        my $lvl = $ibx->{indexlevel};
+        if (defined $lvl) {
+                PublicInbox::Admin::indexlevel_ok_or_die($lvl);
+                next;
+        }
+
+        # Undefined indexlevel, so `full'...
+        # Search::Xapian exists and the DB can be read, at least, fine
+        $ibx->search and next;
+
+        # it's possible for a Xapian directory to exist, but Search::Xapian
+        # to go missing/broken.  Make sure it's purged in that case:
+        $ibx->over or die "no over.sqlite3 in $ibx->{mainrepo}\n";
+
+        # $ibx->{search} is populated by $ibx->over call
+        my $xdir_ro = $ibx->{search}->xdir(1);
+        my $npart = 0;
+        foreach my $part (<$xdir_ro/*>) {
+                if (-d $part && $part =~ m!/\d+\z!) {
+                        my $bytes = 0;
+                        $bytes += -s $_ foreach glob("$part/*");
+                        $npart++ if $bytes;
+                }
+        }
+        if ($npart) {
+                PublicInbox::Admin::require_or_die('-search');
+        } else {
+                # somebody could "rm -r" all the Xapian directories;
+                # let them purge the overview, at least
+                $ibx->{indexlevel} ||= 'basic';
+        }
+}
+
 my $data = do { local $/; scalar <STDIN> };
 $data =~ s/\A[\r\n]*From [^\r\n]*\r?\n//s;
 my $n_purged = 0;