diff options
| -rw-r--r-- | lib/PublicInbox/Admin.pm | 60 | ||||
| -rwxr-xr-x | script/public-inbox-index | 32 | ||||
| -rwxr-xr-x | script/public-inbox-init | 7 | ||||
| -rwxr-xr-x | script/public-inbox-purge | 53 |
4 files changed, 124 insertions, 28 deletions
diff --git a/lib/PublicInbox/Admin.pm b/lib/PublicInbox/Admin.pm index d0a8dd00..3eff5cde 100644 --- a/lib/PublicInbox/Admin.pm +++ b/lib/PublicInbox/Admin.pm @@ -41,4 +41,64 @@ sub resolve_repo_dir { } } +# TODO: make Devel::Peek optional, only used for daemon +my @base_mod = qw(Email::MIME Date::Parse Devel::Peek); +my @over_mod = qw(DBD::SQLite DBI); +my %mod_groups = ( + -index => [ @base_mod, @over_mod ], + -base => \@base_mod, + -search => [ @base_mod, @over_mod, 'Search::Xapian' ], +); + +sub scan_ibx_modules ($$) { + my ($mods, $ibx) = @_; + if (!$ibx->{indexlevel} || $ibx->{indexlevel} ne 'basic') { + $mods->{'Search::Xapian'} = 1; + } else { + $mods->{$_} = 1 foreach @over_mod; + } +} + +sub check_require { + my (@mods) = @_; + my $err = {}; + while (my $mod = shift @mods) { + if (my $groups = $mod_groups{$mod}) { + push @mods, @$groups; + } else { + eval "require $mod"; + $err->{$mod} = $@ if $@; + } + } + scalar keys %$err ? $err : undef; +} + +sub missing_mod_msg { + my ($err) = @_; + my @mods = map { "`$_'" } sort keys %$err; + my $last = pop @mods; + @mods ? (join(', ', @mods)."' and $last") : $last +} + +sub require_or_die { + my $err = check_require(@_) or return; + die missing_mod_msg($err)." required for $0\n"; +} + +sub indexlevel_ok_or_die ($) { + my ($indexlevel) = @_; + my $req; + if ($indexlevel eq 'basic') { + $req = '-index'; + } elsif ($indexlevel =~ /\A(?:medium|full)\z/) { + $req = '-search'; + } else { + die <<""; +invalid indexlevel=$indexlevel (must be `basic', `medium', or `full') + + } + my $err = check_require($req) or return; + die missing_mod_msg($err) ." required for indexlevel=$indexlevel\n"; +} + 1; diff --git a/script/public-inbox-index b/script/public-inbox-index index 53def9a7..cf001cc1 100755 --- a/script/public-inbox-index +++ b/script/public-inbox-index @@ -10,18 +10,14 @@ use strict; use warnings; use Getopt::Long qw(:config gnu_getopt no_ignore_case auto_abbrev); my $usage = "public-inbox-index REPO_DIR"; -use PublicInbox::Config; use PublicInbox::Admin qw(resolve_repo_dir); +PublicInbox::Admin::require_or_die('-index'); +require PublicInbox::Config; my $config = eval { PublicInbox::Config->new } || eval { warn "public-inbox unconfigured for serving, indexing anyways...\n"; undef; }; -eval { require PublicInbox::SearchIdx }; -if ($@) { - print STDERR "Search::Xapian required for $0\n"; - exit 1; -} my $reindex; my $prune; @@ -56,26 +52,34 @@ defined($config) and $config->each_inbox(sub { } }); +my @inboxes; +my $mods = {}; + foreach my $dir (@dirs) { - if (!ref($dir)) { + my $ibx = $dir; + if (!ref($ibx)) { unless (-d $dir) { die "$dir does not appear to be an inbox repository\n"; } - my $ibx = { + $ibx = PublicInbox::Inbox->new({ mainrepo => $dir, name => 'unnamed', indexlevel => $indexlevel, version => -f "$dir/inbox.lock" ? 2 : 1, - }; - $dir = PublicInbox::Inbox->new($ibx); - } elsif (defined $indexlevel && !defined($dir->{indexlevel})) { + }); + } elsif (defined $indexlevel && !defined($ibx->{indexlevel})) { # XXX: users can shoot themselves in the foot, with this... - $dir->{indexlevel} = $indexlevel; + $ibx->{indexlevel} = $indexlevel; } - - index_inbox($dir); + push @inboxes, $ibx; + PublicInbox::Admin::scan_ibx_modules($mods, $ibx); } +PublicInbox::Admin::require_or_die(keys %$mods); + +require PublicInbox::SearchIdx; +index_inbox($_) for @inboxes; + sub index_inbox { my ($repo) = @_; if (ref($repo) && ($repo->{version} || 1) == 2) { diff --git a/script/public-inbox-init b/script/public-inbox-init index 9f0bd1bd..5516e798 100755 --- a/script/public-inbox-init +++ b/script/public-inbox-init @@ -7,8 +7,10 @@ use strict; use warnings; my $usage = "public-inbox-init NAME REPO_DIR HTTP_URL ADDRESS [ADDRESS..]"; use Getopt::Long qw/:config gnu_getopt no_ignore_case auto_abbrev/; -use PublicInbox::Config; -use PublicInbox::Inbox; +use PublicInbox::Admin; +PublicInbox::Admin::require_or_die('-base'); +require PublicInbox::Config; +require PublicInbox::Inbox; use File::Temp qw/tempfile/; use File::Basename qw/dirname/; use File::Path qw/mkpath/; @@ -24,6 +26,7 @@ my %opts = ( 'V|version=i' => \$version, 'S|skip=i' => \$skip, ); GetOptions(%opts) or usage(); +PublicInbox::Admin::indexlevel_ok_or_die($indexlevel) if defined $indexlevel; my $name = shift @ARGV or usage(); my $mainrepo = shift @ARGV or usage(); my $http_url = shift @ARGV or usage(); diff --git a/script/public-inbox-purge b/script/public-inbox-purge index 264bcdef..381826dc 100755 --- a/script/public-inbox-purge +++ b/script/public-inbox-purge @@ -7,16 +7,16 @@ use strict; use warnings; use Getopt::Long qw(:config gnu_getopt no_ignore_case auto_abbrev); -use PublicInbox::Config; -use PublicInbox::MIME; use PublicInbox::Admin qw(resolve_repo_dir); -use PublicInbox::Filter::Base; -*REJECT = *PublicInbox::Filter::Base::REJECT; +PublicInbox::Admin::check_require('-index'); +require PublicInbox::Filter::Base; +require PublicInbox::Config; +require PublicInbox::MIME; +require PublicInbox::V2Writable; -my $usage = "$0 [--all] [INBOX_DIRS] </path/to/message"; +{ no warnings 'once'; *REJECT = *PublicInbox::Filter::Base::REJECT } -eval { require PublicInbox::V2Writable } or die - "DBI, DBD::SQLite and Search::Xapian required for purge\n"; +my $usage = "$0 [--all] [INBOX_DIRS] </path/to/message"; my $config = eval { PublicInbox::Config->new }; my $cfgfile = PublicInbox::Config::default_file(); my ($all, $force); @@ -63,11 +63,6 @@ if ($all) { mainrepo => $dir, }); }; - - # somebody could "rm -r" all the Xapian directories; - # let them purge the overview, at least - $ibx->{indexlevel} ||= 'basic' unless $ibx->search; - push @inboxes, $ibx; } @@ -77,6 +72,40 @@ if ($all) { } } +foreach my $ibx (@inboxes) { + my $lvl = $ibx->{indexlevel}; + if (defined $lvl) { + PublicInbox::Admin::indexlevel_ok_or_die($lvl); + next; + } + + # Undefined indexlevel, so `full'... + # Search::Xapian exists and the DB can be read, at least, fine + $ibx->search and next; + + # it's possible for a Xapian directory to exist, but Search::Xapian + # to go missing/broken. Make sure it's purged in that case: + $ibx->over or die "no over.sqlite3 in $ibx->{mainrepo}\n"; + + # $ibx->{search} is populated by $ibx->over call + my $xdir_ro = $ibx->{search}->xdir(1); + my $npart = 0; + foreach my $part (<$xdir_ro/*>) { + if (-d $part && $part =~ m!/\d+\z!) { + my $bytes = 0; + $bytes += -s $_ foreach glob("$part/*"); + $npart++ if $bytes; + } + } + if ($npart) { + PublicInbox::Admin::require_or_die('-search'); + } else { + # somebody could "rm -r" all the Xapian directories; + # let them purge the overview, at least + $ibx->{indexlevel} ||= 'basic'; + } +} + my $data = do { local $/; scalar <STDIN> }; $data =~ s/\A[\r\n]*From [^\r\n]*\r?\n//s; my $n_purged = 0; |