From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on dcvr.yhbt.net X-Spam-Level: X-Spam-Status: No, score=-4.0 required=3.0 tests=ALL_TRUSTED,BAYES_00 shortcircuit=no autolearn=ham autolearn_force=no version=3.4.2 Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id E2C351F66E for ; Thu, 13 Aug 2020 08:04:37 +0000 (UTC) From: Eric Wong To: meta@public-inbox.org Subject: [PATCH] index|compact|xcpdb: support --all switch Date: Thu, 13 Aug 2020 08:04:37 +0000 Message-Id: <20200813080437.27605-1-e@yhbt.net> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit List-Id: For -index, this is a convenient way to quickly index all inboxes after a grok-pull. Might as well support it for rarely used commands like -compact and -xcpdb, too. --- Documentation/public-inbox-compact.pod | 10 +++++++++- Documentation/public-inbox-index.pod | 8 ++++++++ Documentation/public-inbox-xcpdb.pod | 10 +++++++++- script/public-inbox-compact | 4 ++-- script/public-inbox-index | 5 +++-- script/public-inbox-xcpdb | 5 +++-- 6 files changed, 34 insertions(+), 8 deletions(-) diff --git a/Documentation/public-inbox-compact.pod b/Documentation/public-inbox-compact.pod index d7a7dba0..8e463ab1 100644 --- a/Documentation/public-inbox-compact.pod +++ b/Documentation/public-inbox-compact.pod @@ -4,7 +4,9 @@ public-inbox-compact - compact Xapian DBs in an inbox =head1 SYNOPSIS - public-inbox-compact INBOX_DIR +public-inbox-compact INBOX_DIR + +public-inbox-compact --all =head1 DESCRIPTION @@ -21,6 +23,12 @@ rest of the public-inbox search code. =over +=item --all + +Compact all inboxes configured in ~/.public-inbox/config. +This is an alternative to specifying individual inboxes directories +on the command-line. + =item --blocksize / --no-full / --fuller These options are passed directly to L. diff --git a/Documentation/public-inbox-index.pod b/Documentation/public-inbox-index.pod index b28db773..1ed9f5e7 100644 --- a/Documentation/public-inbox-index.pod +++ b/Documentation/public-inbox-index.pod @@ -6,6 +6,8 @@ public-inbox-index - create and update search indices public-inbox-index [OPTIONS] INBOX_DIR... +public-inbox-index [OPTIONS] --all + =head1 DESCRIPTION public-inbox-index creates and updates the search, overview and @@ -81,6 +83,12 @@ This does not touch the NNTP article number database. It does not affect threading unless C<--rethread> is used. +=item --all + +Index all inboxes configured in ~/.public-inbox/config. +This is an alternative to specifying individual inboxes directories +on the command-line. + =item --rethread Regenerate internal THREADID and message thread associations diff --git a/Documentation/public-inbox-xcpdb.pod b/Documentation/public-inbox-xcpdb.pod index 62a28c0a..2b38f1da 100644 --- a/Documentation/public-inbox-xcpdb.pod +++ b/Documentation/public-inbox-xcpdb.pod @@ -4,7 +4,9 @@ public-inbox-xcpdb - upgrade Xapian DB formats =head1 SYNOPSIS - public-inbox-xcpdb [OPTIONS] INBOX_DIR +public-inbox-xcpdb [OPTIONS] INBOX_DIR + +public-inbox-xcpdb [OPTIONS] --all =head1 DESCRIPTION @@ -19,6 +21,12 @@ L, and L. =over +=item --all + +Copy all inboxes configured in ~/.public-inbox/config. +This is an alternative to specifying individual inboxes directories +on the command-line. + =item -c, --compact In addition to performing the copy operation, run L diff --git a/script/public-inbox-compact b/script/public-inbox-compact index 5c681466..b5fa0086 100755 --- a/script/public-inbox-compact +++ b/script/public-inbox-compact @@ -10,9 +10,9 @@ use PublicInbox::Admin; PublicInbox::Admin::require_or_die('-index'); my $usage = "Usage: public-inbox-compact INBOX_DIR\n"; my $opt = { compact => 1, -coarse_lock => 1 }; -GetOptions($opt, @PublicInbox::Xapcmd::COMPACT_OPT) or +GetOptions($opt, qw(all), @PublicInbox::Xapcmd::COMPACT_OPT) or die "bad command-line args\n$usage"; -my @ibxs = PublicInbox::Admin::resolve_inboxes(\@ARGV) or die $usage; +my @ibxs = PublicInbox::Admin::resolve_inboxes(\@ARGV, $opt) or die $usage; foreach (@ibxs) { my $ibx = PublicInbox::InboxWritable->new($_); PublicInbox::Xapcmd::run($ibx, 'compact', $opt); diff --git a/script/public-inbox-index b/script/public-inbox-index index 14d3afd4..30d24838 100755 --- a/script/public-inbox-index +++ b/script/public-inbox-index @@ -18,6 +18,7 @@ options: --no-fsync speed up indexing, risk corruption on power outage -L LEVEL `basic', `medium', or `full' (default: full) + --all index all configured inboxes --compact | -c run public-inbox-compact(1) after indexing --sequential-shard index Xapian shards sequentially for slow storage --jobs=NUM set or disable parallelization (NUM=0) @@ -38,7 +39,7 @@ GetOptions($opt, qw(verbose|v+ reindex rethread compact|c+ jobs|j=i prune indexlevel|index-level|L=s max_size|max-size=s batch_size|batch-size=s sequential_shard|seq-shard|sequential-shard - help|?)) + all help|?)) or die "bad command-line args\n$usage"; if ($opt->{help}) { print $help; exit 0 }; die "--jobs must be >= 0\n" if defined $opt->{jobs} && $opt->{jobs} < 0; @@ -51,7 +52,7 @@ require PublicInbox::Admin; PublicInbox::Admin::require_or_die('-index'); my $cfg = PublicInbox::Config->new; # Config is loaded by Admin -my @ibxs = PublicInbox::Admin::resolve_inboxes(\@ARGV, undef, $cfg); +my @ibxs = PublicInbox::Admin::resolve_inboxes(\@ARGV, $opt, $cfg); PublicInbox::Admin::require_or_die('-index'); unless (@ibxs) { print STDERR "Usage: $usage\n"; exit 1 } diff --git a/script/public-inbox-xcpdb b/script/public-inbox-xcpdb index 718a34b7..2bfadc09 100755 --- a/script/public-inbox-xcpdb +++ b/script/public-inbox-xcpdb @@ -13,6 +13,7 @@ usage: $usage options: --compact | -c run public-inbox-compact(1) after indexing + --all copy all configured inboxes --reshard=NUM change number the number of shards --jobs=NUM limit parallelism to JOBS count --verbose | -v increase verbosity (may be repeated) @@ -34,7 +35,7 @@ GetOptions($opt, qw( sequential_shard|seq-shard|sequential-shard jobs|j=i quiet|q verbose|v blocksize|b=s no-full|n fuller|F - help|?)) or die "bad command-line args\n$usage"; + all help|?)) or die "bad command-line args\n$usage"; if ($opt->{help}) { print $help; exit 0 }; use PublicInbox::Admin; @@ -42,7 +43,7 @@ PublicInbox::Admin::require_or_die('-search'); require PublicInbox::Config; my $cfg = PublicInbox::Config->new; -my @ibxs = PublicInbox::Admin::resolve_inboxes(\@ARGV, undef, $cfg) or +my @ibxs = PublicInbox::Admin::resolve_inboxes(\@ARGV, $opt, $cfg) or die $usage; my $idx_env = PublicInbox::Admin::index_prepare($opt, $cfg);