* [PATCH 1/7] xapcmd: quietly no-op on indexlevel=basic
2020-08-07 1:13 [PATCH 0/7] index: --sequential-shard and other stuff Eric Wong
@ 2020-08-07 1:14 ` Eric Wong
2020-08-07 1:14 ` [PATCH 2/7] xapcmd: remove redundant searchidx require Eric Wong
` (5 subsequent siblings)
6 siblings, 0 replies; 8+ messages in thread
From: Eric Wong @ 2020-08-07 1:14 UTC (permalink / raw)
To: meta
I find myself mindlessly adding "-c" to public-inbox-index,
and other users may do the same. Instead of erroring out,
we'll just silently ignore it, for now and allow
public-inbox-compact to work on SQLite-only inboxes.
We'll only check for xapian-compact if search exists, since
it won't be needed in case we support SQLite VACUUM.
---
lib/PublicInbox/Xapcmd.pm | 14 ++++++++------
t/indexlevels-mirror.t | 24 +++++++++++++++++-------
2 files changed, 25 insertions(+), 13 deletions(-)
diff --git a/lib/PublicInbox/Xapcmd.pm b/lib/PublicInbox/Xapcmd.pm
index f1c80831..8d09ed27 100644
--- a/lib/PublicInbox/Xapcmd.pm
+++ b/lib/PublicInbox/Xapcmd.pm
@@ -167,9 +167,11 @@ sub prepare_run {
my ($ibx, $opt) = @_;
my $tmp = {}; # old shard dir => File::Temp->newdir object or undef
my @queue; # ([old//src,newdir]) - list of args for cpdb() or compact()
-
- my $old = $ibx->search->xdir(1);
- -d $old or die "$old does not exist\n";
+ my $old;
+ if (my $srch = $ibx->search) {
+ $old = $srch->xdir(1);
+ -d $old or die "$old does not exist\n";
+ }
my $reshard = $opt->{reshard};
if (defined $reshard && $reshard <= 0) {
die "--reshard must be a positive number\n";
@@ -177,7 +179,7 @@ sub prepare_run {
# we want temporary directories to be as deep as possible,
# so v2 shards can keep "xap$SCHEMA_VERSION" on a separate FS.
- if ($ibx->version == 1) {
+ if ($old && $ibx->version == 1) {
if (defined $reshard) {
warn
"--reshard=$reshard ignored for v1 $ibx->{inboxdir}\n";
@@ -189,7 +191,7 @@ sub prepare_run {
$tmp->{$old} = $wip;
nodatacow_dir($wip->dirname);
push @queue, [ $old, $wip ];
- } else {
+ } elsif ($old) {
opendir my $dh, $old or die "Failed to opendir $old: $!\n";
my @old_shards;
while (defined(my $dn = readdir($dh))) {
@@ -255,7 +257,7 @@ sub run {
PublicInbox::Admin::progress_prepare($opt ||= {});
defined(my $dir = $ibx->{inboxdir}) or die "no inboxdir defined\n";
-d $dir or die "inboxdir=$dir does not exist\n";
- check_compact() if $opt->{compact};
+ check_compact() if $opt->{compact} && $ibx->search;
my $reindex; # v1:{ from => $x40 }, v2:{ from => [ $x40, $x40, .. ] } }
if (!$opt->{-coarse_lock}) {
diff --git a/t/indexlevels-mirror.t b/t/indexlevels-mirror.t
index 44313e40..859c2c17 100644
--- a/t/indexlevels-mirror.t
+++ b/t/indexlevels-mirror.t
@@ -6,11 +6,13 @@ use Test::More;
use PublicInbox::Eml;
use PublicInbox::Inbox;
use PublicInbox::InboxWritable;
+use PublicInbox::Spawn qw(which);
require PublicInbox::Admin;
use PublicInbox::TestCommon;
my $PI_TEST_VERSION = $ENV{PI_TEST_VERSION} || 2;
require_git('2.6') if $PI_TEST_VERSION == 2;
require_mods(qw(DBD::SQLite));
+my $have_xapian_compact = which($ENV{XAPIAN_COMPACT} || 'xapian-compact');
my $mime = PublicInbox::Eml->new(<<'EOF');
From: a@example.com
@@ -21,8 +23,9 @@ Date: Fri, 02 Oct 1993 00:00:00 +0000
hello world
EOF
-sub import_index_incremental {
+my $import_index_incremental = sub {
my ($v, $level, $mime) = @_;
+ my $err = '';
my $this = "pi-$v-$level-indexlevels";
my ($tmpdir, $for_destroy) = tmpdir();
local $ENV{PI_CONFIG} = "$tmpdir/config";
@@ -39,8 +42,9 @@ sub import_index_incremental {
$im->done;
# index master (required for v1)
- ok(run_script([qw(-index -j0), $ibx->{inboxdir}, "-L$level"]),
- 'index master OK');
+ my @cmd = (qw(-index -j0), $ibx->{inboxdir}, "-L$level");
+ push @cmd, '-c' if $have_xapian_compact;
+ ok(run_script(\@cmd, undef, { 2 => \$err }), 'index master');
my $ro_master = PublicInbox::Inbox->new({
inboxdir => $ibx->{inboxdir},
indexlevel => $level
@@ -50,7 +54,7 @@ sub import_index_incremental {
is($msgs->[0]->{mid}, 'm@1', 'first message in master indexed');
# clone
- my @cmd = (qw(git clone --mirror -q));
+ @cmd = (qw(git clone --mirror -q));
my $mirror = "$tmpdir/mirror-$v";
if ($v == 1) {
push @cmd, $ibx->{inboxdir}, $mirror;
@@ -157,17 +161,23 @@ sub import_index_incremental {
is(PublicInbox::Admin::detect_indexlevel($ro_mirror), $level,
'indexlevel detectable by Admin '.$v.$level);
-}
+
+ SKIP: {
+ skip 'xapian-compact missing', 1 if !$have_xapian_compact;
+ my $cmd = [ qw(-compact), $mirror ];
+ ok(run_script($cmd, undef, { 2 => \$err}), "compact $level");
+ }
+};
# we can probably cull some other tests
-import_index_incremental($PI_TEST_VERSION, 'basic', $mime);
+$import_index_incremental->($PI_TEST_VERSION, 'basic', $mime);
SKIP: {
require PublicInbox::Search;
PublicInbox::Search::load_xapian() or
skip('Xapian perl binding missing', 2);
foreach my $l (qw(medium full)) {
- import_index_incremental($PI_TEST_VERSION, $l, $mime);
+ $import_index_incremental->($PI_TEST_VERSION, $l, $mime);
}
}
^ permalink raw reply related [flat|nested] 8+ messages in thread* [PATCH 2/7] xapcmd: remove redundant searchidx require
2020-08-07 1:13 [PATCH 0/7] index: --sequential-shard and other stuff Eric Wong
2020-08-07 1:14 ` [PATCH 1/7] xapcmd: quietly no-op on indexlevel=basic Eric Wong
@ 2020-08-07 1:14 ` Eric Wong
2020-08-07 1:14 ` [PATCH 3/7] xapcmd: drop outdated comment Eric Wong
` (4 subsequent siblings)
6 siblings, 0 replies; 8+ messages in thread
From: Eric Wong @ 2020-08-07 1:14 UTC (permalink / raw)
To: meta
We already "use" it starting with commit
cd8dd7b08fddc7c2b5f218c3fcaa5dca5f9ad945
("search: support SWIG-generated Xapian.pm"),
so there's no need to require it redundantly.
---
lib/PublicInbox/Xapcmd.pm | 1 -
1 file changed, 1 deletion(-)
diff --git a/lib/PublicInbox/Xapcmd.pm b/lib/PublicInbox/Xapcmd.pm
index 8d09ed27..47a018d2 100644
--- a/lib/PublicInbox/Xapcmd.pm
+++ b/lib/PublicInbox/Xapcmd.pm
@@ -264,7 +264,6 @@ sub run {
$reindex = $opt->{reindex} = { # per-epoch ranges for v2
from => $ibx->version == 1 ? '' : [],
};
- require PublicInbox::SearchIdx;
PublicInbox::SearchIdx::load_xapian_writable();
}
^ permalink raw reply related [flat|nested] 8+ messages in thread* [PATCH 3/7] xapcmd: drop outdated comment
2020-08-07 1:13 [PATCH 0/7] index: --sequential-shard and other stuff Eric Wong
2020-08-07 1:14 ` [PATCH 1/7] xapcmd: quietly no-op on indexlevel=basic Eric Wong
2020-08-07 1:14 ` [PATCH 2/7] xapcmd: remove redundant searchidx require Eric Wong
@ 2020-08-07 1:14 ` Eric Wong
2020-08-07 1:14 ` [PATCH 4/7] v2writable: fix rethread cleanup Eric Wong
` (3 subsequent siblings)
6 siblings, 0 replies; 8+ messages in thread
From: Eric Wong @ 2020-08-07 1:14 UTC (permalink / raw)
To: meta
We replaced Xtmpdir with File::Temp->newdir in
commit 2a3e3a0469f54f6a4f80bf04614e5ddd794a6c5e
("xapcmd: replace Xtmpdirs with File::Temp->newdir")
but forgot to remove the outdated comment.
---
lib/PublicInbox/Xapcmd.pm | 1 -
1 file changed, 1 deletion(-)
diff --git a/lib/PublicInbox/Xapcmd.pm b/lib/PublicInbox/Xapcmd.pm
index 47a018d2..97a51d1b 100644
--- a/lib/PublicInbox/Xapcmd.pm
+++ b/lib/PublicInbox/Xapcmd.pm
@@ -53,7 +53,6 @@ sub commit_changes ($$$$) {
rename($old, "$new/old") or
die "rename $old => $new/old: $!\n";
}
- # Xtmpdir->DESTROY won't remove $new after this:
rename($new, $old) or die "rename $new => $old: $!\n";
if (@st) {
my $prev = "$old/old";
^ permalink raw reply related [flat|nested] 8+ messages in thread* [PATCH 4/7] v2writable: fix rethread cleanup
2020-08-07 1:13 [PATCH 0/7] index: --sequential-shard and other stuff Eric Wong
` (2 preceding siblings ...)
2020-08-07 1:14 ` [PATCH 3/7] xapcmd: drop outdated comment Eric Wong
@ 2020-08-07 1:14 ` Eric Wong
2020-08-07 1:14 ` [PATCH 5/7] index: v2: --sequential-shard option Eric Wong
` (2 subsequent siblings)
6 siblings, 0 replies; 8+ messages in thread
From: Eric Wong @ 2020-08-07 1:14 UTC (permalink / raw)
To: meta
We need to drop old ghosts properly while inside the
transaction, otherwise it becomes a no-op. This isn't a big
deal, as it only results in a few dangling DB rows and a
small amount of wasted space.
---
lib/PublicInbox/V2Writable.pm | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/PublicInbox/V2Writable.pm b/lib/PublicInbox/V2Writable.pm
index 344edbba..f98afa61 100644
--- a/lib/PublicInbox/V2Writable.pm
+++ b/lib/PublicInbox/V2Writable.pm
@@ -1244,13 +1244,13 @@ sub index_sync {
}
# work forwards through history
index_epoch($self, $sync, $_) for (0..$epoch_max);
+ $self->{over}->rethread_done($opt);
$self->done;
if (my $nr = $sync->{nr}) {
my $pr = $sync->{-opt}->{-progress};
$pr->('all.git '.sprintf($sync->{-regen_fmt}, $$nr)) if $pr;
}
- $self->{over}->rethread_done($opt);
# reindex does not pick up new changes, so we rerun w/o it:
if ($opt->{reindex}) {
^ permalink raw reply related [flat|nested] 8+ messages in thread* [PATCH 5/7] index: v2: --sequential-shard option
2020-08-07 1:13 [PATCH 0/7] index: --sequential-shard and other stuff Eric Wong
` (3 preceding siblings ...)
2020-08-07 1:14 ` [PATCH 4/7] v2writable: fix rethread cleanup Eric Wong
@ 2020-08-07 1:14 ` Eric Wong
2020-08-07 1:14 ` [PATCH 6/7] index: support --xapian-only switch Eric Wong
2020-08-07 1:14 ` [PATCH 7/7] index+xcpdb: rename `--no-sync' to `--no-fsync' Eric Wong
6 siblings, 0 replies; 8+ messages in thread
From: Eric Wong @ 2020-08-07 1:14 UTC (permalink / raw)
To: meta
This gives better page cache utilization for Xapian indexing on
slow storage by improving locality for random I/O activity on
the Xapian DB.
Instead of doing a single-pass to index both SQLite and Xapian;
this indexes them separately. The first pass is identical to
indexlevel=basic: it indexes both over.sqlite3 and msgmap.sqlite3.
Subsequent passes only operate on a single Xapian shard for
documents belonging to that shard. Given enough shards, each
individual shard can be made small enough to fit into the kernel
page cache and avoid HDD seeks for read activity.
Doing rough tests with a busy system with a 7200 RPM HDD with ext4,
full indexing of LKML (9 epochs) goes from ~80 hours (-j0) to
~30 hours (-j8) with 16GB RAM with 7 shards configured and fsync(2)
disabled (--no-sync) and `--batch-size=10m'.
---
Documentation/public-inbox-config.pod | 6 +++
Documentation/public-inbox-index.pod | 53 ++++++++++++++++++++++-
Documentation/public-inbox-v2-format.pod | 11 +++--
lib/PublicInbox/Config.pm | 9 ++--
lib/PublicInbox/V2Writable.pm | 55 ++++++++++++++++++++++--
lib/PublicInbox/WatchMaildir.pm | 2 +-
script/public-inbox-index | 22 +++++++++-
t/config.t | 6 +--
t/v2mirror.t | 14 ++++++
9 files changed, 161 insertions(+), 17 deletions(-)
diff --git a/Documentation/public-inbox-config.pod b/Documentation/public-inbox-config.pod
index e6108c35..05b84819 100644
--- a/Documentation/public-inbox-config.pod
+++ b/Documentation/public-inbox-config.pod
@@ -139,6 +139,10 @@ allow for searching for phrases using quoted text.
Default: C<full>
+=item publicinbox.<name>.indexSequentialShard
+
+See L<public-inbox-index(1)/publicInbox.indexSequentialShard>
+
=item publicinbox.<name>.httpbackendmax
If a digit, the maximum number of parallel
@@ -291,6 +295,8 @@ or /usr/share/cgit/
See L<public-inbox-edit(1)>
=item publicinbox.indexMaxSize
+=item publicinbox.indexBatchSize
+=item publicinbox.indexSequentialShard
See L<public-inbox-index(1)>
diff --git a/Documentation/public-inbox-index.pod b/Documentation/public-inbox-index.pod
index aeb1b3a3..f525ba54 100644
--- a/Documentation/public-inbox-index.pod
+++ b/Documentation/public-inbox-index.pod
@@ -34,12 +34,16 @@ normal search functionality.
=item --jobs=JOBS, -j
-Control the number of Xapian indexing jobs in a
+Influences the number of Xapian indexing shards in a
(L<public-inbox-v2-format(5)>) inbox.
C<--jobs=0> is accepted as of public-inbox 1.6.0 (PENDING)
to disable parallel indexing.
+If the inbox has not been indexed, C<JOBS - 1> shards
+will be created (one job is always needed for indexing
+the overview and article number mapping).
+
Default: the number of existing Xapian shards
=item --compact / -c
@@ -120,6 +124,14 @@ and Xapian. This is only effective with Xapian 1.4+.
Available in public-inbox 1.6.0 (PENDING).
+=item --sequential-shard
+
+Sets or overrides L</publicinbox.indexSequentialShard> on a
+per-invocation basis. See L</publicinbox.indexSequentialShard>
+below.
+
+Available in public-inbox 1.6.0 (PENDING).
+
=back
=head1 FILES
@@ -167,6 +179,45 @@ inbox with 3 shards will flush every 3 megabytes by default.
Default: 1m (one megabyte)
+=item publicinbox.indexBatchSize
+
+Flushes changes to the filesystem and releases locks after
+indexing the given number of bytes. The default value of C<1m>
+(one megabyte) is low to minimize memory use and reduce
+contention with parallel invocations of L<public-inbox-mda(1)>,
+L<public-inbox-learn(1)>, and L<public-inbox-watch(1)>.
+
+Increase this value on powerful systems to improve throughput at
+the expense of memory use. The reduction of lock granularity
+may not be noticeable on fast systems.
+
+This option is available in public-inbox 1.6 or later.
+public-inbox 1.5 and earlier used the current default, C<1m>.
+
+For L<public-inbox-v2-format(5)> inboxes, this value is
+multiplied by the number of Xapian shards. Thus a typical v2
+inbox with 3 shards will flush every 3 megabytes by default.
+
+Default: 1m (one megabyte)
+
+=item publicinbox.indexSequentialShard
+=item publicinbox.<inbox_name>.indexSequentialShard
+
+For L<public-inbox-v2-format(5)> inboxes, setting this to C<true>
+allows indexing Xapian shards in multiple passes. This speeds up
+indexing on rotational storage with high seek latency by allowing
+individual shards to fit into the kernel page cache.
+
+Using a higher-than-normal number of C<--jobs> with
+L<public-inbox-init(1)> may be required to ensure individual
+shards are small enough to fit into cache.
+
+Available in public-inbox 1.6.0 (PENDING).
+
+This is ignored on L<public-inbox-v1-format(5)> inboxes.
+
+Default: false, shards are indexed in parallel
+
=back
=head1 ENVIRONMENT
diff --git a/Documentation/public-inbox-v2-format.pod b/Documentation/public-inbox-v2-format.pod
index 9e284a75..6876989c 100644
--- a/Documentation/public-inbox-v2-format.pod
+++ b/Documentation/public-inbox-v2-format.pod
@@ -113,9 +113,14 @@ improved with high-quality and high-quantity solid-state storage.
Issuing TRIM commands with L<fstrim(8)> was necessary to maintain
consistent performance while developing this feature.
-Rotational storage devices are NOT recommended for indexing of
-large mail archives; but are fine for backup and usable for
-small instances.
+Rotational storage devices perform significantly worse than
+solid state storage for indexing of large mail archives; but are
+fine for backup and usable for small instances.
+
+As of public-inbox 1.6.0, the C<--sequential-shard> option of
+L<public-inbox-index(1)> may be used with a high shard count
+to ensure individual shards fit into page cache when the entire
+Xapian DB cannot.
Our use of the L</OVERVIEW DB> requires Xapian document IDs to
remain stable. Using L<public-inbox-compact(1)> and
diff --git a/lib/PublicInbox/Config.pm b/lib/PublicInbox/Config.pm
index 67199bb3..f9184bd2 100644
--- a/lib/PublicInbox/Config.pm
+++ b/lib/PublicInbox/Config.pm
@@ -369,8 +369,8 @@ sub _fill_code_repo {
$git;
}
-sub _git_config_bool ($) {
- my ($val) = @_;
+sub git_bool {
+ my ($val) = $_[-1]; # $_[0] may be $self, or $val
if ($val =~ /\A(?:false|no|off|[\-\+]?(?:0x)?0+)\z/i) {
0;
} elsif ($val =~ /\A(?:true|yes|on|[\-\+]?(?:0x)?[0-9]+)\z/i) {
@@ -386,7 +386,8 @@ sub _fill {
foreach my $k (qw(inboxdir filter newsgroup
watch httpbackendmax
- replyto feedmax nntpserver indexlevel)) {
+ replyto feedmax nntpserver
+ indexlevel indexsequentialshard)) {
my $v = $self->{"$pfx.$k"};
$ibx->{$k} = $v if defined $v;
}
@@ -400,7 +401,7 @@ sub _fill {
foreach my $k (qw(obfuscate)) {
my $v = $self->{"$pfx.$k"};
defined $v or next;
- if (defined(my $bval = _git_config_bool($v))) {
+ if (defined(my $bval = git_bool($v))) {
$ibx->{$k} = $bval;
} else {
warn "Ignoring $pfx.$k=$v in config, not boolean\n";
diff --git a/lib/PublicInbox/V2Writable.pm b/lib/PublicInbox/V2Writable.pm
index f98afa61..7bc24592 100644
--- a/lib/PublicInbox/V2Writable.pm
+++ b/lib/PublicInbox/V2Writable.pm
@@ -875,7 +875,8 @@ sub reindex_checkpoint ($$) {
$self->{ibx}->git->cleanup; # *async_wait
${$sync->{need_checkpoint}} = 0;
- $sync->{mm_tmp}->atfork_prepare;
+ my $mm_tmp = $sync->{mm_tmp};
+ $mm_tmp->atfork_prepare if $mm_tmp;
$self->done; # release lock
if (my $pr = $sync->{-opt}->{-progress}) {
@@ -884,7 +885,7 @@ sub reindex_checkpoint ($$) {
# allow -watch or -mda to write...
$self->idx_init; # reacquire lock
- $sync->{mm_tmp}->atfork_parent;
+ $mm_tmp->atfork_parent if $mm_tmp;
}
sub index_oid { # cat_async callback
@@ -1085,7 +1086,10 @@ sub sync_prepare ($$$) {
}
$all->cat_async_wait;
}
- return 0 if (!$regen_max && !keys(%{$self->{unindex_range}}));
+ if (!$regen_max && !keys(%{$self->{unindex_range}})) {
+ $sync->{-regen_fmt} = "%u/?\n";
+ return 0;
+ }
# reindex should NOT see new commits anymore, if we do,
# it's a problem and we need to notice it via die()
@@ -1177,6 +1181,36 @@ sub sync_ranges ($$$) {
$ranges;
}
+sub index_xap_only { # git->cat_async callback
+ my ($bref, $oid, $type, $size, $smsg) = @_;
+ my $self = $smsg->{v2w};
+ my $idx = idx_shard($self, $smsg->{num} % $self->{shards});
+ $idx->begin_txn_lazy;
+ $idx->add_message(PublicInbox::Eml->new($bref), $smsg);
+ $self->{transact_bytes} += $size;
+}
+
+sub index_seq_shard ($$$) {
+ my ($self, $sync, $off) = @_;
+ my $ibx = $self->{ibx};
+ my $max = $ibx->mm->max or return;
+ my $all = $ibx->git;
+ my $over = $ibx->over;
+ my $batch_bytes = $PublicInbox::SearchIdx::BATCH_BYTES;
+ if (my $pr = $sync->{-opt}->{-progress}) {
+ $pr->("Xapian indexlevel=$ibx->{indexlevel} % $off\n");
+ }
+ for (my $num = $off; $num <= $max; $num += $self->{shards}) {
+ my $smsg = $over->get_art($num) or next;
+ $smsg->{v2w} = $self;
+ $all->cat_async($smsg->{blob}, \&index_xap_only, $smsg);
+ if ($self->{transact_bytes} >= $batch_bytes) {
+ ${$sync->{nr}} = $num;
+ reindex_checkpoint($self, $sync);
+ }
+ }
+}
+
sub index_epoch ($$$) {
my ($self, $sync, $i) = @_;
@@ -1218,6 +1252,11 @@ sub index_sync {
my $epoch_max;
my $latest = git_dir_latest($self, \$epoch_max);
return unless defined $latest;
+
+ my $seq = $opt->{sequentialshard};
+ my $idxlevel = $self->{ibx}->{indexlevel};
+ local $self->{ibx}->{indexlevel} = 'basic' if $seq;
+
$self->idx_init($opt); # acquire lock
fill_alternates($self, $epoch_max);
$self->{over}->rethread_prepare($opt);
@@ -1252,6 +1291,16 @@ sub index_sync {
$pr->('all.git '.sprintf($sync->{-regen_fmt}, $$nr)) if $pr;
}
+ if ($seq) { # deal with Xapian shards sequentially
+ my $end = $self->{shards} - 1;
+ $self->{ibx}->{indexlevel} = $idxlevel;
+ delete $sync->{mm_tmp};
+ $self->idx_init($opt); # re-acquire lock
+ index_seq_shard($self, $sync, $_) for (0..$end);
+ $self->{ibx}->git->cat_async_wait;
+ $self->done;
+ }
+
# reindex does not pick up new changes, so we rerun w/o it:
if ($opt->{reindex}) {
my %again = %$opt;
diff --git a/lib/PublicInbox/WatchMaildir.pm b/lib/PublicInbox/WatchMaildir.pm
index 142118bd..2ba10a9e 100644
--- a/lib/PublicInbox/WatchMaildir.pm
+++ b/lib/PublicInbox/WatchMaildir.pm
@@ -285,7 +285,7 @@ sub cfg_intvl ($$$) {
sub cfg_bool ($$$) {
my ($cfg, $key, $url) = @_;
my $orig = $cfg->urlmatch($key, $url) // return;
- my $bool = PublicInbox::Config::_git_config_bool($orig);
+ my $bool = $cfg->git_bool($orig);
warn "W: $key=$orig for $url is not boolean\n" unless defined($bool);
$bool;
}
diff --git a/script/public-inbox-index b/script/public-inbox-index
index 5a0ceab7..be518134 100755
--- a/script/public-inbox-index
+++ b/script/public-inbox-index
@@ -16,7 +16,8 @@ use PublicInbox::Xapcmd;
my $compact_opt;
my $opt = { quiet => -1, compact => 0, maxsize => undef, sync => 1 };
GetOptions($opt, qw(verbose|v+ reindex rethread compact|c+ jobs|j=i prune sync!
- indexlevel|L=s maxsize|max-size=s batchsize|batch-size=s))
+ indexlevel|L=s maxsize|max-size=s batchsize|batch-size=s
+ sequentialshard|seq-shard|sequential-shard))
or die "bad command-line args\n$usage";
die "--jobs must be >= 0\n" if defined $opt->{jobs} && $opt->{jobs} < 0;
@@ -46,6 +47,15 @@ if (my $bs = $opt->{batchsize} // $cfg->{lc('publicInbox.indexBatchSize')}) {
$PublicInbox::SearchIdx::BATCH_BYTES = $bs;
}
+my $s = $opt->{sequentialshard} //
+ $cfg->{lc('publicInbox.indexSequentialShard')};
+if (defined $s) {
+ my $v = $cfg->git_bool($s);
+ defined($v) or
+ die "`publicInbox.indexSequentialShard=$s' not boolean\n";
+ $opt->{sequentialshard} = $v;
+}
+
my $mods = {};
foreach my $ibx (@ibxs) {
# XXX: users can shoot themselves in the foot, with opt->{indexlevel}
@@ -63,6 +73,14 @@ for my $ibx (@ibxs) {
PublicInbox::Xapcmd::run($ibx, 'compact', $compact_opt);
}
$ibx->{-no_sync} = 1 if !$opt->{sync};
- PublicInbox::Admin::index_inbox($ibx, undef, $opt);
+
+ my $ibx_opt = $opt;
+ if (defined(my $s = $ibx->{indexsequentialshard})) {
+ defined(my $v = $cfg->git_bool($s)) or die <<EOL;
+publicInbox.$ibx->{name}.indexSequentialShard not boolean
+EOL
+ $ibx_opt = { %$opt, sequentialshard => $v };
+ }
+ PublicInbox::Admin::index_inbox($ibx, undef, $ibx_opt);
PublicInbox::Xapcmd::run($ibx, 'compact', $compact_opt) if $compact_opt;
}
diff --git a/t/config.t b/t/config.t
index d7fd9446..ee51c6cc 100644
--- a/t/config.t
+++ b/t/config.t
@@ -220,18 +220,18 @@ EOF
{
for my $t (qw(TRUE true yes on 1 +1 -1 13 0x1 0x12 0X5)) {
- is(PublicInbox::Config::_git_config_bool($t), 1, "$t is true");
+ is(PublicInbox::Config::git_bool($t), 1, "$t is true");
is(xqx([qw(git -c), "test.val=$t",
qw(config --bool test.val)]),
"true\n", "$t matches git-config behavior");
}
for my $f (qw(FALSE false no off 0 +0 +000 00 0x00 0X0)) {
- is(PublicInbox::Config::_git_config_bool($f), 0, "$f is false");
+ is(PublicInbox::Config::git_bool($f), 0, "$f is false");
is(xqx([qw(git -c), "test.val=$f",
qw(config --bool test.val)]),
"false\n", "$f matches git-config behavior");
}
- is(PublicInbox::Config::_git_config_bool('bogus'), undef,
+ is(PublicInbox::Config::git_bool('bogus'), undef,
'bogus is undef');
}
diff --git a/t/v2mirror.t b/t/v2mirror.t
index b24528fe..a4ac682d 100644
--- a/t/v2mirror.t
+++ b/t/v2mirror.t
@@ -4,6 +4,7 @@ use strict;
use warnings;
use Test::More;
use PublicInbox::TestCommon;
+use File::Path qw(remove_tree);
use Cwd qw(abs_path);
require_git(2.6);
local $ENV{HOME} = abs_path('t');
@@ -189,6 +190,19 @@ is($mibx->git->check($to_purge), undef, 'unindex+prune successful in mirror');
is(scalar($mset->items), 0, '1@example.com no longer visible in mirror');
}
+if ('sequential-shard') {
+ $mset = $mibx->search->query('m:15@example.com', {mset => 1});
+ is(scalar($mset->items), 1, 'large message not indexed');
+ remove_tree(glob("$tmpdir/m/xap*"), glob("$tmpdir/m/msgmap.*"));
+ my $cmd = [ qw(-index -j9 --sequential-shard), "$tmpdir/m" ];
+ ok(run_script($cmd), '--sequential-shard works');
+ my @shards = glob("$tmpdir/m/xap*/?");
+ is(scalar(@shards), 8, 'got expected shard count');
+ PublicInbox::InboxWritable::cleanup($mibx);
+ $mset = $mibx->search->query('m:15@example.com', {mset => 1});
+ is(scalar($mset->items), 1, 'search works after --sequential-shard');
+}
+
if ('max size') {
$mime->header_set('Message-ID', '<2big@a>');
my $max = '2k';
^ permalink raw reply related [flat|nested] 8+ messages in thread* [PATCH 6/7] index: support --xapian-only switch
2020-08-07 1:13 [PATCH 0/7] index: --sequential-shard and other stuff Eric Wong
` (4 preceding siblings ...)
2020-08-07 1:14 ` [PATCH 5/7] index: v2: --sequential-shard option Eric Wong
@ 2020-08-07 1:14 ` Eric Wong
2020-08-07 1:14 ` [PATCH 7/7] index+xcpdb: rename `--no-sync' to `--no-fsync' Eric Wong
6 siblings, 0 replies; 8+ messages in thread
From: Eric Wong @ 2020-08-07 1:14 UTC (permalink / raw)
To: meta
This is useful for speeding up indexing runs when only Xapian
rules change but SQLite indexing doesn't change. This mostly
implies `--reindex', but does NOT pick up new messages (because
SQLite indexing needs to occur for that).
I'm leaving this undocumented in the manpage for now since it's
mainly to speed up development and testing. Users upgrading to
1.6.0 will be advised to `--reindex --rethread', anyways, due to
the threading improvements since 1.1.0-pre1.
It may make sense to document for 1.7+ when there's Xapian-only
indexing changes, though.
---
lib/PublicInbox/SearchIdxShard.pm | 10 ++++--
lib/PublicInbox/V2Writable.pm | 51 +++++++++++++++++++++++--------
script/public-inbox-index | 5 +--
3 files changed, 48 insertions(+), 18 deletions(-)
diff --git a/lib/PublicInbox/SearchIdxShard.pm b/lib/PublicInbox/SearchIdxShard.pm
index cb79f3dc..59b36087 100644
--- a/lib/PublicInbox/SearchIdxShard.pm
+++ b/lib/PublicInbox/SearchIdxShard.pm
@@ -89,16 +89,20 @@ sub shard_worker_loop ($$$$$) {
# called by V2Writable
sub index_raw {
- my ($self, $msgref, $mime, $smsg) = @_;
+ my ($self, $msgref, $eml, $smsg) = @_;
if (my $w = $self->{w}) {
# mid must be last, it can contain spaces (but not LF)
print $w join(' ', @$smsg{qw(raw_bytes bytes
num blob ds ts mid)}),
"\n", $$msgref or die "failed to write shard $!\n";
} else {
- $$msgref = undef;
+ if ($eml) {
+ $$msgref = undef;
+ } else { # --xapian-only + --sequential-shard:
+ $eml = PublicInbox::Eml->new($msgref);
+ }
$self->begin_txn_lazy;
- $self->add_message($mime, $smsg);
+ $self->add_message($eml, $smsg);
}
}
diff --git a/lib/PublicInbox/V2Writable.pm b/lib/PublicInbox/V2Writable.pm
index 7bc24592..6b1effe5 100644
--- a/lib/PublicInbox/V2Writable.pm
+++ b/lib/PublicInbox/V2Writable.pm
@@ -1185,22 +1185,24 @@ sub index_xap_only { # git->cat_async callback
my ($bref, $oid, $type, $size, $smsg) = @_;
my $self = $smsg->{v2w};
my $idx = idx_shard($self, $smsg->{num} % $self->{shards});
- $idx->begin_txn_lazy;
- $idx->add_message(PublicInbox::Eml->new($bref), $smsg);
+ $smsg->{raw_bytes} = $size;
+ $idx->index_raw($bref, undef, $smsg);
$self->{transact_bytes} += $size;
}
-sub index_seq_shard ($$$) {
- my ($self, $sync, $off) = @_;
+sub index_xap_step ($$$;$) {
+ my ($self, $sync, $beg, $step) = @_;
my $ibx = $self->{ibx};
- my $max = $ibx->mm->max or return;
my $all = $ibx->git;
my $over = $ibx->over;
my $batch_bytes = $PublicInbox::SearchIdx::BATCH_BYTES;
+ $step //= $self->{shards};
+ my $end = $sync->{art_end};
if (my $pr = $sync->{-opt}->{-progress}) {
- $pr->("Xapian indexlevel=$ibx->{indexlevel} % $off\n");
+ $pr->("Xapian indexlevel=$ibx->{indexlevel} ".
+ "$beg..$end (% $step)\n");
}
- for (my $num = $off; $num <= $max; $num += $self->{shards}) {
+ for (my $num = $beg; $num <= $end; $num += $step) {
my $smsg = $over->get_art($num) or next;
$smsg->{v2w} = $self;
$all->cat_async($smsg->{blob}, \&index_xap_only, $smsg);
@@ -1244,10 +1246,37 @@ sub index_epoch ($$$) {
update_last_commit($self, $git, $i, $stk->{latest_cmt});
}
+sub xapian_only {
+ my ($self, $opt, $sync) = @_;
+ my $seq = $opt->{sequentialshard};
+ local $self->{parallel} = 0 if $seq;
+ $self->idx_init($opt); # acquire lock
+ if (my $art_end = $self->{ibx}->mm->max) {
+ $sync //= {
+ need_checkpoint => \(my $bool = 0),
+ -opt => $opt,
+ v2w => $self,
+ nr => \(my $nr = 0),
+ -regen_fmt => "%u/?\n",
+ };
+ $sync->{art_end} = $art_end;
+ if ($seq || !$self->{parallel}) {
+ my $shard_end = $self->{shards} - 1;
+ index_xap_step($self, $sync, $_) for (0..$shard_end);
+ } else { # parallel (maybe)
+ index_xap_step($self, $sync, 0, 1);
+ }
+ }
+ $self->{ibx}->git->cat_async_wait;
+ $self->done;
+}
+
# public, called by public-inbox-index
sub index_sync {
my ($self, $opt) = @_;
- $opt ||= {};
+ $opt //= $_[1] //= {};
+ goto \&xapian_only if $opt->{xapianonly};
+
my $pr = $opt->{-progress};
my $epoch_max;
my $latest = git_dir_latest($self, \$epoch_max);
@@ -1292,13 +1321,9 @@ sub index_sync {
}
if ($seq) { # deal with Xapian shards sequentially
- my $end = $self->{shards} - 1;
$self->{ibx}->{indexlevel} = $idxlevel;
delete $sync->{mm_tmp};
- $self->idx_init($opt); # re-acquire lock
- index_seq_shard($self, $sync, $_) for (0..$end);
- $self->{ibx}->git->cat_async_wait;
- $self->done;
+ xapian_only($self, $opt, $sync);
}
# reindex does not pick up new changes, so we rerun w/o it:
diff --git a/script/public-inbox-index b/script/public-inbox-index
index be518134..a52fb1bf 100755
--- a/script/public-inbox-index
+++ b/script/public-inbox-index
@@ -16,6 +16,7 @@ use PublicInbox::Xapcmd;
my $compact_opt;
my $opt = { quiet => -1, compact => 0, maxsize => undef, sync => 1 };
GetOptions($opt, qw(verbose|v+ reindex rethread compact|c+ jobs|j=i prune sync!
+ xapianonly|xapian-only
indexlevel|L=s maxsize|max-size=s batchsize|batch-size=s
sequentialshard|seq-shard|sequential-shard))
or die "bad command-line args\n$usage";
@@ -59,8 +60,8 @@ if (defined $s) {
my $mods = {};
foreach my $ibx (@ibxs) {
# XXX: users can shoot themselves in the foot, with opt->{indexlevel}
- $ibx->{indexlevel} //= $opt->{indexlevel} //
- PublicInbox::Admin::detect_indexlevel($ibx);
+ $ibx->{indexlevel} //= $opt->{indexlevel} // ($opt->{xapianonly} ?
+ 'full' : PublicInbox::Admin::detect_indexlevel($ibx));
$ibx->{index_max_size} = $max_size;
PublicInbox::Admin::scan_ibx_modules($mods, $ibx);
}
^ permalink raw reply related [flat|nested] 8+ messages in thread* [PATCH 7/7] index+xcpdb: rename `--no-sync' to `--no-fsync'
2020-08-07 1:13 [PATCH 0/7] index: --sequential-shard and other stuff Eric Wong
` (5 preceding siblings ...)
2020-08-07 1:14 ` [PATCH 6/7] index: support --xapian-only switch Eric Wong
@ 2020-08-07 1:14 ` Eric Wong
6 siblings, 0 replies; 8+ messages in thread
From: Eric Wong @ 2020-08-07 1:14 UTC (permalink / raw)
To: meta
We'll continue supporting `--no-sync' even if its yet-to-make it
it into a release, but the term `sync' is overloaded in our
codebase which may be confusing to new hackers and users.
None of our our code nor dependencies issue the sync(2) syscall,
either, only fsync(2) and fdatasync(2).
---
Documentation/public-inbox-index.pod | 2 +-
Documentation/public-inbox-xcpdb.pod | 2 +-
lib/PublicInbox/OverIdx.pm | 2 +-
lib/PublicInbox/SearchIdx.pm | 6 +++---
lib/PublicInbox/V2Writable.pm | 4 ++--
lib/PublicInbox/Xapcmd.pm | 2 +-
script/public-inbox-index | 8 ++++----
7 files changed, 13 insertions(+), 13 deletions(-)
diff --git a/Documentation/public-inbox-index.pod b/Documentation/public-inbox-index.pod
index f525ba54..a4edc57a 100644
--- a/Documentation/public-inbox-index.pod
+++ b/Documentation/public-inbox-index.pod
@@ -117,7 +117,7 @@ below.
Available in public-inbox 1.6.0 (PENDING).
-=item --no-sync
+=item --no-fsync
Disables L<fsync(2)> and L<fdatasync(2)> operations on SQLite
and Xapian. This is only effective with Xapian 1.4+.
diff --git a/Documentation/public-inbox-xcpdb.pod b/Documentation/public-inbox-xcpdb.pod
index 7fe1e5fe..89eed079 100644
--- a/Documentation/public-inbox-xcpdb.pod
+++ b/Documentation/public-inbox-xcpdb.pod
@@ -45,7 +45,7 @@ too many shards given the capabilities of the current hardware.
These options are passed directly to L<xapian-compact(1)> when
used with C<--compact>.
-=item --no-sync
+=item --no-fsync
Disable L<fsync(2)> and L<fdatasync(2)>.
diff --git a/lib/PublicInbox/OverIdx.pm b/lib/PublicInbox/OverIdx.pm
index c8f61e01..4543bfa1 100644
--- a/lib/PublicInbox/OverIdx.pm
+++ b/lib/PublicInbox/OverIdx.pm
@@ -21,7 +21,7 @@ use Carp qw(croak);
sub dbh_new {
my ($self) = @_;
- my $dbh = $self->SUPER::dbh_new($self->{-no_sync} ? 2 : 1);
+ my $dbh = $self->SUPER::dbh_new($self->{-no_fsync} ? 2 : 1);
# TRUNCATE reduces I/O compared to the default (DELETE)
# We do not use WAL since we're optimized for read-only ops,
diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm
index a1baa65b..22489731 100644
--- a/lib/PublicInbox/SearchIdx.pm
+++ b/lib/PublicInbox/SearchIdx.pm
@@ -67,7 +67,7 @@ sub new {
$self->{lock_path} = "$inboxdir/ssoma.lock";
my $dir = $self->xdir;
$self->{over} = PublicInbox::OverIdx->new("$dir/over.sqlite3");
- $self->{over}->{-no_sync} = 1 if $ibx->{-no_sync};
+ $self->{over}->{-no_fsync} = 1 if $ibx->{-no_fsync};
$self->{index_max_size} = $ibx->{index_max_size};
} elsif ($version == 2) {
defined $shard or die "shard is required for v2\n";
@@ -138,7 +138,7 @@ sub idx_acquire {
}
}
return unless defined $flag;
- $flag |= $DB_NO_SYNC if $self->{ibx}->{-no_sync};
+ $flag |= $DB_NO_SYNC if $self->{ibx}->{-no_fsync};
my $xdb = eval { ($X->{WritableDatabase})->new($dir, $flag) };
if ($@) {
die "Failed opening $dir: ", $@;
@@ -389,7 +389,7 @@ sub _msgmap_init ($) {
die "BUG: _msgmap_init is only for v1\n" if $self->{ibx_ver} != 1;
$self->{mm} //= eval {
require PublicInbox::Msgmap;
- my $rw = $self->{ibx}->{-no_sync} ? 2 : 1;
+ my $rw = $self->{ibx}->{-no_fsync} ? 2 : 1;
PublicInbox::Msgmap->new($self->{ibx}->{inboxdir}, $rw);
};
}
diff --git a/lib/PublicInbox/V2Writable.pm b/lib/PublicInbox/V2Writable.pm
index 6b1effe5..a029fe4c 100644
--- a/lib/PublicInbox/V2Writable.pm
+++ b/lib/PublicInbox/V2Writable.pm
@@ -122,7 +122,7 @@ sub new {
rotate_bytes => int((1024 * 1024 * 1024) / $PACKING_FACTOR),
last_commit => [], # git epoch -> commit
};
- $self->{over}->{-no_sync} = 1 if $v2ibx->{-no_sync};
+ $self->{over}->{-no_fsync} = 1 if $v2ibx->{-no_fsync};
$self->{shards} = count_shards($self) || nproc_shards($creat);
bless $self, $class;
}
@@ -292,7 +292,7 @@ sub _idx_init { # with_umask callback
# for SQLite:
my $mm = $self->{mm} = PublicInbox::Msgmap->new_file(
"$self->{ibx}->{inboxdir}/msgmap.sqlite3",
- $self->{ibx}->{-no_sync} ? 2 : 1);
+ $self->{ibx}->{-no_fsync} ? 2 : 1);
$mm->{dbh}->begin_work;
}
diff --git a/lib/PublicInbox/Xapcmd.pm b/lib/PublicInbox/Xapcmd.pm
index 97a51d1b..8423194f 100644
--- a/lib/PublicInbox/Xapcmd.pm
+++ b/lib/PublicInbox/Xapcmd.pm
@@ -418,7 +418,7 @@ sub cpdb ($$) {
my $flag = eval($PublicInbox::Search::Xap.'::DB_CREATE()');
die if $@;
my $XapianWritableDatabase = $PublicInbox::Search::X{WritableDatabase};
- $flag |= $PublicInbox::SearchIdx::DB_NO_SYNC if !$opt->{sync};
+ $flag |= $PublicInbox::SearchIdx::DB_NO_SYNC if !$opt->{fsync};
my $dst = $XapianWritableDatabase->new($tmp, $flag);
my $pr = $opt->{-progress};
my $pfx = $opt->{-progress_pfx} = progress_pfx($new);
diff --git a/script/public-inbox-index b/script/public-inbox-index
index a52fb1bf..dc9bdde1 100755
--- a/script/public-inbox-index
+++ b/script/public-inbox-index
@@ -14,9 +14,9 @@ PublicInbox::Admin::require_or_die('-index');
use PublicInbox::Xapcmd;
my $compact_opt;
-my $opt = { quiet => -1, compact => 0, maxsize => undef, sync => 1 };
-GetOptions($opt, qw(verbose|v+ reindex rethread compact|c+ jobs|j=i prune sync!
- xapianonly|xapian-only
+my $opt = { quiet => -1, compact => 0, maxsize => undef, fsync => 1 };
+GetOptions($opt, qw(verbose|v+ reindex rethread compact|c+ jobs|j=i prune
+ fsync|sync! xapianonly|xapian-only
indexlevel|L=s maxsize|max-size=s batchsize|batch-size=s
sequentialshard|seq-shard|sequential-shard))
or die "bad command-line args\n$usage";
@@ -73,7 +73,7 @@ for my $ibx (@ibxs) {
if ($opt->{compact} >= 2) {
PublicInbox::Xapcmd::run($ibx, 'compact', $compact_opt);
}
- $ibx->{-no_sync} = 1 if !$opt->{sync};
+ $ibx->{-no_fsync} = 1 if !$opt->{fsync};
my $ibx_opt = $opt;
if (defined(my $s = $ibx->{indexsequentialshard})) {
^ permalink raw reply related [flat|nested] 8+ messages in thread