diff options
author | Eric Wong <e@80x24.org> | 2022-03-07 10:57:37 +0000 |
---|---|---|
committer | Eric Wong <e@80x24.org> | 2022-03-08 00:30:03 +0000 |
commit | 3eec2f7792040f75f3988c520f308e2445baf645 (patch) | |
tree | e581cdd77765c84ae870999f36193e4302f5df71 /lib/PublicInbox | |
parent | 798bd392d273735a69f0b009e70d8bb2dbc4a7e4 (diff) | |
download | public-inbox-3eec2f7792040f75f3988c520f308e2445baf645.tar.gz |
This enables Xapian::DB_DANGEROUS to support in-place updates. This can speed up the initial index and reduce I/O at the cost of preventing concurrent readers and being unsafe in the face of any abnormal terminations. This is more dangerous than --no-fsync. --no-fsync is only unsafe in the event of a power loss or kernel crash; --dangerous is unsafe even on SIGKILL.
Diffstat (limited to 'lib/PublicInbox')
-rw-r--r-- | lib/PublicInbox/ExtSearchIdx.pm | 3 | ||||
-rw-r--r-- | lib/PublicInbox/MiscIdx.pm | 1 | ||||
-rw-r--r-- | lib/PublicInbox/SearchIdx.pm | 9 |
3 files changed, 11 insertions, 2 deletions
diff --git a/lib/PublicInbox/ExtSearchIdx.pm b/lib/PublicInbox/ExtSearchIdx.pm index 4b46fa16..7c44a1a4 100644 --- a/lib/PublicInbox/ExtSearchIdx.pm +++ b/lib/PublicInbox/ExtSearchIdx.pm @@ -1,4 +1,4 @@ -# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org> +# Copyright (C) all contributors <meta@public-inbox.org> # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt> # Detached/external index cross inbox search indexing support @@ -59,6 +59,7 @@ sub new { nproc_shards({ nproc => $opt->{jobs} }); my $oidx = PublicInbox::OverIdx->new("$self->{xpfx}/over.sqlite3"); $self->{-no_fsync} = $oidx->{-no_fsync} = 1 if !$opt->{fsync}; + $self->{-dangerous} = 1 if $opt->{dangerous}; $self->{oidx} = $oidx; $self } diff --git a/lib/PublicInbox/MiscIdx.pm b/lib/PublicInbox/MiscIdx.pm index dc15442d..5faf5c66 100644 --- a/lib/PublicInbox/MiscIdx.pm +++ b/lib/PublicInbox/MiscIdx.pm @@ -31,6 +31,7 @@ sub new { PublicInbox::Syscall::nodatacow_dir($mi_dir); my $flags = $PublicInbox::SearchIdx::DB_CREATE_OR_OPEN; $flags |= $PublicInbox::SearchIdx::DB_NO_SYNC if $eidx->{-no_fsync}; + $flags |= $PublicInbox::SearchIdx::DB_DANGEROUS if $eidx->{-dangerous}; $json //= PublicInbox::Config::json(); bless { mi_dir => $mi_dir, diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm index 95b14c3a..85fae4ad 100644 --- a/lib/PublicInbox/SearchIdx.pm +++ b/lib/PublicInbox/SearchIdx.pm @@ -30,6 +30,7 @@ our @EXPORT_OK = qw(log2stack is_ancestor check_size prepare_stack my $X = \%PublicInbox::Search::X; our ($DB_CREATE_OR_OPEN, $DB_OPEN); our $DB_NO_SYNC = 0; +our $DB_DANGEROUS = 0; our $BATCH_BYTES = $ENV{XAPIAN_FLUSH_THRESHOLD} ? 0x7fffffff : # assume a typical 64-bit system has 8x more RAM than a # typical 32-bit system: @@ -115,7 +116,10 @@ sub load_xapian_writable () { my $ver = (eval($xap.'::major_version()') << 16) | (eval($xap.'::minor_version()') << 8) | eval($xap.'::revision()'); - $DB_NO_SYNC = 0x4 if $ver >= 0x10400; + if ($ver >= 0x10400) { + $DB_NO_SYNC = 0x4; + $DB_DANGEROUS = 0x10; + } # Xapian v1.2.21..v1.2.24 were missing close-on-exec on OFD locks $X->{CLOEXEC_UNSET} = 1 if $ver >= 0x010215 && $ver <= 0x010218; 1; @@ -142,6 +146,9 @@ sub idx_acquire { require PublicInbox::Syscall; PublicInbox::Syscall::nodatacow_dir($dir); $self->{-set_has_threadid_once} = 1; + if (($self->{ibx} // $self->{eidx})->{-dangerous}) { + $flag |= $DB_DANGEROUS; + } } } return unless defined $flag; |