public-inbox.git  about / heads / tags
an "archives first" approach to mailing lists
blob 1630eb4aa2deba6b7a1908addc5d7707e0e91afa 2227 bytes (raw)
$ git show HEAD:lib/PublicInbox/SearchIdxShard.pm	# shows this blob on the CLI

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
 
# Copyright (C) all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>

# Internal interface for a single Xapian shard in V2 inboxes.
# See L<public-inbox-v2-format(5)> for more info on how we shard Xapian
package PublicInbox::SearchIdxShard;
use v5.12;
use parent qw(PublicInbox::SearchIdx PublicInbox::IPC);
use PublicInbox::OnDestroy;
use PublicInbox::Syscall qw($F_SETPIPE_SZ);

sub new {
	my ($class, $v2w, $shard) = @_; # v2w may be ExtSearchIdx
	my $ibx = $v2w->{ibx};
	my $self = $ibx ? $class->SUPER::new($ibx, 1, $shard)
			: $class->eidx_shard_new($v2w, $shard);
	# create the DB before forking:
	$self->idx_acquire;
	$self->set_metadata_once;
	$self->idx_release;
	if ($v2w->{parallel}) {
		local $self->{-v2w_afc} = $v2w;
		$self->ipc_worker_spawn("shard[$shard]");
		# Increasing the pipe size for requests speeds V2 batch imports
		# across 8 cores by nearly 20%.  Since many of our responses
		# are small, make the response pipe as small as possible
		if ($F_SETPIPE_SZ) {
			fcntl($self->{-ipc_req}, $F_SETPIPE_SZ, 1048576);
			fcntl($self->{-ipc_res}, $F_SETPIPE_SZ, 4096);
		}
	}
	$self;
}

sub _worker_done { # OnDestroy cb
	my ($self) = @_;
	die "BUG: $$ $0 xdb active" if $self->need_xapian && $self->{xdb};
	die "BUG: $$ $0 txn active" if $self->{txn};
}

sub ipc_atfork_child { # called automatically before ipc_worker_loop
	my ($self) = @_;
	my $v2w = delete $self->{-v2w_afc} or die 'BUG: {-v2w_afc} missing';
	$v2w->atfork_child; # calls ipc_sibling_atfork_child on our siblings
	$v2w->{current_info} = "[$self->{shard}]"; # for $SIG{__WARN__}
	$self->begin_txn_lazy;
	# caller (ipc_worker_spawn) must capture this:
	PublicInbox::OnDestroy->new($$, \&_worker_done, $self);
}

sub index_eml {
	my ($self, $eml, $smsg, $eidx_key) = @_;
	$smsg->{eidx_key} = $eidx_key if defined $eidx_key;
	$self->ipc_do('add_xapian', $eml, $smsg);
}

# wait for return to determine when ipc_do('commit_txn_lazy') is done
sub echo {
	shift;
	"@_";
}

sub idx_close {
	my ($self) = @_;
	die "BUG: $$ $0 txn active" if $self->{txn};
	$self->idx_release if $self->{xdb};
}

sub shard_close {
	my ($self) = @_;
	$self->ipc_do('idx_close');
	$self->ipc_worker_stop;
}

1;

git clone https://public-inbox.org/public-inbox.git
git clone http://7fh6tueqddpjyxjmgtdiueylzoqt6pt7hec3pukyptlmohoowvhde4yd.onion/public-inbox.git