From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on dcvr.yhbt.net X-Spam-Level: X-Spam-Status: No, score=-4.0 required=3.0 tests=ALL_TRUSTED,BAYES_00 shortcircuit=no autolearn=ham autolearn_force=no version=3.4.2 Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id 06CE91F66E for ; Wed, 12 Aug 2020 09:34:38 +0000 (UTC) From: Eric Wong To: meta@public-inbox.org Subject: [TESTING] WIP - parallel shards on HDD with sequential flush Date: Wed, 12 Aug 2020 09:34:37 +0000 Message-Id: <20200812093437.9147-1-e@yhbt.net> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit List-Id: Frequent flushing to save RAM with HDD is horrible with random writes Xapian tends to do; especially when parallelized I think just making the Xapian commits in sequence while the random reads + in-memory changes are still parallelized is doable, though... With this, --no-fsync may even be detrimental because Linux defaults to excessive writeback buffering. This could take some time to test (using ext4 on an HDD). So far it's showing decent CPU usage on a 4 core system and sequential shard commits only show one process in D state at a time and it seems to be going at a decent rate... Waiting to see if it slows down as the Xapian DBs get bigger... --- lib/PublicInbox/V2Writable.pm | 34 +++++++++++++++++++++++++++------- 1 file changed, 27 insertions(+), 7 deletions(-) diff --git a/lib/PublicInbox/V2Writable.pm b/lib/PublicInbox/V2Writable.pm index d99e476a..51985a58 100644 --- a/lib/PublicInbox/V2Writable.pm +++ b/lib/PublicInbox/V2Writable.pm @@ -20,6 +20,7 @@ use PublicInbox::Spawn qw(spawn popen_rd); use PublicInbox::SearchIdx qw(log2stack crlf_adjust is_ancestor check_size); use IO::Handle; # ->autoflush use File::Temp (); +use Carp qw(croak); my $OID = qr/[a-f0-9]{40,}/; # an estimate of the post-packed size to the raw uncompressed size @@ -114,6 +115,7 @@ sub new { total_bytes => 0, current_info => '', xpfx => $xpfx, + seq_flush => 1, over => PublicInbox::OverIdx->new("$xpfx/over.sqlite3"), lock_path => "$dir/inbox.lock", # limit each git repo (epoch) to 1GB or so @@ -593,14 +595,20 @@ sub barrier_init { my $barrier = { map { $_ => 1 } (0..$n) }; } +sub barrier_wait_1 ($) { + my ($r) = @_; + defined(my $line = readline($r)) or croak "EOF on barrier_wait: $!"; + $line =~ /\Abarrier (\d+)/ or croak "bad line on barrier_wait: $line"; + $1; +} + sub barrier_wait { my ($self, $barrier) = @_; my $bnote = $self->{bnote} or return; my $r = $bnote->[0]; while (scalar keys %$barrier) { - defined(my $l = readline($r)) or die "EOF on barrier_wait: $!"; - $l =~ /\Abarrier (\d+)/ or die "bad line on barrier_wait: $l"; - delete $barrier->{$1} or die "bad shard[$1] on barrier wait"; + my $i = barrier_wait_1($r); + delete($barrier->{$i}) or die "bad shard[$i] on barrier wait"; } } @@ -609,7 +617,7 @@ sub checkpoint ($;$) { my ($self, $wait) = @_; if (my $im = $self->{im}) { - if ($wait) { + if ($wait || $self->{seq_flush}) { $im->barrier; } else { $im->checkpoint; @@ -626,15 +634,27 @@ sub checkpoint ($;$) { $self->{over}->commit_lazy; # Now deal with Xapian - if ($wait) { + if ($self->{seq_flush} && $self->{parallel}) { + my $r = $self->{bnote}->[0] or + die "BUG: {bnote} missing"; + my $i = 0; + for (@$shards) { + $_->remote_barrier; + my $j = barrier_wait_1($r); + $i == $j or die <barrier_init(scalar @$shards); # each shard needs to issue a barrier command $_->remote_barrier for @$shards; # wait for each Xapian shard - $self->barrier_wait($barrier); - } else { + barrier_wait($self, $barrier); + } else { # sequential flush with 1 process $_->remote_commit for @$shards; }