From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on dcvr.yhbt.net X-Spam-Level: X-Spam-ASN: X-Spam-Status: No, score=-4.0 required=3.0 tests=ALL_TRUSTED,BAYES_00 shortcircuit=no autolearn=ham autolearn_force=no version=3.4.0 Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id 02C031F516 for ; Thu, 5 Apr 2018 21:45:28 +0000 (UTC) From: "Eric Wong (Contractor, The Linux Foundation)" To: meta@public-inbox.org Subject: [PATCH 3/4] over: use only supported and safe SQLite APIs Date: Thu, 5 Apr 2018 21:45:27 +0000 Message-Id: <20180405214528.19403-4-e@80x24.org> In-Reply-To: <20180405214528.19403-1-e@80x24.org> References: <20180405214528.19403-1-e@80x24.org> List-Id: Some of this jankiness was from early performance problems and they turned out to be unnecessary measures. --- lib/PublicInbox/OverIdx.pm | 9 --------- lib/PublicInbox/V2Writable.pm | 3 +-- script/public-inbox-compact | 13 +++++-------- 3 files changed, 6 insertions(+), 19 deletions(-) diff --git a/lib/PublicInbox/OverIdx.pm b/lib/PublicInbox/OverIdx.pm index 0e43aab..5c20f1f 100644 --- a/lib/PublicInbox/OverIdx.pm +++ b/lib/PublicInbox/OverIdx.pm @@ -14,21 +14,12 @@ use DBI qw(:sql_types); # SQL_BLOB sub dbh_new { my ($self) = @_; my $dbh = $self->SUPER::dbh_new; - $dbh->do('PRAGMA synchronous = OFF'); # commit_fsync instead $dbh->do('PRAGMA journal_mode = TRUNCATE'); $dbh->do('PRAGMA cache_size = 80000'); create_tables($dbh); $dbh; } -sub commit_fsync { - my $fn = $_[0]->{filename}; - if (open my $fh, '+<', $fn) { - $fh->sync; - close $fh; - } -} - sub get_counter ($$) { my ($dbh, $key) = @_; my $sth = $dbh->prepare_cached(<<'', undef, 1); diff --git a/lib/PublicInbox/V2Writable.pm b/lib/PublicInbox/V2Writable.pm index b858dcc..a8117aa 100644 --- a/lib/PublicInbox/V2Writable.pm +++ b/lib/PublicInbox/V2Writable.pm @@ -397,7 +397,7 @@ sub checkpoint { # issue a write barrier to ensure all data is visible to other processes # and read-only ops. Order of data importance is: git > SQLite > Xapian sub barrier { - my ($self, $fsync) = @_; + my ($self) = @_; if (my $im = $self->{im}) { $im->barrier; @@ -416,7 +416,6 @@ sub barrier { $_->remote_barrier foreach @$parts; $over->barrier_wait; # wait for each Xapian partition - $over->commit_fsync if $fsync; # last_commit is special, don't commit these until # remote partitions are done: diff --git a/script/public-inbox-compact b/script/public-inbox-compact index b8aaa4b..43e9460 100755 --- a/script/public-inbox-compact +++ b/script/public-inbox-compact @@ -36,15 +36,12 @@ sub commit_changes ($$$) { my ($im, $old, $new) = @_; my @st = stat($old) or die "failed to stat($old): $!\n"; - for my $suf (qw(.pipe.lock -journal)) { - my $orig = "$old/over.sqlite3$suf"; - link($orig, "$new/over.sqlite3$suf") and next; - next if $!{ENOENT}; - die "failed to link $orig => $new/over.sqlite3$suf: $!\n"; + my $over = "$old/over.sqlite3"; + if (-f $over) { + require PublicInbox::Over; + $over = PublicInbox::Over->new($over); + $over->connect->sqlite_backup_to_file("$new/over.sqlite3"); } - link("$old/over.sqlite3", "$new/over.sqlite3") or die - "failed to link {$old => $new}/over.sqlite3: $!\n"; - rename($old, "$new/old") or die "rename $old => $new/old: $!\n"; chmod($st[2] & 07777, $new) or die "chmod $old: $!\n"; rename($new, $old) or die "rename $new => $old: $!\n"; -- EW