user/dev discussion of public-inbox itself
 help / color / mirror / code / Atom feed
Search results ordered by [date|relevance]  view[summary|nested|Atom feed]
thread overview below | download mbox.gz: |
* [PATCH 2/6] xcpdb: support --no-fsync from CLI
  @ 2020-08-12  9:17  6% ` Eric Wong
  0 siblings, 0 replies; 3+ results
From: Eric Wong @ 2020-08-12  9:17 UTC (permalink / raw)
  To: meta

This was omitted in 8b1950055d51d436 :x

Fixes: 8b1950055d51d436 ("index+xcpdb: rename `--no-sync' to `--no-fsync'")
---
 script/public-inbox-xcpdb | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/script/public-inbox-xcpdb b/script/public-inbox-xcpdb
index fcd961488..2c91598cb 100755
--- a/script/public-inbox-xcpdb
+++ b/script/public-inbox-xcpdb
@@ -8,8 +8,9 @@ use PublicInbox::Xapcmd;
 use PublicInbox::Admin;
 PublicInbox::Admin::require_or_die('-search');
 my $usage = "Usage: public-inbox-xcpdb [--compact] INBOX_DIR\n";
-my $opt = { sync => 1 };
-my @opt = (qw(sync! compact reshard|R=i), @PublicInbox::Xapcmd::COMPACT_OPT);
+my $opt = { fsync => 1 };
+my @opt = (qw(fsync|sync! compact reshard|R=i),
+	@PublicInbox::Xapcmd::COMPACT_OPT);
 GetOptions($opt, @opt) or die "bad command-line args\n$usage";
 my @ibxs = PublicInbox::Admin::resolve_inboxes(\@ARGV) or die $usage;
 foreach (@ibxs) {

^ permalink raw reply related	[relevance 6%]

* [PATCH 7/7] index+xcpdb: rename `--no-sync' to `--no-fsync'
  2020-08-07  1:13  7% [PATCH 0/7] index: --sequential-shard and other stuff Eric Wong
@ 2020-08-07  1:14  6% ` Eric Wong
  0 siblings, 0 replies; 3+ results
From: Eric Wong @ 2020-08-07  1:14 UTC (permalink / raw)
  To: meta

We'll continue supporting `--no-sync' even if its yet-to-make it
it into a release, but the term `sync' is overloaded in our
codebase which may be confusing to new hackers and users.

None of our our code nor dependencies issue the sync(2) syscall,
either, only fsync(2) and fdatasync(2).
---
 Documentation/public-inbox-index.pod | 2 +-
 Documentation/public-inbox-xcpdb.pod | 2 +-
 lib/PublicInbox/OverIdx.pm           | 2 +-
 lib/PublicInbox/SearchIdx.pm         | 6 +++---
 lib/PublicInbox/V2Writable.pm        | 4 ++--
 lib/PublicInbox/Xapcmd.pm            | 2 +-
 script/public-inbox-index            | 8 ++++----
 7 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/Documentation/public-inbox-index.pod b/Documentation/public-inbox-index.pod
index f525ba54..a4edc57a 100644
--- a/Documentation/public-inbox-index.pod
+++ b/Documentation/public-inbox-index.pod
@@ -117,7 +117,7 @@ below.
 
 Available in public-inbox 1.6.0 (PENDING).
 
-=item --no-sync
+=item --no-fsync
 
 Disables L<fsync(2)> and L<fdatasync(2)> operations on SQLite
 and Xapian.  This is only effective with Xapian 1.4+.
diff --git a/Documentation/public-inbox-xcpdb.pod b/Documentation/public-inbox-xcpdb.pod
index 7fe1e5fe..89eed079 100644
--- a/Documentation/public-inbox-xcpdb.pod
+++ b/Documentation/public-inbox-xcpdb.pod
@@ -45,7 +45,7 @@ too many shards given the capabilities of the current hardware.
 These options are passed directly to L<xapian-compact(1)> when
 used with C<--compact>.
 
-=item --no-sync
+=item --no-fsync
 
 Disable L<fsync(2)> and L<fdatasync(2)>.
 
diff --git a/lib/PublicInbox/OverIdx.pm b/lib/PublicInbox/OverIdx.pm
index c8f61e01..4543bfa1 100644
--- a/lib/PublicInbox/OverIdx.pm
+++ b/lib/PublicInbox/OverIdx.pm
@@ -21,7 +21,7 @@ use Carp qw(croak);
 
 sub dbh_new {
 	my ($self) = @_;
-	my $dbh = $self->SUPER::dbh_new($self->{-no_sync} ? 2 : 1);
+	my $dbh = $self->SUPER::dbh_new($self->{-no_fsync} ? 2 : 1);
 
 	# TRUNCATE reduces I/O compared to the default (DELETE)
 	# We do not use WAL since we're optimized for read-only ops,
diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm
index a1baa65b..22489731 100644
--- a/lib/PublicInbox/SearchIdx.pm
+++ b/lib/PublicInbox/SearchIdx.pm
@@ -67,7 +67,7 @@ sub new {
 		$self->{lock_path} = "$inboxdir/ssoma.lock";
 		my $dir = $self->xdir;
 		$self->{over} = PublicInbox::OverIdx->new("$dir/over.sqlite3");
-		$self->{over}->{-no_sync} = 1 if $ibx->{-no_sync};
+		$self->{over}->{-no_fsync} = 1 if $ibx->{-no_fsync};
 		$self->{index_max_size} = $ibx->{index_max_size};
 	} elsif ($version == 2) {
 		defined $shard or die "shard is required for v2\n";
@@ -138,7 +138,7 @@ sub idx_acquire {
 		}
 	}
 	return unless defined $flag;
-	$flag |= $DB_NO_SYNC if $self->{ibx}->{-no_sync};
+	$flag |= $DB_NO_SYNC if $self->{ibx}->{-no_fsync};
 	my $xdb = eval { ($X->{WritableDatabase})->new($dir, $flag) };
 	if ($@) {
 		die "Failed opening $dir: ", $@;
@@ -389,7 +389,7 @@ sub _msgmap_init ($) {
 	die "BUG: _msgmap_init is only for v1\n" if $self->{ibx_ver} != 1;
 	$self->{mm} //= eval {
 		require PublicInbox::Msgmap;
-		my $rw = $self->{ibx}->{-no_sync} ? 2 : 1;
+		my $rw = $self->{ibx}->{-no_fsync} ? 2 : 1;
 		PublicInbox::Msgmap->new($self->{ibx}->{inboxdir}, $rw);
 	};
 }
diff --git a/lib/PublicInbox/V2Writable.pm b/lib/PublicInbox/V2Writable.pm
index 6b1effe5..a029fe4c 100644
--- a/lib/PublicInbox/V2Writable.pm
+++ b/lib/PublicInbox/V2Writable.pm
@@ -122,7 +122,7 @@ sub new {
 		rotate_bytes => int((1024 * 1024 * 1024) / $PACKING_FACTOR),
 		last_commit => [], # git epoch -> commit
 	};
-	$self->{over}->{-no_sync} = 1 if $v2ibx->{-no_sync};
+	$self->{over}->{-no_fsync} = 1 if $v2ibx->{-no_fsync};
 	$self->{shards} = count_shards($self) || nproc_shards($creat);
 	bless $self, $class;
 }
@@ -292,7 +292,7 @@ sub _idx_init { # with_umask callback
 	# for SQLite:
 	my $mm = $self->{mm} = PublicInbox::Msgmap->new_file(
 				"$self->{ibx}->{inboxdir}/msgmap.sqlite3",
-				$self->{ibx}->{-no_sync} ? 2 : 1);
+				$self->{ibx}->{-no_fsync} ? 2 : 1);
 	$mm->{dbh}->begin_work;
 }
 
diff --git a/lib/PublicInbox/Xapcmd.pm b/lib/PublicInbox/Xapcmd.pm
index 97a51d1b..8423194f 100644
--- a/lib/PublicInbox/Xapcmd.pm
+++ b/lib/PublicInbox/Xapcmd.pm
@@ -418,7 +418,7 @@ sub cpdb ($$) {
 	my $flag = eval($PublicInbox::Search::Xap.'::DB_CREATE()');
 	die if $@;
 	my $XapianWritableDatabase = $PublicInbox::Search::X{WritableDatabase};
-	$flag |= $PublicInbox::SearchIdx::DB_NO_SYNC if !$opt->{sync};
+	$flag |= $PublicInbox::SearchIdx::DB_NO_SYNC if !$opt->{fsync};
 	my $dst = $XapianWritableDatabase->new($tmp, $flag);
 	my $pr = $opt->{-progress};
 	my $pfx = $opt->{-progress_pfx} = progress_pfx($new);
diff --git a/script/public-inbox-index b/script/public-inbox-index
index a52fb1bf..dc9bdde1 100755
--- a/script/public-inbox-index
+++ b/script/public-inbox-index
@@ -14,9 +14,9 @@ PublicInbox::Admin::require_or_die('-index');
 use PublicInbox::Xapcmd;
 
 my $compact_opt;
-my $opt = { quiet => -1, compact => 0, maxsize => undef, sync => 1 };
-GetOptions($opt, qw(verbose|v+ reindex rethread compact|c+ jobs|j=i prune sync!
-		xapianonly|xapian-only
+my $opt = { quiet => -1, compact => 0, maxsize => undef, fsync => 1 };
+GetOptions($opt, qw(verbose|v+ reindex rethread compact|c+ jobs|j=i prune
+		fsync|sync! xapianonly|xapian-only
 		indexlevel|L=s maxsize|max-size=s batchsize|batch-size=s
 		sequentialshard|seq-shard|sequential-shard))
 	or die "bad command-line args\n$usage";
@@ -73,7 +73,7 @@ for my $ibx (@ibxs) {
 	if ($opt->{compact} >= 2) {
 		PublicInbox::Xapcmd::run($ibx, 'compact', $compact_opt);
 	}
-	$ibx->{-no_sync} = 1 if !$opt->{sync};
+	$ibx->{-no_fsync} = 1 if !$opt->{fsync};
 
 	my $ibx_opt = $opt;
 	if (defined(my $s = $ibx->{indexsequentialshard})) {

^ permalink raw reply related	[relevance 6%]

* [PATCH 0/7] index: --sequential-shard and other stuff
@ 2020-08-07  1:13  7% Eric Wong
  2020-08-07  1:14  6% ` [PATCH 7/7] index+xcpdb: rename `--no-sync' to `--no-fsync' Eric Wong
  0 siblings, 1 reply; 3+ results
From: Eric Wong @ 2020-08-07  1:13 UTC (permalink / raw)
  To: meta

1/7 is a minor usability fix (more on the way)
5/7 is a major improvement for HDDs
6/7 is useful to developers, and may be useful to users
    a few months down the line

And the rest are minor fixes related to indexing...

Eric Wong (7):
  xapcmd: quietly no-op on indexlevel=basic
  xapcmd: remove redundant searchidx require
  xapcmd: drop outdated comment
  v2writable: fix rethread cleanup
  index: v2: indexSequentialShard / --sequential-shard option
  index: support --xapian-only switch
  index+xcpdb: rename `--no-sync' to `--no-fsync'

 Documentation/public-inbox-config.pod    |  6 ++
 Documentation/public-inbox-index.pod     | 55 ++++++++++++++-
 Documentation/public-inbox-v2-format.pod | 11 ++-
 Documentation/public-inbox-xcpdb.pod     |  2 +-
 lib/PublicInbox/Config.pm                |  9 +--
 lib/PublicInbox/OverIdx.pm               |  2 +-
 lib/PublicInbox/SearchIdx.pm             |  6 +-
 lib/PublicInbox/SearchIdxShard.pm        | 10 ++-
 lib/PublicInbox/V2Writable.pm            | 88 ++++++++++++++++++++++--
 lib/PublicInbox/WatchMaildir.pm          |  2 +-
 lib/PublicInbox/Xapcmd.pm                | 18 ++---
 script/public-inbox-index                | 33 +++++++--
 t/config.t                               |  6 +-
 t/indexlevels-mirror.t                   | 24 +++++--
 t/v2mirror.t                             | 14 ++++
 15 files changed, 235 insertions(+), 51 deletions(-)

^ permalink raw reply	[relevance 7%]

Results 1-3 of 3 | reverse | options above
-- pct% links below jump to the message on this page, permalinks otherwise --
2020-08-07  1:13  7% [PATCH 0/7] index: --sequential-shard and other stuff Eric Wong
2020-08-07  1:14  6% ` [PATCH 7/7] index+xcpdb: rename `--no-sync' to `--no-fsync' Eric Wong
2020-08-12  9:17     [PATCH 0/6] xcpdb -index improvements Eric Wong
2020-08-12  9:17  6% ` [PATCH 2/6] xcpdb: support --no-fsync from CLI Eric Wong

Code repositories for project(s) associated with this public inbox

	https://80x24.org/public-inbox.git

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).