From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on dcvr.yhbt.net X-Spam-Level: X-Spam-ASN: X-Spam-Status: No, score=-4.0 required=3.0 tests=ALL_TRUSTED,AWL,BAYES_00 shortcircuit=no autolearn=ham autolearn_force=no version=3.4.2 Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id B74C91FBAC for ; Sat, 15 Jun 2019 08:47:34 +0000 (UTC) From: Eric Wong To: meta@public-inbox.org Subject: [PATCH 18/20] xapcmd: favor 'shard' over 'part' in local variables Date: Sat, 15 Jun 2019 08:47:14 +0000 Message-Id: <20190615084716.3075-19-e@80x24.org> In-Reply-To: <20190615084716.3075-1-e@80x24.org> References: <20190615084716.3075-1-e@80x24.org> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit List-Id: Yet another step to keeping our naming consistent with Xapian terminology. --- lib/PublicInbox/Xapcmd.pm | 70 +++++++++++++++++++-------------------- 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/lib/PublicInbox/Xapcmd.pm b/lib/PublicInbox/Xapcmd.pm index 5e4ac87..819d782 100644 --- a/lib/PublicInbox/Xapcmd.pm +++ b/lib/PublicInbox/Xapcmd.pm @@ -17,13 +17,13 @@ our @COMPACT_OPT = qw(jobs|j=i quiet|q blocksize|b=s no-full|n fuller|F); sub commit_changes ($$$) { my ($ibx, $tmp, $opt) = @_; - my $new_parts = $opt->{reshard}; + my $reshard = $opt->{reshard}; my $reindex = $opt->{reindex}; my $im = $ibx->importer(0); $im->lock_acquire if !$opt->{-coarse_lock}; $SIG{INT} or die 'BUG: $SIG{INT} not handled'; - my @old_part; + my @old_shard; while (my ($old, $new) = each %$tmp) { my @st = stat($old); @@ -41,7 +41,7 @@ sub commit_changes ($$$) { } if (!defined($new)) { # culled shard - push @old_part, $old; + push @old_shard, $old; next; } @@ -58,7 +58,7 @@ sub commit_changes ($$$) { die "failed to remove $prev: $!\n"; } } - remove_tree(@old_part); + remove_tree(@old_shard); $tmp->done; if (!$opt->{-coarse_lock}) { $opt->{-skip_lock} = 1; @@ -66,9 +66,9 @@ sub commit_changes ($$$) { if ($im->can('count_shards')) { my $pr = $opt->{-progress}; my $n = $im->count_shards; - if (defined $new_parts && $n != $new_parts) { + if (defined $reshard && $n != $reshard) { die -"BUG: counted $n shards after resharding to $new_parts"; +"BUG: counted $n shards after resharding to $reshard"; } my $prev = $im->{shards}; if ($pr && $prev != $n) { @@ -171,17 +171,17 @@ sub run { my $tmp = PublicInbox::Xtmpdirs->new; my $v = $ibx->{version} ||= 1; my @q; - my $new_parts = $opt->{reshard}; - if (defined $new_parts && $new_parts <= 0) { + my $reshard = $opt->{reshard}; + if (defined $reshard && $reshard <= 0) { die "--reshard must be a positive number\n"; } # we want temporary directories to be as deep as possible, # so v2 shards can keep "xap$SCHEMA_VERSION" on a separate FS. if ($v == 1) { - if (defined $new_parts) { + if (defined $reshard) { warn -"--reshard=$new_parts ignored for v1 $ibx->{mainrepo}\n"; +"--reshard=$reshard ignored for v1 $ibx->{mainrepo}\n"; } my $old_parent = dirname($old); same_fs_or_die($old_parent, $old); @@ -191,28 +191,28 @@ sub run { push @q, [ $old, $wip ]; } else { opendir my $dh, $old or die "Failed to opendir $old: $!\n"; - my @old_parts; + my @old_shards; while (defined(my $dn = readdir($dh))) { if ($dn =~ /\A[0-9]+\z/) { - push @old_parts, $dn; + push @old_shards, $dn; } elsif ($dn eq '.' || $dn eq '..') { } elsif ($dn =~ /\Aover\.sqlite3/) { } else { warn "W: skipping unknown dir: $old/$dn\n" } } - die "No Xapian parts found in $old\n" unless @old_parts; + die "No Xapian shards found in $old\n" unless @old_shards; - my ($src, $max_part); - if (!defined($new_parts) || $new_parts == scalar(@old_parts)) { + my ($src, $max_shard); + if (!defined($reshard) || $reshard == scalar(@old_shards)) { # 1:1 copy - $max_part = scalar(@old_parts) - 1; + $max_shard = scalar(@old_shards) - 1; } else { # M:N copy - $max_part = $new_parts - 1; - $src = [ map { "$old/$_" } @old_parts ]; + $max_shard = $reshard - 1; + $src = [ map { "$old/$_" } @old_shards ]; } - foreach my $dn (0..$max_part) { + foreach my $dn (0..$max_shard) { my $tmpl = "$dn-XXXXXXXX"; my $wip = tempdir($tmpl, DIR => $old); same_fs_or_die($old, $wip); @@ -220,7 +220,7 @@ sub run { push @q, [ $src // $cur , $wip ]; $tmp->{$cur} = $wip; } - # mark old parts to be unlinked + # mark old shards to be unlinked if ($src) { $tmp->{$_} ||= undef for @$src; } @@ -305,7 +305,7 @@ sub compact ($$) { } sub cpdb_loop ($$$;$$) { - my ($src, $dst, $pr_data, $cur_part, $new_parts) = @_; + my ($src, $dst, $pr_data, $cur_shard, $reshard) = @_; my ($pr, $fmt, $nr, $pfx); if ($pr_data) { $pr = $pr_data->{pr}; @@ -326,9 +326,9 @@ sub cpdb_loop ($$$;$$) { eval { for (; $it != $end; $it++) { my $docid = $it->get_docid; - if (defined $new_parts) { - my $dst_part = $docid % $new_parts; - next if $dst_part != $cur_part; + if (defined $reshard) { + my $dst_shard = $docid % $reshard; + next if $dst_shard != $cur_shard; } my $doc = $src->get_document($docid); $dst->replace_document($docid, $doc); @@ -350,14 +350,14 @@ sub cpdb_loop ($$$;$$) { sub cpdb ($$) { my ($args, $opt) = @_; my ($old, $new) = @$args; - my ($src, $cur_part); - my $new_parts; + my ($src, $cur_shard); + my $reshard; if (ref($old) eq 'ARRAY') { - ($cur_part) = ($new =~ m!xap[0-9]+/([0-9]+)\b!); - defined $cur_part or + ($cur_shard) = ($new =~ m!xap[0-9]+/([0-9]+)\b!); + defined $cur_shard or die "BUG: could not extract shard # from $new"; - $new_parts = $opt->{reshard}; - defined $new_parts or die 'BUG: got array src w/o --reshard'; + $reshard = $opt->{reshard}; + defined $reshard or die 'BUG: got array src w/o --reshard'; # resharding, M:N copy means have full read access foreach (@$old) { @@ -410,8 +410,8 @@ sub cpdb ($$) { # we can only estimate when resharding, # because removed spam causes slight imbalance my $est = ''; - if (defined $cur_part && $new_parts > 1) { - $tot = int($tot/$new_parts); + if (defined $cur_shard && $reshard > 1) { + $tot = int($tot/$reshard); $est = 'around '; } my $fmt = "$pfx % ".length($tot)."u/$tot\n"; @@ -422,15 +422,15 @@ sub cpdb ($$) { }; } while (cpdb_retryable($src, $pfx)); - if (defined $new_parts) { + if (defined $reshard) { # we rely on document IDs matching NNTP article number, - # so we can't have the combined DB support rewriting + # so we can't have the Xapian sharding DB support rewriting # document IDs. Thus we iterate through each shard # individually. $src = undef; foreach (@$old) { my $old = Search::Xapian::Database->new($_); - cpdb_loop($old, $dst, $pr_data, $cur_part, $new_parts); + cpdb_loop($old, $dst, $pr_data, $cur_shard, $reshard); } } else { cpdb_loop($src, $dst, $pr_data); -- EW