From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on dcvr.yhbt.net X-Spam-Level: X-Spam-ASN: X-Spam-Status: No, score=-4.0 required=3.0 tests=ALL_TRUSTED,BAYES_00 shortcircuit=no autolearn=ham autolearn_force=no version=3.4.0 Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id 444AB1F517; Wed, 18 Apr 2018 09:13:19 +0000 (UTC) From: "Eric Wong (Contractor, The Linux Foundation)" To: meta@public-inbox.org Cc: "Eric Wong (Contractor, The Linux Foundation)" Subject: [PATCH 03/12] compact: do not merge v2 repos by default Date: Wed, 18 Apr 2018 09:13:07 +0000 Message-Id: <20180418091316.29114-4-e@80x24.org> In-Reply-To: <20180418091316.29114-1-e@80x24.org> References: <20180418091316.29114-1-e@80x24.org> List-Id: --no-renumber does not allow merging, and merging is not ideal for reindexing, either. --- script/public-inbox-compact | 15 ++++++++++----- t/convert-compact.t | 2 -- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/script/public-inbox-compact b/script/public-inbox-compact index 9f33265..5f18497 100755 --- a/script/public-inbox-compact +++ b/script/public-inbox-compact @@ -10,6 +10,7 @@ use PublicInbox::InboxWritable; use Cwd 'abs_path'; use File::Temp qw(tempdir); use File::Path qw(remove_tree); +use PublicInbox::Spawn qw(spawn); my $usage = "Usage: public-inbox-compact REPO_DIR\n"; my $dir = shift or die $usage; my $config = PublicInbox::Config->new; @@ -58,10 +59,11 @@ if ($v == 2) { my $new = tempdir('compact-XXXXXXXX', CLEANUP => 1, DIR => $dir); $ibx->with_umask(sub { $v2w->lock_acquire; - my @parts; + my %pids; while (defined(my $dn = readdir($dh))) { if ($dn =~ /\A\d+\z/) { - push @parts, "$old/$dn"; + my $cmd = [ @compact, "$old/$dn", "$new/$dn" ]; + $pids{spawn($cmd)} = join(' ', @$cmd); } elsif ($dn eq '.' || $dn eq '..') { } elsif ($dn =~ /\Aover\.sqlite3/) { } else { @@ -69,9 +71,12 @@ if ($v == 2) { } } close $dh; - die "No Xapian parts found in $old\n" unless @parts; - my $cmd = [@compact, @parts, "$new/0" ]; - PublicInbox::Import::run_die($cmd); + die "No Xapian parts found in $old\n" unless keys %pids; + while (scalar keys %pids) { + my $pid = waitpid(-1, 0); + my $desc = delete $pids{$pid}; + die "$desc failed: $?\n" if $?; + } commit_changes($v2w, $old, $new); }); } elsif ($v == 1) { diff --git a/t/convert-compact.t b/t/convert-compact.t index 5caa0ac..ced4541 100644 --- a/t/convert-compact.t +++ b/t/convert-compact.t @@ -80,8 +80,6 @@ my $env = { NPROC => 2 }; ok(PublicInbox::Import::run_die($cmd, $env, $rdr), 'v2 compact works'); $ibx->{mainrepo} = "$tmpdir/v2"; $ibx->{version} = 2; -my $v2w = PublicInbox::V2Writable->new($ibx); -is($v2w->{partitions}, 1, "only one partition in compacted repo"); @xdir = glob("$tmpdir/v2/xap*/*"); foreach (@xdir) { -- EW