From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on dcvr.yhbt.net X-Spam-Level: X-Spam-ASN: X-Spam-Status: No, score=-4.0 required=3.0 tests=ALL_TRUSTED,BAYES_00 shortcircuit=no autolearn=ham autolearn_force=no version=3.4.2 Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id 4156D1F5A2 for ; Thu, 30 Jan 2020 07:51:06 +0000 (UTC) From: Eric Wong To: meta@public-inbox.org Subject: [PATCH] convert: preserve highwater mark from v1 msgmap Date: Thu, 30 Jan 2020 07:51:06 +0000 Message-Id: <20200130075106.5853-1-e@yhbt.net> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit List-Id: If we're reusing the msgmap from a v1 inbox, we also need to ensure the highwater mark doesn't get doubled in the v1->v2 conversion by internally triggering the equivalent of "--reindex" on a fresh v2 inbox. This was needed to convert an indexed v1 inbox which featured messages with multiple Message-IDs in it. Fresh, unindexed clones of v1 inboxes would not have been affected by this. --- script/public-inbox-convert | 16 +++++++++++++--- t/convert-compact.t | 8 ++++++++ 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/script/public-inbox-convert b/script/public-inbox-convert index 56a810eb..8ac111a2 100755 --- a/script/public-inbox-convert +++ b/script/public-inbox-convert @@ -7,6 +7,7 @@ use Getopt::Long qw(:config gnu_getopt no_ignore_case auto_abbrev); use PublicInbox::InboxWritable; use PublicInbox::Config; use PublicInbox::V2Writable; +use PublicInbox::Git; use PublicInbox::Spawn qw(spawn); use Cwd 'abs_path'; use File::Copy 'cp'; # preserves permissions: @@ -144,10 +145,19 @@ $w = $r = undef; close $rd or die "close fast-export: $!\n"; waitpid($pid, 0) or die "waitpid failed: $!\n"; $? == 0 or die "fast-export failed: $?\n"; -my $mm = $old->mm; -$mm->{dbh}->sqlite_backup_to_file("$new_dir/msgmap.sqlite3") if $mm; $v2w->done; +if (my $mm = $old->mm) { + $old->cleanup; + $mm->{dbh}->sqlite_backup_to_file("$new_dir/msgmap.sqlite3"); + + # we want to trigger a reindex, not a from scratch index if + # we're reusing the msgmap from an existing v1 installation. + $v2w->idx_init; + my $epoch0 = PublicInbox::Git->new($v2w->git_init(0)); + chop(my $cmt = $epoch0->qx(qw(rev-parse --verify), $head)); + $v2w->last_epoch_commit(0, $cmt); +} if ($index) { - $v2w->index_sync; + $v2w->index_sync({reindex => 1}); $v2w->done; } diff --git a/t/convert-compact.t b/t/convert-compact.t index 6e4a8268..fc46083b 100644 --- a/t/convert-compact.t +++ b/t/convert-compact.t @@ -69,6 +69,13 @@ is(scalar(@xdir), 1, 'got one xapian directory after compact'); is(((stat($xdir[0]))[2]) & 07777, 0755, 'sharedRepository respected on v1 compact'); +my $hwm = do { + my $mm = $ibx->mm; + $ibx->cleanup; + $mm->num_highwater; +}; +ok(defined($hwm) && $hwm > 0, "highwater mark set #$hwm"); + $cmd = [ '-convert', $ibx->{inboxdir}, "$tmpdir/v2" ]; ok(run_script($cmd, undef, $rdr), 'convert works'); @xdir = glob("$tmpdir/v2/xap*/*"); @@ -83,6 +90,7 @@ my $env = { NPROC => 2 }; ok(run_script($cmd, $env, $rdr), 'v2 compact works'); $ibx->{inboxdir} = "$tmpdir/v2"; $ibx->{version} = 2; +is($ibx->mm->num_highwater, $hwm, 'highwater mark unchanged in v2 inbox'); @xdir = glob("$tmpdir/v2/xap*/*"); foreach (@xdir) {