From 1bc79688ecf6bc79951c2c6ad6b1dbff1586af3f Mon Sep 17 00:00:00 2001 From: Eric Wong Date: Mon, 27 May 2019 18:45:45 +0000 Subject: v2: fix reindex skipping NNTP article numbers `public-inbox-index --reindex' could cause NNTP article number gaps to form when it also has to deal with new, never-before-seen commits in mirrors running off `git fetch'. Fix this by running two distinct invocations of ->index_sync; once to only reindex old commits, and a second time to index new commits. This does not appear to be a problem on v1 at the moment, but I'll need more time to analyze this. --- lib/PublicInbox/V2Writable.pm | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) (limited to 'lib/PublicInbox') diff --git a/lib/PublicInbox/V2Writable.pm b/lib/PublicInbox/V2Writable.pm index cd08acd7..331c4f4f 100644 --- a/lib/PublicInbox/V2Writable.pm +++ b/lib/PublicInbox/V2Writable.pm @@ -850,11 +850,19 @@ sub index_prepare { my $pr = $opts->{-progress}; my $regen_max = 0; my $head = $self->{-inbox}->{ref_head} || 'refs/heads/master'; + + # reindex stops at the current heads and we later rerun index_sync + # without {reindex} + my $reindex_heads = last_commits($self, $epoch_max) if $opts->{reindex}; + for (my $i = $epoch_max; $i >= 0; $i--) { die 'BUG: already indexing!' if $self->{reindex_pipe}; my $git_dir = git_dir_n($self, $i); -d $git_dir or next; # missing parts are fine my $git = PublicInbox::Git->new($git_dir); + if ($reindex_heads) { + $head = $reindex_heads->[$i] or next; + } chomp(my $tip = $git->qx(qw(rev-parse -q --verify), $head)); next if $?; # new repo @@ -959,7 +967,14 @@ sub index_sync { my $high = $self->{mm}->num_highwater(); my $regen = $self->index_prepare($opts, $epoch_max, $ranges); - $$regen += $high if $high; + if ($opts->{reindex}) { + # reindex should NOT see new commits anymore, if we do, + # it's a problem and we need to notice it via die() + $$regen = -1; + } else { + $$regen += $high; + } + my $D = {}; # "$mid\0$cid" => $oid my @cmd = qw(log --raw -r --pretty=tformat:%H --no-notes --no-color --no-abbrev --no-renames); @@ -1001,6 +1016,14 @@ sub index_sync { $git->cleanup; } $self->done; + + # reindex does not pick up new changes, so we rerun w/o it: + if ($opts->{reindex}) { + my %again = %$opts; + $mm_tmp = undef; + delete @again{qw(reindex -skip_lock)}; + index_sync($self, \%again); + } } 1; -- cgit v1.2.3-24-ge0c7