From 375b3ccfd3ca978281cb3869b62fc91eebc60d6e Mon Sep 17 00:00:00 2001 From: Eric Wong Date: Wed, 15 Sep 2021 11:26:17 +0000 Subject: multi_git: hoist out common epoch/alternates handling IMHO, this greatly improves code sharing and organization between v2, extindex, and lei/store. Common git-related logic for these is lightly-refactored and easier to reason about. The impetus for this big change was to ensure inboxes created+managed by public-inbox-{clone,fetch} could have alternates and configs setup properly without depending on SQLite (via V2Writable). This change does that while making old code shorter and better factored. --- MANIFEST | 1 + lib/PublicInbox/ExtSearchIdx.pm | 85 +++++++++---------------- lib/PublicInbox/Fetch.pm | 17 +++-- lib/PublicInbox/LeiMirror.pm | 15 +++-- lib/PublicInbox/LeiStore.pm | 32 ++-------- lib/PublicInbox/MultiGit.pm | 136 ++++++++++++++++++++++++++++++++++++++++ lib/PublicInbox/V2Writable.pm | 87 ++++--------------------- script/public-inbox-convert | 2 +- t/lei-mirror.t | 15 ++++- t/v2mirror.t | 9 ++- t/v2writable.t | 2 +- 11 files changed, 222 insertions(+), 179 deletions(-) create mode 100644 lib/PublicInbox/MultiGit.pm diff --git a/MANIFEST b/MANIFEST index a1450880..640eabd1 100644 --- a/MANIFEST +++ b/MANIFEST @@ -270,6 +270,7 @@ lib/PublicInbox/MiscSearch.pm lib/PublicInbox/MsgIter.pm lib/PublicInbox/MsgTime.pm lib/PublicInbox/Msgmap.pm +lib/PublicInbox/MultiGit.pm lib/PublicInbox/NDC_PP.pm lib/PublicInbox/NNTP.pm lib/PublicInbox/NNTPD.pm diff --git a/lib/PublicInbox/ExtSearchIdx.pm b/lib/PublicInbox/ExtSearchIdx.pm index 8cdad23d..e0ba6c32 100644 --- a/lib/PublicInbox/ExtSearchIdx.pm +++ b/lib/PublicInbox/ExtSearchIdx.pm @@ -21,6 +21,7 @@ use Carp qw(croak carp); use Sys::Hostname qw(hostname); use POSIX qw(strftime); use File::Glob qw(bsd_glob GLOB_NOSORT); +use PublicInbox::MultiGit; use PublicInbox::Search; use PublicInbox::SearchIdx qw(prepare_stack is_ancestor is_bad_blob); use PublicInbox::OverIdx; @@ -1133,88 +1134,60 @@ sub idx_init { # similar to V2Writable $self->git->cleanup; my $mode = 0644; - my $ALL = $self->git->{git_dir}; # ALL.git - my $old = -d $ALL; + my $ALL = $self->git->{git_dir}; # topdir/ALL.git + my ($has_new, $alt, $seen); if ($opt->{-private}) { # LeiStore + my $local = "$self->{topdir}/local"; # lei/store + $self->{mg} //= PublicInbox::MultiGit->new($self->{topdir}, + 'ALL.git', 'local'); $mode = 0600; - if (!$old) { - umask 077; # don't bother restoring + unless (-d $ALL) { + umask 077; # don't bother restoring for lei PublicInbox::Import::init_bare($ALL); $self->git->qx(qw(config core.sharedRepository 0600)); } - } else { - PublicInbox::Import::init_bare($ALL) unless $old; - } - my $info_dir = "$ALL/objects/info"; - my $alt = "$info_dir/alternates"; - my (@old, @new, %seen); # seen: st_dev + st_ino - if (-e $alt) { - open(my $fh, '<', $alt) or die "open $alt: $!"; - $mode = (stat($fh))[2] & 07777; - while (my $line = <$fh>) { - chomp(my $d = $line); - - # expand relative path (/local/ stuff) - substr($d, 0, 3) eq '../' and - $d = "$ALL/objects/$d"; - if (my @st = stat($d)) { - next if $seen{"$st[0]\0$st[1]"}++; - } else { - warn "W: stat($d) failed (from $alt): $!\n"; - next if $opt->{-idx_gc}; - } - push @old, $line; - } + ($alt, $seen) = $self->{mg}->read_alternates(\$mode); + $has_new = $self->{mg}->merge_epochs($alt, $seen); + } else { # extindex has no epochs + $self->{mg} //= PublicInbox::MultiGit->new($self->{topdir}, + 'ALL.git'); + ($alt, $seen) = $self->{mg}->read_alternates(\$mode, + $opt->{-idx_gc}); + PublicInbox::Import::init_bare($ALL); } - # for LeiStore, and possibly some mirror-only state - if (opendir(my $dh, my $local = "$self->{topdir}/local")) { - # highest numbered epoch first - for my $n (sort { $b <=> $a } map { substr($_, 0, -4) + 0 } - grep(/\A[0-9]+\.git\z/, readdir($dh))) { - my $d = "$local/$n.git/objects"; # absolute path - if (my @st = stat($d)) { - next if $seen{"$st[0]\0$st[1]"}++; - # favor relative paths for rename-friendliness - push @new, "../../local/$n.git/objects\n"; - } else { - warn "W: stat($d) failed: $!\n"; - } - } - } # git-multi-pack-index(1) can speed up "git cat-file" startup slightly - my $dh; my $git_midx = 0; my $pd = "$ALL/objects/pack"; - if (!mkdir($pd) && $!{EEXIST} && opendir($dh, $pd)) { - # drop stale symlinks + if (opendir(my $dh, $pd)) { # drop stale symlinks while (defined(my $dn = readdir($dh))) { if ($dn =~ /\.(?:idx|pack|promisor|bitmap|rev)\z/) { my $f = "$pd/$dn"; unlink($f) if -l $f && !-e $f; } } - undef $dh; + } elsif ($!{ENOENT}) { + mkdir($pd) or die "mkdir($pd): $!"; + } else { + die "opendir($pd): $!"; } + my $new = ''; for my $ibx (@{ibx_sorted($self, 'active')}) { # create symlinks for multi-pack-index $git_midx += symlink_packs($ibx, $pd); # add new lines to our alternates file - my $line = $ibx->git->{git_dir} . "/objects\n"; - chomp(my $d = $line); + my $d = $ibx->git->{git_dir} . '/objects'; + next if exists $alt->{$d}; if (my @st = stat($d)) { - next if $seen{"$st[0]\0$st[1]"}++; + next if $seen->{"$st[0]\0$st[1]"}++; } else { warn "W: stat($d) failed (from $ibx->{inboxdir}): $!\n"; next if $opt->{-idx_gc}; } - push @new, $line; - } - if (scalar @new) { - push @old, @new; - my $o = \@old; - PublicInbox::V2Writable::write_alternates($info_dir, $mode, $o); + $new .= "$d\n"; } + ($has_new || $new ne '') and + $self->{mg}->write_alternates($mode, $alt, $new); $git_midx and $self->with_umask(sub { my @cmd = ('multi-pack-index'); push @cmd, '--no-progress' if ($opt->{quiet}//0) > 1; @@ -1226,7 +1199,7 @@ sub idx_init { # similar to V2Writable $self->with_umask(\&_idx_init, $self, $opt); $self->{oidx}->begin_lazy; $self->{oidx}->eidx_prep; - $self->{midx}->create_xdb if @new; + $self->{midx}->create_xdb if $new ne ''; } sub _watch_commit { # PublicInbox::DS::add_timer callback diff --git a/lib/PublicInbox/Fetch.pm b/lib/PublicInbox/Fetch.pm index 6a6daee6..9ea55e9d 100644 --- a/lib/PublicInbox/Fetch.pm +++ b/lib/PublicInbox/Fetch.pm @@ -6,12 +6,11 @@ use strict; use v5.10.1; use parent qw(PublicInbox::IPC); use URI (); -use PublicInbox::Spawn qw(popen_rd); +use PublicInbox::Spawn qw(popen_rd run_die); use PublicInbox::Admin; use PublicInbox::LEI; use PublicInbox::LeiCurl; use PublicInbox::LeiMirror; -use IO::Uncompress::Gunzip qw(gunzip $GunzipError); use File::Temp (); sub new { bless {}, __PACKAGE__ } @@ -87,15 +86,15 @@ sub do_fetch { my $ibx_ver; $lei->{curl} //= PublicInbox::LeiCurl->new($lei) or return; my $dir = PublicInbox::Admin::resolve_inboxdir($cd, \$ibx_ver); - my ($ibx_uri, @git_dir, @epochs); + my ($ibx_uri, @git_dir, @epochs, $mg, @new_epoch); if ($ibx_ver == 1) { my $url = remote_url($lei, $dir) // die "E: $dir missing remote.origin.url\n"; $ibx_uri = URI->new($url); } else { # v2: - opendir my $dh, "$dir/git" or die "opendir $dir/git: $!"; - @epochs = sort { $b <=> $a } map { substr($_, 0, -4) + 0 } - grep(/\A[0-9]+\.git\z/, readdir($dh)); + require PublicInbox::MultiGit; + $mg = PublicInbox::MultiGit->new($dir, 'all.git', 'git'); + my @epochs = $mg->git_epochs; my ($git_url, $epoch); for my $nr (@epochs) { # try newest epoch, first my $edir = "$dir/git/$nr.git"; @@ -121,9 +120,7 @@ EOM if ($code == 404) { # any pre-manifest.js.gz instances running? Just fetch all # existing ones and unconditionally try cloning the next - $v2_epochs = [ map {; - "$dir/git/$_.git"; - } @epochs ]; + $v2_epochs = [ map { "$dir/git/$_.git" } @epochs ]; push @$v2_epochs, "$dir/git/".($epochs[-1] + 1) if @epochs; } else { $code == 200 or die "BUG unexpected code $code\n"; @@ -154,6 +151,7 @@ EOM $cmd = [ @$torsocks, PublicInbox::LeiMirror::clone_cmd($lei, $opt), $$e_uri, $d]; + push @new_epoch, substr($epath, 5, -4) + 0; } my $cerr = PublicInbox::LeiMirror::run_reap($lei, $cmd, $opt); # do not bail on clone failure if we didn't have a manifest @@ -162,6 +160,7 @@ EOM return; } } + for my $i (@new_epoch) { $mg->epoch_cfg_set($i) } if ($ft) { my $fn = $ft->filename; rename($fn, $mf) or die "E: rename($fn, $mf): $!\n"; diff --git a/lib/PublicInbox/LeiMirror.pm b/lib/PublicInbox/LeiMirror.pm index bc2e749c..c113c9de 100644 --- a/lib/PublicInbox/LeiMirror.pm +++ b/lib/PublicInbox/LeiMirror.pm @@ -1,13 +1,13 @@ # Copyright (C) 2021 all contributors # License: AGPL-3.0+ -# "lei add-external --mirror" support +# "lei add-external --mirror" support (also "public-inbox-clone"); package PublicInbox::LeiMirror; use strict; use v5.10.1; use parent qw(PublicInbox::IPC); use IO::Uncompress::Gunzip qw(gunzip $GunzipError); -use PublicInbox::Spawn qw(popen_rd spawn); +use PublicInbox::Spawn qw(popen_rd spawn run_die); use File::Temp (); use Fcntl qw(SEEK_SET); @@ -209,7 +209,6 @@ sub clone_v2 { my $lei = $self->{lei}; my $curl = $self->{curl} //= PublicInbox::LeiCurl->new($lei) or return; my $pfx //= $curl->torsocks($lei, $v2_uris->[0]) or return; - my @epochs; my $dst = $self->{dst}; my @src_edst; for my $uri (@$v2_uris) { @@ -220,17 +219,21 @@ failed to extract epoch number from $src my $nr = $1 + 0; $edst .= "/git/$nr.git"; - push @src_edst, [ $src, $edst ]; + push @src_edst, $src, $edst; } my $lk = bless { lock_path => "$dst/inbox.lock" }, 'PublicInbox::Lock'; _try_config($self); my $on_destroy = $lk->lock_for_scope($$); my @cmd = clone_cmd($lei, my $opt = {}); - while (my $pair = shift(@src_edst)) { - my $cmd = [ @$pfx, @cmd, @$pair ]; + while (my ($src, $edst) = splice(@src_edst, 0, 2)) { + my $cmd = [ @$pfx, @cmd, $src, $edst ]; my $cerr = run_reap($lei, $cmd, $opt); return $lei->child_error($cerr, "@$cmd failed") if $cerr; } + require PublicInbox::MultiGit; + my $mg = PublicInbox::MultiGit->new($dst, 'all.git', 'git'); + $mg->fill_alternates; + for my $i ($mg->git_epochs) { $mg->epoch_cfg_set($i) } undef $on_destroy; # unlock index_cloned_inbox($self, 2); } diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm index f81a8dae..42f574f2 100644 --- a/lib/PublicInbox/LeiStore.pm +++ b/lib/PublicInbox/LeiStore.pm @@ -27,7 +27,6 @@ use PublicInbox::MDA; use PublicInbox::Spawn qw(spawn); use PublicInbox::MdirReader; use PublicInbox::LeiToMail; -use List::Util qw(max); use File::Temp (); use POSIX (); use IO::Handle (); # ->autoflush @@ -50,19 +49,6 @@ sub rotate_bytes { $_[0]->{rotate_bytes} // ((1024 * 1024 * 1024) / $_[0]->packing_factor) } -sub git_pfx { "$_[0]->{priv_eidx}->{topdir}/local" }; - -sub git_epoch_max { - my ($self) = @_; - if (opendir(my $dh, $self->git_pfx)) { - max(map { - substr($_, 0, -4) + 0; # drop ".git" suffix - } grep(/\A[0-9]+\.git\z/, readdir($dh))) // 0; - } else { - $!{ENOENT} ? 0 : die("opendir ${\$self->git_pfx}: $!\n"); - } -} - sub git_ident ($) { my ($git) = @_; my $rdr = {}; @@ -91,22 +77,16 @@ sub importer { $im->done; undef $im; $self->checkpoint; - $max = $self->git_epoch_max + 1; + $max = $self->{priv_eidx}->{mg}->git_epochs + 1; } my (undef, $tl) = eidx_init($self); # acquire lock - my $pfx = $self->git_pfx; - $max //= $self->git_epoch_max; + $max //= $self->{priv_eidx}->{mg}->git_epochs; while (1) { - my $latest = "$pfx/$max.git"; - my $old = -e $latest; - PublicInbox::Import::init_bare($latest); + my $latest = $self->{priv_eidx}->{mg}->add_epoch($max); my $git = PublicInbox::Git->new($latest); - if (!$old) { - $git->qx(qw(config core.sharedRepository 0600)); - $self->done; # unlock - # re-acquire lock, update alternates for new epoch - (undef, $tl) = eidx_init($self); - } + $self->done; # unlock + # re-acquire lock, update alternates for new epoch + (undef, $tl) = eidx_init($self); my $packed_bytes = $git->packed_bytes; my $unpacked_bytes = $packed_bytes / $self->packing_factor; if ($unpacked_bytes >= $self->rotate_bytes) { diff --git a/lib/PublicInbox/MultiGit.pm b/lib/PublicInbox/MultiGit.pm new file mode 100644 index 00000000..91d7998a --- /dev/null +++ b/lib/PublicInbox/MultiGit.pm @@ -0,0 +1,136 @@ +# Copyright (C) all contributors +# License: AGPL-3.0+ + +# common git alternates + all.git||ALL.git management code +package PublicInbox::MultiGit; +use strict; +use v5.10.1; +use PublicInbox::Spawn qw(run_die); +use PublicInbox::Import; +use File::Temp 0.19; +use List::Util qw(max); + +sub new { + my ($cls, $topdir, $all, $epfx) = @_; + bless { + topdir => $topdir, # inboxdir || extindex.*.topdir + all => $all, # all.git or ALL.git + epfx => $epfx, # "git" (inbox) or "local" (lei/store) + }, $cls; +} + +sub read_alternates { + my ($self, $moderef, $prune) = @_; + my $objpfx = "$self->{topdir}/$self->{all}/objects/"; + my $f = "${objpfx}info/alternates"; + my %alt; # line => score + my %seen; # $st_dev\0$st_ino => count + my $other = 0; + if (open(my $fh, '<', $f)) { + my $is_edir = defined($self->{epfx}) ? + qr!\A\Q../../$self->{epfx}\E/([0-9]+)\.git/objects\z! : + undef; + $$moderef = (stat($fh))[2] & 07777; + for my $rel (split(/^/m, do { local $/; <$fh> })) { + chomp(my $dir = $rel); + my $score; + if (defined($is_edir) && $dir =~ $is_edir) { + $score = $1 + 0; + substr($dir, 0, 0) = $objpfx; + } else { # absolute paths, if any (extindex) + $score = --$other; + } + if (my @st = stat($dir)) { + next if $seen{"$st[0]\0$st[1]"}++; + $alt{$rel} = $score; + } else { + warn "W: stat($dir) failed: $! ($f)"; + $alt{$rel} = $score unless $prune; + } + } + } elsif (!$!{ENOENT}) { + die "E: open($f): $!"; + } + (\%alt, \%seen); +} + +sub epoch_dir { "$_[0]->{topdir}/$_[0]->{epfx}" } + +sub write_alternates { + my ($self, $mode, $alt, @new) = @_; + my $all_dir = "$self->{topdir}/$self->{all}"; + PublicInbox::Import::init_bare($all_dir); + my $out = join('', sort { $alt->{$b} <=> $alt->{$a} } keys %$alt); + my $info_dir = "$all_dir/objects/info"; + my $fh = File::Temp->new(TEMPLATE => 'alt-XXXX', DIR => $info_dir); + my $f = $fh->filename; + print $fh $out, @new or die "print($f): $!"; + chmod($mode, $fh) or die "fchmod($f): $!"; + close $fh or die "close($f): $!"; + my $fn = "$info_dir/alternates"; + rename($f, $fn) or die "rename($f, $fn): $!"; + $fh->unlink_on_destroy(0); +} + +# returns true if new epochs exist +sub merge_epochs { + my ($self, $alt, $seen) = @_; + my $epoch_dir = epoch_dir($self); + if (opendir my $dh, $epoch_dir) { + my $has_new; + for my $bn (grep(/\A[0-9]+\.git\z/, readdir($dh))) { + my $rel = "../../$self->{epfx}/$bn/objects\n"; + next if exists($alt->{$rel}); + if (my @st = stat("$epoch_dir/$bn/objects")) { + next if $seen->{"$st[0]\0$st[1]"}++; + $alt->{$rel} = substr($bn, 0, -4) + 0; + $has_new = 1; + } else { + warn "E: stat($epoch_dir/$bn/objects): $!"; + } + } + $has_new; + } else { + $!{ENOENT} ? undef : die "opendir($epoch_dir): $!"; + } +} + +sub fill_alternates { + my ($self) = @_; + my ($alt, $seen) = read_alternates($self, \(my $mode = 0644)); + merge_epochs($self, $alt, $seen) and + write_alternates($self, $mode, $alt); +} + +sub epoch_cfg_set { + my ($self, $epoch_nr) = @_; + run_die([qw(git config -f), epoch_dir($self)."/$epoch_nr.git/config", + 'include.path', "../../$self->{all}/config" ]); +} + +sub add_epoch { + my ($self, $epoch_nr) = @_; + my $git_dir = epoch_dir($self)."/$epoch_nr.git"; + my $f = "$git_dir/config"; + my $existing = -f $f; + PublicInbox::Import::init_bare($git_dir); + epoch_cfg_set($self, $epoch_nr) unless $existing; + fill_alternates($self); + $git_dir; +} + +sub git_epochs { + my ($self) = @_; + if (opendir(my $dh, epoch_dir($self))) { + my @epochs = map { + substr($_, 0, -4) + 0; # drop ".git" suffix + } grep(/\A[0-9]+\.git\z/, readdir($dh)); + wantarray ? sort { $b <=> $a } @epochs : (max(@epochs) // 0); + } elsif ($!{ENOENT}) { + wantarray ? () : 0; + } else { + die(epoch_dir($self).": $!"); + } +} + +1; diff --git a/lib/PublicInbox/V2Writable.pm b/lib/PublicInbox/V2Writable.pm index 1288f47b..971b007b 100644 --- a/lib/PublicInbox/V2Writable.pm +++ b/lib/PublicInbox/V2Writable.pm @@ -12,6 +12,7 @@ use PublicInbox::IPC; use PublicInbox::Eml; use PublicInbox::Git; use PublicInbox::Import; +use PublicInbox::MultiGit; use PublicInbox::MID qw(mids references); use PublicInbox::ContentHash qw(content_hash content_digest git_sha); use PublicInbox::InboxWritable; @@ -72,16 +73,14 @@ sub new { $v2ibx = PublicInbox::InboxWritable->new($v2ibx); my $dir = $v2ibx->assert_usable_dir; unless (-d $dir) { - if ($creat) { - require File::Path; - File::Path::mkpath($dir); - } else { - die "$dir does not exist\n"; - } + die "$dir does not exist\n" if !$creat; + require File::Path; + File::Path::mkpath($dir); } my $xpfx = "$dir/xap" . PublicInbox::Search::SCHEMA_VERSION; my $self = { ibx => $v2ibx, + mg => PublicInbox::MultiGit->new($dir, 'all.git', 'git'), im => undef, # PublicInbox::Import parallel => 1, transact_bytes => 0, @@ -110,7 +109,7 @@ sub init_inbox { $self->{mm}->skip_artnum($skip_artnum) if defined $skip_artnum; my $max = $self->{ibx}->max_git_epoch; $max = $skip_epoch if (defined($skip_epoch) && !defined($max)); - $self->git_init($max // 0); + $self->{mg}->add_epoch($max // 0); $self->done; } @@ -641,70 +640,6 @@ sub done { die $err if $err; } -sub write_alternates ($$$) { - my ($info_dir, $mode, $out) = @_; - my $fh = File::Temp->new(TEMPLATE => 'alt-XXXX', DIR => $info_dir); - my $tmp = $fh->filename; - print $fh @$out or die "print $tmp: $!\n"; - chmod($mode, $fh) or die "fchmod $tmp: $!\n"; - close $fh or die "close $tmp $!\n"; - my $alt = "$info_dir/alternates"; - rename($tmp, $alt) or die "rename $tmp => $alt: $!\n"; - $fh->unlink_on_destroy(0); -} - -sub fill_alternates ($$) { - my ($self, $epoch) = @_; - - my $pfx = "$self->{ibx}->{inboxdir}/git"; - my $all = "$self->{ibx}->{inboxdir}/all.git"; - PublicInbox::Import::init_bare($all) unless -d $all; - my $info_dir = "$all/objects/info"; - my $alt = "$info_dir/alternates"; - my (%alt, $new); - my $mode = 0644; - if (-e $alt) { - open(my $fh, '<', $alt) or die "open < $alt: $!\n"; - $mode = (stat($fh))[2] & 07777; - - # we assign a sort score to every alternate and favor - # the newest (highest numbered) one because loose objects - # require scanning epochs and only the latest epoch is - # expected to see loose objects - my $score; - my $other = 0; # in case admin adds non-epoch repos - %alt = map {; - if (m!\A\Q../../\E([0-9]+)\.git/objects\z!) { - $score = $1 + 0; - } else { - $score = --$other; - } - $_ => $score; - } split(/\n+/, do { local $/; <$fh> }); - } - - foreach my $i (0..$epoch) { - my $dir = "../../git/$i.git/objects"; - if (!exists($alt{$dir}) && -d "$pfx/$i.git") { - $alt{$dir} = $i; - $new = 1; - } - } - return unless $new; - write_alternates($info_dir, $mode, - [join("\n", sort { $alt{$b} <=> $alt{$a} } keys %alt), "\n"]); -} - -sub git_init { - my ($self, $epoch) = @_; - my $git_dir = "$self->{ibx}->{inboxdir}/git/$epoch.git"; - PublicInbox::Import::init_bare($git_dir); - run_die([qw(git config), "--file=$git_dir/config", - qw(include.path ../../all.git/config)]); - fill_alternates($self, $epoch); - $git_dir -} - sub importer { my ($self) = @_; my $im = $self->{im}; @@ -716,8 +651,8 @@ sub importer { $im->done; $im = undef; $self->checkpoint; - my $git_dir = $self->git_init(++$self->{epoch_max}); - my $git = PublicInbox::Git->new($git_dir); + my $dir = $self->{mg}->add_epoch(++$self->{epoch_max}); + my $git = PublicInbox::Git->new($dir); return $self->import_init($git, 0); } } @@ -737,8 +672,8 @@ sub importer { } } $self->{epoch_max} = $epoch; - $latest = $self->git_init($epoch); - $self->import_init(PublicInbox::Git->new($latest), 0); + my $dir = $self->{mg}->add_epoch($epoch); + $self->import_init(PublicInbox::Git->new($dir), 0); } sub import_init { @@ -1335,7 +1270,7 @@ sub index_sync { local $self->{ibx}->{indexlevel} = 'basic' if $seq; $self->idx_init($opt); # acquire lock - fill_alternates($self, $epoch_max); + $self->{mg}->fill_alternates; $self->{oidx}->rethread_prepare($opt); my $sync = { need_checkpoint => \(my $bool = 0), diff --git a/script/public-inbox-convert b/script/public-inbox-convert index fec6b624..01af846a 100755 --- a/script/public-inbox-convert +++ b/script/public-inbox-convert @@ -179,7 +179,7 @@ if (my $old_mm = $old->mm) { $v2w->idx_init($opt); $v2w->{mm}->{dbh}->sqlite_backup_from_file($old_mm); - my $epoch0 = PublicInbox::Git->new($v2w->git_init(0)); + my $epoch0 = PublicInbox::Git->new($v2w->{mg}->add_epoch(0)); chop(my $cmt = $epoch0->qx(qw(rev-parse --verify), $head)); $v2w->last_epoch_commit(0, $cmt); } diff --git a/t/lei-mirror.t b/t/lei-mirror.t index 44acbe95..5238b67c 100644 --- a/t/lei-mirror.t +++ b/t/lei-mirror.t @@ -95,7 +95,20 @@ SKIP: { ok(run_script([qw(-clone -q -C), $d, "$http/t2"], undef, $opt), '-clone succeeds on v2'); - ok(-d "$d/t2/git/0.git", 'epoch cloned'); + ok(-f "$d/t2/git/0.git/config", 'epoch cloned'); + + # writeBitmaps is the default for bare repos in git 2.22+, + # so we may stop setting it ourselves. + 0 and is(xqx(['git', "--git-dir=$d/t2/git/0.git", 'config', + qw(--bool repack.writeBitmaps)]), "true\n", + 'write bitmaps set (via include.path=all.git/config'); + + is(xqx(['git', "--git-dir=$d/t2/git/0.git", 'config', + qw(include.path)]), "../../all.git/config\n", + 'include.path set'); + + ok(-s "$d/t2/all.git/objects/info/alternates", + 'all.git alternates created'); ok(-f "$d/t2/manifest.js.gz", 'manifest saved'); ok(!-e "$d/t2/mirror.done", 'no leftover mirror.done'); ok(run_script([qw(-fetch -C), "$d/t2"], undef, $opt), diff --git a/t/v2mirror.t b/t/v2mirror.t index 8bcffc29..54ad6945 100644 --- a/t/v2mirror.t +++ b/t/v2mirror.t @@ -228,10 +228,13 @@ EOF is(scalar($mset->items), 0, 'large message not re-indexed'); } ok(scalar(@new_epochs), 'new epochs were created and fetched'); +for my $d (@new_epochs) { + is(xqx(['git', "--git-dir=$d", 'config', qw(include.path)]), + "../../all.git/config\n", + 'include.path set'); +} ok($td->kill, 'killed httpd'); $td->join; -done_testing(); - -1; +done_testing; diff --git a/t/v2writable.t b/t/v2writable.t index d9e7b980..477621e2 100644 --- a/t/v2writable.t +++ b/t/v2writable.t @@ -308,7 +308,7 @@ ok($@, 'V2Writable fails on non-existent dir'); open $fh, '<', $alt or die $!; my $before = do { local $/; <$fh> }; - ok($v2w->git_init(3), 'init a new epoch'); + ok($v2w->{mg}->add_epoch(3), 'init a new epoch'); open $fh, '<', $alt or die $!; my $after = do { local $/; <$fh> }; ok(index($after, $before) > 0, -- cgit v1.2.3-24-ge0c7