diff options
Diffstat (limited to 'lib/PublicInbox/Gcf2.pm')
| -rw-r--r-- | lib/PublicInbox/Gcf2.pm | 108 |
1 files changed, 45 insertions, 63 deletions
diff --git a/lib/PublicInbox/Gcf2.pm b/lib/PublicInbox/Gcf2.pm index 41ee0715..78392990 100644 --- a/lib/PublicInbox/Gcf2.pm +++ b/lib/PublicInbox/Gcf2.pm @@ -1,84 +1,74 @@ -# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org> +# Copyright (C) all contributors <meta@public-inbox.org> # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt> # backend for a git-cat-file-workalike based on libgit2, # other libgit2 stuff may go here, too. package PublicInbox::Gcf2; -use strict; -use v5.10.1; -use PublicInbox::Spawn qw(which popen_rd); # may set PERL_INLINE_DIRECTORY -use Fcntl qw(LOCK_EX SEEK_SET); +use v5.12; +use PublicInbox::Spawn qw(which run_qx); # may set PERL_INLINE_DIRECTORY +use Fcntl qw(SEEK_SET); use Time::HiRes qw(clock_gettime CLOCK_MONOTONIC); use IO::Handle; # autoflush +use PublicInbox::Git qw($ck_unlinked_packs); +use PublicInbox::Lock; +use autodie qw(close open seek truncate); + BEGIN { my (%CFG, $c_src); # PublicInbox::Spawn will set PERL_INLINE_DIRECTORY - # to ~/.cache/public-inbox/inline-c if it exists + # to ~/.cache/public-inbox/inline-c if it exists and Inline::C works my $inline_dir = $ENV{PERL_INLINE_DIRECTORY} // die 'PERL_INLINE_DIRECTORY not defined'; - my $f = "$inline_dir/.public-inbox.lock"; - open my $fh, '+>', $f or die "open($f): $!"; # CentOS 7.x ships Inline 0.53, 0.64+ has built-in locking - flock($fh, LOCK_EX) or die "LOCK_EX($f): $!\n"; + my $lk = PublicInbox::Lock->new("$inline_dir/.public-inbox.lock"); + my $fh = $lk->lock_acquire; my $pc = which($ENV{PKG_CONFIG} // 'pkg-config') // die "pkg-config missing for libgit2"; my ($dir) = (__FILE__ =~ m!\A(.+?)/[^/]+\z!); - my $ef = "$inline_dir/.public-inbox.pkg-config.err"; - open my $err, '+>', $ef or die "open($ef): $!"; - for my $x (qw(libgit2)) { - my $rdr = { 2 => $err }; - my ($l, $pid) = popen_rd([$pc, '--libs', $x], undef, $rdr); - $l = do { local $/; <$l> }; - waitpid($pid, 0); - next if $?; - (my $c, $pid) = popen_rd([$pc, '--cflags', $x], undef, $rdr); - $c = do { local $/; <$c> }; - waitpid($pid, 0); - next if $?; - - # note: we name C source files .h to prevent - # ExtUtils::MakeMaker from automatically trying to - # build them. - my $f = "$dir/gcf2_$x.h"; - open(my $src, '<', $f) or die "E: open($f): $!"; - chomp($l, $c); - local $/; - defined($c_src = <$src>) or die "read $f: $!"; - $CFG{LIBS} = $l; - $CFG{CCFLAGSEX} = $c; - last; - } - unless ($c_src) { - seek($err, 0, SEEK_SET); - $err = do { local $/; <$err> }; - die "E: libgit2 not installed: $err\n"; + my $vals = {}; + my $rdr = { 2 => \(my $err) }; + my @switches = qw(modversion cflags libs); + for my $k (@switches) { + chomp(my $val = run_qx([$pc, "--$k", 'libgit2'], undef, $rdr)); + die "E: libgit2 not installed: $err\n" if $?; + $vals->{$k} = $val; } - open my $oldout, '>&', \*STDOUT or die "dup(1): $!"; - open my $olderr, '>&', \*STDERR or die "dup(2): $!"; - open STDOUT, '>&', $fh or die "1>$f: $!"; - open STDERR, '>&', $fh or die "2>$f: $!"; + my $f = "$dir/gcf2_libgit2.h"; + $c_src = PublicInbox::IO::try_cat $f or die "cat $f: $!"; + # append pkg-config results to the source to ensure Inline::C + # can rebuild if there's changes (it doesn't seem to detect + # $CFG{CCFLAGSEX} nor $CFG{CPPFLAGS} changes) + $c_src .= "/* $pc --$_ libgit2 => $vals->{$_} */\n" for @switches; + open my $oldout, '>&', \*STDOUT; + open my $olderr, '>&', \*STDERR; + open STDOUT, '>&', $fh; + open STDERR, '>&', $fh; STDERR->autoflush(1); STDOUT->autoflush(1); + $CFG{CCFLAGSEX} = $vals->{cflags}; + $CFG{LIBS} = $vals->{libs}; # we use Capitalized and ALLCAPS for compatibility with old Inline::C - eval <<'EOM'; + CORE::eval <<'EOM'; use Inline C => Config => %CFG, BOOT => q[git_libgit2_init();]; use Inline C => $c_src, BUILD_NOISY => 1; EOM $err = $@; - open(STDERR, '>&', $olderr) or warn "restore stderr: $!"; - open(STDOUT, '>&', $oldout) or warn "restore stdout: $!"; + open(STDERR, '>&', $olderr); + open(STDOUT, '>&', $oldout); if ($err) { seek($fh, 0, SEEK_SET); my @msg = <$fh>; + truncate($fh, 0); die "Inline::C Gcf2 build failed:\n", $err, "\n", @msg; } } sub add_alt ($$) { - my ($gcf2, $objdir) = @_; + my ($gcf2, $git_dir) = @_; + my $objdir = PublicInbox::Git->new($git_dir)->git_path('objects'); # libgit2 (tested 0.27.7+dfsg.1-0.2 and 0.28.3+dfsg.1-1~bpo10+1 # in Debian) doesn't handle relative epochs properly when nested @@ -89,23 +79,13 @@ sub add_alt ($$) { # to refer to $V2INBOX_DIR/git/$EPOCH.git/objects # # See https://bugs.debian.org/975607 - if (open(my $fh, '<', "$objdir/info/alternates")) { - chomp(my @abs_alt = grep(m!^/!, <$fh>)); - $gcf2->add_alternate($_) for @abs_alt; + if (my $s = PublicInbox::IO::try_cat("$objdir/info/alternates")) { + $gcf2->add_alternate($_) for ($s =~ m!^(/[^\n]+)\n!gms); } $gcf2->add_alternate($objdir); 1; } -sub have_unlinked_files () { - # FIXME: port gcf2-like over to git.git so we won't need to - # deal with libgit2 - return 1 if $^O ne 'linux'; - open my $fh, '<', "/proc/$$/maps" or return; - while (<$fh>) { return 1 if /\.(?:idx|pack) \(deleted\)$/ } - undef; -} - # Usage: $^X -MPublicInbox::Gcf2 -e PublicInbox::Gcf2::loop [EXPIRE-TIMEOUT] # (see lib/PublicInbox/Gcf2Client.pm) sub loop (;$) { @@ -114,23 +94,24 @@ sub loop (;$) { my (%seen, $check_at); STDERR->autoflush(1); STDOUT->autoflush(1); + my $pid = $$; while (<STDIN>) { chomp; my ($oid, $git_dir) = split(/ /, $_, 2); - $seen{$git_dir} //= add_alt($gcf2, "$git_dir/objects"); + $seen{$git_dir} //= add_alt($gcf2, $git_dir); if (!$gcf2->cat_oid(1, $oid)) { # retry once if missing. We only get unabbreviated OIDs # from SQLite or Xapian DBs, here, so malicious clients # can't trigger excessive retries: - warn "I: $$ $oid missing, retrying in $git_dir\n"; + warn "# $$ $oid missing, retrying in $git_dir\n"; $gcf2 = new(); - %seen = ($git_dir => add_alt($gcf2,"$git_dir/objects")); + %seen = ($git_dir => add_alt($gcf2, $git_dir)); $check_at = clock_gettime(CLOCK_MONOTONIC) + $exp; if ($gcf2->cat_oid(1, $oid)) { - warn "I: $$ $oid found after retry\n"; + warn "# $$ $oid found after retry\n"; } else { warn "W: $$ $oid missing after retry\n"; print "$oid missing\n"; # mimic git-cat-file @@ -140,7 +121,8 @@ sub loop (;$) { $check_at //= $now + $exp; if ($now > $check_at) { undef $check_at; - if (have_unlinked_files()) { + if (!$ck_unlinked_packs || + $ck_unlinked_packs->($pid)) { $gcf2 = new(); %seen = (); } |