1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
| | # Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
# backend for a git-cat-file-workalike based on libgit2,
# other libgit2 stuff may go here, too.
package PublicInbox::Gcf2;
use strict;
use v5.10.1;
use PublicInbox::Spawn qw(which popen_rd); # may set PERL_INLINE_DIRECTORY
use Fcntl qw(LOCK_EX SEEK_SET);
use Time::HiRes qw(clock_gettime CLOCK_MONOTONIC);
use IO::Handle; # autoflush
use File::Path qw(make_path);
BEGIN {
my (%CFG, $c_src);
# PublicInbox::Spawn will set PERL_INLINE_DIRECTORY
# to ~/.cache/public-inbox/inline-c if it exists
my $inline_dir = $ENV{PERL_INLINE_DIRECTORY} //
die 'PERL_INLINE_DIRECTORY not defined';
make_path($inline_dir);
my $f = "$inline_dir/.public-inbox.lock";
open my $fh, '+>', $f or die "open($f): $!";
# CentOS 7.x ships Inline 0.53, 0.64+ has built-in locking
flock($fh, LOCK_EX) or die "LOCK_EX($f): $!\n";
my $pc = which($ENV{PKG_CONFIG} // 'pkg-config') //
die "pkg-config missing for libgit2";
my ($dir) = (__FILE__ =~ m!\A(.+?)/[^/]+\z!);
my $ef = "$inline_dir/.public-inbox.pkg-config.err";
open my $err, '+>', $ef or die "open($ef): $!";
for my $x (qw(libgit2)) {
my $rdr = { 2 => $err };
my ($l, $pid) = popen_rd([$pc, '--libs', $x], undef, $rdr);
$l = do { local $/; <$l> };
waitpid($pid, 0);
next if $?;
(my $c, $pid) = popen_rd([$pc, '--cflags', $x], undef, $rdr);
$c = do { local $/; <$c> };
waitpid($pid, 0);
next if $?;
# note: we name C source files .h to prevent
# ExtUtils::MakeMaker from automatically trying to
# build them.
my $f = "$dir/gcf2_$x.h";
open(my $src, '<', $f) or die "E: open($f): $!";
chomp($l, $c);
local $/;
defined($c_src = <$src>) or die "read $f: $!";
$CFG{LIBS} = $l;
$CFG{CCFLAGSEX} = $c;
last;
}
unless ($c_src) {
seek($err, 0, SEEK_SET);
$err = do { local $/; <$err> };
die "E: libgit2 not installed: $err\n";
}
open my $oldout, '>&', \*STDOUT or die "dup(1): $!";
open my $olderr, '>&', \*STDERR or die "dup(2): $!";
open STDOUT, '>&', $fh or die "1>$f: $!";
open STDERR, '>&', $fh or die "2>$f: $!";
STDERR->autoflush(1);
STDOUT->autoflush(1);
# we use Capitalized and ALLCAPS for compatibility with old Inline::C
eval <<'EOM';
use Inline C => Config => %CFG, BOOT => q[git_libgit2_init();];
use Inline C => $c_src, BUILD_NOISY => 1;
EOM
$err = $@;
open(STDERR, '>&', $olderr) or warn "restore stderr: $!";
open(STDOUT, '>&', $oldout) or warn "restore stdout: $!";
if ($err) {
seek($fh, 0, SEEK_SET);
my @msg = <$fh>;
die "Inline::C Gcf2 build failed:\n", $err, "\n", @msg;
}
}
sub add_alt ($$) {
my ($gcf2, $git_dir) = @_;
my $objdir = PublicInbox::Git->new($git_dir)->git_path('objects');
# libgit2 (tested 0.27.7+dfsg.1-0.2 and 0.28.3+dfsg.1-1~bpo10+1
# in Debian) doesn't handle relative epochs properly when nested
# multiple levels. Add all the absolute paths to workaround it,
# since $EXTINDEX_DIR/ALL.git/objects/info/alternates uses absolute
# paths to reference $V2INBOX_DIR/all.git/objects and
# $V2INBOX_DIR/all.git/objects/info/alternates uses relative paths
# to refer to $V2INBOX_DIR/git/$EPOCH.git/objects
#
# See https://bugs.debian.org/975607
if (open(my $fh, '<', "$objdir/info/alternates")) {
chomp(my @abs_alt = grep(m!^/!, <$fh>));
$gcf2->add_alternate($_) for @abs_alt;
}
$gcf2->add_alternate($objdir);
1;
}
sub have_unlinked_files () {
# FIXME: port gcf2-like over to git.git so we won't need to
# deal with libgit2
return 1 if $^O ne 'linux';
open my $fh, '<', "/proc/$$/maps" or return;
while (<$fh>) { return 1 if /\.(?:idx|pack) \(deleted\)$/ }
undef;
}
# Usage: $^X -MPublicInbox::Gcf2 -e PublicInbox::Gcf2::loop [EXPIRE-TIMEOUT]
# (see lib/PublicInbox/Gcf2Client.pm)
sub loop (;$) {
my $exp = $_[0] || $ARGV[0] || 60; # seconds
my $gcf2 = new();
my (%seen, $check_at);
STDERR->autoflush(1);
STDOUT->autoflush(1);
while (<STDIN>) {
chomp;
my ($oid, $git_dir) = split(/ /, $_, 2);
$seen{$git_dir} //= add_alt($gcf2, $git_dir);
if (!$gcf2->cat_oid(1, $oid)) {
# retry once if missing. We only get unabbreviated OIDs
# from SQLite or Xapian DBs, here, so malicious clients
# can't trigger excessive retries:
warn "I: $$ $oid missing, retrying in $git_dir\n";
$gcf2 = new();
%seen = ($git_dir => add_alt($gcf2, $git_dir);
$check_at = clock_gettime(CLOCK_MONOTONIC) + $exp;
if ($gcf2->cat_oid(1, $oid)) {
warn "I: $$ $oid found after retry\n";
} else {
warn "W: $$ $oid missing after retry\n";
print "$oid missing\n"; # mimic git-cat-file
}
} else { # check expiry to deal with deleted pack files
my $now = clock_gettime(CLOCK_MONOTONIC);
$check_at //= $now + $exp;
if ($now > $check_at) {
undef $check_at;
if (have_unlinked_files()) {
$gcf2 = new();
%seen = ();
}
}
}
}
}
1;
|