user/dev discussion of public-inbox itself
 help / color / mirror / Atom feed
From: Eric Wong <e@80x24.org>
To: meta@public-inbox.org
Cc: Eric Wong <e@yhbt.net>
Subject: [PATCH 1/7] gcf2: libgit2-based git cat-file alternative
Date: Sat, 19 Sep 2020 09:37:08 +0000
Message-ID: <20200919093714.21776-2-e@80x24.org> (raw)
In-Reply-To: <20200919093714.21776-1-e@80x24.org>

From: Eric Wong <e@yhbt.net>

Having tens of thousands of inboxes and associated git processes
won't work well, so we'll use libgit2 to access the object DB
directly.  We only care about OID lookups and won't need to rely
on per-repo revision names or paths.

The Git::Raw XS package won't be used since its manpages don't
promise a stable API.  Since we already use Inline::C and have
experience with I::C when it comes to compatibility, this only
introduces libgit2 itself as a source of new incompatibilities.

This also provides an excuse for me to writev(2) to reduce
syscalls, but liburing is on the horizon for next year.
---
 MANIFEST                       |   3 +
 lib/PublicInbox/Gcf2.pm        |  56 +++++++++++++
 lib/PublicInbox/gcf2_libgit2.h | 139 +++++++++++++++++++++++++++++++++
 t/gcf2.t                       | 112 ++++++++++++++++++++++++++
 4 files changed, 310 insertions(+)
 create mode 100644 lib/PublicInbox/Gcf2.pm
 create mode 100644 lib/PublicInbox/gcf2_libgit2.h
 create mode 100644 t/gcf2.t

diff --git a/MANIFEST b/MANIFEST
index 04a3744f..0d3a7073 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -128,6 +128,7 @@ lib/PublicInbox/Filter/Mirror.pm
 lib/PublicInbox/Filter/RubyLang.pm
 lib/PublicInbox/Filter/SubjectTag.pm
 lib/PublicInbox/Filter/Vger.pm
+lib/PublicInbox/Gcf2.pm
 lib/PublicInbox/GetlineBody.pm
 lib/PublicInbox/Git.pm
 lib/PublicInbox/GitAsyncCat.pm
@@ -212,6 +213,7 @@ lib/PublicInbox/WwwStatic.pm
 lib/PublicInbox/WwwStream.pm
 lib/PublicInbox/WwwText.pm
 lib/PublicInbox/Xapcmd.pm
+lib/PublicInbox/gcf2_libgit2.h
 sa_config/Makefile
 sa_config/README
 sa_config/root/etc/spamassassin/public-inbox.pre
@@ -275,6 +277,7 @@ t/filter_mirror.t
 t/filter_rubylang.t
 t/filter_subjecttag.t
 t/filter_vger.t
+t/gcf2.t
 t/git-http-backend.psgi
 t/git.fast-import-data
 t/git.t
diff --git a/lib/PublicInbox/Gcf2.pm b/lib/PublicInbox/Gcf2.pm
new file mode 100644
index 00000000..6ac3aa18
--- /dev/null
+++ b/lib/PublicInbox/Gcf2.pm
@@ -0,0 +1,56 @@
+# Copyright (C) 2020 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+#
+# git-cat-file based on libgit2
+package PublicInbox::Gcf2;
+use strict;
+use PublicInbox::Spawn qw(which popen_rd);
+use Fcntl qw(LOCK_EX);
+my (%CFG, $c_src, $lockfh);
+BEGIN {
+	# PublicInbox::Spawn will set PERL_INLINE_DIRECTORY
+	# to ~/.cache/public-inbox/inline-c if it exists
+	my $inline_dir = $ENV{PERL_INLINE_DIRECTORY} //
+		die 'PERL_INLINE_DIRECTORY not defined';
+	my $f = "$inline_dir/.public-inbox.lock";
+	open $lockfh, '>', $f or die "failed to open $f: $!\n";
+	my $pc = which($ENV{PKG_CONFIG} // 'pkg-config');
+	my ($dir) = (__FILE__ =~ m!\A(.+?)/[^/]+\z!);
+	my $rdr = {};
+	open $rdr->{2}, '>', '/dev/null' or die "open /dev/null: $!";
+	for my $x (qw(libgit2)) {
+		my $l = popen_rd([$pc, '--libs', $x], undef, $rdr);
+		$l = do { local $/; <$l> };
+		next if $?;
+		my $c = popen_rd([$pc, '--cflags', $x], undef, $rdr);
+		$c = do { local $/; <$c> };
+		next if $?;
+
+		# note: we name C source files .h to prevent
+		# ExtUtils::MakeMaker from automatically trying to
+		# build them.
+		my $f = "$dir/gcf2_$x.h";
+		if (open(my $fh, '<', $f)) {
+			chomp($l, $c);
+			local $/;
+			$c_src = <$fh>;
+			$CFG{LIBS} = $l;
+			$CFG{CCFLAGSEX} = $c;
+			last;
+		} else {
+			die "E: $f: $!\n";
+		}
+	}
+	die "E: libgit2 not installed\n" unless $c_src;
+
+	# CentOS 7.x ships Inline 0.53, 0.64+ has built-in locking
+	flock($lockfh, LOCK_EX) or die "LOCK_EX failed on $f: $!\n";
+}
+
+# we use Capitalized and ALLCAPS for compatibility with old Inline::C
+use Inline C => Config => %CFG, BOOT => 'git_libgit2_init();';
+use Inline C => $c_src;
+undef $c_src;
+undef %CFG;
+undef $lockfh;
+1;
diff --git a/lib/PublicInbox/gcf2_libgit2.h b/lib/PublicInbox/gcf2_libgit2.h
new file mode 100644
index 00000000..d9c79cf9
--- /dev/null
+++ b/lib/PublicInbox/gcf2_libgit2.h
@@ -0,0 +1,139 @@
+/*
+ * Copyright (C) 2020 all contributors <meta@public-inbox.org>
+ * License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+ *
+ * libgit2 for Inline::C
+ * Avoiding Git::Raw since it doesn't guarantee a stable API,
+ * while libgit2 itself seems reasonably stable.
+ */
+#include <git2.h>
+#include <sys/uio.h>
+#include <errno.h>
+#include <poll.h>
+
+static void croak_if_err(int rc, const char *msg)
+{
+	if (rc != GIT_OK) {
+		const git_error *e = giterr_last();
+
+		croak("%d %s (%s)", rc, msg, e ? e->message : "unknown");
+	}
+}
+
+SV *new()
+{
+	git_odb *odb;
+	SV *ref, *self;
+	int rc = git_odb_new(&odb);
+	croak_if_err(rc, "git_odb_new");
+
+	ref = newSViv((IV)odb);
+	self = newRV_noinc(ref);
+	sv_bless(self, gv_stashpv("PublicInbox::Gcf2", GV_ADD));
+	SvREADONLY_on(ref);
+
+	return self;
+}
+
+static git_odb *odb_ptr(SV *self)
+{
+	return (git_odb *)SvIV(SvRV(self));
+}
+
+void DESTROY(SV *self)
+{
+	git_odb_free(odb_ptr(self));
+}
+
+/* needs "$GIT_DIR/objects", not $GIT_DIR */
+void add_alternate(SV *self, const char *objects_path)
+{
+	int rc = git_odb_add_disk_alternate(odb_ptr(self), objects_path);
+	croak_if_err(rc, "git_odb_add_disk_alternate");
+}
+
+/* this requires an unabbreviated git OID */
+#define CAPA(v) (sizeof(v) / sizeof((v)[0]))
+void cat_oid(SV *self, int fd, SV *oidsv)
+{
+	/*
+	 * adjust when libgit2 gets SHA-256 support, we return the
+	 * same header as git-cat-file --batch "$OID $TYPE $SIZE\n"
+	 */
+	char hdr[GIT_OID_HEXSZ + sizeof(" commit 18446744073709551615")];
+	struct iovec vec[3];
+	size_t nvec = CAPA(vec);
+	git_oid oid;
+	git_odb_object *object = NULL;
+	int rc, err = 0;
+	STRLEN oidlen;
+	char *oidptr = SvPV(oidsv, oidlen);
+
+	/* same trailer as git-cat-file --batch */
+	vec[2].iov_len = 1;
+	vec[2].iov_base = "\n";
+
+	rc = git_oid_fromstrn(&oid, oidptr, oidlen);
+	if (rc == GIT_OK)
+		rc = git_odb_read(&object, odb_ptr(self), &oid);
+	if (rc == GIT_OK) {
+		vec[0].iov_base = hdr;
+		vec[1].iov_base = (void *)git_odb_object_data(object);
+		vec[1].iov_len = git_odb_object_size(object);
+
+		git_oid_nfmt(hdr, GIT_OID_HEXSZ, git_odb_object_id(object));
+		vec[0].iov_len = GIT_OID_HEXSZ +
+				snprintf(hdr + GIT_OID_HEXSZ,
+					sizeof(hdr) - GIT_OID_HEXSZ,
+					" %s %zu\n",
+					git_object_type2string(
+						git_odb_object_type(object)),
+					vec[1].iov_len);
+	} else {
+		vec[0].iov_base = oidptr;
+		vec[0].iov_len = oidlen;
+		vec[1].iov_base = " missing";
+		vec[1].iov_len = strlen(vec[1].iov_base);
+	}
+	while (nvec && !err) {
+		ssize_t w = writev(fd, vec + CAPA(vec) - nvec, nvec);
+
+		if (w > 0) {
+			size_t done = 0;
+			size_t i;
+
+			for (i = CAPA(vec) - nvec; i < CAPA(vec); i++) {
+				if (w >= vec[i].iov_len) {
+					/* fully written vec */
+					w -= vec[i].iov_len;
+					done++;
+				} else { /* partially written vec */
+					char *p = vec[i].iov_base;
+					vec[i].iov_base = p + w;
+					vec[i].iov_len -= w;
+					break;
+				}
+			}
+			nvec -= done;
+		} else if (w < 0) {
+			err = errno;
+			switch (err) {
+			case EAGAIN: {
+				struct pollfd pfd;
+				pfd.events = POLLOUT;
+				pfd.fd = fd;
+				poll(&pfd, 1, -1);
+			}
+				/* fall-through */
+			case EINTR:
+				err = 0;
+			}
+		} else { /* w == 0 */
+			err = ENOSPC;
+		}
+	}
+	if (object)
+		git_odb_object_free(object);
+	if (err)
+		croak("writev error: %s", strerror(err));
+}
diff --git a/t/gcf2.t b/t/gcf2.t
new file mode 100644
index 00000000..c67efb6c
--- /dev/null
+++ b/t/gcf2.t
@@ -0,0 +1,112 @@
+#!perl -w
+# Copyright (C) 2020 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+use strict;
+use PublicInbox::TestCommon;
+use Test::More;
+use Fcntl qw(:seek);
+use IO::Handle ();
+use POSIX qw(_exit);
+require_mods('PublicInbox::Gcf2');
+use_ok 'PublicInbox::Gcf2';
+my $gcf2 = PublicInbox::Gcf2::new();
+is(ref($gcf2), 'PublicInbox::Gcf2', '::new works');
+chomp(my $objdir = xqx([qw(git rev-parse --git-path objects)]));
+if ($objdir =~ /\A--git-path\n/) { # git <2.5
+	chomp($objdir = xqx([qw(git rev-parse --git-dir)]));
+	$objdir .= '/objects';
+	$objdir = undef unless -d $objdir;
+}
+
+my $COPYING = 'dba13ed2ddf783ee8118c6a581dbf75305f816a3';
+open my $agpl, '<', 'COPYING' or BAIL_OUT "AGPL-3 missing: $!";
+$agpl = do { local $/; <$agpl> };
+
+SKIP: {
+	skip 'not in git worktree', 15 unless defined($objdir);
+	$gcf2->add_alternate($objdir);
+	open my $fh, '+>', undef or BAIL_OUT "open: $!";
+	my $fd = fileno($fh);
+	$fh->autoflush(1);
+
+	$gcf2->cat_oid($fd, 'invalid');
+	seek($fh, 0, SEEK_SET) or BAIL_OUT "seek: $!";
+	is(do { local $/; <$fh> }, "invalid missing\n", 'got missing message');
+
+	seek($fh, 0, SEEK_SET) or BAIL_OUT "seek: $!";
+	$gcf2->cat_oid($fd, '0'x40);
+	seek($fh, 0, SEEK_SET) or BAIL_OUT "seek: $!";
+	is(do { local $/; <$fh> }, ('0'x40)." missing\n",
+		'got missing message for 0x40');
+
+	seek($fh, 0, SEEK_SET) or BAIL_OUT "seek: $!";
+	$gcf2->cat_oid($fd, $COPYING);
+	my $buf;
+	my $ck_copying = sub {
+		my ($desc) = @_;
+		seek($fh, 0, SEEK_SET) or BAIL_OUT "seek: $!";
+		is(<$fh>, "$COPYING blob 34520\n", 'got expected header');
+		$buf = do { local $/; <$fh> };
+		is(chop($buf), "\n", 'got trailing \\n');
+		is($buf, $agpl, "AGPL matches ($desc)");
+	};
+	$ck_copying->('regular file');
+
+	$^O eq 'linux' or skip('pipe tests are Linux-only', 12);
+	my $size = -s $fh;
+	for my $blk (1, 0) {
+		my ($r, $w);
+		pipe($r, $w) or BAIL_OUT $!;
+		fcntl($w, 1031, 4096) or
+			skip('Linux too old for F_SETPIPE_SZ', 12);
+		$w->blocking($blk);
+		seek($fh, 0, SEEK_SET) or BAIL_OUT "seek: $!";
+		truncate($fh, 0) or BAIL_OUT "truncate: $!";
+		defined(my $pid = fork) or BAIL_OUT "fork: $!";
+		if ($pid == 0) {
+			close $w;
+			tick; # wait for parent to block on writev
+			$buf = do { local $/; <$r> };
+			print $fh $buf or _exit(1);
+			_exit(0);
+		}
+		$gcf2->cat_oid(fileno($w), $COPYING);
+		close $w or BAIL_OUT "close: $!";
+		is(waitpid($pid, 0), $pid, 'child exited');
+		is($?, 0, 'no error in child');
+		$ck_copying->("pipe blocking($blk)");
+
+		pipe($r, $w) or BAIL_OUT $!;
+		fcntl($w, 1031, 4096) or BAIL_OUT $!;
+		$w->blocking($blk);
+		close $r;
+		local $SIG{PIPE} = 'IGNORE';
+		eval { $gcf2->cat_oid(fileno($w), $COPYING) };
+		like($@, qr/writev error:/, 'got writev error');
+	}
+}
+
+if (my $nr = $ENV{TEST_LEAK_NR}) {
+	open my $null, '>', '/dev/null' or BAIL_OUT "open /dev/null: $!";
+	my $fd = fileno($null);
+	my $cat = $ENV{TEST_LEAK_CAT} // 10;
+	diag "checking for leaks... (TEST_LEAK_NR=$nr TEST_LEAK_CAT=$cat)";
+	local $SIG{PIPE} = 'IGNORE';
+	my ($r, $w);
+	pipe($r, $w);
+	close $r;
+	my $broken = fileno($w);
+	for (1..$nr) {
+		my $obj = PublicInbox::Gcf2::new();
+		if (defined($objdir)) {
+			$obj->add_alternate($objdir);
+			for (1..$cat) {
+				$obj->cat_oid($fd, $COPYING);
+				eval { $obj->cat_oid($broken, $COPYING) };
+				$obj->cat_oid($fd, '0'x40);
+				$obj->cat_oid($fd, 'invalid');
+			}
+		}
+	}
+}
+done_testing;

  reply	other threads:[~2020-09-19  9:37 UTC|newest]

Thread overview: 8+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2020-09-19  9:37 [PATCH 0/7] gcf2: libgit2-based " Eric Wong
2020-09-19  9:37 ` Eric Wong [this message]
2020-09-19  9:37 ` [PATCH 2/7] t/gcf2: test changes to alternates Eric Wong
2020-09-19  9:37 ` [PATCH 3/7] add gcf2 client and executable script Eric Wong
2020-09-19  9:37 ` [PATCH 4/7] gcf2: transparently retry on missing OID Eric Wong
2020-09-19  9:37 ` [PATCH 5/7] gcf2*: more descriptive package descriptions Eric Wong
2020-09-19  9:37 ` [PATCH 6/7] gcf2: require git dir with OID Eric Wong
2020-09-19  9:37 ` [PATCH 7/7] gcf2: wire up read-only daemons and rm -gcf2 script Eric Wong

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

  List information: http://public-inbox.org/README

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20200919093714.21776-2-e@80x24.org \
    --to=e@80x24.org \
    --cc=e@yhbt.net \
    --cc=meta@public-inbox.org \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link

user/dev discussion of public-inbox itself

This inbox may be cloned and mirrored by anyone:

	git clone --mirror http://public-inbox.org/meta
	git clone --mirror http://czquwvybam4bgbro.onion/meta
	git clone --mirror http://hjrcffqmbrq6wope.onion/meta
	git clone --mirror http://ou63pmih66umazou.onion/meta

	# If you have public-inbox 1.1+ installed, you may
	# initialize and index your mirror using the following commands:
	public-inbox-init -V1 meta meta/ http://public-inbox.org/meta \
		meta@public-inbox.org
	public-inbox-index meta

Example config snippet for mirrors.
Newsgroups are available over NNTP:
	nntp://news.public-inbox.org/inbox.comp.mail.public-inbox.meta
	nntp://ou63pmih66umazou.onion/inbox.comp.mail.public-inbox.meta
	nntp://czquwvybam4bgbro.onion/inbox.comp.mail.public-inbox.meta
	nntp://hjrcffqmbrq6wope.onion/inbox.comp.mail.public-inbox.meta
	nntp://news.gmane.io/gmane.mail.public-inbox.general
 note: .onion URLs require Tor: https://www.torproject.org/

code repositories for the project(s) associated with this inbox:

	https://80x24.org/public-inbox.git

AGPL code for this site: git clone https://public-inbox.org/public-inbox.git