From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on dcvr.yhbt.net X-Spam-Level: X-Spam-ASN: X-Spam-Status: No, score=-4.0 required=3.0 tests=ALL_TRUSTED,BAYES_00 shortcircuit=no autolearn=ham autolearn_force=no version=3.4.2 Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id 05B801F464 for ; Fri, 29 Nov 2019 12:25:09 +0000 (UTC) From: Eric Wong To: meta@public-inbox.org Subject: [PATCH 1/4] git: async batch interface Date: Fri, 29 Nov 2019 12:25:05 +0000 Message-Id: <20191129122508.7708-2-e@80x24.org> In-Reply-To: <20191129122508.7708-1-e@80x24.org> References: <20191129122508.7708-1-e@80x24.org> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit List-Id: This is a transitionary interface which does NOT require an event loop. It can be plugged into in current synchronous code without major surgery. It allows HTTP/1.1 pipelining-like functionality by taking advantage of predictable and well-specified POSIX pipe semantics by stuffing multiple git cat-file requests into the --batch pipe With xt/git_async_cmp.t and GIANT_GIT_DIR=git.git, the async interface is 10-25% faster than the synchronous interface since it can keep the "git cat-file" process busier. This is expected to improve performance on systems with slower storage (but multiple cores). --- MANIFEST | 1 + lib/PublicInbox/Git.pm | 94 ++++++++++++++++++++++++++++++++---------- xt/git_async_cmp.t | 61 +++++++++++++++++++++++++++ 3 files changed, 135 insertions(+), 21 deletions(-) create mode 100644 xt/git_async_cmp.t diff --git a/MANIFEST b/MANIFEST index 098e656d..ee7dd3e8 100644 --- a/MANIFEST +++ b/MANIFEST @@ -290,6 +290,7 @@ t/watch_maildir_v2.t t/www_listing.t t/xcpdb-reshard.t xt/git-http-backend.t +xt/git_async_cmp.t xt/nntpd-validate.t xt/perf-msgview.t xt/perf-nntpd.t diff --git a/lib/PublicInbox/Git.pm b/lib/PublicInbox/Git.pm index 218846f3..90595840 100644 --- a/lib/PublicInbox/Git.pm +++ b/lib/PublicInbox/Git.pm @@ -16,6 +16,11 @@ use PublicInbox::Tmpfile; use base qw(Exporter); our @EXPORT_OK = qw(git_unquote git_quote); +use constant MAX_INFLIGHT => + ($^O eq 'linux' ? 4096 : POSIX::_POSIX_PIPE_BUF()) + / + 65; # SHA-256 hex size + "\n" in preparation for git using non-SHA1 + my %GIT_ESC = ( a => "\a", b => "\b", @@ -124,10 +129,50 @@ sub _bidi_pipe { $self->{$in} = $in_r; } +sub read_cat_in_full ($$$) { + my ($self, $in, $left) = @_; + my $offset = 0; + my $buf = ''; + while ($left > 0) { + my $r = read($in, $buf, $left, $offset); + defined($r) or fail($self, "read failed: $!"); + $r == 0 and fail($self, 'exited unexpectedly'); + $left -= $r; + $offset += $r; + } + my $r = read($in, my $lf, 1); + defined($r) or fail($self, "read failed: $!"); + fail($self, 'newline missing after blob') if ($r != 1 || $lf ne "\n"); + \$buf; +} + +sub _cat_async_step ($$$) { + my ($self, $inflight, $in) = @_; + my $cb = shift @$inflight or die 'BUG: inflight empty'; + local $/ = "\n"; + my $head = $in->getline; + return eval { $cb->(undef) } if $head =~ / missing$/; + + $head =~ /^([0-9a-f]{40}) (\S+) ([0-9]+)$/ or + fail($self, "Unexpected result from async git cat-file: $head"); + my ($oid_hex, $type, $size) = ($1, $2, $3 + 0); + my $bref = read_cat_in_full($self, $in, $size); + eval { $cb->($bref, $oid_hex, $type, $size) }; +} + +sub cat_async_wait ($) { + my ($self) = @_; + my $inflight = delete $self->{inflight} or return; + my $in = $self->{in}; + while (scalar(@$inflight)) { + _cat_async_step($self, $inflight, $in); + } +} + sub cat_file { my ($self, $obj, $ref) = @_; my ($retried, $in, $head); - + cat_async_wait($self); again: batch_prepare($self); $self->{out}->print($obj, "\n") or fail($self, "write error: $!"); @@ -147,26 +192,8 @@ again: fail($self, "Unexpected result from git cat-file: $head"); my $size = $1; - my $rv; - my $left = $size; $$ref = $size if $ref; - - my $offset = 0; - my $buf = ''; - while ($left > 0) { - my $r = read($in, $buf, $left, $offset); - defined($r) or fail($self, "read failed: $!"); - $r == 0 and fail($self, 'exited unexpectedly'); - $left -= $r; - $offset += $r; - } - $rv = \$buf; - - my $r = read($in, my $lf, 1); - defined($r) or fail($self, "read failed: $!"); - fail($self, 'newline missing after blob') if ($r != 1 || $lf ne "\n"); - - $rv; + read_cat_in_full($self, $in, $size); } sub batch_prepare ($) { _bidi_pipe($_[0], qw(--batch in out pid)) } @@ -206,9 +233,15 @@ sub _destroy { waitpid($p, 0) if $@; # wait synchronously if not in event loop } +sub cat_async_abort ($) { + my ($self) = @_; + my $inflight = delete $self->{inflight} or die 'BUG: not in async'; + cleanup($self); +} + sub fail { my ($self, $msg) = @_; - cleanup($self); + $self->{inflight} ? cat_async_abort($self) : cleanup($self); die $msg; } @@ -278,6 +311,25 @@ sub pub_urls { local_nick($self); } +sub cat_async_begin { + my ($self) = @_; + cleanup($self) if alternates_changed($self); + batch_prepare($self); + die 'BUG: already in async' if $self->{inflight}; + $self->{inflight} = []; +} + +sub cat_async ($$$) { + my ($self, $oid, $cb) = @_; + my $inflight = $self->{inflight} or die 'BUG: not in async'; + if (scalar(@$inflight) >= MAX_INFLIGHT) { + _cat_async_step($self, $inflight, $self->{in}); + } + + $self->{out}->print($oid, "\n") or fail($self, "write error: $!"); + push @$inflight, $cb; +} + sub commit_title ($$) { my ($self, $oid) = @_; # PublicInbox::Git, $sha1hex my $buf = cat_file($self, $oid) or return; diff --git a/xt/git_async_cmp.t b/xt/git_async_cmp.t new file mode 100644 index 00000000..f8ffe3d9 --- /dev/null +++ b/xt/git_async_cmp.t @@ -0,0 +1,61 @@ +#!perl -w +# Copyright (C) 2019 all contributors +# License: AGPL-3.0+ +use strict; +use Test::More; +use Benchmark qw(:all); +use Digest::SHA; +use PublicInbox::TestCommon; +my $git_dir = $ENV{GIANT_GIT_DIR}; +plan 'skip_all' => "GIANT_GIT_DIR not defined for $0" unless defined($git_dir); +use_ok 'PublicInbox::Git'; +my $git = PublicInbox::Git->new($git_dir); +my @cat = qw(cat-file --buffer --batch-check --batch-all-objects); +if (require_git(2.19, 1)) { + push @cat, '--unordered'; +} else { + warn "git <2.19, cat-file lacks --unordered, locality suffers\n"; +} +my @dig; +my $nr = $ENV{NR} || 1; +my $async = timeit($nr, sub { + my $dig = Digest::SHA->new(1); + my $cb = sub { + my ($bref) = @_; + $dig->add($$bref); + }; + my $cat = $git->popen(@cat); + $git->cat_async_begin; + + foreach (<$cat>) { + my ($oid, undef, undef) = split(/ /); + $git->cat_async($oid, $cb); + } + close $cat or die "cat: $?"; + $git->cat_async_wait; + push @dig, ['async', $dig->hexdigest ]; +}); + +my $sync = timeit($nr, sub { + my $dig = Digest::SHA->new(1); + my $cat = $git->popen(@cat); + foreach (<$cat>) { + my ($oid, undef, undef) = split(/ /); + my $bref = $git->cat_file($oid); + $dig->add($$bref); + } + close $cat or die "cat: $?"; + push @dig, ['sync', $dig->hexdigest ]; +}); + +ok(scalar(@dig) >= 2, 'got some digests'); +my $ref = shift @dig; +my $exp = $ref->[1]; +isnt($exp, Digest::SHA->new(1)->hexdigest, 'not empty'); +foreach (@dig) { + is($_->[1], $exp, "digest matches $_->[0] <=> $ref->[0]"); +} +diag "sync=".timestr($sync); +diag "async=".timestr($async); +done_testing; +1;