* [PATCH 11/14] solver: reduce "git apply" invocations
2019-01-27 4:03 7% [PATCH 00/14] convert solver to use pi-httpd.async Eric Wong
@ 2019-01-27 4:03 6% ` Eric Wong
0 siblings, 0 replies; 2+ results
From: Eric Wong @ 2019-01-27 4:03 UTC (permalink / raw)
To: meta
"git apply" is capable of applying multiple patches in one
invocation, so give it multiple patches on the command-line
now that we no longer rely on anonymous file handles to hold
patches.
This cuts down a 64-patch series on git@vger from ~1s to ~800ms
with vfork spawn enabled using Inline::C.
---
lib/PublicInbox/SolverGit.pm | 61 +++++++++++++++++++++++-------------
1 file changed, 39 insertions(+), 22 deletions(-)
diff --git a/lib/PublicInbox/SolverGit.pm b/lib/PublicInbox/SolverGit.pm
index e307202..dfc28d2 100644
--- a/lib/PublicInbox/SolverGit.pm
+++ b/lib/PublicInbox/SolverGit.pm
@@ -18,6 +18,12 @@ use PublicInbox::MsgIter qw(msg_iter msg_part_text);
use PublicInbox::Qspawn;
use URI::Escape qw(uri_escape_utf8);
+# POSIX requires _POSIX_ARG_MAX >= 4096, and xargs is required to
+# subtract 2048 bytes. We also don't factor in environment variable
+# headroom into this.
+use POSIX qw(sysconf _SC_ARG_MAX);
+my $ARG_SIZE_MAX = (sysconf(_SC_ARG_MAX) || 4096) - 2048;
+
# di = diff info / a hashref with information about a diff ($di):
# {
# oid_a => abbreviated pre-image oid,
@@ -75,8 +81,8 @@ sub solve_existing ($$) {
scalar(@ambiguous) ? \@ambiguous : undef;
}
-sub extract_diff ($$$$) {
- my ($p, $re, $ibx, $smsg) = @_;
+sub extract_diff ($$$$$) {
+ my ($self, $p, $re, $ibx, $smsg) = @_;
my ($part) = @$p; # ignore $depth and @idx;
my $hdr_lines; # diff --git a/... b/...
my $tmp;
@@ -103,9 +109,11 @@ sub extract_diff ($$$$) {
}
}
+
# start writing the diff out to a tempfile
- open($tmp, '+>', undef) or die "open(tmp): $!";
- $di->{tmp} = $tmp;
+ my $pn = ++$self->{tot};
+ open($tmp, '>', $self->{tmp}->dirname . "/$pn") or
+ die "open(tmp): $!";
push @$hdr_lines, $l;
$di->{hdr_lines} = $hdr_lines;
@@ -115,7 +123,7 @@ sub extract_diff ($$$$) {
$di->{ibx} = $ibx;
$di->{smsg} = $smsg;
} elsif ($l =~ m!\Adiff --git ("?a/.+) ("?b/.+)$!) {
- return $di if $tmp; # got our blob, done!
+ last if $tmp; # got our blob, done!
my ($path_a, $path_b) = ($1, $2);
@@ -145,7 +153,9 @@ sub extract_diff ($$$$) {
}
}
}
- $tmp ? $di : undef;
+ return undef unless $tmp;
+ close $tmp or die "close(tmp): $!";
+ $di;
}
sub path_searchable ($) { defined($_[0]) && $_[0] =~ m!\A[\w/\. \-]+\z! }
@@ -182,7 +192,7 @@ sub find_extract_diff ($$$) {
foreach my $smsg (@$msgs) {
$ibx->smsg_mime($smsg) or next;
msg_iter(delete($smsg->{mime}), sub {
- $di ||= extract_diff($_[0], $re, $ibx, $smsg);
+ $di ||= extract_diff($self, $_[0], $re, $ibx, $smsg);
});
return $di if $di;
}
@@ -192,7 +202,6 @@ sub prepare_index ($) {
my ($self) = @_;
my $patches = $self->{patches};
$self->{nr} = 0;
- $self->{tot} = scalar @$patches;
my $di = $patches->[0] or die 'no patches';
my $oid_a = $di->{oid_a} or die '{oid_a} unset';
@@ -368,24 +377,31 @@ sub start_ls_files ($$) {
sub do_git_apply ($) {
my ($self) = @_;
-
- my $di = shift @{$self->{patches}} or die 'empty {patches}';
- my $tmp = delete $di->{tmp} or die 'no tmp ', di_url($self, $di);
- $tmp->flush or die "tmp->flush failed: $!";
- sysseek($tmp, 0, SEEK_SET) or die "sysseek(tmp) failed: $!";
-
- my $i = ++$self->{nr};
- dbg($self, "\napplying [$i/$self->{tot}] " . di_url($self, $di) .
- "\n" . join('', @{$di->{hdr_lines}}));
+ my $dn = $self->{tmp}->dirname;
+ my $patches = $self->{patches};
# we need --ignore-whitespace because some patches are CRLF
- my $cmd = [ qw(git apply --cached --ignore-whitespace
- --whitespace=warn --verbose) ];
- my $rdr = { 0 => fileno($tmp), 2 => 1 };
- my $qsp = PublicInbox::Qspawn->new($cmd, $self->{git_env}, $rdr);
+ my @cmd = qw(git apply --cached --ignore-whitespace
+ --whitespace=warn --verbose);
+ my $len = length(join(' ', @cmd));
+ my $total = $self->{tot};
+ my $di; # keep track of the last one for "git ls-files"
+
+ do {
+ my $i = ++$self->{nr};
+ $di = shift @$patches;
+ dbg($self, "\napplying [$i/$total] " . di_url($self, $di) .
+ "\n" . join('', @{$di->{hdr_lines}}));
+ my $pn = $total + 1 - $i;
+ my $path = "$dn/$pn";
+ $len += length($path) + 1;
+ push @cmd, $path;
+ } while (@$patches && $len < $ARG_SIZE_MAX);
+
+ my $rdr = { 2 => 1 };
+ my $qsp = PublicInbox::Qspawn->new(\@cmd, $self->{git_env}, $rdr);
$qsp->psgi_qx($self->{psgi_env}, undef, sub {
my ($bref) = @_;
- close $tmp;
dbg($self, $$bref);
if (my $err = $qsp->{err}) {
ERR($self, "git apply error: $err");
@@ -481,6 +497,7 @@ sub solve ($$$$$) {
$self->{oid_want} = $oid_want;
$self->{out} = $out;
+ $self->{tot} = 0;
$self->{psgi_env} = $env;
$self->{todo} = [ { %$hints, oid_b => $oid_want } ];
$self->{patches} = []; # [ $di, $di, ... ]
--
EW
^ permalink raw reply related [relevance 6%]
* [PATCH 00/14] convert solver to use pi-httpd.async
@ 2019-01-27 4:03 7% Eric Wong
2019-01-27 4:03 6% ` [PATCH 11/14] solver: reduce "git apply" invocations Eric Wong
0 siblings, 1 reply; 2+ results
From: Eric Wong @ 2019-01-27 4:03 UTC (permalink / raw)
To: meta
Much of the groundwork for this was laid in the now-abandoned
"repobrowse" branch. The goal was to improves fairness as we no
longer wait synchronously on git (apply|update-index|ls-files)
processes and can requests for other clients.
The end result was slightly (2-3%?) slower with all the
callbacks, but reducing "git apply" invocations by relying on
pathnames (instead of stdin) made the end result ~20% faster for
a large (64) patch series.
Email::Simple (via Email::MIME/PublicInbox::MIME) remains a
performance bottleneck, as it does a lot of unnecessary header
parsing and hash-table populating we don't care about; but I'm
not sure if I'll have time to address that.
Eric Wong (14):
httpd/async: remove needless sysread wrapper
qspawn: implement psgi_return and use it for githttpbackend
qspawn|getlinebody: support streaming filters
qspawn|httpd/async: improve and fix out-of-date comments
httpd/async: stop running command if client disconnects
qspawn: implement psgi_qx
t/qspawn.t: psgi_qx stderr test
view: swap CRLF for LF in HTML output
solver: rewrite to use Qspawn->psgi_qx and pi-httpd.async
solver: hold patches in temporary directory
solver: reduce "git apply" invocations
qspawn: decode $? for user-friendliness
viewvcs: do not show final error message twice
solver: crank up max patches to 9999
lib/PublicInbox/GetlineBody.pm | 16 +-
lib/PublicInbox/Git.pm | 2 +-
lib/PublicInbox/GitHTTPBackend.pm | 64 +---
lib/PublicInbox/HTTPD/Async.pm | 27 +-
lib/PublicInbox/Qspawn.pm | 143 +++++++-
lib/PublicInbox/SolverGit.pm | 532 +++++++++++++++++-------------
lib/PublicInbox/View.pm | 4 +
lib/PublicInbox/ViewVCS.pm | 50 ++-
t/qspawn.t | 12 +-
t/solver_git.t | 22 +-
10 files changed, 543 insertions(+), 329 deletions(-)
--
EW
^ permalink raw reply [relevance 7%]
Results 1-2 of 2 | reverse | options above
-- pct% links below jump to the message on this page, permalinks otherwise --
2019-01-27 4:03 7% [PATCH 00/14] convert solver to use pi-httpd.async Eric Wong
2019-01-27 4:03 6% ` [PATCH 11/14] solver: reduce "git apply" invocations Eric Wong
Code repositories for project(s) associated with this public inbox
https://80x24.org/public-inbox.git
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).