From: Eric Wong <e@80x24.org>
To: meta@public-inbox.org
Subject: [PATCH] searchidx: run `git patch-id' in parallel
Date: Mon, 20 Nov 2023 19:22:32 +0000 [thread overview]
Message-ID: <20231120192232.1701486-1-e@80x24.org> (raw)
Informal benchmarks show a rough 5% indexing improvement on an
SMP system when there are idle cores due to Xapian shards being
I/O bound (since `git patch-id' is mainly CPU bound).
This is only parallelized on a per-patch basis. Further
increasing parallelism would increase complexity and probably
not be worth it since `git patch-id' is reasonably fast while
our text indexing tends to be slow.
---
lib/PublicInbox/CodeSearchIdx.pm | 2 +-
lib/PublicInbox/SearchIdx.pm | 24 +++++++++++-------------
2 files changed, 12 insertions(+), 14 deletions(-)
diff --git a/lib/PublicInbox/CodeSearchIdx.pm b/lib/PublicInbox/CodeSearchIdx.pm
index 3a30b5e5..54a2ba22 100644
--- a/lib/PublicInbox/CodeSearchIdx.pm
+++ b/lib/PublicInbox/CodeSearchIdx.pm
@@ -675,7 +675,7 @@ sub get_roots ($$) {
{ 0 => $refs, 1 => $roots_fh }, \&index_repo, $self, $git)
}
-# for PublicInbox::SearchIdx::patch_id and with_umask
+# for PublicInbox::SearchIdx `git patch-id' call and with_umask
sub git { $_[0]->{git} }
sub load_existing ($) { # for -u/--update
diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm
index 9566b14d..32598b7c 100644
--- a/lib/PublicInbox/SearchIdx.pm
+++ b/lib/PublicInbox/SearchIdx.pm
@@ -22,7 +22,7 @@ use POSIX qw(strftime);
use Fcntl qw(SEEK_SET);
use Time::Local qw(timegm);
use PublicInbox::OverIdx;
-use PublicInbox::Spawn qw(run_wait run_qx);
+use PublicInbox::Spawn qw(run_wait popen_rd);
use PublicInbox::Git qw(git_unquote);
use PublicInbox::MsgTime qw(msg_timestamp msg_datestamp);
use PublicInbox::Address;
@@ -350,20 +350,13 @@ sub index_diff ($$$) {
index_text($self, join("\n", @$xnq), 1, 'XNQ');
}
-sub patch_id {
- my ($self, $sref) = @_;
- my $git = ($self->{ibx} // $self->{eidx} // $self)->git;
- my $opt = { 0 => [ ':utf8', $sref ], 2 => \(my $err) };
- my $id = run_qx($git->cmd(qw(patch-id --stable)), undef, $opt);
- warn $err if $err;
- $id =~ /\A([a-f0-9]{40,})/ ? $1 : undef;
-}
-
sub index_body_text {
my ($self, $doc, $sref) = @_;
- if ($$sref =~ /^(?:diff|---|\+\+\+) /ms) {
- my $id = patch_id($self, $sref);
- $doc->add_term('XDFID'.$id) if defined($id);
+ my $rd;
+ if ($$sref =~ /^(?:diff|---|\+\+\+) /ms) { # start patch-id in parallel
+ my $git = ($self->{ibx} // $self->{eidx} // $self)->git;
+ $rd = popen_rd($git->cmd(qw(patch-id --stable)), undef,
+ { 0 => [ ':utf8', $sref ] });
}
# split off quoted and unquoted blocks:
@@ -387,6 +380,11 @@ sub index_body_text {
}
undef $txt; # free memory
}
+ if (defined $rd) { # reap `git patch-id'
+ (readline($rd) // '') =~ /\A([a-f0-9]{40,})/ and
+ $doc->add_term('XDFID'.$1);
+ $rd->close or warn "W: git patch-id failed: \$?=$? (non-fatal)"
+ }
}
sub index_xapian { # msg_iter callback
reply other threads:[~2023-11-20 19:22 UTC|newest]
Thread overview: [no followups] expand[flat|nested] mbox.gz Atom feed
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
List information: http://public-inbox.org/README
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20231120192232.1701486-1-e@80x24.org \
--to=e@80x24.org \
--cc=meta@public-inbox.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
Code repositories for project(s) associated with this public inbox
https://80x24.org/public-inbox.git
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).