diff options
author | Eric Wong <e@80x24.org> | 2023-11-20 19:22:32 +0000 |
---|---|---|
committer | Eric Wong <e@80x24.org> | 2023-11-21 12:07:44 +0000 |
commit | abcdd9217ba5cdb03629b0d17ef3d8ee42ab62f7 (patch) | |
tree | 98505564d790329aa0f92cfa63f8710597da8de7 | |
parent | d9a3750473e2e2fb13c8a4002be890772ee54330 (diff) | |
download | public-inbox-abcdd9217ba5cdb03629b0d17ef3d8ee42ab62f7.tar.gz |
Informal benchmarks show a rough 5% indexing improvement on an SMP system when there are idle cores due to Xapian shards being I/O bound (since `git patch-id' is mainly CPU bound). This is only parallelized on a per-patch basis. Further increasing parallelism would increase complexity and probably not be worth it since `git patch-id' is reasonably fast while our text indexing tends to be slow.
-rw-r--r-- | lib/PublicInbox/CodeSearchIdx.pm | 2 | ||||
-rw-r--r-- | lib/PublicInbox/SearchIdx.pm | 24 |
2 files changed, 12 insertions, 14 deletions
diff --git a/lib/PublicInbox/CodeSearchIdx.pm b/lib/PublicInbox/CodeSearchIdx.pm index 3a30b5e5..54a2ba22 100644 --- a/lib/PublicInbox/CodeSearchIdx.pm +++ b/lib/PublicInbox/CodeSearchIdx.pm @@ -675,7 +675,7 @@ sub get_roots ($$) { { 0 => $refs, 1 => $roots_fh }, \&index_repo, $self, $git) } -# for PublicInbox::SearchIdx::patch_id and with_umask +# for PublicInbox::SearchIdx `git patch-id' call and with_umask sub git { $_[0]->{git} } sub load_existing ($) { # for -u/--update diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm index 9566b14d..32598b7c 100644 --- a/lib/PublicInbox/SearchIdx.pm +++ b/lib/PublicInbox/SearchIdx.pm @@ -22,7 +22,7 @@ use POSIX qw(strftime); use Fcntl qw(SEEK_SET); use Time::Local qw(timegm); use PublicInbox::OverIdx; -use PublicInbox::Spawn qw(run_wait run_qx); +use PublicInbox::Spawn qw(run_wait popen_rd); use PublicInbox::Git qw(git_unquote); use PublicInbox::MsgTime qw(msg_timestamp msg_datestamp); use PublicInbox::Address; @@ -350,20 +350,13 @@ sub index_diff ($$$) { index_text($self, join("\n", @$xnq), 1, 'XNQ'); } -sub patch_id { - my ($self, $sref) = @_; - my $git = ($self->{ibx} // $self->{eidx} // $self)->git; - my $opt = { 0 => [ ':utf8', $sref ], 2 => \(my $err) }; - my $id = run_qx($git->cmd(qw(patch-id --stable)), undef, $opt); - warn $err if $err; - $id =~ /\A([a-f0-9]{40,})/ ? $1 : undef; -} - sub index_body_text { my ($self, $doc, $sref) = @_; - if ($$sref =~ /^(?:diff|---|\+\+\+) /ms) { - my $id = patch_id($self, $sref); - $doc->add_term('XDFID'.$id) if defined($id); + my $rd; + if ($$sref =~ /^(?:diff|---|\+\+\+) /ms) { # start patch-id in parallel + my $git = ($self->{ibx} // $self->{eidx} // $self)->git; + $rd = popen_rd($git->cmd(qw(patch-id --stable)), undef, + { 0 => [ ':utf8', $sref ] }); } # split off quoted and unquoted blocks: @@ -387,6 +380,11 @@ sub index_body_text { } undef $txt; # free memory } + if (defined $rd) { # reap `git patch-id' + (readline($rd) // '') =~ /\A([a-f0-9]{40,})/ and + $doc->add_term('XDFID'.$1); + $rd->close or warn "W: git patch-id failed: \$?=$? (non-fatal)" + } } sub index_xapian { # msg_iter callback |