From 27277902f3ddb10af08c9870e3e9af1285e390ce Mon Sep 17 00:00:00 2001 From: Eric Wong Date: Sun, 26 Jul 2020 06:57:30 +0000 Subject: imap: introduce and use Git->async_prefetch We can keep the git process more active by sending another request to it while fetch_run_ops() is running. This parallelization speeds up mutt's initial FETCH for headers by around ~35%(!). --- lib/PublicInbox/Git.pm | 16 ++++++++++++++++ lib/PublicInbox/IMAP.pm | 7 ++++++- 2 files changed, 22 insertions(+), 1 deletion(-) (limited to 'lib/PublicInbox') diff --git a/lib/PublicInbox/Git.pm b/lib/PublicInbox/Git.pm index ffc464eb..7b2ada24 100644 --- a/lib/PublicInbox/Git.pm +++ b/lib/PublicInbox/Git.pm @@ -410,6 +410,22 @@ sub cat_async ($$$;$) { push(@$inflight, $oid, $cb, $arg); } +# this is safe to call inside $cb, but not guaranteed to enqueue +# returns true if successful, undef if not. +sub async_prefetch { + my ($self, $oid, $cb, $arg) = @_; + if (defined($self->{async_cat}) && (my $inflight = $self->{inflight})) { + # we could use MAX_INFLIGHT here w/o the halving, + # but lets not allow one client to monopolize a git process + if (scalar(@$inflight) < int(MAX_INFLIGHT/2)) { + print { $self->{out} } $oid, "\n" or + fail($self, "write error: $!"); + return push(@$inflight, $oid, $cb, $arg); + } + } + undef; +} + sub extract_cmt_time { my ($bref, undef, undef, undef, $modified) = @_; diff --git a/lib/PublicInbox/IMAP.pm b/lib/PublicInbox/IMAP.pm index 8ab4b1e7..3d66f930 100644 --- a/lib/PublicInbox/IMAP.pm +++ b/lib/PublicInbox/IMAP.pm @@ -626,8 +626,13 @@ sub fetch_blob_cb { # called by git->cat_async via git_async_cat } else { $smsg->{blob} eq $oid or die "BUG: $smsg->{blob} != $oid"; } + my $pre; + if (!$self->{wbuf} && (my $nxt = $msgs->[0])) { + $pre = $self->{ibx}->git->async_prefetch($nxt->{blob}, + \&fetch_blob_cb, $fetch_arg); + } fetch_run_ops($self, $smsg, $bref, $ops, $partial); - requeue_once($self); + $pre ? $self->zflush : requeue_once($self); } sub emit_rfc822 { -- cgit v1.2.3-24-ge0c7