searchmsg: add git object ID to doc_data

Doing git tree lookups based on the SHA-1 of the Message-ID
is expensive as trees get larger, instead, use the SHA-1
object ID directly.  This drastically reduces the amount
of time spent in the "git cat-file --batch" process for
fetching the /$INBOX/all.mbox.gz endpoint on the ~800MB
git@vger.kernel.org mirror

This retains backwards compatibility and allows existing
indices to be transparently upgraded without performance
degradation.

1 files changed, 12 insertions, 0 deletions

diff --git a/lib/PublicInbox/Inbox.pm b/lib/PublicInbox/Inbox.pm
index 4fbbb522..e552cd4f 100644
--- a/lib/PublicInbox/Inbox.pm
+++ b/lib/PublicInbox/Inbox.pm
@@ -162,6 +162,18 @@ sub msg_by_path ($$;$) {
         $str;
 }
 
+sub msg_by_smsg ($$;$) {
+        my ($self, $smsg, $ref) = @_;
+
+        # backwards compat to fallback to msg_by_mid
+        # TODO: remove if we bump SCHEMA_VERSION in Search.pm:
+        defined(my $blob = $smsg->blob) or return msg_by_mid($self, $smsg->mid);
+
+        my $str = git($self)->cat_file($blob, $ref);
+        $$str =~ s/\A[\r\n]*From [^\r\n]*\r?\n//s if $str;
+        $str;
+}
+
 sub path_check {
         my ($self, $path) = @_;
         git($self)->check('HEAD:'.$path);