searchmsg: add git object ID to doc_data

Doing git tree lookups based on the SHA-1 of the Message-ID
is expensive as trees get larger, instead, use the SHA-1
object ID directly.  This drastically reduces the amount
of time spent in the "git cat-file --batch" process for
fetching the /$INBOX/all.mbox.gz endpoint on the ~800MB
git@vger.kernel.org mirror

This retains backwards compatibility and allows existing
indices to be transparently upgraded without performance
degradation.

1 files changed, 16 insertions, 4 deletions

diff --git a/lib/PublicInbox/SearchMsg.pm b/lib/PublicInbox/SearchMsg.pm
index 4b0b645e..9d873c4a 100644
--- a/lib/PublicInbox/SearchMsg.pm
+++ b/lib/PublicInbox/SearchMsg.pm
@@ -38,7 +38,7 @@ sub load_doc {
         my $data = $doc->get_data or return;
         my $ts = get_val($doc, &PublicInbox::Search::TS);
         utf8::decode($data);
-        my ($subj, $from, $refs, $to, $cc) = split(/\n/, $data);
+        my ($subj, $from, $refs, $to, $cc, $blob) = split(/\n/, $data);
         bless {
                 doc => $doc,
                 subject => $subj,
@@ -47,6 +47,7 @@ sub load_doc {
                 references => $refs,
                 to => $to,
                 cc => $cc,
+                blob => $blob,
         }, $class;
 }
 
@@ -105,9 +106,11 @@ sub ts {
 }
 
 sub to_doc_data {
-        my ($self) = @_;
-        join("\n", $self->subject, $self->from, $self->references,
-                $self->to, $self->cc);
+        my ($self, $blob) = @_;
+        my @rows = ($self->subject, $self->from, $self->references,
+                        $self->to, $self->cc);
+        push @rows, $blob if defined $blob;
+        join("\n", @rows);
 }
 
 sub references {
@@ -185,6 +188,15 @@ sub mid ($;$) {
 
 sub _extract_mid { mid_clean(mid_mime($_[0]->mime)) }
 
+sub blob {
+        my ($self, $x40) = @_;
+        if (defined $x40) {
+                $self->{blob} = $x40;
+        } else {
+                $self->{blob};
+        }
+}
+
 sub mime {
         my ($self, $mime) = @_;
         if (defined $mime) {