From 95acd5901491e4f333f5d2bbeed6fb5e6b53e07c Mon Sep 17 00:00:00 2001 From: Eric Wong Date: Tue, 2 Aug 2016 10:02:54 +0000 Subject: searchmsg: add git object ID to doc_data Doing git tree lookups based on the SHA-1 of the Message-ID is expensive as trees get larger, instead, use the SHA-1 object ID directly. This drastically reduces the amount of time spent in the "git cat-file --batch" process for fetching the /$INBOX/all.mbox.gz endpoint on the ~800MB git@vger.kernel.org mirror This retains backwards compatibility and allows existing indices to be transparently upgraded without performance degradation. --- lib/PublicInbox/SearchMsg.pm | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) (limited to 'lib/PublicInbox/SearchMsg.pm') diff --git a/lib/PublicInbox/SearchMsg.pm b/lib/PublicInbox/SearchMsg.pm index 4b0b645e..9d873c4a 100644 --- a/lib/PublicInbox/SearchMsg.pm +++ b/lib/PublicInbox/SearchMsg.pm @@ -38,7 +38,7 @@ sub load_doc { my $data = $doc->get_data or return; my $ts = get_val($doc, &PublicInbox::Search::TS); utf8::decode($data); - my ($subj, $from, $refs, $to, $cc) = split(/\n/, $data); + my ($subj, $from, $refs, $to, $cc, $blob) = split(/\n/, $data); bless { doc => $doc, subject => $subj, @@ -47,6 +47,7 @@ sub load_doc { references => $refs, to => $to, cc => $cc, + blob => $blob, }, $class; } @@ -105,9 +106,11 @@ sub ts { } sub to_doc_data { - my ($self) = @_; - join("\n", $self->subject, $self->from, $self->references, - $self->to, $self->cc); + my ($self, $blob) = @_; + my @rows = ($self->subject, $self->from, $self->references, + $self->to, $self->cc); + push @rows, $blob if defined $blob; + join("\n", @rows); } sub references { @@ -185,6 +188,15 @@ sub mid ($;$) { sub _extract_mid { mid_clean(mid_mime($_[0]->mime)) } +sub blob { + my ($self, $x40) = @_; + if (defined $x40) { + $self->{blob} = $x40; + } else { + $self->{blob}; + } +} + sub mime { my ($self, $mime) = @_; if (defined $mime) { -- cgit v1.2.3-24-ge0c7