about summary refs log tree commit homepage
path: root/lib
diff options
context:
space:
mode:
authorEric Wong <e@80x24.org>2016-08-02 10:02:54 +0000
committerEric Wong <e@80x24.org>2016-08-04 00:04:53 +0000
commit95acd5901491e4f333f5d2bbeed6fb5e6b53e07c (patch)
treef8e53e9f38ea3710499ca7799936bfd1af352601 /lib
parent4caa17ef5aa8554d78b8c013096f12429c58e050 (diff)
downloadpublic-inbox-95acd5901491e4f333f5d2bbeed6fb5e6b53e07c.tar.gz
Doing git tree lookups based on the SHA-1 of the Message-ID
is expensive as trees get larger, instead, use the SHA-1
object ID directly.  This drastically reduces the amount
of time spent in the "git cat-file --batch" process for
fetching the /$INBOX/all.mbox.gz endpoint on the ~800MB
git@vger.kernel.org mirror

This retains backwards compatibility and allows existing
indices to be transparently upgraded without performance
degradation.
Diffstat (limited to 'lib')
-rw-r--r--lib/PublicInbox/Inbox.pm12
-rw-r--r--lib/PublicInbox/Mbox.pm2
-rw-r--r--lib/PublicInbox/SearchIdx.pm24
-rw-r--r--lib/PublicInbox/SearchMsg.pm20
4 files changed, 42 insertions, 16 deletions
diff --git a/lib/PublicInbox/Inbox.pm b/lib/PublicInbox/Inbox.pm
index 4fbbb522..e552cd4f 100644
--- a/lib/PublicInbox/Inbox.pm
+++ b/lib/PublicInbox/Inbox.pm
@@ -162,6 +162,18 @@ sub msg_by_path ($$;$) {
         $str;
 }
 
+sub msg_by_smsg ($$;$) {
+        my ($self, $smsg, $ref) = @_;
+
+        # backwards compat to fallback to msg_by_mid
+        # TODO: remove if we bump SCHEMA_VERSION in Search.pm:
+        defined(my $blob = $smsg->blob) or return msg_by_mid($self, $smsg->mid);
+
+        my $str = git($self)->cat_file($blob, $ref);
+        $$str =~ s/\A[\r\n]*From [^\r\n]*\r?\n//s if $str;
+        $str;
+}
+
 sub path_check {
         my ($self, $path) = @_;
         git($self)->check('HEAD:'.$path);
diff --git a/lib/PublicInbox/Mbox.pm b/lib/PublicInbox/Mbox.pm
index d2c09544..1e3de5b0 100644
--- a/lib/PublicInbox/Mbox.pm
+++ b/lib/PublicInbox/Mbox.pm
@@ -129,7 +129,7 @@ sub getline {
         my $gz = $self->{gz};
         do {
                 while (defined(my $smsg = shift @{$self->{msgs}})) {
-                        my $msg = eval { $ibx->msg_by_mid($smsg->mid) } or next;
+                        my $msg = eval { $ibx->msg_by_smsg($smsg) } or next;
                         $msg = Email::Simple->new($msg);
                         $gz->write(PublicInbox::Mbox::msg_str($ctx, $msg));
                         my $bref = $self->{buf};
diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm
index f0a3687d..f8249c50 100644
--- a/lib/PublicInbox/SearchIdx.pm
+++ b/lib/PublicInbox/SearchIdx.pm
@@ -91,7 +91,7 @@ sub add_val {
 }
 
 sub add_message {
-        my ($self, $mime, $bytes, $num) = @_; # mime = Email::MIME object
+        my ($self, $mime, $bytes, $num, $blob) = @_; # mime = Email::MIME object
         my $db = $self->{xdb};
 
         my ($doc_id, $old_tid);
@@ -170,7 +170,7 @@ sub add_message {
                 });
 
                 link_message($self, $smsg, $old_tid);
-                $doc->set_data($smsg->to_doc_data);
+                $doc->set_data($smsg->to_doc_data($blob));
                 if (defined $doc_id) {
                         $db->replace_document($doc_id, $doc);
                 } else {
@@ -279,8 +279,8 @@ sub link_message {
 }
 
 sub index_blob {
-        my ($self, $git, $mime, $bytes, $num) = @_;
-        $self->add_message($mime, $bytes, $num);
+        my ($self, $git, $mime, $bytes, $num, $blob) = @_;
+        $self->add_message($mime, $bytes, $num, $blob);
 }
 
 sub unindex_blob {
@@ -300,9 +300,9 @@ sub unindex_mm {
 }
 
 sub index_mm2 {
-        my ($self, $git, $mime, $bytes) = @_;
+        my ($self, $git, $mime, $bytes, $blob) = @_;
         my $num = $self->{mm}->num_for(mid_clean(mid_mime($mime)));
-        index_blob($self, $git, $mime, $bytes, $num);
+        index_blob($self, $git, $mime, $bytes, $num, $blob);
 }
 
 sub unindex_mm2 {
@@ -312,9 +312,9 @@ sub unindex_mm2 {
 }
 
 sub index_both {
-        my ($self, $git, $mime, $bytes) = @_;
+        my ($self, $git, $mime, $bytes, $blob) = @_;
         my $num = index_mm($self, $git, $mime);
-        index_blob($self, $git, $mime, $bytes, $num);
+        index_blob($self, $git, $mime, $bytes, $num, $blob);
 }
 
 sub unindex_both {
@@ -355,10 +355,12 @@ sub rlog {
         my $line;
         while (defined($line = <$log>)) {
                 if ($line =~ /$addmsg/o) {
-                        my $mime = do_cat_mail($git, $1, \$bytes) or next;
-                        $add_cb->($self, $git, $mime, $bytes);
+                        my $blob = $1;
+                        my $mime = do_cat_mail($git, $blob, \$bytes) or next;
+                        $add_cb->($self, $git, $mime, $bytes, $blob);
                 } elsif ($line =~ /$delmsg/o) {
-                        my $mime = do_cat_mail($git, $1) or next;
+                        my $blob = $1;
+                        my $mime = do_cat_mail($git, $blob) or next;
                         $del_cb->($self, $git, $mime);
                 } elsif ($line =~ /^commit ($h40)/o) {
                         if (defined $max && --$max <= 0) {
diff --git a/lib/PublicInbox/SearchMsg.pm b/lib/PublicInbox/SearchMsg.pm
index 4b0b645e..9d873c4a 100644
--- a/lib/PublicInbox/SearchMsg.pm
+++ b/lib/PublicInbox/SearchMsg.pm
@@ -38,7 +38,7 @@ sub load_doc {
         my $data = $doc->get_data or return;
         my $ts = get_val($doc, &PublicInbox::Search::TS);
         utf8::decode($data);
-        my ($subj, $from, $refs, $to, $cc) = split(/\n/, $data);
+        my ($subj, $from, $refs, $to, $cc, $blob) = split(/\n/, $data);
         bless {
                 doc => $doc,
                 subject => $subj,
@@ -47,6 +47,7 @@ sub load_doc {
                 references => $refs,
                 to => $to,
                 cc => $cc,
+                blob => $blob,
         }, $class;
 }
 
@@ -105,9 +106,11 @@ sub ts {
 }
 
 sub to_doc_data {
-        my ($self) = @_;
-        join("\n", $self->subject, $self->from, $self->references,
-                $self->to, $self->cc);
+        my ($self, $blob) = @_;
+        my @rows = ($self->subject, $self->from, $self->references,
+                        $self->to, $self->cc);
+        push @rows, $blob if defined $blob;
+        join("\n", @rows);
 }
 
 sub references {
@@ -185,6 +188,15 @@ sub mid ($;$) {
 
 sub _extract_mid { mid_clean(mid_mime($_[0]->mime)) }
 
+sub blob {
+        my ($self, $x40) = @_;
+        if (defined $x40) {
+                $self->{blob} = $x40;
+        } else {
+                $self->{blob};
+        }
+}
+
 sub mime {
         my ($self, $mime) = @_;
         if (defined $mime) {