From 71461c67fee940b05309baa8c67bac10c8c51ac6 Mon Sep 17 00:00:00 2001
From: Eric Wong <e@80x24.org>
Date: Sun, 3 Jan 2021 02:06:15 +0000
Subject: use Eml (or MIME) objects for all indexing paths

We don't need to be keeping the raw message around after it hits
git.  Shard work now relies on Storable (or Sereal) and all of
the indexing code relies on the Email::MIME-like API of Eml to
access interesting parts of the message.

Similarly, smsg->{raw_bytes} is no longer carried around and we
do the CRLF adjustment when setting smsg->{bytes}.

There's also a small simplification to t/import.t while
we're in the area to use xqx instead of spawn/popen_rd.
---
 lib/PublicInbox/SearchIdx.pm | 17 +++--------------
 1 file changed, 3 insertions(+), 14 deletions(-)

(limited to 'lib/PublicInbox/SearchIdx.pm')

diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm
index da3ac2e3..a7005051 100644
--- a/lib/PublicInbox/SearchIdx.pm
+++ b/lib/PublicInbox/SearchIdx.pm
@@ -22,7 +22,7 @@ use PublicInbox::OverIdx;
 use PublicInbox::Spawn qw(spawn nodatacow_dir);
 use PublicInbox::Git qw(git_unquote);
 use PublicInbox::MsgTime qw(msg_timestamp msg_datestamp);
-our @EXPORT_OK = qw(crlf_adjust log2stack is_ancestor check_size prepare_stack
+our @EXPORT_OK = qw(log2stack is_ancestor check_size prepare_stack
 	index_text term_generator add_val is_bad_blob);
 my $X = \%PublicInbox::Search::X;
 our ($DB_CREATE_OR_OPEN, $DB_OPEN);
@@ -613,17 +613,6 @@ sub index_mm {
 	}
 }
 
-# returns the number of bytes to add if given a non-CRLF arg
-sub crlf_adjust ($) {
-	if (index($_[0], "\r\n") < 0) {
-		# common case is LF-only, every \n needs an \r;
-		# so favor a cheap tr// over an expensive m//g
-		$_[0] =~ tr/\n/\n/;
-	} else { # count number of '\n' w/o '\r', expensive:
-		scalar(my @n = ($_[0] =~ m/(?<!\r)\n/g));
-	}
-}
-
 sub is_bad_blob ($$$$) {
 	my ($oid, $type, $size, $expect_oid) = @_;
 	if ($type ne 'blob') {
@@ -640,8 +629,8 @@ sub index_both { # git->cat_async callback
 	my ($nr, $max) = @$sync{qw(nr max)};
 	++$$nr;
 	$$max -= $size;
-	$size += crlf_adjust($$bref);
-	my $smsg = bless { bytes => $size, blob => $oid }, 'PublicInbox::Smsg';
+	my $smsg = bless { blob => $oid }, 'PublicInbox::Smsg';
+	$smsg->set_bytes($$bref, $size);
 	my $self = $sync->{sidx};
 	local $self->{current_info} = "$self->{current_info}: $oid";
 	my $eml = PublicInbox::Eml->new($bref);
-- 
cgit v1.2.3-24-ge0c7