index: account for CRLF conversion when storing bytes

NNTP and IMAP both require CRLF conversions on the wire.
They're also the only components which care about
$smsg->{bytes}, so store the CRLF-adjusted value in over.sqlite3
and Xapian DBs..

This will allow us to optimize RFC822.SIZE fetch item in IMAP
without triggering size mismatch errors in some clients' default
configurations (e.g. Mail::IMAPClient), but not most others.

It could also fix hypothetical problems with NNTP clients that
report discrepancies between overview and article data.

3 files changed, 15 insertions, 3 deletions

diff --git a/t/import.t b/t/import.t
index f987b114..abbc8229 100644
--- a/t/import.t
+++ b/t/import.t
@@ -32,8 +32,9 @@ like($im->add($mime, undef, $smsg), qr/\A:[0-9]+\z/, 'added one message');
 
 if ($v2) {
         like($smsg->{blob}, qr/\A[a-f0-9]{40}\z/, 'got last object_id');
-        is($mime->as_string, ${$smsg->{-raw_email}}, 'string matches');
-        is($smsg->{bytes}, length(${$smsg->{-raw_email}}), 'length matches');
+        my $raw_email = $smsg->{-raw_email};
+        is($mime->as_string, $$raw_email, 'string matches');
+        is($smsg->{raw_bytes}, length($$raw_email), 'length matches');
         my @cmd = ('git', "--git-dir=$git->{git_dir}", qw(hash-object --stdin));
         my $in = tempfile();
         print $in $mime->as_string or die "write failed: $!";
diff --git a/t/nntpd.t b/t/nntpd.t
index eee67ea6..d2f31323 100644
--- a/t/nntpd.t
+++ b/t/nntpd.t
@@ -73,7 +73,10 @@ EOF
                 my $list_id = $addr;
                 $list_id =~ s/@/./;
                 $mime->header_set('List-Id', "<$list_id>");
-                $len = length($mime->as_string);
+                my $str = $mime->as_string;
+                $str =~ s/(?<!\r)\n/\r\n/sg;
+                $len = length($str);
+                undef $str;
                 $im->add($mime);
                 $im->done;
                 if ($version == 1) {
diff --git a/t/search.t b/t/search.t
index d4ca28c7..82caf9e4 100644
--- a/t/search.t
+++ b/t/search.t
@@ -59,6 +59,14 @@ sub oct_is ($$$) {
         }
 }
 
+{
+        my $crlf_adjust = \&PublicInbox::SearchIdx::crlf_adjust;
+        is($crlf_adjust->("hi\r\nworld\r\n"), 0, 'no adjustment needed');
+        is($crlf_adjust->("hi\nworld\n"), 2, 'LF-only counts two CR');
+        is($crlf_adjust->("hi\r\nworld\n"), 1, 'CRLF/LF-mix 1 counts 1 CR');
+        is($crlf_adjust->("hi\nworld\r\n"), 1, 'CRLF/LF-mix 2 counts 1 CR');
+}
+
 $ibx->with_umask(sub {
         my $root = PublicInbox::Eml->new(<<'EOF');
 Date: Fri, 02 Oct 1993 00:00:00 +0000