From 2833f2adeb224a83d731deaa717e1969e5bf9e62 Mon Sep 17 00:00:00 2001
From: Eric Wong <e@80x24.org>
Date: Mon, 25 Oct 2021 08:59:19 +0000
Subject: lei_to_mail: write directly to mail_sync.sqlite3

No need to go through the lei/store process when we write
mail_sync.sqlite3.  This ought to reduce ENOBUFS errors (and the
sleep workaround) on RAM-starved systems.
---
 lib/PublicInbox/LeiQuery.pm  |  9 ++++++---
 lib/PublicInbox/LeiStore.pm  |  7 +------
 lib/PublicInbox/LeiToMail.pm | 23 ++++++++++-------------
 3 files changed, 17 insertions(+), 22 deletions(-)

(limited to 'lib/PublicInbox')

diff --git a/lib/PublicInbox/LeiQuery.pm b/lib/PublicInbox/LeiQuery.pm
index effc572f..3d0e5b14 100644
--- a/lib/PublicInbox/LeiQuery.pm
+++ b/lib/PublicInbox/LeiQuery.pm
@@ -32,10 +32,13 @@ sub _start_query { # used by "lei q" and "lei up"
 	if ($l2m && grep { $opt->{$_} //= \1 } (qw(mail-sync import-remote
 							import-before))) {
 		$self->_lei_store(1)->write_prepare($self);
+		if ($opt->{'mail-sync'}) {
+			my $lms = $l2m->{-lms_rw} = $self->lms(1);
+			$lms->lms_write_prepare->lms_pause; # just create
+		}
 	}
-	$l2m and $l2m->{-wq_nr_workers} = $mj // do {
-		$mj = int($nproc * 0.75 + 0.5); # keep some CPU for git
-	};
+	$l2m and $l2m->{-wq_nr_workers} = $mj //
+		int($nproc * 0.75 + 0.5); # keep some CPU for git
 
 	# descending docid order is cheapest, MUA controls sorting order
 	$self->{mset_opt}->{relevance} //= -2 if $l2m || $opt->{threads};
diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm
index f1316229..66049dfe 100644
--- a/lib/PublicInbox/LeiStore.pm
+++ b/lib/PublicInbox/LeiStore.pm
@@ -265,11 +265,6 @@ sub _lms_rw ($) { # it is important to have eidx processes open before lms
 	};
 }
 
-sub set_sync_info {
-	my ($self, $oidhex, $folder, $id) = @_;
-	_lms_rw($self)->set_src(pack('H*', $oidhex), $folder, $id);
-}
-
 sub _remove_if_local { # git->cat_async arg
 	my ($bref, $oidhex, $type, $size, $self) = @_;
 	$self->{im}->remove($bref) if $bref;
@@ -349,7 +344,7 @@ sub add_eml {
 	$smsg->{-eidx_git} = $eidx->git if !$self->{-fake_im};
 	my $im_mark = $im->add($eml, undef, $smsg);
 	if ($vmd && $vmd->{sync_info}) {
-		set_sync_info($self, $smsg->{blob}, @{$vmd->{sync_info}});
+		_lms_rw($self)->set_src($smsg->oidbin, @{$vmd->{sync_info}});
 	}
 	unless ($im_mark) { # duplicate blob returns undef
 		return unless wantarray;
diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm
index d33d27ae..e7220bf0 100644
--- a/lib/PublicInbox/LeiToMail.pm
+++ b/lib/PublicInbox/LeiToMail.pm
@@ -135,8 +135,8 @@ sub git_to_mail { # git->cat_async callback
 	my ($bref, $oid, $type, $size, $arg) = @_;
 	$type // return; # called by git->async_abort
 	my ($write_cb, $smsg) = @$arg;
-	if ($type eq 'missing' && $smsg->{-lms_ro}) {
-		if ($bref = $smsg->{-lms_ro}->local_blob($oid, 1)) {
+	if ($type eq 'missing' && $smsg->{-lms_rw}) {
+		if ($bref = $smsg->{-lms_rw}->local_blob($oid, 1)) {
 			$type = 'blob';
 			$size = length($$bref);
 		}
@@ -280,8 +280,8 @@ sub _maildir_write_cb ($$) {
 	$dedupe->prepare_dedupe if $dedupe;
 	my $dst = $lei->{ovv}->{dst};
 	my $lse = $lei->{lse}; # may be undef
-	my $sto = $lei->{opt}->{'mail-sync'} ? $lei->{sto} : undef;
-	my $out = $sto ? 'maildir:'.$lei->abs_path($dst) : undef;
+	my $lms = $self->{-lms_rw};
+	my $out = $lms ? 'maildir:'.$lei->abs_path($dst) : undef;
 
 	# Favor cur/ and only write to new/ when augmenting.  This
 	# saves MUAs from having to do a mass rename when the initial
@@ -296,7 +296,7 @@ sub _maildir_write_cb ($$) {
 		$lse->xsmsg_vmd($smsg) if $lse;
 		my $n = _buf2maildir($dst, $bref // \($eml->as_string),
 					$smsg, $dir);
-		$sto->wq_do('set_sync_info', $smsg->{blob}, $out, $n) if $sto;
+		$lms->set_src($smsg->oidbin, $out, $n) if $lms;
 		++$lei->{-nr_write};
 	}
 }
@@ -311,7 +311,7 @@ sub _imap_write_cb ($$) {
 	my $folder = $uri->mailbox;
 	$uri->uidvalidity($mic->uidvalidity($folder));
 	my $lse = $lei->{lse}; # may be undef
-	my $sto = $lei->{opt}->{'mail-sync'} ? $lei->{sto} : undef;
+	my $lms = $self->{-lms_rw};
 	sub { # for git_to_mail
 		my ($bref, $smsg, $eml) = @_;
 		$mic // return $lei->fail; # mic may be undef-ed in last run
@@ -325,9 +325,8 @@ sub _imap_write_cb ($$) {
 			die $err;
 		}
 		# imap_append returns UID if IMAP server has UIDPLUS extension
-		($sto && $uid =~ /\A[0-9]+\z/) and
-			$sto->wq_do('set_sync_info',
-					$smsg->{blob}, $$uri, $uid + 0);
+		($lms && $uid =~ /\A[0-9]+\z/) and
+			$lms->set_src($smsg->oidbin, $$uri, $uid + 0);
 		++$lei->{-nr_write};
 	}
 }
@@ -751,9 +750,7 @@ sub ipc_atfork_child {
 	my ($self) = @_;
 	my $lei = $self->{lei};
 	$lei->_lei_atfork_child;
-	if (my $lse = $lei->{lse}) {
-		$self->{-lms_ro} = $lse->{-lms_ro} //= $lse->lms;
-	}
+	$self->{-lms_rw}->lms_write_prepare if $self->{-lms_rw};
 	$lei->{auth}->do_auth_atfork($self) if $lei->{auth};
 	$SIG{__WARN__} = PublicInbox::Eml::warn_ignore_cb();
 	$self->SUPER::ipc_atfork_child;
@@ -775,7 +772,7 @@ sub poke_dst {
 sub write_mail { # via ->wq_io_do
 	my ($self, $smsg, $eml) = @_;
 	return $self->{wcb}->(undef, $smsg, $eml) if $eml;
-	$smsg->{-lms_ro} = $self->{-lms_ro};
+	$smsg->{-lms_rw} = $self->{-lms_rw};
 	$self->{lei}->{ale}->git->cat_async($smsg->{blob}, \&git_to_mail,
 				[$self->{wcb}, $smsg]);
 }
-- 
cgit v1.2.3-24-ge0c7