* [PATCH] lei_to_mail: write directly to mail_sync.sqlite3
@ 2021-10-25 8:59 7% Eric Wong
0 siblings, 0 replies; 1+ results
From: Eric Wong @ 2021-10-25 8:59 UTC (permalink / raw)
To: meta
No need to go through the lei/store process when we write
mail_sync.sqlite3. This ought to reduce ENOBUFS errors (and the
sleep workaround) on RAM-starved systems.
---
lib/PublicInbox/LeiQuery.pm | 9 ++++++---
lib/PublicInbox/LeiStore.pm | 7 +------
lib/PublicInbox/LeiToMail.pm | 23 ++++++++++-------------
3 files changed, 17 insertions(+), 22 deletions(-)
diff --git a/lib/PublicInbox/LeiQuery.pm b/lib/PublicInbox/LeiQuery.pm
index effc572f6aaf..3d0e5b144d3a 100644
--- a/lib/PublicInbox/LeiQuery.pm
+++ b/lib/PublicInbox/LeiQuery.pm
@@ -32,10 +32,13 @@ sub _start_query { # used by "lei q" and "lei up"
if ($l2m && grep { $opt->{$_} //= \1 } (qw(mail-sync import-remote
import-before))) {
$self->_lei_store(1)->write_prepare($self);
+ if ($opt->{'mail-sync'}) {
+ my $lms = $l2m->{-lms_rw} = $self->lms(1);
+ $lms->lms_write_prepare->lms_pause; # just create
+ }
}
- $l2m and $l2m->{-wq_nr_workers} = $mj // do {
- $mj = int($nproc * 0.75 + 0.5); # keep some CPU for git
- };
+ $l2m and $l2m->{-wq_nr_workers} = $mj //
+ int($nproc * 0.75 + 0.5); # keep some CPU for git
# descending docid order is cheapest, MUA controls sorting order
$self->{mset_opt}->{relevance} //= -2 if $l2m || $opt->{threads};
diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm
index f1316229bb32..66049dfeac84 100644
--- a/lib/PublicInbox/LeiStore.pm
+++ b/lib/PublicInbox/LeiStore.pm
@@ -265,11 +265,6 @@ sub _lms_rw ($) { # it is important to have eidx processes open before lms
};
}
-sub set_sync_info {
- my ($self, $oidhex, $folder, $id) = @_;
- _lms_rw($self)->set_src(pack('H*', $oidhex), $folder, $id);
-}
-
sub _remove_if_local { # git->cat_async arg
my ($bref, $oidhex, $type, $size, $self) = @_;
$self->{im}->remove($bref) if $bref;
@@ -349,7 +344,7 @@ sub add_eml {
$smsg->{-eidx_git} = $eidx->git if !$self->{-fake_im};
my $im_mark = $im->add($eml, undef, $smsg);
if ($vmd && $vmd->{sync_info}) {
- set_sync_info($self, $smsg->{blob}, @{$vmd->{sync_info}});
+ _lms_rw($self)->set_src($smsg->oidbin, @{$vmd->{sync_info}});
}
unless ($im_mark) { # duplicate blob returns undef
return unless wantarray;
diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm
index d33d27aec006..e7220bf02954 100644
--- a/lib/PublicInbox/LeiToMail.pm
+++ b/lib/PublicInbox/LeiToMail.pm
@@ -135,8 +135,8 @@ sub git_to_mail { # git->cat_async callback
my ($bref, $oid, $type, $size, $arg) = @_;
$type // return; # called by git->async_abort
my ($write_cb, $smsg) = @$arg;
- if ($type eq 'missing' && $smsg->{-lms_ro}) {
- if ($bref = $smsg->{-lms_ro}->local_blob($oid, 1)) {
+ if ($type eq 'missing' && $smsg->{-lms_rw}) {
+ if ($bref = $smsg->{-lms_rw}->local_blob($oid, 1)) {
$type = 'blob';
$size = length($$bref);
}
@@ -280,8 +280,8 @@ sub _maildir_write_cb ($$) {
$dedupe->prepare_dedupe if $dedupe;
my $dst = $lei->{ovv}->{dst};
my $lse = $lei->{lse}; # may be undef
- my $sto = $lei->{opt}->{'mail-sync'} ? $lei->{sto} : undef;
- my $out = $sto ? 'maildir:'.$lei->abs_path($dst) : undef;
+ my $lms = $self->{-lms_rw};
+ my $out = $lms ? 'maildir:'.$lei->abs_path($dst) : undef;
# Favor cur/ and only write to new/ when augmenting. This
# saves MUAs from having to do a mass rename when the initial
@@ -296,7 +296,7 @@ sub _maildir_write_cb ($$) {
$lse->xsmsg_vmd($smsg) if $lse;
my $n = _buf2maildir($dst, $bref // \($eml->as_string),
$smsg, $dir);
- $sto->wq_do('set_sync_info', $smsg->{blob}, $out, $n) if $sto;
+ $lms->set_src($smsg->oidbin, $out, $n) if $lms;
++$lei->{-nr_write};
}
}
@@ -311,7 +311,7 @@ sub _imap_write_cb ($$) {
my $folder = $uri->mailbox;
$uri->uidvalidity($mic->uidvalidity($folder));
my $lse = $lei->{lse}; # may be undef
- my $sto = $lei->{opt}->{'mail-sync'} ? $lei->{sto} : undef;
+ my $lms = $self->{-lms_rw};
sub { # for git_to_mail
my ($bref, $smsg, $eml) = @_;
$mic // return $lei->fail; # mic may be undef-ed in last run
@@ -325,9 +325,8 @@ sub _imap_write_cb ($$) {
die $err;
}
# imap_append returns UID if IMAP server has UIDPLUS extension
- ($sto && $uid =~ /\A[0-9]+\z/) and
- $sto->wq_do('set_sync_info',
- $smsg->{blob}, $$uri, $uid + 0);
+ ($lms && $uid =~ /\A[0-9]+\z/) and
+ $lms->set_src($smsg->oidbin, $$uri, $uid + 0);
++$lei->{-nr_write};
}
}
@@ -751,9 +750,7 @@ sub ipc_atfork_child {
my ($self) = @_;
my $lei = $self->{lei};
$lei->_lei_atfork_child;
- if (my $lse = $lei->{lse}) {
- $self->{-lms_ro} = $lse->{-lms_ro} //= $lse->lms;
- }
+ $self->{-lms_rw}->lms_write_prepare if $self->{-lms_rw};
$lei->{auth}->do_auth_atfork($self) if $lei->{auth};
$SIG{__WARN__} = PublicInbox::Eml::warn_ignore_cb();
$self->SUPER::ipc_atfork_child;
@@ -775,7 +772,7 @@ sub poke_dst {
sub write_mail { # via ->wq_io_do
my ($self, $smsg, $eml) = @_;
return $self->{wcb}->(undef, $smsg, $eml) if $eml;
- $smsg->{-lms_ro} = $self->{-lms_ro};
+ $smsg->{-lms_rw} = $self->{-lms_rw};
$self->{lei}->{ale}->git->cat_async($smsg->{blob}, \&git_to_mail,
[$self->{wcb}, $smsg]);
}
^ permalink raw reply related [relevance 7%]
Results 1-1 of 1 | reverse | options above
-- pct% links below jump to the message on this page, permalinks otherwise --
2021-10-25 8:59 7% [PATCH] lei_to_mail: write directly to mail_sync.sqlite3 Eric Wong
Code repositories for project(s) associated with this public inbox
https://80x24.org/public-inbox.git
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).