[PATCH 0/3] lei: some mail sync stuff

[PATCH 0/3] lei: some mail sync stuff

[Eric Wong]

Still trying to wrap my head around some other stuff related to
mail_sync.sqlite3, but 3/3 is a necessary fix and 2/3 is an obvious
cleanup.  The ipc_do('done') stuff could probably be made
faster, but probably after the 1.7 release...

Eric Wong (3):
  lei: add ->lms shortcut for LeiMailSync
  ipc: remove WQ_MAX_WORKERS
  lei forget-mail-sync: rely on lei/store process

 lib/PublicInbox/IPC.pm               |  5 -----
 lib/PublicInbox/LEI.pm               |  9 +++++++++
 lib/PublicInbox/LeiBlob.pm           |  3 +--
 lib/PublicInbox/LeiForgetMailSync.pm | 11 +++++------
 lib/PublicInbox/LeiInspect.pm        |  3 +--
 lib/PublicInbox/LeiLcat.pm           | 10 ++++------
 lib/PublicInbox/LeiLsMailSource.pm   |  3 +--
 lib/PublicInbox/LeiStore.pm          |  6 ++++++
 8 files changed, 27 insertions(+), 23 deletions(-)

[PATCH 1/3] lei: add ->lms shortcut for LeiMailSync

[Eric Wong]

We access this read-only in many places (and will in more),
so provide a shortcut to simplify callers.
---
 lib/PublicInbox/LEI.pm               |  9 +++++++++
 lib/PublicInbox/LeiBlob.pm           |  3 +--
 lib/PublicInbox/LeiForgetMailSync.pm |  3 +--
 lib/PublicInbox/LeiInspect.pm        |  3 +--
 lib/PublicInbox/LeiLcat.pm           | 10 ++++------
 lib/PublicInbox/LeiLsMailSource.pm   |  3 +--
 6 files changed, 17 insertions(+), 14 deletions(-)

diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 347dd280..e5232f1b 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -1425,4 +1425,13 @@ sub git_blob_id {
 	($lei->{sto} // _lei_store($lei, 1))->git_blob_id($eml);
 }
 
+sub lms { # read-only LeiMailSync
+	my ($lei) = @_;
+	my $lse = $lei->{lse} // do {
+		my $sto = $lei->{sto} // _lei_store($lei);
+		$sto ? $sto->search : undef
+	};
+	$lse ? $lse->lms : undef;
+}
+
 1;
diff --git a/lib/PublicInbox/LeiBlob.pm b/lib/PublicInbox/LeiBlob.pm
index 3158ca3b..21003894 100644
--- a/lib/PublicInbox/LeiBlob.pm
+++ b/lib/PublicInbox/LeiBlob.pm
@@ -133,8 +133,7 @@ sub lei_blob {
 		}
 		my $ce = $?;
 		return if $ce == 0;
-		my $sto = $lei->_lei_store;
-		my $lms = $sto ? $sto->search->lms : undef;
+		my $lms = $lei->lms;
 		if (my $bref = $lms ? $lms->local_blob($blob, 1) : undef) {
 			defined($lei->{-attach_idx}) and
 				return extract_attach($lei, $blob, $bref);
diff --git a/lib/PublicInbox/LeiForgetMailSync.pm b/lib/PublicInbox/LeiForgetMailSync.pm
index 46dde1a7..c74ba25d 100644
--- a/lib/PublicInbox/LeiForgetMailSync.pm
+++ b/lib/PublicInbox/LeiForgetMailSync.pm
@@ -14,8 +14,7 @@ use PublicInbox::LeiExportKw;
 
 sub lei_forget_mail_sync {
 	my ($lei, @folders) = @_;
-	my $sto = $lei->_lei_store or return;
-	my $lms = $sto->search->lms or return;
+	my $lms = $lei->lms or return;
 	my $err = $lms->arg2folder($lei, \@folders);
 	$lei->qerr(@{$err->{qerr}}) if $err->{qerr};
 	return $lei->fail($err->{fail}) if $err->{fail};
diff --git a/lib/PublicInbox/LeiInspect.pm b/lib/PublicInbox/LeiInspect.pm
index bf7a4836..2d2ff1a0 100644
--- a/lib/PublicInbox/LeiInspect.pm
+++ b/lib/PublicInbox/LeiInspect.pm
@@ -41,8 +41,7 @@ sub inspect_imap_uid ($$) {
 sub inspect_sync_folder ($$) {
 	my ($lei, $folder) = @_;
 	my $ent = {};
-	my $lse = $lei->{lse} or return $ent;
-	my $lms = $lse->lms or return $ent;
+	my $lms = $lei->lms or return $ent;
 	my $folders = [ $folder ];
 	my $err = $lms->arg2folder($lei, $folders);
 	if ($err) {
diff --git a/lib/PublicInbox/LeiLcat.pm b/lib/PublicInbox/LeiLcat.pm
index 4a0c24a9..9d95e899 100644
--- a/lib/PublicInbox/LeiLcat.pm
+++ b/lib/PublicInbox/LeiLcat.pm
@@ -13,7 +13,7 @@ use PublicInbox::MID qw($MID_EXTRACT);
 
 sub lcat_folder ($$$) {
 	my ($lei, $lms, $folder) = @_;
-	$lms //= $lei->{lse}->lms // return;
+	$lms //= $lei->lms or return;
 	my $folders = [ $folder];
 	my $err = $lms->arg2folder($lei, $folders);
 	$lei->qerr(@{$err->{qerr}}) if $err && $err->{qerr};
@@ -29,7 +29,7 @@ sub lcat_folder ($$$) {
 
 sub lcat_imap_uri ($$) {
 	my ($lei, $uri) = @_;
-	my $lms = $lei->{lse}->lms or return;
+	my $lms = $lei->lms or return;
 	# cf. LeiXsearch->lcat_dump
 	if (defined $uri->uid) {
 		my $oidhex = $lms->imap_oid($lei, $uri);
@@ -129,8 +129,7 @@ sub lei_lcat {
 	my ($lei, @argv) = @_;
 	my $lxs = $lei->lxs_prepare or return;
 	$lei->ale->refresh_externals($lxs, $lei);
-	my $sto = $lei->_lei_store(1);
-	$lei->{lse} = $sto->search;
+	$lei->_lei_store(1);
 	my $opt = $lei->{opt};
 	my %mset_opt = map { $_ => $opt->{$_} } qw(threads limit offset);
 	$mset_opt{asc} = $opt->{'reverse'} ? 1 : 0;
@@ -153,8 +152,7 @@ no args allowed on command-line with --stdin
 
 sub _complete_lcat {
 	my ($lei, @argv) = @_;
-	my $sto = $lei->_lei_store or return;
-	my $lms = $sto->search->lms or return;
+	my $lms = $lei->lms or return;
 	my $match_cb = $lei->complete_url_prepare(\@argv);
 	map { $match_cb->($_) } $lms->folders;
 }
diff --git a/lib/PublicInbox/LeiLsMailSource.pm b/lib/PublicInbox/LeiLsMailSource.pm
index cadc61ed..2d8913ac 100644
--- a/lib/PublicInbox/LeiLsMailSource.pm
+++ b/lib/PublicInbox/LeiLsMailSource.pm
@@ -103,8 +103,7 @@ sub _complete_ls_mail_source {
 	my $match_cb = $lei->complete_url_prepare(\@argv);
 	my @m = map { $match_cb->($_) } $lei->url_folder_cache->keys;
 	my %f = map { $_ => 1 } @m;
-	my $sto = $lei->_lei_store;
-	if (my $lms = $sto ? $sto->search->lms : undef) {
+	if (my $lms = $lei->lms) {
 		@m = map { $match_cb->($_) } grep(
 			m!\A(?:imaps?|nntps?|s?news)://!, $lms->folders);
 		@f{@m} = @m;

[PATCH 2/3] ipc: remove WQ_MAX_WORKERS

[Eric Wong]

We no longer rely on IO::FDPass, so there's no longer a reason
to limit this internally.
---
 lib/PublicInbox/IPC.pm | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/lib/PublicInbox/IPC.pm b/lib/PublicInbox/IPC.pm
index d909dc1c..9efe551b 100644
--- a/lib/PublicInbox/IPC.pm
+++ b/lib/PublicInbox/IPC.pm
@@ -20,7 +20,6 @@ use Socket qw(AF_UNIX MSG_EOR SOCK_STREAM);
 my $MY_MAX_ARG_STRLEN = 4096 * 33; # extra 4K for serialization
 my $SEQPACKET = eval { Socket::SOCK_SEQPACKET() }; # portable enough?
 our @EXPORT_OK = qw(ipc_freeze ipc_thaw);
-my $WQ_MAX_WORKERS = 4096;
 my ($enc, $dec);
 # ->imports at BEGIN turns sereal_*_with_object into custom ops on 5.14+
 # and eliminate method call overhead
@@ -354,7 +353,6 @@ sub wq_workers_start {
 		die "socketpair: $!";
 	$self->ipc_atfork_prepare;
 	$nr_workers //= $self->{-wq_nr_workers};
-	$nr_workers = $WQ_MAX_WORKERS if $nr_workers > $WQ_MAX_WORKERS;
 	my $sigset = $oldset // PublicInbox::DS::block_signals();
 	$self->{-wq_workers} = {};
 	$self->{-wq_ident} = $ident;
@@ -367,7 +365,6 @@ sub wq_worker_incr { # SIGTTIN handler
 	my ($self, $oldset, $fields) = @_;
 	$self->{-wq_s2} or return;
 	die "-wq_nr_workers locked" if defined $self->{-wq_nr_workers};
-	return if wq_workers($self) >= $WQ_MAX_WORKERS;
 	$self->ipc_atfork_prepare;
 	my $sigset = $oldset // PublicInbox::DS::block_signals();
 	_wq_worker_start($self, $sigset, $fields);
@@ -443,8 +440,6 @@ sub wq_kill {
 	kill($sig // 'TERM', keys %$workers);
 }
 
-sub WQ_MAX_WORKERS { $WQ_MAX_WORKERS }
-
 sub DESTROY {
 	my ($self) = @_;
 	my $ppid = $self->{-wq_ppid};

[PATCH 3/3] lei forget-mail-sync: rely on lei/store process

[Eric Wong]

As implied in commit 6ff03ba2be9247f1
("lei export-kw: do not write directly to mail_sync.sqlite3"),
modifying mail_sync.sqlite3 directly can lead to conflicts
and making everything go through lei/store is easier.
---
 lib/PublicInbox/LeiForgetMailSync.pm | 8 ++++----
 lib/PublicInbox/LeiStore.pm          | 6 ++++++
 2 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/lib/PublicInbox/LeiForgetMailSync.pm b/lib/PublicInbox/LeiForgetMailSync.pm
index c74ba25d..940ca1b6 100644
--- a/lib/PublicInbox/LeiForgetMailSync.pm
+++ b/lib/PublicInbox/LeiForgetMailSync.pm
@@ -15,13 +15,13 @@ use PublicInbox::LeiExportKw;
 sub lei_forget_mail_sync {
 	my ($lei, @folders) = @_;
 	my $lms = $lei->lms or return;
+	my $sto = $lei->_lei_store or return; # may disappear due to race
+	$sto->write_prepare;
 	my $err = $lms->arg2folder($lei, \@folders);
 	$lei->qerr(@{$err->{qerr}}) if $err->{qerr};
 	return $lei->fail($err->{fail}) if $err->{fail};
-	delete $lms->{dbh};
-	$lms->lms_begin;
-	$lms->forget_folder($_) for @folders;
-	$lms->lms_commit;
+	$sto->ipc_do('lms_forget_folders', @folders);
+	my $wait = $sto->ipc_do('done');
 }
 
 *_complete_forget_mail_sync = \&PublicInbox::LeiExportKw::_complete_export_kw;
diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm
index 3f33d114..e8334187 100644
--- a/lib/PublicInbox/LeiStore.pm
+++ b/lib/PublicInbox/LeiStore.pm
@@ -216,6 +216,12 @@ sub lms_mv_src {
 	_lms_rw($self)->mv_src($folder, $oidbin, $id, $newbn);
 }
 
+sub lms_forget_folders {
+	my ($self, @folders) = @_;
+	my $lms = _lms_rw($self);
+	for my $f (@folders) { $lms->forget_folder($f) }
+}
+
 sub set_sync_info {
 	my ($self, $oidhex, $folder, $id) = @_;
 	_lms_rw($self)->set_src($oidhex, $folder, $id);