user/dev discussion of public-inbox itself
 help / color / mirror / code / Atom feed
From: Eric Wong <e@80x24.org>
To: meta@public-inbox.org
Subject: [PATCH 6/6] lei <q|up>: writes to Maildirs and IMAP use mail-sync
Date: Sun,  2 May 2021 06:05:42 +0000	[thread overview]
Message-ID: <20210502060542.11598-7-e@80x24.org> (raw)
In-Reply-To: <20210502060542.11598-1-e@80x24.org>

This will allow keyword updates from other folders to propagate
to folders where search results may be duplicated.
---
 lib/PublicInbox/LEI.pm       |  2 +-
 lib/PublicInbox/LeiQuery.pm  |  7 +++----
 lib/PublicInbox/LeiStore.pm  |  8 ++++----
 lib/PublicInbox/LeiToMail.pm | 24 +++++++++++++++++-------
 4 files changed, 25 insertions(+), 16 deletions(-)

diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index d5c6bd52..599cfab2 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -145,7 +145,7 @@ our %CMD = ( # sorted in order of importance/use:
 	qw(save output|mfolder|o=s format|f=s dedupe|d=s threads|t+
 	sort|s=s reverse|r offset=i pretty jobs|j=s globoff|g augment|a
 	import-before! lock=s@ rsyncable alert=s@ mua=s verbose|v+
-	color!), @c_opt, opt_dash('limit|n=i', '[0-9]+') ],
+	color! mail-sync!), @c_opt, opt_dash('limit|n=i', '[0-9]+') ],
 
 'up' => [ 'OUTPUT|--all', 'update saved search',
 	qw(jobs|j=s lock=s@ alert=s@ mua=s verbose|v+ all:s), @c_opt ],
diff --git a/lib/PublicInbox/LeiQuery.pm b/lib/PublicInbox/LeiQuery.pm
index efe328cc..1999a534 100644
--- a/lib/PublicInbox/LeiQuery.pm
+++ b/lib/PublicInbox/LeiQuery.pm
@@ -29,10 +29,9 @@ sub _start_query { # used by "lei q" and "lei up"
 		return $self->fail("`$mj' writer jobs must be >= 1");
 	}
 	my $l2m = $self->{l2m};
-	if ($l2m && ($opt->{'import-remote'} //= 1) |
-				# we use \1 (a ref) to distinguish between
-				# user-supplied and default value
-				(($opt->{'import-before'} //= \1) ? 1 : 0)) {
+	# we use \1 (a ref) to distinguish between default vs. user-supplied
+	if ($l2m && grep { $opt->{$_} //= \1 } (qw(mail-sync import-remote
+							import-before))) {
 		$self->_lei_store(1)->write_prepare($self);
 	}
 	$l2m and $l2m->{-wq_nr_workers} = $mj // do {
diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm
index 8af740fd..29362b2e 100644
--- a/lib/PublicInbox/LeiStore.pm
+++ b/lib/PublicInbox/LeiStore.pm
@@ -193,15 +193,15 @@ sub remove_eml_vmd {
 	\@docids;
 }
 
-sub set_sync_info ($$$) {
-	my ($self, $oidhex, $sync_info) = @_;
+sub set_sync_info {
+	my ($self, $oidhex, $folder, $id) = @_;
 	($self->{lms} //= do {
 		require PublicInbox::LeiMailSync;
 		my $f = "$self->{priv_eidx}->{topdir}/mail_sync.sqlite3";
 		my $lms = PublicInbox::LeiMailSync->new($f);
 		$lms->lms_begin;
 		$lms;
-	})->set_src($oidhex, @$sync_info);
+	})->set_src($oidhex, $folder, $id);
 }
 
 sub add_eml {
@@ -212,7 +212,7 @@ sub add_eml {
 	my $smsg = bless { -oidx => $oidx }, 'PublicInbox::Smsg';
 	my $im_mark = $im->add($eml, undef, $smsg);
 	if ($vmd && $vmd->{sync_info}) {
-		set_sync_info($self, $smsg->{blob}, $vmd->{sync_info});
+		set_sync_info($self, $smsg->{blob}, @{$vmd->{sync_info}});
 	}
 	$im_mark or return; # duplicate blob returns undef
 
diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm
index ab4de378..71acf952 100644
--- a/lib/PublicInbox/LeiToMail.pm
+++ b/lib/PublicInbox/LeiToMail.pm
@@ -242,7 +242,7 @@ sub _buf2maildir {
 	my $kw = $smsg->{kw} // [];
 	my $sfx = join('', sort(map { $kw2char{$_} // () } @$kw));
 	my $rand = ''; # chosen by die roll :P
-	my ($tmp, $fh, $final, $ok);
+	my ($tmp, $fh, $base, $ok);
 	my $common = $smsg->{blob} // _rand;
 	if (defined(my $pct = $smsg->{pct})) { $common .= "=$pct" }
 	do {
@@ -257,11 +257,12 @@ sub _buf2maildir {
 		$dst .= 'cur/';
 		$rand = '';
 		do {
-			$final = $dst.$rand.$common.':2,'.$sfx;
-		} while (!($ok = link($tmp, $final)) && $!{EEXIST} &&
+			$base = $rand.$common.':2,'.$sfx
+		} while (!($ok = link($tmp, $dst.$base)) && $!{EEXIST} &&
 			($rand = _rand.','));
-		die "link($tmp, $final): $!" unless $ok;
+		die "link($tmp, $dst$base): $!" unless $ok;
 		unlink($tmp) or warn "W: failed to unlink $tmp: $!\n";
+		\$base;
 	} else {
 		my $err = "Error writing $smsg->{blob} to $dst: $!\n";
 		$_[0] = undef; # clobber dst
@@ -276,13 +277,16 @@ sub _maildir_write_cb ($$) {
 	$dedupe->prepare_dedupe if $dedupe;
 	my $dst = $lei->{ovv}->{dst};
 	my $lse = $lei->{lse}; # may be undef
+	my $sto = $lei->{opt}->{'mail-sync'} ? $lei->{sto} : undef;
+	my $out = $sto ? 'maildir:'.$lei->rel2abs($dst) : undef;
 	sub { # for git_to_mail
 		my ($bref, $smsg, $eml) = @_;
 		$dst // return $lei->fail; # dst may be undef-ed in last run
 		return if $dedupe && $dedupe->is_dup($eml //
 						PublicInbox::Eml->new($$bref));
 		$lse->xsmsg_vmd($smsg) if $lse;
-		_buf2maildir($dst, $bref // \($eml->as_string), $smsg);
+		my $n = _buf2maildir($dst, $bref // \($eml->as_string), $smsg);
+		$sto->ipc_do('set_sync_info', $smsg->{blob}, $out, $n) if $sto;
 		++$lei->{-nr_write};
 	}
 }
@@ -291,21 +295,27 @@ sub _imap_write_cb ($$) {
 	my ($self, $lei) = @_;
 	my $dedupe = $lei->{dedupe};
 	$dedupe->prepare_dedupe if $dedupe;
-	my $imap_append = $lei->{net}->can('imap_append');
+	my $append = $lei->{net}->can('imap_append');
 	my $mic = $lei->{net}->mic_get($self->{uri});
 	my $folder = $self->{uri}->mailbox;
 	my $lse = $lei->{lse}; # may be undef
+	my $sto = $lei->{opt}->{'mail-sync'} ? $lei->{sto} : undef;
+	my $out = $lei->{ovv}->{dst};
 	sub { # for git_to_mail
 		my ($bref, $smsg, $eml) = @_;
 		$mic // return $lei->fail; # mic may be undef-ed in last run
 		return if $dedupe && $dedupe->is_dup($eml //
 						PublicInbox::Eml->new($$bref));
 		$lse->xsmsg_vmd($smsg) if $lse;
-		eval { $imap_append->($mic, $folder, $bref, $smsg, $eml) };
+		my $uid = eval { $append->($mic, $folder, $bref, $smsg, $eml) };
 		if (my $err = $@) {
 			undef $mic;
 			die $err;
 		}
+		# imap_append returns UID if IMAP server has UIDPLUS extension
+		($sto && $uid =~ /\A[0-9]+\z/) and
+			$sto->ipc_do('set_sync_info',
+					$smsg->{blob}, $out, $uid + 0);
 		++$lei->{-nr_write};
 	}
 }

      parent reply	other threads:[~2021-05-02  6:05 UTC|newest]

Thread overview: 7+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2021-05-02  6:05 [PATCH 0/6] lei: more steps towards kw sync Eric Wong
2021-05-02  6:05 ` [PATCH 1/6] lei <q|up>: combine written/results into one line Eric Wong
2021-05-02  6:05 ` [PATCH 2/6] lei_input: common net_merge_all_done for lei <import|tag> Eric Wong
2021-05-02  6:05 ` [PATCH 3/6] lei_input: reject --mail-sync if using HTTP(S) for now Eric Wong
2021-05-02  6:05 ` [PATCH 4/6] lei: simplify workers_start API Eric Wong
2021-05-02  6:05 ` [PATCH 5/6] net_writer: use "FLAGS.SILENT" to set keywords Eric Wong
2021-05-02  6:05 ` Eric Wong [this message]

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

  List information: https://public-inbox.org/README

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20210502060542.11598-7-e@80x24.org \
    --to=e@80x24.org \
    --cc=meta@public-inbox.org \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
Code repositories for project(s) associated with this public inbox

	https://80x24.org/public-inbox.git

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).