user/dev discussion of public-inbox itself
 help / color / mirror / code / Atom feed
From: Eric Wong <e@80x24.org>
To: meta@public-inbox.org
Subject: [PATCH v2 1/6] lei refresh-mail-sync: replace prune-mail-sync
Date: Thu, 16 Sep 2021 20:56:39 -0500	[thread overview]
Message-ID: <20210917015644.32746-2-e@80x24.org> (raw)
In-Reply-To: <20210917015644.32746-1-e@80x24.org>

Merely pruning mail synchronization information was
insufficient for Maildir: renames are common in Maildir
and we need to detect them after-the-fact when lei-daemon
isn't running.

Running this command could make "lei index" far more
useful...

v2: close R/O mail_sync.sqlite3 dbh before fork
  Keeping the DB file handle open across fork can cause bad things
  to happen even if we don't use it since sqlite3 itself still knows
  about it (but doesn't know Perl code doesn't know about it).
---
 MANIFEST                                      |  3 +-
 lib/PublicInbox/LEI.pm                        |  3 +-
 ...PruneMailSync.pm => LeiRefreshMailSync.pm} | 54 +++++++++------
 lib/PublicInbox/LeiStore.pm                   |  5 ++
 t/lei-export-kw.t                             |  1 -
 t/lei-refresh-mail-sync.t                     | 67 +++++++++++++++++++
 6 files changed, 111 insertions(+), 22 deletions(-)
 rename lib/PublicInbox/{LeiPruneMailSync.pm => LeiRefreshMailSync.pm} (62%)
 create mode 100644 t/lei-refresh-mail-sync.t

diff --git a/MANIFEST b/MANIFEST
index 640eabd1..9f11f2f9 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -235,9 +235,9 @@ lib/PublicInbox/LeiNoteEvent.pm
 lib/PublicInbox/LeiOverview.pm
 lib/PublicInbox/LeiP2q.pm
 lib/PublicInbox/LeiPmdir.pm
-lib/PublicInbox/LeiPruneMailSync.pm
 lib/PublicInbox/LeiQuery.pm
 lib/PublicInbox/LeiRediff.pm
+lib/PublicInbox/LeiRefreshMailSync.pm
 lib/PublicInbox/LeiRemote.pm
 lib/PublicInbox/LeiRm.pm
 lib/PublicInbox/LeiRmWatch.pm
@@ -450,6 +450,7 @@ t/lei-q-kw.t
 t/lei-q-remote-import.t
 t/lei-q-save.t
 t/lei-q-thread.t
+t/lei-refresh-mail-sync.t
 t/lei-sigpipe.t
 t/lei-tag.t
 t/lei-up.t
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index ec103231..9794497b 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -263,7 +263,7 @@ our %CMD = ( # sorted in order of importance/use:
 	@net_opt, @c_opt ],
 'forget-mail-sync' => [ 'LOCATION...',
 	'forget sync information for a mail folder', @c_opt ],
-'prune-mail-sync' => [ 'LOCATION...|--all',
+'refresh-mail-sync' => [ 'LOCATION...|--all',
 	'prune dangling sync data for a mail folder', 'all:s', @c_opt ],
 'export-kw' => [ 'LOCATION...|--all',
 	'one-time export of keywords of sync sources',
@@ -616,6 +616,7 @@ sub pkt_ops {
 	$ops->{x_it} = [ \&x_it, $lei ];
 	$ops->{child_error} = [ \&child_error, $lei ];
 	$ops->{incr} = [ \&incr, $lei ];
+	$ops->{sto_done_request} = [ \&sto_done_request, $lei, $lei->{sock} ];
 	$ops;
 }
 
diff --git a/lib/PublicInbox/LeiPruneMailSync.pm b/lib/PublicInbox/LeiRefreshMailSync.pm
similarity index 62%
rename from lib/PublicInbox/LeiPruneMailSync.pm
rename to lib/PublicInbox/LeiRefreshMailSync.pm
index 3678bd04..3c083965 100644
--- a/lib/PublicInbox/LeiPruneMailSync.pm
+++ b/lib/PublicInbox/LeiRefreshMailSync.pm
@@ -1,16 +1,20 @@
 # Copyright (C) 2021 all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
-# "lei prune-mail-sync" drops dangling sync information
-package PublicInbox::LeiPruneMailSync;
+# "lei refresh-mail-sync" drops dangling sync information
+# and attempts to detect moved files
+package PublicInbox::LeiRefreshMailSync;
 use strict;
 use v5.10.1;
 use parent qw(PublicInbox::IPC PublicInbox::LeiInput);
 use PublicInbox::LeiExportKw;
 use PublicInbox::InboxWritable qw(eml_from_path);
+use PublicInbox::ContentHash qw(git_sha);
+use PublicInbox::Import;
 
 sub eml_match ($$) {
 	my ($eml, $oidbin) = @_;
+	$eml->header_set($_) for @PublicInbox::Import::UNWANTED_HEADERS;
 	$oidbin eq git_sha(length($oidbin) == 20 ? 1 : 256, $eml)->digest;
 }
 
@@ -20,7 +24,7 @@ sub prune_mdir { # lms->each_src callback
 	for my $d (@try) {
 		my $src = "$mdir/$d/$$id";
 		if ($self->{verify}) {
-			my $eml = eml_from_path($src) or next;
+			my $eml = eml_from_path($src) // next;
 			return if eml_match($eml, $oidbin);
 		} elsif (-f $src) {
 			return;
@@ -38,12 +42,27 @@ sub prune_imap { # lms->each_src callback
 	$self->{lei}->{sto}->ipc_do('lms_clear_src', $url, $uid);
 }
 
+# detects missed file moves
+sub pmdir_cb { # called via LeiPmdir->each_mdir_fn
+	my ($self, $f, $fl) = @_;
+	my ($folder, $bn) = ($f =~ m!\A(.+?)/(?:new|cur)/([^/]+)\z!) or
+		die "BUG: $f was not from a Maildir?";
+	substr($folder, 0, 0) = 'maildir:'; # add prefix
+	my $lms = $self->{-lms_ro} //= $self->{lei}->lms;
+	return if defined($lms->name_oidbin($folder, $bn));
+	my $eml = eml_from_path($f) // return;
+	my $oidbin = $self->{lei}->git_oid($eml)->digest;
+	$self->{lei}->{sto}->ipc_do('lms_set_src', $oidbin, $folder, \$bn);
+}
+
 sub input_path_url { # overrides PublicInbox::LeiInput::input_path_url
 	my ($self, $input, @args) = @_;
 	my $lms = $self->{-lms_ro} //= $self->{lei}->lms;
 	if ($input =~ /\Amaildir:(.+)/i) {
-		my $mdir = $1;
-		$lms->each_src($input, \&prune_mdir, $self, $mdir);
+		$lms->each_src($input, \&prune_mdir, $self, my $mdir = $1);
+		$self->{lse} //= $self->{lei}->{sto}->search;
+		# call pmdir_cb (via maildir_each_file -> each_mdir_fn)
+		PublicInbox::LeiInput::input_path_url($self, $input);
 	} elsif ($input =~ m!\Aimaps?://!i) {
 		my $uri = PublicInbox::URIimap->new($input);
 		my $mic = $self->{lei}->{net}->mic_for_folder($uri);
@@ -51,34 +70,31 @@ sub input_path_url { # overrides PublicInbox::LeiInput::input_path_url
 		$uids = +{ map { $_ => undef } @$uids };
 		$lms->each_src($$uri, \&prune_imap, $self, $uids, $$uri);
 	} else { die "BUG: $input not supported" }
-	my $wait = $self->{lei}->{sto}->ipc_do('done');
+	$self->{lei}->{pkt_op_p}->pkt_do('sto_done_request');
 }
 
-sub lei_prune_mail_sync {
+sub lei_refresh_mail_sync {
 	my ($lei, @folders) = @_;
 	my $sto = $lei->_lei_store or return $lei->fail(<<EOM);
 lei/store uninitialized, see lei-import(1)
 EOM
-	if (my $lms = $lei->lms) {
-		if (defined(my $all = $lei->{opt}->{all})) {
-			$lms->group2folders($lei, $all, \@folders) or return;
-		} else {
-			my $err = $lms->arg2folder($lei, \@folders);
-			$lei->qerr(@{$err->{qerr}}) if $err->{qerr};
-			return $lei->fail($err->{fail}) if $err->{fail};
-		}
-	} else {
-		return $lei->fail(<<EOM);
+	my $lms = $lei->lms or return $lei->fail(<<EOM);
 lei mail_sync.sqlite3 uninitialized, see lei-import(1)
 EOM
+	if (defined(my $all = $lei->{opt}->{all})) {
+		$lms->group2folders($lei, $all, \@folders) or return;
+	} else {
+		my $err = $lms->arg2folder($lei, \@folders);
+		$lei->qerr(@{$err->{qerr}}) if $err->{qerr};
+		return $lei->fail($err->{fail}) if $err->{fail};
 	}
+	undef $lms; # must be done before fork
 	$sto->write_prepare($lei);
 	my $self = bless { missing_ok => 1 }, __PACKAGE__;
 	$lei->{opt}->{'mail-sync'} = 1; # for prepare_inputs
 	$self->prepare_inputs($lei, \@folders) or return;
 	my $j = $lei->{opt}->{jobs} || scalar(@{$self->{inputs}}) || 1;
 	my $ops = {};
-	$sto->write_prepare($lei);
 	$lei->{auth}->op_merge($ops, $self) if $lei->{auth};
 	$self->{-wq_nr_workers} = $j // 1; # locked
 	(my $op_c, $ops) = $lei->workers_start($self, $j, $ops);
@@ -89,7 +105,7 @@ EOM
 }
 
 no warnings 'once';
-*_complete_prune_mail_sync = \&PublicInbox::LeiExportKw::_complete_export_kw;
+*_complete_refresh_mail_sync = \&PublicInbox::LeiExportKw::_complete_export_kw;
 *ipc_atfork_child = \&PublicInbox::LeiInput::input_only_atfork_child;
 *net_merge_all_done = \&PublicInbox::LeiInput::input_only_net_merge_all_done;
 
diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm
index e8bcb04e..32f55abd 100644
--- a/lib/PublicInbox/LeiStore.pm
+++ b/lib/PublicInbox/LeiStore.pm
@@ -293,6 +293,11 @@ sub set_sync_info {
 	_lms_rw($self)->set_src(pack('H*', $oidhex), $folder, $id);
 }
 
+sub lms_set_src {
+	my ($self, $oidbin, $folder, $id) = @_;
+	_lms_rw($self)->set_src($oidbin, $folder, $id);
+}
+
 sub _remove_if_local { # git->cat_async arg
 	my ($bref, $oidhex, $type, $size, $self) = @_;
 	$self->{im}->remove($bref) if $bref;
diff --git a/t/lei-export-kw.t b/t/lei-export-kw.t
index 9531949a..1fe940bb 100644
--- a/t/lei-export-kw.t
+++ b/t/lei-export-kw.t
@@ -6,7 +6,6 @@ use File::Copy qw(cp);
 use File::Path qw(make_path);
 require_mods(qw(lei -imapd Mail::IMAPClient));
 my ($tmpdir, $for_destroy) = tmpdir;
-my ($ro_home, $cfg_path) = setup_public_inboxes;
 my $expect = eml_load('t/data/0001.patch');
 test_lei({ tmpdir => $tmpdir }, sub {
 	my $home = $ENV{HOME};
diff --git a/t/lei-refresh-mail-sync.t b/t/lei-refresh-mail-sync.t
new file mode 100644
index 00000000..ff558277
--- /dev/null
+++ b/t/lei-refresh-mail-sync.t
@@ -0,0 +1,67 @@
+#!perl -w
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+use strict; use v5.10.1; use PublicInbox::TestCommon;
+require_mods(qw(lei));
+
+my $stop_daemon = sub { # needed since we don't have inotify
+	lei_ok qw(daemon-pid);
+	chomp(my $pid = $lei_out);
+	$pid > 0 or xbail "bad pid: $pid";
+	kill('TERM', $pid) or xbail "kill: $!";
+	for (0..10) {
+		tick;
+		kill(0, $pid) or last;
+	}
+	kill(0, $pid) and xbail "daemon still running (PID:$pid)";
+};
+
+test_lei({ daemon_only => 1 }, sub {
+	my $d = "$ENV{HOME}/d";
+	my ($ro_home, $cfg_path) = setup_public_inboxes;
+	lei_ok qw(daemon-pid);
+	lei_ok qw(add-external), "$ro_home/t2";
+	lei_ok qw(q mid:testmessage@example.com -o), "Maildir:$d";
+	my (@o) = glob("$d/*/*");
+	scalar(@o) == 1 or xbail('multiple results', \@o);
+	my ($bn0) = ($o[0] =~ m!/([^/]+)\z!);
+
+	my $oid = '9bf1002c49eb075df47247b74d69bcd555e23422';
+	lei_ok 'inspect', "blob:$oid";
+	my $before = json_utf8->decode($lei_out);
+	my $exp0 = { 'mail-sync' => { "maildir:$d" => [ $bn0 ] } };
+	is_deeply($before, $exp0, 'inspect shows expected');
+
+	$stop_daemon->();
+	my $dst = $o[0];
+	$dst =~ s/:2,.*\z// and $dst =~ s!/cur/!/new/! and
+		rename($o[0], $dst) or xbail "rename($o[0] => $dst): $!";
+
+	lei_ok 'inspect', "blob:$oid";
+	is_deeply(json_utf8->decode($lei_out),
+		$before, 'inspect unchanged immediately after restart');
+	lei_ok 'refresh-mail-sync', '--all';
+	lei_ok 'inspect', "blob:$oid";
+	my ($bn1) = ($dst =~ m!/([^/]+)\z!);
+	my $exp1 = { 'mail-sync' => { "maildir:$d" => [ $bn1 ] } };
+	is_deeply(json_utf8->decode($lei_out), $exp1,
+		'refresh-mail-sync updated location');
+
+	$stop_daemon->();
+	rename($dst, "$d/unwatched") or xbail "rename $dst out-of-the-way $!";
+
+	lei_ok 'refresh-mail-sync', $d;
+	lei_ok 'inspect', "blob:$oid";
+	is($lei_out, '{}', 'no known locations after "removal"');
+	lei_ok 'refresh-mail-sync', "Maildir:$d";
+
+	$stop_daemon->();
+	rename("$d/unwatched", $dst) or xbail "rename $dst back";
+
+	lei_ok 'refresh-mail-sync', "Maildir:$d";
+	lei_ok 'inspect', "blob:$oid";
+	is_deeply(json_utf8->decode($lei_out), $exp1,
+		'replaced file noted again');
+});
+
+done_testing;

  reply	other threads:[~2021-09-17  1:56 UTC|newest]

Thread overview: 7+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2021-09-17  1:56 [PATCH v2 0/6] lei refresh-mail-sync: another try Eric Wong
2021-09-17  1:56 ` Eric Wong [this message]
2021-09-17  1:56 ` [PATCH v2 2/6] lei_mail_sync: don't hold statement handle into callback Eric Wong
2021-09-17  1:56 ` [PATCH v2 3/6] lei refresh-mail-sync: remove "gone" notices Eric Wong
2021-09-17  1:56 ` [PATCH v2 4/6] lei refresh-mail-sync: drop unused {verify} code path Eric Wong
2021-09-17  1:56 ` [PATCH v2 5/6] lei refresh-mail-sync: implicitly remove missing folders Eric Wong
2021-09-17  1:56 ` [PATCH v2 6/6] lei refresh-mail-sync: drop old IMAP folder info Eric Wong

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

  List information: https://public-inbox.org/README

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20210917015644.32746-2-e@80x24.org \
    --to=e@80x24.org \
    --cc=meta@public-inbox.org \
    --subject='Re: [PATCH v2 1/6] lei refresh-mail-sync: replace prune-mail-sync' \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link

Code repositories for project(s) associated with this inbox:

	https://80x24.org/public-inbox.git

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).