From 2c472f1f571ae55155c78bcbc4d420d06266ba63 Mon Sep 17 00:00:00 2001
From: Eric Wong <e@80x24.org>
Date: Wed, 15 Nov 2023 09:21:43 +0000
Subject: lei convert: fix repeat and idempotent v2 output

We should be able to treat v2 outputs just like any other mail
format, with the exception that content dedupe is always
enforced by the v2 format.

This allows users hosting v2 public-inboxes to catch up broken
synchronization from alternate archives such as the mbox
archives hosted by https://lists.gnu.org/

Link: https://public-inbox.org/meta/20231114-hypersonic-papaya-starling-e1cfc8@nitro/
---
 lib/PublicInbox/LeiConvert.pm  | 8 ++++++--
 lib/PublicInbox/LeiOverview.pm | 4 ++--
 lib/PublicInbox/LeiToMail.pm   | 3 +--
 lib/PublicInbox/LeiXSearch.pm  | 4 ++--
 lib/PublicInbox/V2Writable.pm  | 3 ++-
 5 files changed, 13 insertions(+), 9 deletions(-)

(limited to 'lib/PublicInbox')

diff --git a/lib/PublicInbox/LeiConvert.pm b/lib/PublicInbox/LeiConvert.pm
index 22aba81a..4a1f8323 100644
--- a/lib/PublicInbox/LeiConvert.pm
+++ b/lib/PublicInbox/LeiConvert.pm
@@ -34,9 +34,13 @@ sub process_inputs { # via wq_do
 	$self->SUPER::process_inputs;
 	my $lei = $self->{lei};
 	delete $lei->{1};
-	my $l2m = delete $self->{l2m};
-	delete $self->{wcb}; # commit
+	my $l2m = delete $lei->{l2m};
 	my $nr_w = delete($l2m->{-nr_write}) // 0;
+	delete $self->{wcb}; # commit
+	if (my $v2w = delete $lei->{v2w}) {
+		$nr_w = $v2w->wq_do('done'); # may die
+		$v2w->wq_close;
+	}
 	my $d = (delete($l2m->{-nr_seen}) // 0) - $nr_w;
 	$d = $d ? " ($d duplicates)" : '';
 	$lei->qerr("# converted $nr_w messages$d");
diff --git a/lib/PublicInbox/LeiOverview.pm b/lib/PublicInbox/LeiOverview.pm
index 129dabf8..0529bbe4 100644
--- a/lib/PublicInbox/LeiOverview.pm
+++ b/lib/PublicInbox/LeiOverview.pm
@@ -41,8 +41,8 @@ sub detect_fmt ($) {
 	my ($dst) = @_;
 	if ($dst =~ m!\A([:/]+://)!) {
 		die "$1 support not implemented, yet\n";
-	} elsif (!-e $dst || -d _) {
-		'maildir'; # the default TODO: MH?
+	} elsif (!-e $dst || -d _) { # maildir is the default TODO: MH
+		-e "$dst/inbox.lock" ? 'v2' : 'maildir';
 	} elsif (-f _ || -p _) {
 		die "unable to determine mbox family of $dst\n";
 	} else {
diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm
index 2928be45..2d9b7061 100644
--- a/lib/PublicInbox/LeiToMail.pm
+++ b/lib/PublicInbox/LeiToMail.pm
@@ -375,7 +375,6 @@ sub _v2_write_cb ($$) {
 		++$self->{-nr_seen};
 		return if $dedupe && $dedupe->is_dup($eml, $smsg);
 		$lei->{v2w}->wq_do('add', $eml); # V2Writable->add
-		++$self->{-nr_write};
 	}
 }
 
@@ -435,7 +434,7 @@ sub new {
 			($lei->{opt}->{dedupe}//'') eq 'oid';
 		$self->{base_type} = 'v2';
 		$self->{-wq_nr_workers} = 1; # v2 has shards
-		$lei->{opt}->{save} = \1;
+		$lei->{opt}->{save} //= \1 if $lei->{cmd} eq 'q';
 		$dst = $lei->{ovv}->{dst} = $lei->abs_path($dst);
 		@conflict = qw(mua sort);
 	} else {
diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index e85fd3c4..7eda6f9e 100644
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -391,8 +391,9 @@ sub query_done { # EOF callback for main daemon
 	($lei->{opt}->{'mail-sync'} && !$lei->{sto}) and
 		warn "BUG: {sto} missing with --mail-sync";
 	$lei->sto_done_request;
+	my $nr_w = delete($lei->{-nr_write}) // 0;
 	if (my $v2w = delete $lei->{v2w}) {
-		my $wait = $v2w->wq_do('done'); # may die
+		$nr_w = $v2w->wq_do('done'); # may die
 		$v2w->wq_close;
 	}
 	$lei->{ovv}->ovv_end($lei);
@@ -412,7 +413,6 @@ Error closing $lei->{ovv}->{dst}: \$!=$! \$?=$?
 			delete $l2m->{mbl}; # drop dotlock
 		}
 	}
-	my $nr_w = delete($lei->{-nr_write}) // 0;
 	my $nr_dup = (delete($lei->{-nr_seen}) // 0) - $nr_w;
 	if ($lei->{-progress}) {
 		my $tot = $lei->{-mset_total} // 0;
diff --git a/lib/PublicInbox/V2Writable.pm b/lib/PublicInbox/V2Writable.pm
index 4d606dfe..231ed516 100644
--- a/lib/PublicInbox/V2Writable.pm
+++ b/lib/PublicInbox/V2Writable.pm
@@ -135,7 +135,7 @@ sub add {
 	if (do_idx($self, $mime, $smsg)) {
 		$self->checkpoint;
 	}
-
+	++$self->{-nr_add}; # for lei convert
 	$cmt;
 }
 
@@ -611,6 +611,7 @@ sub done {
 	$self->lock_release(!!$nbytes) if $shards;
 	$self->git->cleanup;
 	die $err if $err;
+	delete $self->{-nr_add}; # for lei-convert
 }
 
 sub importer {
-- 
cgit v1.2.3-24-ge0c7