user/dev discussion of public-inbox itself
 help / color / mirror / code / Atom feed
From: Eric Wong <e@80x24.org>
To: meta@public-inbox.org
Subject: [PATCH 3/5] lei import|convert: support mbox locking on reads
Date: Thu, 25 Feb 2021 22:41:39 -1100	[thread overview]
Message-ID: <20210226094141.11514-4-e@80x24.org> (raw)
In-Reply-To: <20210226094141.11514-1-e@80x24.org>

In case somebody is writing non-atomically, ensure we
take read locks when opening mbox files for reading.
---
 lib/PublicInbox/LEI.pm        | 13 +++++++++----
 lib/PublicInbox/LeiConvert.pm |  9 ++++++---
 lib/PublicInbox/LeiImport.pm  | 13 +++++++------
 3 files changed, 22 insertions(+), 13 deletions(-)

diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index b5bdda21..e133b357 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -172,12 +172,12 @@ our %CMD = ( # sorted in order of importance/use:
 'import' => [ 'LOCATION...|--stdin',
 	'one-time import/update from URL or filesystem',
 	qw(stdin| offset=i recursive|r exclude=s include|I=s
-	in-format|F=s kw|keywords|flags! C=s@),
+	lock=s@ in-format|F=s kw|keywords|flags! C=s@),
 	],
 'convert' => [ 'LOCATION...|--stdin',
 	'one-time conversion from URL or filesystem to another format',
 	qw(stdin| in-format|F=s out-format|f=s output|mfolder|o=s quiet|q
-	kw|keywords|flags! C=s@),
+	lock=s@ kw|keywords|flags! C=s@),
 	],
 'config' => [ '[...]', sub {
 		'git-config(1) wrapper for '._config_path($_[0]);
@@ -218,6 +218,9 @@ my %OPTDESC = (
 'help|h' => 'show this built-in help',
 'C=s@' => [ 'DIR', 'chdir to specify to directory' ],
 'quiet|q' => 'be quiet',
+'lock=s@' => [ 'METHOD|dotlock|fcntl|flock|none',
+	'mbox(5) locking method(s) to use (default: fcntl,dotlock)' ],
+
 'globoff|g' => "do not match locations using '*?' wildcards ".
 		"and\xa0'[]'\x{a0}ranges",
 'verbose|v+' => 'be more verbose',
@@ -410,8 +413,10 @@ sub check_input_format ($;$) {
 	return 1 if $fmt eq 'eml';
 	# XXX: should this handle {gz,bz2,xz}? that's currently in LeiToMail
 	require PublicInbox::MboxReader;
-	PublicInbox::MboxReader->can($fmt) ||
-				fail($self, "--$opt_key=$fmt unrecognized");
+	PublicInbox::MboxReader->can($fmt) or
+		return fail($self, "--$opt_key=$fmt unrecognized");
+	require PublicInbox::MboxLock if $files;
+	1;
 }
 
 sub out ($;@) {
diff --git a/lib/PublicInbox/LeiConvert.pm b/lib/PublicInbox/LeiConvert.pm
index 45d42c9c..4c0bbd88 100644
--- a/lib/PublicInbox/LeiConvert.pm
+++ b/lib/PublicInbox/LeiConvert.pm
@@ -62,9 +62,11 @@ sub do_convert { # via wq_do
 			$ifmt = lc $1;
 		}
 		if (-f $input) {
-			open my $fh, '<', $input or
-					return $lei->fail("open $input: $!");
-			convert_fh($self, $ifmt, $fh, $input);
+			my $m = $lei->{opt}->{'lock'} //
+					($ifmt eq 'eml' ? ['none'] :
+					PublicInbox::MboxLock->defaults);
+			my $mbl = PublicInbox::MboxLock->acq($input, 0, $m);
+			convert_fh($self, $ifmt, $mbl->{fh}, $input);
 		} elsif (-d _) {
 			PublicInbox::MdirReader::maildir_each_eml($input,
 							\&mdir_cb, $self);
@@ -109,6 +111,7 @@ sub call { # the main "lei convert" method
 
 			}
 			if (-f $input_path) {
+				require PublicInbox::MboxLock;
 				require PublicInbox::MboxReader;
 				PublicInbox::MboxReader->can($ifmt) or return
 					$lei->fail("$ifmt not supported");
diff --git a/lib/PublicInbox/LeiImport.pm b/lib/PublicInbox/LeiImport.pm
index 7f247b64..c2c98030 100644
--- a/lib/PublicInbox/LeiImport.pm
+++ b/lib/PublicInbox/LeiImport.pm
@@ -80,10 +80,11 @@ sub call { # the main "lei import" method
 			my $ifmt = lc $1;
 			if (($fmt // $ifmt) ne $ifmt) {
 				return $lei->fail(<<"");
---format=$fmt and `$ifmt:' conflict
+--in-format=$fmt and `$ifmt:' conflict
 
 			}
 			if (-f $input_path) {
+				require PublicInbox::MboxLock;
 				require PublicInbox::MboxReader;
 				PublicInbox::MboxReader->can($ifmt) or return
 					$lei->fail("$ifmt not supported");
@@ -142,7 +143,7 @@ error reading $input: $!
 			$cb->(undef, $fh, \&_import_eml, $lei->{sto}, $set_kw);
 		}
 	};
-	$lei->child_error(1 << 8, "<stdin>: $@") if $@;
+	$lei->child_error(1 << 8, "$input: $@") if $@;
 }
 
 sub _import_maildir { # maildir_each_file cb
@@ -171,10 +172,10 @@ sub import_path_url {
 		$ifmt = lc $1;
 	}
 	if (-f $input) {
-		open my $fh, '<', $input or return $lei->child_error(1 << 8, <<"");
-unable to open $input: $!
-
-		_import_fh($lei, $fh, $input, $ifmt);
+		my $m = $lei->{opt}->{'lock'} // ($ifmt eq 'eml' ? ['none'] :
+				PublicInbox::MboxLock->defaults);
+		my $mbl = PublicInbox::MboxLock->acq($input, 0, $m);
+		_import_fh($lei, $mbl->{fh}, $input, $ifmt);
 	} elsif (-d _ && (-d "$input/cur" || -d "$input/new")) {
 		return $lei->fail(<<EOM) if $ifmt && $ifmt ne 'maildir';
 $input appears to a be a maildir, not $ifmt

  parent reply	other threads:[~2021-02-26  9:41 UTC|newest]

Thread overview: 7+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2021-02-26  9:41 [PATCH 0/5] lei mbox locking Eric Wong
2021-02-26  9:41 ` [PATCH 1/5] lei: style fix for $oldset declaration Eric Wong
2021-02-26  9:41 ` [PATCH 2/5] lei q: support mbox locking by default Eric Wong
2021-02-26  9:41 ` Eric Wong [this message]
2021-02-26 21:03   ` [SQUASH 6/5] require MboxLock even for .eml files Eric Wong
2021-02-26  9:41 ` [PATCH 4/5] t/lei_store: rename $lst to $sto Eric Wong
2021-02-26  9:41 ` [PATCH 5/5] lei_xsearch: more detail about ->xdb call chain Eric Wong

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

  List information: https://public-inbox.org/README

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20210226094141.11514-4-e@80x24.org \
    --to=e@80x24.org \
    --cc=meta@public-inbox.org \
    --subject='Re: [PATCH 3/5] lei import|convert: support mbox locking on reads' \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link

Code repositories for project(s) associated with this inbox:

	https://80x24.org/public-inbox.git

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).