* [PATCH 0/5] lei mbox locking
@ 2021-02-26 9:41 7% Eric Wong
2021-02-26 9:41 5% ` [PATCH 3/5] lei import|convert: support mbox locking on reads Eric Wong
0 siblings, 1 reply; 2+ results
From: Eric Wong @ 2021-02-26 9:41 UTC (permalink / raw)
To: meta
mbox locking is in preparation for inotify/EVFILT_VNODE
mbox monitoring and keyword storage updating. And some
other odds and ends...
Anyways, still not sure how I want to store keywords
for read-only externals:
https://public-inbox.org/meta/20210224204950.GA2076@dcvr/
Eric Wong (5):
lei: style fix for $oldset declaration
lei q: support mbox locking by default
lei import|convert: support mbox locking on reads
t/lei_store: rename $lst to $sto
lei_xsearch: more detail about ->xdb call chain
MANIFEST | 2 +
lib/PublicInbox/LEI.pm | 19 ++++--
lib/PublicInbox/LeiConvert.pm | 9 ++-
lib/PublicInbox/LeiImport.pm | 13 ++--
lib/PublicInbox/LeiToMail.pm | 16 +++--
lib/PublicInbox/LeiXSearch.pm | 3 +-
lib/PublicInbox/MboxLock.pm | 121 ++++++++++++++++++++++++++++++++++
t/lei-q-remote-import.t | 12 ++++
t/lei_store.t | 102 ++++++++++++++--------------
t/mbox_lock.t | 90 +++++++++++++++++++++++++
10 files changed, 315 insertions(+), 72 deletions(-)
create mode 100644 lib/PublicInbox/MboxLock.pm
create mode 100644 t/mbox_lock.t
^ permalink raw reply [relevance 7%]
* [PATCH 3/5] lei import|convert: support mbox locking on reads
2021-02-26 9:41 7% [PATCH 0/5] lei mbox locking Eric Wong
@ 2021-02-26 9:41 5% ` Eric Wong
0 siblings, 0 replies; 2+ results
From: Eric Wong @ 2021-02-26 9:41 UTC (permalink / raw)
To: meta
In case somebody is writing non-atomically, ensure we
take read locks when opening mbox files for reading.
---
lib/PublicInbox/LEI.pm | 13 +++++++++----
lib/PublicInbox/LeiConvert.pm | 9 ++++++---
lib/PublicInbox/LeiImport.pm | 13 +++++++------
3 files changed, 22 insertions(+), 13 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index b5bdda21..e133b357 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -172,12 +172,12 @@ our %CMD = ( # sorted in order of importance/use:
'import' => [ 'LOCATION...|--stdin',
'one-time import/update from URL or filesystem',
qw(stdin| offset=i recursive|r exclude=s include|I=s
- in-format|F=s kw|keywords|flags! C=s@),
+ lock=s@ in-format|F=s kw|keywords|flags! C=s@),
],
'convert' => [ 'LOCATION...|--stdin',
'one-time conversion from URL or filesystem to another format',
qw(stdin| in-format|F=s out-format|f=s output|mfolder|o=s quiet|q
- kw|keywords|flags! C=s@),
+ lock=s@ kw|keywords|flags! C=s@),
],
'config' => [ '[...]', sub {
'git-config(1) wrapper for '._config_path($_[0]);
@@ -218,6 +218,9 @@ my %OPTDESC = (
'help|h' => 'show this built-in help',
'C=s@' => [ 'DIR', 'chdir to specify to directory' ],
'quiet|q' => 'be quiet',
+'lock=s@' => [ 'METHOD|dotlock|fcntl|flock|none',
+ 'mbox(5) locking method(s) to use (default: fcntl,dotlock)' ],
+
'globoff|g' => "do not match locations using '*?' wildcards ".
"and\xa0'[]'\x{a0}ranges",
'verbose|v+' => 'be more verbose',
@@ -410,8 +413,10 @@ sub check_input_format ($;$) {
return 1 if $fmt eq 'eml';
# XXX: should this handle {gz,bz2,xz}? that's currently in LeiToMail
require PublicInbox::MboxReader;
- PublicInbox::MboxReader->can($fmt) ||
- fail($self, "--$opt_key=$fmt unrecognized");
+ PublicInbox::MboxReader->can($fmt) or
+ return fail($self, "--$opt_key=$fmt unrecognized");
+ require PublicInbox::MboxLock if $files;
+ 1;
}
sub out ($;@) {
diff --git a/lib/PublicInbox/LeiConvert.pm b/lib/PublicInbox/LeiConvert.pm
index 45d42c9c..4c0bbd88 100644
--- a/lib/PublicInbox/LeiConvert.pm
+++ b/lib/PublicInbox/LeiConvert.pm
@@ -62,9 +62,11 @@ sub do_convert { # via wq_do
$ifmt = lc $1;
}
if (-f $input) {
- open my $fh, '<', $input or
- return $lei->fail("open $input: $!");
- convert_fh($self, $ifmt, $fh, $input);
+ my $m = $lei->{opt}->{'lock'} //
+ ($ifmt eq 'eml' ? ['none'] :
+ PublicInbox::MboxLock->defaults);
+ my $mbl = PublicInbox::MboxLock->acq($input, 0, $m);
+ convert_fh($self, $ifmt, $mbl->{fh}, $input);
} elsif (-d _) {
PublicInbox::MdirReader::maildir_each_eml($input,
\&mdir_cb, $self);
@@ -109,6 +111,7 @@ sub call { # the main "lei convert" method
}
if (-f $input_path) {
+ require PublicInbox::MboxLock;
require PublicInbox::MboxReader;
PublicInbox::MboxReader->can($ifmt) or return
$lei->fail("$ifmt not supported");
diff --git a/lib/PublicInbox/LeiImport.pm b/lib/PublicInbox/LeiImport.pm
index 7f247b64..c2c98030 100644
--- a/lib/PublicInbox/LeiImport.pm
+++ b/lib/PublicInbox/LeiImport.pm
@@ -80,10 +80,11 @@ sub call { # the main "lei import" method
my $ifmt = lc $1;
if (($fmt // $ifmt) ne $ifmt) {
return $lei->fail(<<"");
---format=$fmt and `$ifmt:' conflict
+--in-format=$fmt and `$ifmt:' conflict
}
if (-f $input_path) {
+ require PublicInbox::MboxLock;
require PublicInbox::MboxReader;
PublicInbox::MboxReader->can($ifmt) or return
$lei->fail("$ifmt not supported");
@@ -142,7 +143,7 @@ error reading $input: $!
$cb->(undef, $fh, \&_import_eml, $lei->{sto}, $set_kw);
}
};
- $lei->child_error(1 << 8, "<stdin>: $@") if $@;
+ $lei->child_error(1 << 8, "$input: $@") if $@;
}
sub _import_maildir { # maildir_each_file cb
@@ -171,10 +172,10 @@ sub import_path_url {
$ifmt = lc $1;
}
if (-f $input) {
- open my $fh, '<', $input or return $lei->child_error(1 << 8, <<"");
-unable to open $input: $!
-
- _import_fh($lei, $fh, $input, $ifmt);
+ my $m = $lei->{opt}->{'lock'} // ($ifmt eq 'eml' ? ['none'] :
+ PublicInbox::MboxLock->defaults);
+ my $mbl = PublicInbox::MboxLock->acq($input, 0, $m);
+ _import_fh($lei, $mbl->{fh}, $input, $ifmt);
} elsif (-d _ && (-d "$input/cur" || -d "$input/new")) {
return $lei->fail(<<EOM) if $ifmt && $ifmt ne 'maildir';
$input appears to a be a maildir, not $ifmt
^ permalink raw reply related [relevance 5%]
Results 1-2 of 2 | reverse | options above
-- pct% links below jump to the message on this page, permalinks otherwise --
2021-02-26 9:41 7% [PATCH 0/5] lei mbox locking Eric Wong
2021-02-26 9:41 5% ` [PATCH 3/5] lei import|convert: support mbox locking on reads Eric Wong
Code repositories for project(s) associated with this public inbox
https://80x24.org/public-inbox.git
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).