* [PATCH 3/3] lei q: mbox*: disable changing parallelism, add --rsyncable
@ 2021-03-12 10:39 7% ` Eric Wong
0 siblings, 0 replies; 3+ results
From: Eric Wong @ 2021-03-12 10:39 UTC (permalink / raw)
To: meta
Unfortunately, being mairix-compatible with --threads means we
can't change thread-count of gzip, bzip2, or xz when writing to
compressed mbox with a --threads= parameter. It's probably not
worth changing, anyways, so another switch or additional value
for --jobs= won't be added.
While we're in the area, add --rsyncable support since
most installations of gzip support it nowadays.
Fixes: 5beb4a5f6585acd ("lei: replace --thread with --threads")
---
lib/PublicInbox/LEI.pm | 2 +-
lib/PublicInbox/LeiToMail.pm | 6 +++---
2 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 9bf60ad4..59a3338c 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -113,7 +113,7 @@ our %CMD = ( # sorted in order of importance/use:
qw(save-as=s output|mfolder|o=s format|f=s dedupe|d=s threads|t+
sort|s=s reverse|r offset=i remote! local! external! pretty
include|I=s@ exclude=s@ only=s@ jobs|j=s globoff|g augment|a
- import-remote! import-before! lock=s@
+ import-remote! import-before! lock=s@ rsyncable
alert=s@ mua=s no-torsocks torsocks=s verbose|v+ quiet|q C=s@),
PublicInbox::LeiQuery::curl_opt(), opt_dash('limit|n=i', '[0-9]+') ],
diff --git a/lib/PublicInbox/LeiToMail.pm b/lib/PublicInbox/LeiToMail.pm
index 13b4f672..13764d79 100644
--- a/lib/PublicInbox/LeiToMail.pm
+++ b/lib/PublicInbox/LeiToMail.pm
@@ -170,9 +170,9 @@ sub reap_compress { # dwaitpid callback
# { foo => '' } means "--foo" is passed to the command-line,
# otherwise { foo => '--bar' } passes "--bar"
our %zsfx2cmd = (
- gz => [ qw(GZIP pigz gzip), { rsyncable => '', threads => '-p' } ],
+ gz => [ qw(GZIP pigz gzip), { rsyncable => '' } ],
bz2 => [ 'bzip2', {} ],
- xz => [ 'xz', { threads => '-T' } ],
+ xz => [ 'xz', {} ],
# XXX does anybody care for these? I prefer zstd on entire FSes,
# so it's probably not necessary on a per-file basis
# zst => [ 'zstd', { -default => [ qw(-q) ], # it's noisy by default
@@ -203,7 +203,7 @@ sub zsfx2cmd ($$$) {
my $switch = $cmd_opt->{rsyncable} // next;
push @cmd, '--'.($switch || $bool);
}
- for my $key (qw(threads)) { # support compression level?
+ for my $key (qw(rsyncable)) { # support compression level?
my $switch = $cmd_opt->{$key} // next;
my $val = $lei->{opt}->{$key} // next;
push @cmd, $switch, $val;
^ permalink raw reply related [relevance 7%]
* [PATCH 14/19] lei: replace --thread with --threads
2021-02-07 8:51 6% [PATCH 00/19] lei import Maildir, remote mboxrd fixes Eric Wong
@ 2021-02-07 8:51 4% ` Eric Wong
0 siblings, 0 replies; 3+ results
From: Eric Wong @ 2021-02-07 8:51 UTC (permalink / raw)
To: meta
Nobody is expected to use long options, but for consistency
with mairix(1), we'll use the pluralized option throughout
(including existing PublicInbox::{Search,SearchView}).
Link: https://public-inbox.org/meta/20210206090119.GA14519@dcvr/
---
Documentation/lei-q.pod | 2 +-
lib/PublicInbox/LEI.pm | 16 ++++++++--------
lib/PublicInbox/LeiHelp.pm | 4 ++--
lib/PublicInbox/LeiQuery.pm | 4 ++--
lib/PublicInbox/LeiXSearch.pm | 12 ++++++------
lib/PublicInbox/Mbox.pm | 2 +-
lib/PublicInbox/Search.pm | 2 +-
lib/PublicInbox/SearchView.pm | 2 +-
8 files changed, 22 insertions(+), 22 deletions(-)
diff --git a/Documentation/lei-q.pod b/Documentation/lei-q.pod
index 07c742d2..8f053a55 100644
--- a/Documentation/lei-q.pod
+++ b/Documentation/lei-q.pod
@@ -47,7 +47,7 @@ or C<neomutt>.
Augment output destination instead of clobbering it.
-=item -t, --thread
+=item -t, --threads
Return all messages in the same thread as the actual match(es).
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 818f2cfb..31e6b4a8 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -109,14 +109,14 @@ sub index_opt {
# command => [ positional_args, 1-line description, Getopt::Long option spec ]
our %CMD = ( # sorted in order of importance/use:
'q' => [ '--stdin|SEARCH_TERMS...', 'search for messages matching terms', qw(
- save-as=s output|mfolder|o=s format|f=s dedupe|d=s thread|t augment|a
+ save-as=s output|mfolder|o=s format|f=s dedupe|d=s threads|t augment|a
sort|s=s reverse|r offset=i remote! local! external! pretty
include|I=s@ exclude=s@ only=s@ jobs|j=s globoff|g stdin|
mua=s no-torsocks torsocks=s verbose|v+ quiet|q),
PublicInbox::LeiQuery::curl_opt(), opt_dash('limit|n=i', '[0-9]+') ],
'show' => [ 'MID|OID', 'show a given object (Message-ID or object ID)',
- qw(type=s solve! format|f=s dedupe|d=s thread|t remote local!),
+ qw(type=s solve! format|f=s dedupe|d=s threads|t remote local!),
pass_through('git show') ],
'add-external' => [ 'LOCATION',
@@ -135,9 +135,9 @@ our %CMD = ( # sorted in order of importance/use:
'rm-query' => [ 'QUERY_NAME', 'remove a saved search' ],
'mv-query' => [ qw(OLD_NAME NEW_NAME), 'rename a saved search' ],
-'plonk' => [ '--thread|--from=IDENT',
- 'exclude mail matching From: or thread from non-Message-ID searches',
- qw(stdin| thread|t from|f=s mid=s oid=s) ],
+'plonk' => [ '--threads|--from=IDENT',
+ 'exclude mail matching From: or threads from non-Message-ID searches',
+ qw(stdin| threads|t from|f=s mid=s oid=s) ],
'mark' => [ 'MESSAGE_FLAGS...',
'set/unset keywords on message(s) from stdin',
qw(stdin| oid=s exact by-mid|mid:s) ],
@@ -224,9 +224,9 @@ my %OPTDESC = (
'dedupe|d=s' => ['STRATEGY|content|oid|mid|none',
'deduplication strategy'],
-'show thread|t' => 'display entire thread a message belongs to',
-'q thread|t' =>
- 'return all messages in the same thread as the actual match(es)',
+'show threads|t' => 'display entire thread a message belongs to',
+'q threads|t' =>
+ 'return all messages in the same threads as the actual match(es)',
'augment|a' => 'augment --output destination instead of clobbering',
'output|mfolder|o=s' => [ 'MFOLDER',
diff --git a/lib/PublicInbox/LeiHelp.pm b/lib/PublicInbox/LeiHelp.pm
index e62298f7..a654e1c2 100644
--- a/lib/PublicInbox/LeiHelp.pm
+++ b/lib/PublicInbox/LeiHelp.pm
@@ -40,7 +40,7 @@ sub call {
@vals = (' [', undef, ']');
} elsif ($x =~ s/=.+//) { # required arg: $x = "type=s"
@vals = (' ', undef);
- } # else: no args $x = 'thread|t'
+ } # else: no args $x = 'threads|t'
# we support underscore options from public-inbox-* commands;
# but they've never been documented and will likely go away.
@@ -48,7 +48,7 @@ sub call {
for (grep { !/_/ && !$NOHELP{$_} } split(/\|/, $x)) {
length($_) > 1 ? push(@l, "--$_") : push(@s, "-$_");
}
- if (!scalar(@vals)) { # no args 'thread|t'
+ if (!scalar(@vals)) { # no args 'threads|t'
} elsif ($arg_vals =~ s/\A([A-Z_]+)\b//) { # "NAME"
$vals[1] = $1;
} else {
diff --git a/lib/PublicInbox/LeiQuery.pm b/lib/PublicInbox/LeiQuery.pm
index 0346498f..9a6fa718 100644
--- a/lib/PublicInbox/LeiQuery.pm
+++ b/lib/PublicInbox/LeiQuery.pm
@@ -81,7 +81,7 @@ sub lei_q {
$self->{l2m}->{jobs} = ($mj // $nproc) if $self->{l2m};
PublicInbox::LeiOverview->new($self) or return;
- my %mset_opt = map { $_ => $opt->{$_} } qw(thread limit offset);
+ my %mset_opt = map { $_ => $opt->{$_} } qw(threads limit offset);
$mset_opt{asc} = $opt->{'reverse'} ? 1 : 0;
$mset_opt{limit} //= 10000;
if (defined(my $sort = $opt->{'sort'})) {
@@ -96,7 +96,7 @@ sub lei_q {
}
}
# descending docid order
- $mset_opt{relevance} //= -2 if $opt->{thread};
+ $mset_opt{relevance} //= -2 if $opt->{threads};
$self->{mset_opt} = \%mset_opt;
if ($opt->{stdin}) {
diff --git a/lib/PublicInbox/LeiXSearch.pm b/lib/PublicInbox/LeiXSearch.pm
index 1024b020..2794140a 100644
--- a/lib/PublicInbox/LeiXSearch.pm
+++ b/lib/PublicInbox/LeiXSearch.pm
@@ -118,7 +118,7 @@ sub mset_progress {
}
}
-sub query_thread_mset { # for --thread
+sub query_thread_mset { # for --threads
my ($self, $ibxish) = @_;
local $0 = "$0 query_thread_mset";
my $lei = $self->{lei};
@@ -151,7 +151,7 @@ sub query_thread_mset { # for --thread
$lei->{ovv}->ovv_atexit_child($lei);
}
-sub query_mset { # non-parallel for non-"--thread" users
+sub query_mset { # non-parallel for non-"--threads" users
my ($self) = @_;
local $0 = "$0 query_mset";
my $lei = $self->{lei};
@@ -204,7 +204,7 @@ sub query_remote_mboxrd {
my $lei = $self->{lei};
my ($opt, $env) = @$lei{qw(opt env)};
my @qform = (q => $lei->{mset_opt}->{qstr}, x => 'm');
- push(@qform, t => 1) if $opt->{thread};
+ push(@qform, t => 1) if $opt->{threads};
my $verbose = $opt->{verbose};
my ($reap_tail, $reap_curl);
my $cerr = File::Temp->new(TEMPLATE => 'curl.err-XXXX', TMPDIR => 1);
@@ -326,7 +326,7 @@ my $MAX_PER_HOST = 4;
sub concurrency {
my ($self, $opt) = @_;
- my $nl = $opt->{thread} ? locals($self) : 1;
+ my $nl = $opt->{threads} ? locals($self) : 1;
my $nr = remotes($self);
$nr = $MAX_PER_HOST if $nr > $MAX_PER_HOST;
$nl + $nr;
@@ -337,7 +337,7 @@ sub start_query { # always runs in main (lei-daemon) process
if (my $l2m = $lei->{l2m}) {
$lei->start_mua if $l2m->lock_free;
}
- if ($lei->{opt}->{thread}) {
+ if ($lei->{opt}->{threads}) {
for my $ibxish (locals($self)) {
$self->wq_io_do('query_thread_mset', [], $ibxish);
}
@@ -393,7 +393,7 @@ sub do_query {
# 1031: F_SETPIPE_SZ
fcntl($lei->{startq}, 1031, 4096) if $^O eq 'linux';
}
- if (!$lei->{opt}->{thread} && locals($self)) { # for query_mset
+ if (!$lei->{opt}->{threads} && locals($self)) { # for query_mset
# lei->{git_tmp} is set for wq_wait_old so we don't
# delete until all lei2mail + lei_xsearch workers are reaped
$lei->{git_tmp} = $self->{git_tmp} = git_tmp($self);
diff --git a/lib/PublicInbox/Mbox.pm b/lib/PublicInbox/Mbox.pm
index 964147fa..1fca356b 100644
--- a/lib/PublicInbox/Mbox.pm
+++ b/lib/PublicInbox/Mbox.pm
@@ -236,7 +236,7 @@ sub mbox_all {
return PublicInbox::WWW::need($ctx, 'Overview');
my $qopts = $ctx->{qopts} = { relevance => -1 }; # ORDER BY docid ASC
- $qopts->{thread} = 1 if $q->{t};
+ $qopts->{threads} = 1 if $q->{t};
my $mset = $srch->mset($q_string, $qopts);
$qopts->{offset} = $mset->size or
return [404, [qw(Content-Type text/plain)],
diff --git a/lib/PublicInbox/Search.pm b/lib/PublicInbox/Search.pm
index 7c6a16be..dbae3bc5 100644
--- a/lib/PublicInbox/Search.pm
+++ b/lib/PublicInbox/Search.pm
@@ -336,7 +336,7 @@ sub _enquire_once { # retry_reopen callback
}
# `mairix -t / --threads' or JMAP collapseThreads
- if ($opts->{thread} && has_threadid($self)) {
+ if ($opts->{threads} && has_threadid($self)) {
$enquire->set_collapse_key(THREADID);
}
$enquire->get_mset($opts->{offset} || 0, $opts->{limit} || 50);
diff --git a/lib/PublicInbox/SearchView.pm b/lib/PublicInbox/SearchView.pm
index d50d3cf6..08c77f35 100644
--- a/lib/PublicInbox/SearchView.pm
+++ b/lib/PublicInbox/SearchView.pm
@@ -48,7 +48,7 @@ sub sres_top_html {
limit => $q->{l},
offset => $o,
relevance => $q->{r},
- thread => $q->{t},
+ threads => $q->{t},
asc => $asc,
};
my ($mset, $total, $err, $html);
^ permalink raw reply related [relevance 4%]
* [PATCH 00/19] lei import Maildir, remote mboxrd fixes
@ 2021-02-07 8:51 6% Eric Wong
2021-02-07 8:51 4% ` [PATCH 14/19] lei: replace --thread with --threads Eric Wong
0 siblings, 1 reply; 3+ results
From: Eric Wong @ 2021-02-07 8:51 UTC (permalink / raw)
To: meta
"lei q" with remote mboxrd + early MUA spawning is
nicer, too. Several risky constructs eliminated,
Interrupting "add-external --mirror" is less bad, now;
though it could probably support indexlevel=none in
case somebody wants to run index themselves.
Eric Wong (19):
spawn: pi_fork_exec: restore parent sigmask in child
spawn: pi_fork_exec: support "pgid"
lei add-external: handle interrupts with --mirror
spawn_pp: die more consistently in child
ipc: do not die inside wq_worker child process
ipc: trim down the Storable checks
Makefile.PL: depend on IO::Uncompress::Gunzip
xapcmd: avoid potential die surprise in children
tests: guard setup_public_inboxes for SQLite and Xapian
Revert "ipc: add support for asynchronous callbacks"
ipc: wq_do => wq_io_do
lei: more consistent IPC exit and error handling
lei: remove --mua-cmd alias for --mua
lei: replace --thread with --threads
lei q: improve remote mboxrd UX
lei q: SIGWINCH process group with the terminal
lei import: support Maildirs
imap: avoid unnecessary delete on stack
httpd/async: avoid unnecessary on-stack delete
Documentation/lei-q.pod | 4 +-
MANIFEST | 1 +
Makefile.PL | 1 +
lib/PublicInbox/HTTPD/Async.pm | 2 +-
lib/PublicInbox/IMAP.pm | 6 +-
lib/PublicInbox/IPC.pm | 105 +++++++-----------------
lib/PublicInbox/LEI.pm | 49 +++++++----
lib/PublicInbox/LeiCurl.pm | 11 ++-
lib/PublicInbox/LeiHelp.pm | 6 +-
lib/PublicInbox/LeiImport.pm | 38 ++++++---
lib/PublicInbox/LeiMirror.pm | 75 ++++++++++-------
lib/PublicInbox/LeiOverview.pm | 7 +-
lib/PublicInbox/LeiQuery.pm | 4 +-
lib/PublicInbox/LeiStore.pm | 8 +-
lib/PublicInbox/LeiToMail.pm | 37 ++++-----
lib/PublicInbox/LeiXSearch.pm | 143 ++++++++++++++++++++-------------
lib/PublicInbox/Mbox.pm | 2 +-
lib/PublicInbox/OnDestroy.pm | 2 +-
lib/PublicInbox/Search.pm | 2 +-
lib/PublicInbox/SearchView.pm | 2 +-
lib/PublicInbox/Spawn.pm | 63 +++++++++------
lib/PublicInbox/SpawnPP.pm | 44 +++++-----
lib/PublicInbox/Xapcmd.pm | 11 +--
script/lei | 8 +-
t/ipc.t | 39 ++-------
t/lei-externals.t | 2 +
t/lei-import-maildir.t | 33 ++++++++
t/lei-mirror.t | 14 ++++
t/lei.t | 2 +-
t/lei_to_mail.t | 6 +-
t/spawn.t | 18 +++++
xt/stress-sharedkv.t | 6 +-
32 files changed, 433 insertions(+), 318 deletions(-)
create mode 100644 t/lei-import-maildir.t
^ permalink raw reply [relevance 6%]
Results 1-3 of 3 | reverse | options above
-- pct% links below jump to the message on this page, permalinks otherwise --
2021-02-07 8:51 6% [PATCH 00/19] lei import Maildir, remote mboxrd fixes Eric Wong
2021-02-07 8:51 4% ` [PATCH 14/19] lei: replace --thread with --threads Eric Wong
2021-03-12 10:39 [PATCH 0/3] lei CLI option updates Eric Wong
2021-03-12 10:39 7% ` [PATCH 3/3] lei q: mbox*: disable changing parallelism, add --rsyncable Eric Wong
Code repositories for project(s) associated with this public inbox
https://80x24.org/public-inbox.git
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).