From: Eric Wong <e@80x24.org>
To: meta@public-inbox.org
Subject: [PATCH 1/9] lei: lock worker counts
Date: Sat, 18 Sep 2021 09:33:24 +0000 [thread overview]
Message-ID: <20210918093332.16054-2-e@80x24.org> (raw)
In-Reply-To: <20210918093332.16054-1-e@80x24.org>
It doesn't seem worthwhile to change worker counts dynamically
on a per-command-basis with lei, and I don't know how such an
interface would even work...
---
lib/PublicInbox/LEI.pm | 3 ++-
lib/PublicInbox/LeiExportKw.pm | 1 -
lib/PublicInbox/LeiImport.pm | 1 -
lib/PublicInbox/LeiLsMailSource.pm | 3 +--
lib/PublicInbox/LeiLsSearch.pm | 2 +-
lib/PublicInbox/LeiRefreshMailSync.pm | 4 +---
lib/PublicInbox/LeiRm.pm | 2 +-
lib/PublicInbox/LeiTag.pm | 3 +--
8 files changed, 7 insertions(+), 12 deletions(-)
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 9794497b..41e761f8 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -258,7 +258,7 @@ our %CMD = ( # sorted in order of importance/use:
@c_opt ],
'import' => [ 'LOCATION...|--stdin',
'one-time import/update from URL or filesystem',
- qw(stdin| offset=i recursive|r exclude=s include|I=s jobs=s new-only
+ qw(stdin| offset=i recursive|r exclude=s include|I=s new-only
lock=s@ in-format|F=s kw! verbose|v+ incremental! mail-sync!),
@net_opt, @c_opt ],
'forget-mail-sync' => [ 'LOCATION...',
@@ -627,6 +627,7 @@ sub workers_start {
my $end = $lei->pkt_op_pair;
my $ident = $wq->{-wq_ident} // "lei-$lei->{cmd} worker";
$flds->{lei} = $lei;
+ $wq->{-wq_nr_workers} //= $jobs; # lock, no incrementing
$wq->wq_workers_start($ident, $jobs, $lei->oldset, $flds);
delete $lei->{pkt_op_p};
my $op_c = delete $lei->{pkt_op_c};
diff --git a/lib/PublicInbox/LeiExportKw.pm b/lib/PublicInbox/LeiExportKw.pm
index d37f3768..8b8aa373 100644
--- a/lib/PublicInbox/LeiExportKw.pm
+++ b/lib/PublicInbox/LeiExportKw.pm
@@ -124,7 +124,6 @@ EOM
my $ops = {};
$sto->write_prepare($lei);
$lei->{auth}->op_merge($ops, $self) if $lei->{auth};
- $self->{-wq_nr_workers} = $j // 1; # locked
(my $op_c, $ops) = $lei->workers_start($self, $j, $ops);
$lei->{wq1} = $self;
$lei->{-err_type} = 'non-fatal';
diff --git a/lib/PublicInbox/LeiImport.pm b/lib/PublicInbox/LeiImport.pm
index 7c563bd8..b1cb3940 100644
--- a/lib/PublicInbox/LeiImport.pm
+++ b/lib/PublicInbox/LeiImport.pm
@@ -101,7 +101,6 @@ sub do_import_index ($$@) {
}
my $ops = {};
$lei->{auth}->op_merge($ops, $self) if $lei->{auth};
- $self->{-wq_nr_workers} = $j // 1; # locked
$lei->{-eml_noisy} = 1;
(my $op_c, $ops) = $lei->workers_start($self, $j, $ops);
$lei->{wq1} = $self;
diff --git a/lib/PublicInbox/LeiLsMailSource.pm b/lib/PublicInbox/LeiLsMailSource.pm
index f012e10e..bcb1838e 100644
--- a/lib/PublicInbox/LeiLsMailSource.pm
+++ b/lib/PublicInbox/LeiLsMailSource.pm
@@ -96,8 +96,7 @@ sub lei_ls_mail_source {
$lei->start_pager if -t $lei->{1};
my $ops = {};
$lei->{auth}->op_merge($ops, $self);
- my $j = $self->{-wq_nr_workers} = 1; # locked
- (my $op_c, $ops) = $lei->workers_start($self, $j, $ops);
+ (my $op_c, $ops) = $lei->workers_start($self, 1, $ops);
$lei->{wq1} = $self;
$lei->{-err_type} = 'non-fatal';
net_merge_all_done($self) unless $lei->{auth};
diff --git a/lib/PublicInbox/LeiLsSearch.pm b/lib/PublicInbox/LeiLsSearch.pm
index 70136135..aebf0184 100644
--- a/lib/PublicInbox/LeiLsSearch.pm
+++ b/lib/PublicInbox/LeiLsSearch.pm
@@ -71,7 +71,7 @@ sub do_ls_search_long {
sub bg_worker ($$$) {
my ($lei, $pfx, $json) = @_;
- my $self = bless { -wq_nr_workers => 1, json => $json }, __PACKAGE__;
+ my $self = bless { json => $json }, __PACKAGE__;
my ($op_c, $ops) = $lei->workers_start($self, 1);
$lei->{wq1} = $self;
$self->wq_io_do('do_ls_search_long', [], $pfx);
diff --git a/lib/PublicInbox/LeiRefreshMailSync.pm b/lib/PublicInbox/LeiRefreshMailSync.pm
index 09a7ead0..cdd99725 100644
--- a/lib/PublicInbox/LeiRefreshMailSync.pm
+++ b/lib/PublicInbox/LeiRefreshMailSync.pm
@@ -84,11 +84,9 @@ EOM
my $self = bless { missing_ok => 1 }, __PACKAGE__;
$lei->{opt}->{'mail-sync'} = 1; # for prepare_inputs
$self->prepare_inputs($lei, \@folders) or return;
- my $j = $lei->{opt}->{jobs} || scalar(@{$self->{inputs}}) || 1;
my $ops = {};
$lei->{auth}->op_merge($ops, $self) if $lei->{auth};
- $self->{-wq_nr_workers} = $j // 1; # locked
- (my $op_c, $ops) = $lei->workers_start($self, $j, $ops);
+ (my $op_c, $ops) = $lei->workers_start($self, 1, $ops);
$lei->{wq1} = $self;
$lei->{-err_type} = 'non-fatal';
net_merge_all_done($self) unless $lei->{auth};
diff --git a/lib/PublicInbox/LeiRm.pm b/lib/PublicInbox/LeiRm.pm
index 778fa1de..3371f3ed 100644
--- a/lib/PublicInbox/LeiRm.pm
+++ b/lib/PublicInbox/LeiRm.pm
@@ -32,7 +32,7 @@ sub lei_rm {
my ($lei, @inputs) = @_;
$lei->_lei_store(1)->write_prepare($lei);
$lei->{opt}->{'in-format'} //= 'eml';
- my $self = bless { -wq_nr_workers => 1 }, __PACKAGE__;
+ my $self = bless {}, __PACKAGE__;
$self->prepare_inputs($lei, \@inputs) or return;
my ($op_c, $ops) = $lei->workers_start($self, 1);
$lei->{wq1} = $self;
diff --git a/lib/PublicInbox/LeiTag.pm b/lib/PublicInbox/LeiTag.pm
index 44d77b88..c4f5ecff 100644
--- a/lib/PublicInbox/LeiTag.pm
+++ b/lib/PublicInbox/LeiTag.pm
@@ -50,8 +50,7 @@ sub lei_tag { # the "lei tag" method
return $lei->fail('no keywords or labels specified');
my $ops = {};
$lei->{auth}->op_merge($ops, $self) if $lei->{auth};
- my $j = $self->{-wq_nr_workers} = 1; # locked for now
- (my $op_c, $ops) = $lei->workers_start($self, $j, $ops);
+ (my $op_c, $ops) = $lei->workers_start($self, 1, $ops);
$lei->{wq1} = $self;
$lei->{-err_type} = 'non-fatal';
net_merge_all_done($self) unless $lei->{auth};
next prev parent reply other threads:[~2021-09-18 9:33 UTC|newest]
Thread overview: 13+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-09-18 9:33 [PATCH 0/9] lei: a bunch of random stuff Eric Wong
2021-09-18 9:33 ` Eric Wong [this message]
2021-09-18 9:33 ` [PATCH 2/9] lei_mail_sync: rely on flock(2), avoid IPC Eric Wong
2021-09-18 9:33 ` [PATCH 3/9] lei_mail_sync: set nodatacow on btrfs Eric Wong
2021-09-18 9:33 ` [PATCH 4/9] ds: support add unique timers Eric Wong
2021-09-18 9:33 ` [PATCH 5/9] net_reader: tie SocksDebug to {imap,nntp}.Debug Eric Wong
2021-09-18 9:33 ` [PATCH 6/9] net_reader: detect IMAP failures earlier Eric Wong
2021-09-18 9:33 ` [PATCH 7/9] net_reader: support imaps:// w/ socks5h:// proxy Eric Wong
2021-09-18 10:41 ` Eric Wong
2021-09-18 9:33 ` [PATCH 8/9] net_reader: set SO_KEEPALIVE on all Net::NNTP sockets Eric Wong
2021-09-18 9:33 ` [PATCH 9/9] lei up: automatically use dt: for remote externals Eric Wong
2021-09-18 14:31 ` Kyle Meyer
2021-09-18 20:30 ` Eric Wong
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
List information: https://public-inbox.org/README
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20210918093332.16054-2-e@80x24.org \
--to=e@80x24.org \
--cc=meta@public-inbox.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
Code repositories for project(s) associated with this public inbox
https://80x24.org/public-inbox.git
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).