user/dev discussion of public-inbox itself
 help / color / mirror / code / Atom feed
From: Eric Wong <e@80x24.org>
To: meta@public-inbox.org
Subject: [PATCH 1/9] lei: lock worker counts
Date: Sat, 18 Sep 2021 09:33:24 +0000	[thread overview]
Message-ID: <20210918093332.16054-2-e@80x24.org> (raw)
In-Reply-To: <20210918093332.16054-1-e@80x24.org>

It doesn't seem worthwhile to change worker counts dynamically
on a per-command-basis with lei, and I don't know how such an
interface would even work...
---
 lib/PublicInbox/LEI.pm                | 3 ++-
 lib/PublicInbox/LeiExportKw.pm        | 1 -
 lib/PublicInbox/LeiImport.pm          | 1 -
 lib/PublicInbox/LeiLsMailSource.pm    | 3 +--
 lib/PublicInbox/LeiLsSearch.pm        | 2 +-
 lib/PublicInbox/LeiRefreshMailSync.pm | 4 +---
 lib/PublicInbox/LeiRm.pm              | 2 +-
 lib/PublicInbox/LeiTag.pm             | 3 +--
 8 files changed, 7 insertions(+), 12 deletions(-)

diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index 9794497b..41e761f8 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -258,7 +258,7 @@ our %CMD = ( # sorted in order of importance/use:
 	 @c_opt ],
 'import' => [ 'LOCATION...|--stdin',
 	'one-time import/update from URL or filesystem',
-	qw(stdin| offset=i recursive|r exclude=s include|I=s jobs=s new-only
+	qw(stdin| offset=i recursive|r exclude=s include|I=s new-only
 	lock=s@ in-format|F=s kw! verbose|v+ incremental! mail-sync!),
 	@net_opt, @c_opt ],
 'forget-mail-sync' => [ 'LOCATION...',
@@ -627,6 +627,7 @@ sub workers_start {
 	my $end = $lei->pkt_op_pair;
 	my $ident = $wq->{-wq_ident} // "lei-$lei->{cmd} worker";
 	$flds->{lei} = $lei;
+	$wq->{-wq_nr_workers} //= $jobs; # lock, no incrementing
 	$wq->wq_workers_start($ident, $jobs, $lei->oldset, $flds);
 	delete $lei->{pkt_op_p};
 	my $op_c = delete $lei->{pkt_op_c};
diff --git a/lib/PublicInbox/LeiExportKw.pm b/lib/PublicInbox/LeiExportKw.pm
index d37f3768..8b8aa373 100644
--- a/lib/PublicInbox/LeiExportKw.pm
+++ b/lib/PublicInbox/LeiExportKw.pm
@@ -124,7 +124,6 @@ EOM
 	my $ops = {};
 	$sto->write_prepare($lei);
 	$lei->{auth}->op_merge($ops, $self) if $lei->{auth};
-	$self->{-wq_nr_workers} = $j // 1; # locked
 	(my $op_c, $ops) = $lei->workers_start($self, $j, $ops);
 	$lei->{wq1} = $self;
 	$lei->{-err_type} = 'non-fatal';
diff --git a/lib/PublicInbox/LeiImport.pm b/lib/PublicInbox/LeiImport.pm
index 7c563bd8..b1cb3940 100644
--- a/lib/PublicInbox/LeiImport.pm
+++ b/lib/PublicInbox/LeiImport.pm
@@ -101,7 +101,6 @@ sub do_import_index ($$@) {
 	}
 	my $ops = {};
 	$lei->{auth}->op_merge($ops, $self) if $lei->{auth};
-	$self->{-wq_nr_workers} = $j // 1; # locked
 	$lei->{-eml_noisy} = 1;
 	(my $op_c, $ops) = $lei->workers_start($self, $j, $ops);
 	$lei->{wq1} = $self;
diff --git a/lib/PublicInbox/LeiLsMailSource.pm b/lib/PublicInbox/LeiLsMailSource.pm
index f012e10e..bcb1838e 100644
--- a/lib/PublicInbox/LeiLsMailSource.pm
+++ b/lib/PublicInbox/LeiLsMailSource.pm
@@ -96,8 +96,7 @@ sub lei_ls_mail_source {
 	$lei->start_pager if -t $lei->{1};
 	my $ops = {};
 	$lei->{auth}->op_merge($ops, $self);
-	my $j = $self->{-wq_nr_workers} = 1; # locked
-	(my $op_c, $ops) = $lei->workers_start($self, $j, $ops);
+	(my $op_c, $ops) = $lei->workers_start($self, 1, $ops);
 	$lei->{wq1} = $self;
 	$lei->{-err_type} = 'non-fatal';
 	net_merge_all_done($self) unless $lei->{auth};
diff --git a/lib/PublicInbox/LeiLsSearch.pm b/lib/PublicInbox/LeiLsSearch.pm
index 70136135..aebf0184 100644
--- a/lib/PublicInbox/LeiLsSearch.pm
+++ b/lib/PublicInbox/LeiLsSearch.pm
@@ -71,7 +71,7 @@ sub do_ls_search_long {
 
 sub bg_worker ($$$) {
 	my ($lei, $pfx, $json) = @_;
-	my $self = bless { -wq_nr_workers => 1, json => $json }, __PACKAGE__;
+	my $self = bless { json => $json }, __PACKAGE__;
 	my ($op_c, $ops) = $lei->workers_start($self, 1);
 	$lei->{wq1} = $self;
 	$self->wq_io_do('do_ls_search_long', [], $pfx);
diff --git a/lib/PublicInbox/LeiRefreshMailSync.pm b/lib/PublicInbox/LeiRefreshMailSync.pm
index 09a7ead0..cdd99725 100644
--- a/lib/PublicInbox/LeiRefreshMailSync.pm
+++ b/lib/PublicInbox/LeiRefreshMailSync.pm
@@ -84,11 +84,9 @@ EOM
 	my $self = bless { missing_ok => 1 }, __PACKAGE__;
 	$lei->{opt}->{'mail-sync'} = 1; # for prepare_inputs
 	$self->prepare_inputs($lei, \@folders) or return;
-	my $j = $lei->{opt}->{jobs} || scalar(@{$self->{inputs}}) || 1;
 	my $ops = {};
 	$lei->{auth}->op_merge($ops, $self) if $lei->{auth};
-	$self->{-wq_nr_workers} = $j // 1; # locked
-	(my $op_c, $ops) = $lei->workers_start($self, $j, $ops);
+	(my $op_c, $ops) = $lei->workers_start($self, 1, $ops);
 	$lei->{wq1} = $self;
 	$lei->{-err_type} = 'non-fatal';
 	net_merge_all_done($self) unless $lei->{auth};
diff --git a/lib/PublicInbox/LeiRm.pm b/lib/PublicInbox/LeiRm.pm
index 778fa1de..3371f3ed 100644
--- a/lib/PublicInbox/LeiRm.pm
+++ b/lib/PublicInbox/LeiRm.pm
@@ -32,7 +32,7 @@ sub lei_rm {
 	my ($lei, @inputs) = @_;
 	$lei->_lei_store(1)->write_prepare($lei);
 	$lei->{opt}->{'in-format'} //= 'eml';
-	my $self = bless { -wq_nr_workers => 1 }, __PACKAGE__;
+	my $self = bless {}, __PACKAGE__;
 	$self->prepare_inputs($lei, \@inputs) or return;
 	my ($op_c, $ops) = $lei->workers_start($self, 1);
 	$lei->{wq1} = $self;
diff --git a/lib/PublicInbox/LeiTag.pm b/lib/PublicInbox/LeiTag.pm
index 44d77b88..c4f5ecff 100644
--- a/lib/PublicInbox/LeiTag.pm
+++ b/lib/PublicInbox/LeiTag.pm
@@ -50,8 +50,7 @@ sub lei_tag { # the "lei tag" method
 		return $lei->fail('no keywords or labels specified');
 	my $ops = {};
 	$lei->{auth}->op_merge($ops, $self) if $lei->{auth};
-	my $j = $self->{-wq_nr_workers} = 1; # locked for now
-	(my $op_c, $ops) = $lei->workers_start($self, $j, $ops);
+	(my $op_c, $ops) = $lei->workers_start($self, 1, $ops);
 	$lei->{wq1} = $self;
 	$lei->{-err_type} = 'non-fatal';
 	net_merge_all_done($self) unless $lei->{auth};

  reply	other threads:[~2021-09-18  9:33 UTC|newest]

Thread overview: 13+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2021-09-18  9:33 [PATCH 0/9] lei: a bunch of random stuff Eric Wong
2021-09-18  9:33 ` Eric Wong [this message]
2021-09-18  9:33 ` [PATCH 2/9] lei_mail_sync: rely on flock(2), avoid IPC Eric Wong
2021-09-18  9:33 ` [PATCH 3/9] lei_mail_sync: set nodatacow on btrfs Eric Wong
2021-09-18  9:33 ` [PATCH 4/9] ds: support add unique timers Eric Wong
2021-09-18  9:33 ` [PATCH 5/9] net_reader: tie SocksDebug to {imap,nntp}.Debug Eric Wong
2021-09-18  9:33 ` [PATCH 6/9] net_reader: detect IMAP failures earlier Eric Wong
2021-09-18  9:33 ` [PATCH 7/9] net_reader: support imaps:// w/ socks5h:// proxy Eric Wong
2021-09-18 10:41   ` Eric Wong
2021-09-18  9:33 ` [PATCH 8/9] net_reader: set SO_KEEPALIVE on all Net::NNTP sockets Eric Wong
2021-09-18  9:33 ` [PATCH 9/9] lei up: automatically use dt: for remote externals Eric Wong
2021-09-18 14:31   ` Kyle Meyer
2021-09-18 20:30     ` Eric Wong

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

  List information: https://public-inbox.org/README

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20210918093332.16054-2-e@80x24.org \
    --to=e@80x24.org \
    --cc=meta@public-inbox.org \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
Code repositories for project(s) associated with this public inbox

	https://80x24.org/public-inbox.git

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).