From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on dcvr.yhbt.net X-Spam-Level: X-Spam-ASN: X-Spam-Status: No, score=-3.8 required=3.0 tests=ALL_TRUSTED,AWL,BAYES_00, DKIM_SIGNED,DKIM_VALID,DKIM_VALID_AU,DKIM_VALID_EF shortcircuit=no autolearn=ham autolearn_force=no version=3.4.2 Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id 94CBE1FAF9 for ; Mon, 28 Nov 2022 05:32:34 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=80x24.org; s=selector1; t=1669613554; bh=WDhGSSNoaEo6KFpIXiBvcDcv3CTO62nSKMcxWArxQg0=; h=From:To:Subject:Date:In-Reply-To:References:From; b=fJ4oA//N+MnEp51iaa8pbA0Roee426CuDOkMS9OlTL9uQ357ckGdv55K3FgEbS7bv eOSdPf1hJPGbbkB5abUQI7A0hMNIPicbt7ZlA+6DHJOJ6F4WGflm03kY8WS0OsUm1U ssUcYVEgOw/HfJdhjYcolTdpLjDT9R9dKkuRg8ys= From: Eric Wong To: meta@public-inbox.org Subject: [PATCH 10/95] lei_mirror: retrieve description text asynchronously, too Date: Mon, 28 Nov 2022 05:31:07 +0000 Message-Id: <20221128053232.291618-11-e@80x24.org> In-Reply-To: <20221128053232.291618-1-e@80x24.org> References: <20221128053232.291618-1-e@80x24.org> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit List-Id: We can easily parallelize this, so do it. --- lib/PublicInbox/LeiMirror.pm | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/lib/PublicInbox/LeiMirror.pm b/lib/PublicInbox/LeiMirror.pm index 6d72f15d..0696c2d9 100644 --- a/lib/PublicInbox/LeiMirror.pm +++ b/lib/PublicInbox/LeiMirror.pm @@ -122,13 +122,13 @@ sub _get_txt_start { # non-fatal spawn($cmd, undef, $opt); } -sub _get_txt_done { +sub _get_txt_done { # returns true on error (non-fatal), undef on success my ($self) = @_; my ($ft, $cmd, $uri, $file, $mode) = @{delete $self->{-get_txt}}; my $cerr = $?; $? = 0; - return "$uri missing" if ($cerr >> 8) == 22; - return "# @$cmd failed (non-fatal)" if $cerr; + return warn("$uri missing\n") if ($cerr >> 8) == 22; + return warn("# @$cmd failed (non-fatal)\n") if $cerr; my $dst = $self->{cur_dst} // $self->{dst}; ft_rename($ft, "$dst/$file", $mode); undef; # success @@ -142,8 +142,7 @@ sub _try_config_start { sub _try_config_done { my ($self) = @_; - my $err = _get_txt_done($self); - return warn($err, "\n") if $err; + _get_txt_done($self) and return; my $dst = $self->{cur_dst} // $self->{dst}; my $f = "$dst/inbox.config.example"; my $cfg = PublicInbox::Config->git_config_dump($f, $self->{lei}->{2}); @@ -155,11 +154,6 @@ sub _try_config_done { } } -sub _get_txt { # non-fatal temporary compat function - waitpid(_get_txt_start(@_), 0) > 0 or die "waitpid: $!"; - _get_txt_done($_[0]); -} - sub set_description ($) { my ($self) = @_; my $dst = $self->{cur_dst} // $self->{dst}; @@ -180,8 +174,6 @@ sub set_description ($) { sub index_cloned_inbox { my ($self, $iv) = @_; my $lei = $self->{lei}; - my $err = _get_txt($self, qw(description description), 0666); - warn($err, "\n") if $err; # non fatal eval { set_description($self) }; warn $@ if $@; @@ -240,6 +232,9 @@ sub clone_v1 { # wait for `git clone' to mkdir $dst (TODO: inotify/kevent?) select(undef, undef, undef, 0.011) until -d $dst; $LIVE{_try_config_start($self)} = [ \&_try_config_done, $self, $fini ]; + reap_live() while keys(%LIVE) >= $jobs; + $LIVE{_get_txt_start($self, qw(description description), 0666)} = + [ \&_get_txt_done, $self, $fini ]; reap_live() until ($nohang || !keys(%LIVE)); } @@ -383,10 +378,13 @@ failed to extract epoch number from $src } my $lk = bless { lock_path => "$dst/inbox.lock" }, 'PublicInbox::Lock'; my $fini = PublicInbox::OnDestroy->new($$, \&v2_done, $task); + my $jobs = $self->{lei}->{opt}->{jobs} // 1; $LIVE{_try_config_start($task)} = [ \&_try_config_done, $task, $fini ]; + reap_live() while keys(%LIVE) >= $jobs; + $LIVE{_get_txt_start($self, qw(description description), 0666)} = + [ \&_get_txt_done, $self, $fini ]; $task->{-locked} = $lk->lock_for_scope($$); my @cmd = clone_cmd($lei, my $opt = {}); - my $jobs = $self->{lei}->{opt}->{jobs} // 1; do { reap_live() while keys(%LIVE) >= $jobs; while (keys(%LIVE) < $jobs && @src_edst &&