From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on dcvr.yhbt.net X-Spam-Level: X-Spam-ASN: X-Spam-Status: No, score=-3.8 required=3.0 tests=ALL_TRUSTED,AWL,BAYES_00, DKIM_SIGNED,DKIM_VALID,DKIM_VALID_AU,DKIM_VALID_EF shortcircuit=no autolearn=ham autolearn_force=no version=3.4.2 Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id 2AE761FAF7 for ; Mon, 28 Nov 2022 05:32:34 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=80x24.org; s=selector1; t=1669613554; bh=7IpSixteS4oC6mT79yTNKio8yMgDVtBngBbBuEO82XQ=; h=From:To:Subject:Date:In-Reply-To:References:From; b=enXABphhqS7CK/fy3KZT/1TMBlvvb+Xnhn0VOMDp/w1/FTCujgCEz6tub82qe9wiU 6oSzFppYGABHMwwak46MLiOS2FsuqmrFwPCksmkVkhv8HeBbhCILM3LFGZfybH0LF1 sMgmeemgR4WNKyShCQvVAuRAw8QZgas+JpF/+MAs= From: Eric Wong To: meta@public-inbox.org Subject: [PATCH 08/95] lei_mirror: default to single job by default Date: Mon, 28 Nov 2022 05:31:05 +0000 Message-Id: <20221128053232.291618-9-e@80x24.org> In-Reply-To: <20221128053232.291618-1-e@80x24.org> References: <20221128053232.291618-1-e@80x24.org> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit List-Id: Parallel git clones are expensive on the server-side, and smaller machines (which we encourage) can't handle them, well. We'll also set `-q' since parallel clones will have output step all over each other. --- lib/PublicInbox/LeiMirror.pm | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/lib/PublicInbox/LeiMirror.pm b/lib/PublicInbox/LeiMirror.pm index a0b197a7..285c64d8 100644 --- a/lib/PublicInbox/LeiMirror.pm +++ b/lib/PublicInbox/LeiMirror.pm @@ -87,7 +87,8 @@ sub clone_cmd { # e.g.: git -c http.proxy=socks5h://127.0.0.1:9050 push(@cmd, '-c', $_) for @{$lei->{opt}->{c} // []}; push @cmd, qw(clone --mirror); - push @cmd, '-q' if $lei->{opt}->{quiet}; + push @cmd, '-q' if $lei->{opt}->{quiet} || + ($lei->{opt}->{jobs} // 1) > 1; push @cmd, '-v' if $lei->{opt}->{verbose}; # XXX any other options to support? # --reference is tricky with multiple epochs... @@ -236,7 +237,7 @@ sub clone_v1 { my $pfx = $curl->torsocks($lei, $uri) or return; my $dst = $self->{cur_dst} // $self->{dst}; my $fini = PublicInbox::OnDestroy->new($$, \&v1_done, $self); - my $jobs = $self->{lei}->{opt}->{jobs} // 2; + my $jobs = $self->{lei}->{opt}->{jobs} // 1; my $cmd = [ @$pfx, clone_cmd($lei, my $opt = {}), "$uri", $dst ]; $lei->qerr("# @$cmd"); $LIVE{spawn($cmd, undef, $opt)} = [ \&reap_clone, $lei, $cmd, $fini ]; @@ -385,7 +386,7 @@ failed to extract epoch number from $src $LIVE{_try_config_start($task)} = [ \&_try_config_done, $task, $fini ]; $task->{-locked} = $lk->lock_for_scope($$); my @cmd = clone_cmd($lei, my $opt = {}); - my $jobs = $self->{lei}->{opt}->{jobs} // 2; + my $jobs = $self->{lei}->{opt}->{jobs} // 1; do { reap_live() while keys(%LIVE) >= $jobs; while (keys(%LIVE) < $jobs && @src_edst && @@ -498,7 +499,7 @@ sub try_manifest { } my ($path_pfx, $n, $multi) = multi_inbox($self, \$path, $m); return $lei->child_error(1, $multi) if !ref($multi); - my $jobs = $self->{lei}->{opt}->{jobs} // 2; + my $jobs = $self->{lei}->{opt}->{jobs} // 1; if (my $v2 = delete $multi->{v2}) { for my $name (sort keys %$v2) { my $epochs = delete $v2->{$name};