From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on dcvr.yhbt.net X-Spam-Level: X-Spam-ASN: X-Spam-Status: No, score=-4.1 required=3.0 tests=ALL_TRUSTED,AWL,BAYES_00, DKIM_SIGNED,DKIM_VALID,DKIM_VALID_AU,DKIM_VALID_EF shortcircuit=no autolearn=ham autolearn_force=no version=3.4.2 Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id 64962203BC for ; Mon, 28 Nov 2022 05:32:49 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=80x24.org; s=selector1; t=1669613569; bh=8OZqZ8zCNoIvQsTTj9Yg//009GIC3dAiHDSHaqRO7Gg=; h=From:To:Subject:Date:In-Reply-To:References:From; b=v0oOBTjwYugf1ipWTEFlh4fj+CVvbaN4ouQPvTYLfkKEIDpXDbQ7qArK5/Py8PIxV p5rLbZm6VPsAQrOVpdxTvvVoLE0eQA1ExqdpHIU0dGoqx4cTd5eD5CKl5BR0CysFt3 iZblG52NHxLASbqjtn7iL0VnFw295ixhxjXOngOM= From: Eric Wong To: meta@public-inbox.org Subject: [PATCH 78/95] lei_mirror: --manifest= affects destination, too Date: Mon, 28 Nov 2022 05:32:15 +0000 Message-Id: <20221128053232.291618-79-e@80x24.org> In-Reply-To: <20221128053232.291618-1-e@80x24.org> References: <20221128053232.291618-1-e@80x24.org> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit List-Id: This probably makes the most sense, if a user wants to use an alternate path to read from, it's likely they want to write it there, too. --- Documentation/public-inbox-clone.pod | 4 ++++ lib/PublicInbox/LeiMirror.pm | 16 +++++++++++----- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/Documentation/public-inbox-clone.pod b/Documentation/public-inbox-clone.pod index 9bcb9967..2a7081ac 100644 --- a/Documentation/public-inbox-clone.pod +++ b/Documentation/public-inbox-clone.pod @@ -99,6 +99,10 @@ is specified where C is an empty string (C<"">), then C When incrementally updating an existing mirror, load the given manifest (typically C) to speed up updates. +By default, public-inbox writes the retrieved manifest to +C<$DESTINATION/manifest.js.gz>, this directive also +changes the destination to the specified C + If C does not start with C, C<./>, or C<../>, it is treated as relative to the C directory. If only C<--manifest=> is specified where C is an empty string (C<"">), then C diff --git a/lib/PublicInbox/LeiMirror.pm b/lib/PublicInbox/LeiMirror.pm index ec2ed557..2e94a2fa 100644 --- a/lib/PublicInbox/LeiMirror.pm +++ b/lib/PublicInbox/LeiMirror.pm @@ -826,7 +826,7 @@ sub load_current_manifest ($) { if (open(my $fh, '<', $fn)) { decode_manifest($fh, $fn, $fn); } elsif ($!{ENOENT}) { # non-fatal, we can just do it slowly - warn "open($fn): $!\n" if -d $self->{dst}; + warn "open($fn): $!\n" if !$self->{-initial_clone}; undef; } else { die "open($fn): $!\n"; @@ -955,10 +955,15 @@ sub try_manifest { my $path = $uri->path; chop($path) eq '/' or die "BUG: $uri not canonicalized"; $uri->path($path . '/manifest.js.gz'); - my $ft = File::Temp->new(TEMPLATE => '.manifest-XXXX', - UNLINK => 1, TMPDIR => 1, SUFFIX => '.tmp'); + my $manifest = $self->{-manifest} // "$self->{dst}/manifest.js.gz"; + my %opt = (UNLINK => 1, SUFFIX => '.tmp', TMPDIR => 1); + if (!$self->{dry_run} && $manifest =~ m!\A(.+?)/[^/]+\z! and -d $1) { + $opt{DIR} = $1; # allows fast rename(2) w/o EXDEV + delete $opt{TMPDIR}; + } + my $ft = File::Temp->new(TEMPLATE => '.manifest-XXXX', %opt); my $cmd = $curl->for_uri($lei, $uri, qw(-f -R -o), $ft->filename); - my %opt = map { $_ => $lei->{$_} } (0..2); + %opt = map { $_ => $lei->{$_} } (0..2); my $cerr = run_reap($lei, $cmd, \%opt); if ($cerr) { return try_scrape($self) if ($cerr >> 8) == 22; # 404 missing @@ -1030,7 +1035,7 @@ EOM # set by clone_v2_prep/-I/--exclude dump_manifest($m => $ft) if delete $self->{-culled_manifest}; - ft_rename($ft, "$self->{dst}/manifest.js.gz", 0666); + ft_rename($ft, $manifest, 0666); open my $x, '>', "$self->{dst}/mirror.done"; # for _wq_done_wait } @@ -1045,6 +1050,7 @@ sub do_mirror { # via wq_io_do or public-inbox-clone my $lei = $self->{lei}; $self->{dry_run} = 1 if $lei->{opt}->{'dry-run'}; umask($lei->{client_umask}) if defined $lei->{client_umask}; + $self->{-initial_clone} = 1 if !-d $self->{dst}; eval { my $ic = $lei->{opt}->{'inbox-config'} //= 'always'; $ic =~ /\A(?:v1|v2|always|never)\z/s or die <<"";