user/dev discussion of public-inbox itself
 help / color / mirror / code / Atom feed
From: Eric Wong <e@80x24.org>
To: meta@public-inbox.org
Subject: [PATCH 13/17] xapcmd: replace Xtmpdirs with File::Temp->newdir
Date: Sun, 24 Nov 2019 00:22:33 +0000	[thread overview]
Message-ID: <20191124002237.15713-14-e@80x24.org> (raw)
In-Reply-To: <20191124002237.15713-1-e@80x24.org>

Since we're using Perl 5.10.1 and File::Temp 0.19+, we don't
need Xtmpdirs at all for cleaning up tempdirs on failure and
can just rely on the DESTROY handler provided by File::Temp.
---
 lib/PublicInbox/Xapcmd.pm | 73 +++++++++++++++------------------------
 1 file changed, 27 insertions(+), 46 deletions(-)

diff --git a/lib/PublicInbox/Xapcmd.pm b/lib/PublicInbox/Xapcmd.pm
index 28285898..78b3a9ac 100644
--- a/lib/PublicInbox/Xapcmd.pm
+++ b/lib/PublicInbox/Xapcmd.pm
@@ -6,7 +6,7 @@ use warnings;
 use PublicInbox::Spawn qw(which spawn);
 use PublicInbox::Over;
 use PublicInbox::Search;
-use File::Temp qw(tempdir);
+use File::Temp ();
 use File::Path qw(remove_tree);
 use File::Basename qw(dirname);
 use POSIX ();
@@ -24,13 +24,14 @@ sub commit_changes ($$$$) {
 	$SIG{INT} or die 'BUG: $SIG{INT} not handled';
 	my @old_shard;
 
-	while (my ($old, $new) = each %$tmp) {
+	while (my ($old, $newdir) = each %$tmp) {
 		next if $old eq ''; # no invalid paths
 		my @st = stat($old);
 		if (!@st && !defined($opt->{reshard})) {
 			die "failed to stat($old): $!";
 		}
 
+		my $new = $newdir->dirname if defined($newdir);
 		my $over = "$old/over.sqlite3";
 		if (-f $over) { # only for v1, v2 over is untouched
 			defined $new or die "BUG: $over exists when culling v2";
@@ -145,6 +146,12 @@ sub process_queue {
 	}
 }
 
+sub setup_signals () {
+	# http://www.tldp.org/LDP/abs/html/exitcodes.html
+	$SIG{INT} = sub { exit(130) };
+	$SIG{HUP} = $SIG{PIPE} = $SIG{TERM} = sub { exit(1) };
+}
+
 sub run {
 	my ($ibx, $task, $opt) = @_; # task = 'cpdb' or 'compact'
 	my $cb = \&${\"PublicInbox::Xapcmd::$task"};
@@ -164,7 +171,7 @@ sub run {
 	my $old = $ibx->search->xdir(1);
 	-d $old or die "$old does not exist\n";
 
-	my $tmp = PublicInbox::Xtmpdirs->new;
+	my $tmp = {};
 	my $v = $ibx->{version} ||= 1;
 	my @q;
 	my $reshard = $opt->{reshard};
@@ -173,7 +180,7 @@ sub run {
 	}
 
 	local %SIG = %SIG;
-	$tmp->setup_signals;
+	setup_signals();
 
 	# we want temporary directories to be as deep as possible,
 	# so v2 shards can keep "xap$SCHEMA_VERSION" on a separate FS.
@@ -182,10 +189,10 @@ sub run {
 			warn
 "--reshard=$reshard ignored for v1 $ibx->{inboxdir}\n";
 		}
-		my $old_parent = dirname($old);
-		same_fs_or_die($old_parent, $old);
+		my $dir = dirname($old);
+		same_fs_or_die($dir, $old);
 		my $v = PublicInbox::Search::SCHEMA_VERSION();
-		my $wip = tempdir("xapian$v-XXXXXXXX", DIR => $old_parent);
+		my $wip = File::Temp->newdir("xapian$v-XXXXXXXX", DIR => $dir);
 		$tmp->{$old} = $wip;
 		push @q, [ $old, $wip ];
 	} else {
@@ -213,8 +220,8 @@ sub run {
 		}
 		foreach my $dn (0..$max_shard) {
 			my $tmpl = "$dn-XXXXXXXX";
-			my $wip = tempdir($tmpl, DIR => $old);
-			same_fs_or_die($old, $wip);
+			my $wip = File::Temp->newdir($tmpl, DIR => $old);
+			same_fs_or_die($old, $wip->dirname);
 			my $cur = "$old/$dn";
 			push @q, [ $src // $cur , $wip ];
 			$tmp->{$cur} = $wip;
@@ -267,7 +274,8 @@ sub progress_pfx ($) {
 # xapian-compact wrapper
 sub compact ($$) {
 	my ($args, $opt) = @_;
-	my ($src, $dst) = @$args;
+	my ($src, $newdir) = @$args;
+	my $dst = ref($newdir) ? $newdir->dirname : $newdir;
 	my ($r, $w);
 	my $pfx = $opt->{-progress_pfx} ||= progress_pfx($src);
 	my $pr = $opt->{-progress};
@@ -349,7 +357,8 @@ sub cpdb_loop ($$$;$$) {
 # to the overhead of Perl.
 sub cpdb ($$) {
 	my ($args, $opt) = @_;
-	my ($old, $new) = @$args;
+	my ($old, $newdir) = @$args;
+	my $new = $newdir->dirname;
 	my ($src, $cur_shard);
 	my $reshard;
 	if (ref($old) eq 'ARRAY') {
@@ -372,15 +381,14 @@ sub cpdb ($$) {
 		$src = Search::Xapian::Database->new($old);
 	}
 
-	my ($xtmp, $tmp);
+	my ($tmp, $ft);
 	local %SIG = %SIG;
 	if ($opt->{compact}) {
-		my $newdir = dirname($new);
-		same_fs_or_die($newdir, $new);
-		$tmp = tempdir("$new.compact-XXXXXX", DIR => $newdir);
-		$xtmp = PublicInbox::Xtmpdirs->new;
-		$xtmp->setup_signals;
-		$xtmp->{$new} = $tmp;
+		my $dir = dirname($new);
+		same_fs_or_die($dir, $new);
+		$ft = File::Temp->newdir("$new.compact-XXXXXX", DIR => $dir);
+		setup_signals();
+		$tmp = $ft->dirname;
 	} else {
 		$tmp = $new;
 	}
@@ -439,7 +447,7 @@ sub cpdb ($$) {
 	}
 
 	$pr->(sprintf($pr_data->{fmt}, $pr_data->{nr})) if $pr;
-	return unless $xtmp;
+	return unless $opt->{compact};
 
 	$src = $dst = undef; # flushes and closes
 
@@ -447,33 +455,6 @@ sub cpdb ($$) {
 	# since $dst isn't readable by HTTP or NNTP clients, yet:
 	compact([ $tmp, $new ], $opt);
 	remove_tree($tmp) or die "failed to remove $tmp: $!\n";
-	$xtmp = undef;
-}
-
-# slightly easier-to-manage manage than END{} blocks
-package PublicInbox::Xtmpdirs;
-use strict;
-use warnings;
-use File::Path qw(remove_tree);
-
-sub setup_signals () {
-	# http://www.tldp.org/LDP/abs/html/exitcodes.html
-	$SIG{INT} = sub { exit(130) };
-	$SIG{HUP} = $SIG{PIPE} = $SIG{TERM} = sub { exit(1) };
-}
-
-sub new {
-	bless { '' => $$ }, $_[0]; # old shard => new (WIP) shard
-}
-
-sub DESTROY {
-	my ($self) = @_;
-	my $owner_pid = delete($self->{''}) or return;
-	return if $owner_pid != $$;
-	foreach my $new (values %$self) {
-		defined $new or next; # may be undef if resharding
-		remove_tree($new) unless -d "$new/old";
-	}
 }
 
 1;

  parent reply	other threads:[~2019-11-24  0:22 UTC|newest]

Thread overview: 54+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2019-11-15  9:50 [PATCH 00/29] speed up tests by preloading Eric Wong
2019-11-15  9:50 ` [PATCH 01/29] edit: pass global variables into subs Eric Wong
2019-11-15  9:50 ` [PATCH 02/29] edit: use OO API of File::Temp to shorten lifetime Eric Wong
2019-11-15  9:50 ` [PATCH 03/29] admin: get rid of singleton $CFG var Eric Wong
2019-11-15  9:50 ` [PATCH 04/29] index: pass global variables into subs Eric Wong
2019-11-15  9:50 ` [PATCH 05/29] init: " Eric Wong
2019-11-15  9:50 ` [PATCH 06/29] mda: " Eric Wong
2019-11-15  9:50 ` [PATCH 07/29] learn: " Eric Wong
2019-11-15  9:50 ` [PATCH 08/29] inboxwritable: add ->cleanup method Eric Wong
2019-11-15  9:50 ` [PATCH 09/29] import: only pass Inbox object to SearchIdx->new Eric Wong
2019-11-15  9:50 ` [PATCH 10/29] xapcmd: do not fire END and DESTROY handlers in child Eric Wong
2019-11-15  9:50 ` [PATCH 11/29] spawn: which: allow embedded slash for relative path Eric Wong
2019-11-15  9:50 ` [PATCH 12/29] t/common: introduce run_script wrapper for t/cgi.t Eric Wong
2019-11-15  9:50 ` [PATCH 13/29] t/edit: switch to use run_script Eric Wong
2019-11-15  9:50 ` [PATCH 14/29] t/init: convert to using run_script Eric Wong
2019-11-15  9:50 ` [PATCH 15/29] t/purge: convert to run_script Eric Wong
2019-11-15  9:50 ` [PATCH 16/29] t/v2mirror: get rid of IPC::Run dependency Eric Wong
2019-11-15  9:50 ` [PATCH 17/29] t/mda: switch to run_script for testing Eric Wong
2019-11-15  9:50 ` [PATCH 18/29] t/mda_filter_rubylang: drop IPC::Run dependency Eric Wong
2019-11-15  9:50 ` [PATCH 19/29] doc: remove IPC::Run as a dev and test dependency Eric Wong
2019-11-15  9:50 ` [PATCH 20/29] t/v2mirror: switch to default run_mode for speedup Eric Wong
2019-11-15  9:50 ` [PATCH 21/29] t/convert-compact: convert to run_script Eric Wong
2019-11-15  9:50 ` [PATCH 22/29] t/httpd: use run_script for -init Eric Wong
2019-11-15  9:50 ` [PATCH 23/29] t/watch_maildir_v2: " Eric Wong
2019-11-15  9:50 ` [PATCH 24/29] t/nntpd: " Eric Wong
2019-11-15  9:50 ` [PATCH 25/29] t/watch_filter_rubylang: run_script for -init and -index Eric Wong
2019-11-15  9:50 ` [PATCH 26/29] t/v2mda: switch to run_script in many places Eric Wong
2019-11-15  9:50 ` [PATCH 27/29] t/indexlevels-mirror*: switch to run_script Eric Wong
2019-11-15  9:50 ` [PATCH 28/29] t/xcpdb-reshard: use run_script for -xcpdb Eric Wong
2019-11-15  9:51 ` [PATCH 29/29] t/common: start_script replaces spawn_listener Eric Wong
2019-11-16  6:52   ` Eric Wong
2019-11-16 11:43     ` Eric Wong
2019-11-24  0:22       ` [PATCH 00/17] test fixes and cleanups Eric Wong
2019-11-24  0:22         ` [PATCH 01/17] tests: disable daemon workers in a few more places Eric Wong
2019-11-24  0:22         ` [PATCH 02/17] tests: use strict everywhere Eric Wong
2019-11-24  0:22         ` [PATCH 03/17] t/v1-add-remove-add: quiet down "git init" Eric Wong
2019-11-24  0:22         ` [PATCH 04/17] t/xcpdb-reshard: test xcpdb --compact Eric Wong
2019-11-24  0:22         ` [PATCH 05/17] t/httpd-corner: wait for worker process death Eric Wong
2019-11-24  0:22         ` [PATCH 06/17] t/nntpd-tls: sometimes SSL_connect succeeds quickly Eric Wong
2019-11-24  0:22         ` [PATCH 07/17] .gitignore: ignore local prove(1) files Eric Wong
2019-11-24  0:22         ` [PATCH 08/17] daemon: use sigprocmask to block signals at startup Eric Wong
2019-11-24  0:22         ` [PATCH 09/17] daemon: use sigprocmask when respawning workers Eric Wong
2019-11-24  0:22         ` [PATCH 10/17] daemon: avoid race when quitting workers Eric Wong
2019-11-25  8:59           ` Eric Wong
2019-11-27  1:33             ` [PATCH 0/2] fix kqueue support and missed signal wakeups Eric Wong
2019-11-27  1:33               ` [PATCH 1/2] dskqxs: fix missing EV_DISPATCH define Eric Wong
2019-11-27  1:33               ` [PATCH 2/2] httpd|nntpd: avoid missed signal wakeups Eric Wong
2019-11-24  0:22         ` [PATCH 11/17] t/common: start_script replaces spawn_listener Eric Wong
2019-11-24  0:22         ` [PATCH 12/17] t/nntpd-validate: get rid of threads dependency Eric Wong
2019-11-24  0:22         ` Eric Wong [this message]
2019-11-24  0:22         ` [PATCH 14/17] tests: use File::Temp->newdir instead of tempdir() Eric Wong
2019-11-24  0:22         ` [PATCH 15/17] tests: quiet down commit graph Eric Wong
2019-11-24  0:22         ` [PATCH 16/17] t/perf-*.t: use $ENV{GIANT_INBOX_DIR} consistently Eric Wong
2019-11-24  0:22         ` [PATCH 17/17] tests: move giant inbox/git dependent tests to xt/ Eric Wong

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

  List information: http://public-inbox.org/README

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20191124002237.15713-14-e@80x24.org \
    --to=e@80x24.org \
    --cc=meta@public-inbox.org \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
Code repositories for project(s) associated with this public inbox

	https://80x24.org/public-inbox.git

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).