user/dev discussion of public-inbox itself
 help / color / mirror / code / Atom feed
Search results ordered by [date|relevance]  view[summary|nested|Atom feed]
thread overview below | download mbox.gz: |
* [PATCH 24/36] spawn: move run_die here from PublicInbox::Import
  2020-12-31 13:51  7% [PATCH 00/36] another round of lei stuff Eric Wong
@ 2020-12-31 13:51  7% ` Eric Wong
  0 siblings, 0 replies; 2+ results
From: Eric Wong @ 2020-12-31 13:51 UTC (permalink / raw)
  To: meta

It seems like a more logical place for it, but we'll favor the
newly-added xsys_e() in tests for BAIL_OUT use.
---
 lib/PublicInbox/Import.pm     |  9 +--------
 lib/PublicInbox/LEI.pm        |  5 ++---
 lib/PublicInbox/Spawn.pm      |  9 ++++++++-
 lib/PublicInbox/TestCommon.pm | 25 ++++++++++++++++++-------
 lib/PublicInbox/V2Writable.pm | 10 ++++------
 script/public-inbox-init      | 13 ++++++++-----
 t/convert-compact.t           |  4 ++--
 t/index-git-times.t           |  3 +--
 t/plack.t                     |  4 ++--
 9 files changed, 46 insertions(+), 36 deletions(-)

diff --git a/lib/PublicInbox/Import.pm b/lib/PublicInbox/Import.pm
index 60cff9c2..6f387b77 100644
--- a/lib/PublicInbox/Import.pm
+++ b/lib/PublicInbox/Import.pm
@@ -9,7 +9,7 @@ package PublicInbox::Import;
 use strict;
 use parent qw(PublicInbox::Lock);
 use v5.10.1;
-use PublicInbox::Spawn qw(spawn popen_rd);
+use PublicInbox::Spawn qw(run_die popen_rd);
 use PublicInbox::MID qw(mids mid2path);
 use PublicInbox::Address;
 use PublicInbox::Smsg;
@@ -442,13 +442,6 @@ sub add {
 	$self->{tip} = ":$commit";
 }
 
-sub run_die ($;$$) {
-	my ($cmd, $env, $rdr) = @_;
-	my $pid = spawn($cmd, $env, $rdr);
-	waitpid($pid, 0) == $pid or die join(' ', @$cmd) .' did not finish';
-	$? == 0 or die join(' ', @$cmd) . " failed: $?\n";
-}
-
 my @INIT_FILES = ('HEAD' => undef, # filled in at runtime
 		'description' => <<EOD,
 Unnamed repository; edit this file 'description' to name the repository.
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index bb77198e..1ba9eff3 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -19,7 +19,7 @@ use PublicInbox::Config;
 use PublicInbox::Syscall qw($SFD_NONBLOCK EPOLLIN EPOLLONESHOT);
 use PublicInbox::Sigfd;
 use PublicInbox::DS qw(now);
-use PublicInbox::Spawn qw(spawn);
+use PublicInbox::Spawn qw(spawn run_die);
 use PublicInbox::OnDestroy;
 use Text::Wrap qw(wrap);
 use File::Path qw(mkpath);
@@ -482,8 +482,7 @@ sub lei_config {
 	my $cfg = _lei_cfg($self, 1);
 	my $cmd = [ qw(git config -f), $cfg->{'-f'}, @argv ];
 	my %rdr = map { $_ => $self->{$_} } (0..2);
-	require PublicInbox::Import;
-	PublicInbox::Import::run_die($cmd, $env, \%rdr);
+	run_die($cmd, $env, \%rdr);
 }
 
 sub lei_init {
diff --git a/lib/PublicInbox/Spawn.pm b/lib/PublicInbox/Spawn.pm
index cb16fcf6..5e2495d2 100644
--- a/lib/PublicInbox/Spawn.pm
+++ b/lib/PublicInbox/Spawn.pm
@@ -19,7 +19,7 @@ use strict;
 use parent qw(Exporter);
 use Symbol qw(gensym);
 use PublicInbox::ProcessPipe;
-our @EXPORT_OK = qw/which spawn popen_rd nodatacow_dir/;
+our @EXPORT_OK = qw(which spawn popen_rd run_die nodatacow_dir);
 our @RLIMITS = qw(RLIMIT_CPU RLIMIT_CORE RLIMIT_DATA);
 
 my $vfork_spawn = <<'VFORK_SPAWN';
@@ -306,4 +306,11 @@ sub popen_rd {
 	$ret;
 }
 
+sub run_die ($;$$) {
+	my ($cmd, $env, $rdr) = @_;
+	my $pid = spawn($cmd, $env, $rdr);
+	waitpid($pid, 0) == $pid or die "@$cmd did not finish";
+	$? == 0 or die "@$cmd failed: \$?=$?\n";
+}
+
 1;
diff --git a/lib/PublicInbox/TestCommon.pm b/lib/PublicInbox/TestCommon.pm
index 338e760c..b833984d 100644
--- a/lib/PublicInbox/TestCommon.pm
+++ b/lib/PublicInbox/TestCommon.pm
@@ -10,8 +10,14 @@ use Fcntl qw(FD_CLOEXEC F_SETFD F_GETFD :seek);
 use POSIX qw(dup2);
 use IO::Socket::INET;
 our @EXPORT = qw(tmpdir tcp_server tcp_connect require_git require_mods
-	run_script start_script key2sub xsys xqx eml_load tick
+	run_script start_script key2sub xsys xsys_e xqx eml_load tick
 	have_xapian_compact);
+BEGIN {
+	require Test::More;
+	*BAIL_OUT = \&Test::More::BAIL_OUT;
+	*plan = \&Test::More::plan;
+	*skip = \&Test::More::skip;
+}
 
 sub eml_load ($) {
 	my ($path, $cb) = @_;
@@ -38,7 +44,7 @@ sub tcp_server () {
 		Type => Socket::SOCK_STREAM(),
 		Listen => 1024,
 		Blocking => 0,
-	) or Test::More::BAIL_OUT("failed to create TCP server: $!");
+	) or BAIL_OUT "failed to create TCP server: $!";
 }
 
 sub tcp_connect {
@@ -49,7 +55,7 @@ sub tcp_connect {
 		Type => Socket::SOCK_STREAM(),
 		PeerAddr => $addr,
 		%opt,
-	) or Test::More::BAIL_OUT("failed to connect to $addr: $!");
+	) or BAIL_OUT "failed to connect to $addr: $!";
 	$s->autoflush(1);
 	$s;
 }
@@ -64,8 +70,8 @@ sub require_git ($;$) {
 	my $cur_int = ($cur_maj << 24) | ($cur_min << 16) | ($cur_sub // 0);
 	if ($cur_int < $req_int) {
 		return 0 if $maybe;
-		Test::More::plan(skip_all =>
-			"git $req+ required, have $cur_maj.$cur_min.$cur_sub");
+		plan skip_all =>
+			"git $req+ required, have $cur_maj.$cur_min.$cur_sub";
 	}
 	1;
 }
@@ -113,8 +119,8 @@ sub require_mods {
 	}
 	return unless @need;
 	my $m = join(', ', @need)." missing for $0";
-	Test::More::skip($m, $maybe) if $maybe;
-	Test::More::plan(skip_all => $m)
+	skip($m, $maybe) if $maybe;
+	plan(skip_all => $m)
 }
 
 sub key2script ($) {
@@ -323,6 +329,11 @@ sub xsys {
 	$? >> 8
 }
 
+sub xsys_e { # like "/bin/sh -e"
+	xsys(@_) == 0 or
+		BAIL_OUT (ref $_[0] ? "@{$_[0]}" : "@_"). " failed \$?=$?"
+}
+
 # like `backtick` or qx{} op, but uses spawn() for env/rdr + vfork
 sub xqx {
 	my ($cmd, $env, $rdr) = @_;
diff --git a/lib/PublicInbox/V2Writable.pm b/lib/PublicInbox/V2Writable.pm
index edb8ba57..5a8caa08 100644
--- a/lib/PublicInbox/V2Writable.pm
+++ b/lib/PublicInbox/V2Writable.pm
@@ -16,7 +16,7 @@ use PublicInbox::ContentHash qw(content_hash content_digest);
 use PublicInbox::InboxWritable;
 use PublicInbox::OverIdx;
 use PublicInbox::Msgmap;
-use PublicInbox::Spawn qw(spawn popen_rd);
+use PublicInbox::Spawn qw(spawn popen_rd run_die);
 use PublicInbox::SearchIdx qw(log2stack crlf_adjust is_ancestor check_size
 	is_bad_blob);
 use IO::Handle; # ->autoflush
@@ -745,9 +745,8 @@ sub git_init {
 	my ($self, $epoch) = @_;
 	my $git_dir = "$self->{ibx}->{inboxdir}/git/$epoch.git";
 	PublicInbox::Import::init_bare($git_dir);
-	my @cmd = (qw/git config/, "--file=$git_dir/config",
-			'include.path', '../../all.git/config');
-	PublicInbox::Import::run_die(\@cmd);
+	run_die([qw(git config), "--file=$git_dir/config",
+		qw(include.path ../../all.git/config)]);
 	fill_alternates($self, $epoch);
 	$git_dir
 }
@@ -1222,8 +1221,7 @@ sub unindex_todo ($$$) {
 	return if $before == $after;
 
 	# ensure any blob can not longer be accessed via dumb HTTP
-	PublicInbox::Import::run_die(['git',
-		"--git-dir=$unit->{git}->{git_dir}",
+	run_die(['git', "--git-dir=$unit->{git}->{git_dir}",
 		qw(-c gc.reflogExpire=now gc --prune=all --quiet)]);
 }
 
diff --git a/script/public-inbox-init b/script/public-inbox-init
index 7ac77830..85d14377 100755
--- a/script/public-inbox-init
+++ b/script/public-inbox-init
@@ -185,17 +185,20 @@ $ibx->init_inbox(0, $skip_epoch, $skip_artnum);
 # needed for git prior to v2.1.0
 umask(0077) if defined $perm;
 
+require PublicInbox::Spawn;
+PublicInbox::Spawn->import(qw(run_die));
+
 foreach my $addr (@address) {
 	next if $seen{lc($addr)};
-	PublicInbox::Import::run_die([@x, "--add", "$pfx.address", $addr]);
+	run_die([@x, "--add", "$pfx.address", $addr]);
 }
-PublicInbox::Import::run_die([@x, "$pfx.url", $http_url]);
-PublicInbox::Import::run_die([@x, "$pfx.inboxdir", $inboxdir]);
+run_die([@x, "$pfx.url", $http_url]);
+run_die([@x, "$pfx.inboxdir", $inboxdir]);
 
 if (defined($indexlevel)) {
-	PublicInbox::Import::run_die([@x, "$pfx.indexlevel", $indexlevel]);
+	run_die([@x, "$pfx.indexlevel", $indexlevel]);
 }
-PublicInbox::Import::run_die([@x, "$pfx.newsgroup", $ng]) if $ng ne '';
+run_die([@x, "$pfx.newsgroup", $ng]) if $ng ne '';
 
 # needed for git prior to v2.1.0
 if (defined $perm) {
diff --git a/t/convert-compact.t b/t/convert-compact.t
index e479476d..fe8a7ec2 100644
--- a/t/convert-compact.t
+++ b/t/convert-compact.t
@@ -21,8 +21,8 @@ my $ibx = {
 
 PublicInbox::Import::init_bare($ibx->{inboxdir});
 ok(umask(077), 'set restrictive umask');
-ok(PublicInbox::Import::run_die([qw(git) , "--git-dir=$ibx->{inboxdir}",
-	qw(config core.sharedRepository 0644)]), 'set sharedRepository');
+xsys_e(qw(git) , "--git-dir=$ibx->{inboxdir}",
+	qw(config core.sharedRepository 0644));
 $ibx = PublicInbox::Inbox->new($ibx);
 my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
 my $mime = PublicInbox::Eml->new(<<'EOF');
diff --git a/t/index-git-times.t b/t/index-git-times.t
index f9869cfa..9b869f94 100644
--- a/t/index-git-times.t
+++ b/t/index-git-times.t
@@ -5,7 +5,6 @@ use strict;
 use v5.10.1;
 use Test::More;
 use PublicInbox::TestCommon;
-use PublicInbox::Import;
 use PublicInbox::Config;
 use PublicInbox::Admin;
 use File::Path qw(remove_tree);
@@ -48,7 +47,7 @@ EOF
 	print $w $data or die;
 	close $w or die;
 	my $cmd = ['git', "--git-dir=$v1dir", 'fast-import', '--quiet'];
-	PublicInbox::Import::run_die($cmd, undef, { 0 => $r });
+	xsys_e($cmd, undef, { 0 => $r });
 }
 
 run_script(['-index', '--skip-docdata', $v1dir]) or die 'v1 index failed';
diff --git a/t/plack.t b/t/plack.t
index 1fedf426..5d65cd05 100644
--- a/t/plack.t
+++ b/t/plack.t
@@ -21,8 +21,8 @@ ok(-f $psgi, "psgi example file found");
 my $pfx = 'http://example.com/test';
 ok(run_script(['-init', 'test', $inboxdir, "$pfx/", $addr]),
 	'initialized repo');
-PublicInbox::Import::run_die([qw(git config -f), $pi_config,
-	'publicinbox.test.newsgroup', 'inbox.test']);
+xsys_e(qw(git config -f), $pi_config,
+	qw(publicinbox.test.newsgroup inbox.test));
 open my $fh, '>', "$inboxdir/description" or die "open: $!\n";
 print $fh "test for public-inbox\n";
 close $fh or die "close: $!\n";

^ permalink raw reply	[relevance 7%]

* [PATCH 00/36] another round of lei stuff
@ 2020-12-31 13:51  7% Eric Wong
  2020-12-31 13:51  7% ` [PATCH 24/36] spawn: move run_die here from PublicInbox::Import Eric Wong
  0 siblings, 1 reply; 2+ results
From: Eric Wong @ 2020-12-31 13:51 UTC (permalink / raw)
  To: meta

This is against lei branch @ commit
0c8106d44f317175e122744b43407bf067183175 in
https://public-inbox.org/public-inbox.git

Infrastructure stuff for reading + writing local Maildirs and a
bunch of mbox formats are done (including gz/bz2/xz support)
and it's usage should be familiar to mairix(1) users.

Infrastructure for deduplication + augmenting search results
in place and tested.

Going to skip MH and MMDF for now; but IMAP/JMAP might happen
sooner but deduplication needs low-latency.

"extinbox" renamed "external"

Basic infrastructure like PublicInbox::IPC and SharedKV
should've been done and in use ages ago...  I look forward to
using them, at least.

Some DS safety fixes since lei will use it in stranger ways
than current.

Bad enough we have messages with duplicate Message-IDs, lei will
need to deal with Unsent/Drafts messages w/o Message-IDs at all!

Eric Wong (36):
  import: respect init.defaultBranch
  lei_store: use per-machine refname as git HEAD
  revert "lei_store: use per-machine refname as git HEAD"
  lei_to_mail: initial implementation for writing mbox formats
  sharedkv: fork()-friendly key-value store
  sharedkv: split out index_values
  lei_to_mail: start atomic and compressed mbox writing
  mboxreader: new class for reading various mbox formats
  lei_to_mail: start --augment, dedupe, bz2 and xz
  lei: implement various deduplication strategies
  lei_to_mail: lazy-require LeiDedupe
  lei_to_mail: support for non-seekable outputs
  lei_to_mail: support Maildir, fix+test --augment
  ipc: generic IPC dispatch based on Storable
  ipc: support Sereal
  lei_store: add ->set_eml, ->add_eml can return smsg
  lei: rename "extinbox" => "external"
  mid: use defined-or with `push' for uniqueness check
  mid: hoist out mids_in sub
  lei_store: handle messages without Message-ID at all
  ipc: use shutdown(2), base atfork* callback
  lei_to_mail: unlink mboxes if not augmenting
  lei: add --mfolder as an option
  spawn: move run_die here from PublicInbox::Import
  init: remove embedded UnlinkMe package
  t/run.perl: avoid uninitialized var on incomplete test
  gcf2client: reap process on DESTROY
  lei_to_mail: open FIFOs O_WRONLY so we block
  searchidxshard: call DS->Reset at worker start
  t/ipc.t: test for references via `die'
  use PublicInbox::DS for dwaitpid
  syscall: SFD_NONBLOCK can be a constant, again
  lei: avoid Spawn package when starting daemon
  avoid calling waitpid from children in DESTROY
  ds: clobber $in_loop first at reset
  on_destroy: support PID owner guard

 MANIFEST                                      |  12 +-
 lib/PublicInbox/DS.pm                         |  42 +-
 lib/PublicInbox/DSKQXS.pm                     |   4 +-
 lib/PublicInbox/Daemon.pm                     |   4 +-
 lib/PublicInbox/Gcf2Client.pm                 |  18 +-
 lib/PublicInbox/Git.pm                        |   7 +-
 lib/PublicInbox/IPC.pm                        | 165 ++++++++
 lib/PublicInbox/Import.pm                     |  36 +-
 lib/PublicInbox/LEI.pm                        |  44 +--
 lib/PublicInbox/LeiDedupe.pm                  | 100 +++++
 .../{LeiExtinbox.pm => LeiExternal.pm}        |  18 +-
 lib/PublicInbox/LeiStore.pm                   |  32 +-
 lib/PublicInbox/LeiToMail.pm                  | 361 ++++++++++++++++++
 lib/PublicInbox/LeiXSearch.pm                 |   2 +-
 lib/PublicInbox/Lock.pm                       |  17 +-
 lib/PublicInbox/MID.pm                        |  15 +-
 lib/PublicInbox/MboxReader.pm                 | 127 ++++++
 lib/PublicInbox/OnDestroy.pm                  |   5 +
 lib/PublicInbox/OverIdx.pm                    |   2 +
 lib/PublicInbox/ProcessPipe.pm                |  34 +-
 lib/PublicInbox/Qspawn.pm                     |  43 +--
 lib/PublicInbox/SearchIdxShard.pm             |   1 +
 lib/PublicInbox/SharedKV.pm                   | 148 +++++++
 lib/PublicInbox/Sigfd.pm                      |   4 +-
 lib/PublicInbox/Smsg.pm                       |   6 +-
 lib/PublicInbox/Spawn.pm                      |   9 +-
 lib/PublicInbox/Syscall.pm                    |   4 +-
 lib/PublicInbox/TestCommon.pm                 |  25 +-
 lib/PublicInbox/V2Writable.pm                 |  10 +-
 script/lei                                    |  17 +-
 script/public-inbox-init                      |  32 +-
 script/public-inbox-watch                     |   4 +-
 t/convert-compact.t                           |   4 +-
 t/index-git-times.t                           |   3 +-
 t/ipc.t                                       |  80 ++++
 t/lei.t                                       |  22 +-
 t/lei_dedupe.t                                |  59 +++
 t/lei_store.t                                 |  47 ++-
 t/lei_to_mail.t                               | 246 ++++++++++++
 t/lei_xsearch.t                               |   2 +-
 t/mbox_reader.t                               |  75 ++++
 t/on_destroy.t                                |   9 +
 t/plack.t                                     |   4 +-
 t/run.perl                                    |   3 +-
 t/shared_kv.t                                 |  58 +++
 t/sigfd.t                                     |   6 +-
 46 files changed, 1755 insertions(+), 211 deletions(-)
 create mode 100644 lib/PublicInbox/IPC.pm
 create mode 100644 lib/PublicInbox/LeiDedupe.pm
 rename lib/PublicInbox/{LeiExtinbox.pm => LeiExternal.pm} (75%)
 create mode 100644 lib/PublicInbox/LeiToMail.pm
 create mode 100644 lib/PublicInbox/MboxReader.pm
 create mode 100644 lib/PublicInbox/SharedKV.pm
 create mode 100644 t/ipc.t
 create mode 100644 t/lei_dedupe.t
 create mode 100644 t/lei_to_mail.t
 create mode 100644 t/mbox_reader.t
 create mode 100644 t/shared_kv.t


^ permalink raw reply	[relevance 7%]

Results 1-2 of 2 | reverse | sort options + mbox downloads above
-- links below jump to the message on this page --
2020-12-31 13:51  7% [PATCH 00/36] another round of lei stuff Eric Wong
2020-12-31 13:51  7% ` [PATCH 24/36] spawn: move run_die here from PublicInbox::Import Eric Wong

Code repositories for project(s) associated with this inbox:

	https://80x24.org/public-inbox.git

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).