* [PATCH 24/36] spawn: move run_die here from PublicInbox::Import
2020-12-31 13:51 7% [PATCH 00/36] another round of lei stuff Eric Wong
@ 2020-12-31 13:51 6% ` Eric Wong
0 siblings, 0 replies; 2+ results
From: Eric Wong @ 2020-12-31 13:51 UTC (permalink / raw)
To: meta
It seems like a more logical place for it, but we'll favor the
newly-added xsys_e() in tests for BAIL_OUT use.
---
lib/PublicInbox/Import.pm | 9 +--------
lib/PublicInbox/LEI.pm | 5 ++---
lib/PublicInbox/Spawn.pm | 9 ++++++++-
lib/PublicInbox/TestCommon.pm | 25 ++++++++++++++++++-------
lib/PublicInbox/V2Writable.pm | 10 ++++------
script/public-inbox-init | 13 ++++++++-----
t/convert-compact.t | 4 ++--
t/index-git-times.t | 3 +--
t/plack.t | 4 ++--
9 files changed, 46 insertions(+), 36 deletions(-)
diff --git a/lib/PublicInbox/Import.pm b/lib/PublicInbox/Import.pm
index 60cff9c2..6f387b77 100644
--- a/lib/PublicInbox/Import.pm
+++ b/lib/PublicInbox/Import.pm
@@ -9,7 +9,7 @@ package PublicInbox::Import;
use strict;
use parent qw(PublicInbox::Lock);
use v5.10.1;
-use PublicInbox::Spawn qw(spawn popen_rd);
+use PublicInbox::Spawn qw(run_die popen_rd);
use PublicInbox::MID qw(mids mid2path);
use PublicInbox::Address;
use PublicInbox::Smsg;
@@ -442,13 +442,6 @@ sub add {
$self->{tip} = ":$commit";
}
-sub run_die ($;$$) {
- my ($cmd, $env, $rdr) = @_;
- my $pid = spawn($cmd, $env, $rdr);
- waitpid($pid, 0) == $pid or die join(' ', @$cmd) .' did not finish';
- $? == 0 or die join(' ', @$cmd) . " failed: $?\n";
-}
-
my @INIT_FILES = ('HEAD' => undef, # filled in at runtime
'description' => <<EOD,
Unnamed repository; edit this file 'description' to name the repository.
diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index bb77198e..1ba9eff3 100644
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -19,7 +19,7 @@ use PublicInbox::Config;
use PublicInbox::Syscall qw($SFD_NONBLOCK EPOLLIN EPOLLONESHOT);
use PublicInbox::Sigfd;
use PublicInbox::DS qw(now);
-use PublicInbox::Spawn qw(spawn);
+use PublicInbox::Spawn qw(spawn run_die);
use PublicInbox::OnDestroy;
use Text::Wrap qw(wrap);
use File::Path qw(mkpath);
@@ -482,8 +482,7 @@ sub lei_config {
my $cfg = _lei_cfg($self, 1);
my $cmd = [ qw(git config -f), $cfg->{'-f'}, @argv ];
my %rdr = map { $_ => $self->{$_} } (0..2);
- require PublicInbox::Import;
- PublicInbox::Import::run_die($cmd, $env, \%rdr);
+ run_die($cmd, $env, \%rdr);
}
sub lei_init {
diff --git a/lib/PublicInbox/Spawn.pm b/lib/PublicInbox/Spawn.pm
index cb16fcf6..5e2495d2 100644
--- a/lib/PublicInbox/Spawn.pm
+++ b/lib/PublicInbox/Spawn.pm
@@ -19,7 +19,7 @@ use strict;
use parent qw(Exporter);
use Symbol qw(gensym);
use PublicInbox::ProcessPipe;
-our @EXPORT_OK = qw/which spawn popen_rd nodatacow_dir/;
+our @EXPORT_OK = qw(which spawn popen_rd run_die nodatacow_dir);
our @RLIMITS = qw(RLIMIT_CPU RLIMIT_CORE RLIMIT_DATA);
my $vfork_spawn = <<'VFORK_SPAWN';
@@ -306,4 +306,11 @@ sub popen_rd {
$ret;
}
+sub run_die ($;$$) {
+ my ($cmd, $env, $rdr) = @_;
+ my $pid = spawn($cmd, $env, $rdr);
+ waitpid($pid, 0) == $pid or die "@$cmd did not finish";
+ $? == 0 or die "@$cmd failed: \$?=$?\n";
+}
+
1;
diff --git a/lib/PublicInbox/TestCommon.pm b/lib/PublicInbox/TestCommon.pm
index 338e760c..b833984d 100644
--- a/lib/PublicInbox/TestCommon.pm
+++ b/lib/PublicInbox/TestCommon.pm
@@ -10,8 +10,14 @@ use Fcntl qw(FD_CLOEXEC F_SETFD F_GETFD :seek);
use POSIX qw(dup2);
use IO::Socket::INET;
our @EXPORT = qw(tmpdir tcp_server tcp_connect require_git require_mods
- run_script start_script key2sub xsys xqx eml_load tick
+ run_script start_script key2sub xsys xsys_e xqx eml_load tick
have_xapian_compact);
+BEGIN {
+ require Test::More;
+ *BAIL_OUT = \&Test::More::BAIL_OUT;
+ *plan = \&Test::More::plan;
+ *skip = \&Test::More::skip;
+}
sub eml_load ($) {
my ($path, $cb) = @_;
@@ -38,7 +44,7 @@ sub tcp_server () {
Type => Socket::SOCK_STREAM(),
Listen => 1024,
Blocking => 0,
- ) or Test::More::BAIL_OUT("failed to create TCP server: $!");
+ ) or BAIL_OUT "failed to create TCP server: $!";
}
sub tcp_connect {
@@ -49,7 +55,7 @@ sub tcp_connect {
Type => Socket::SOCK_STREAM(),
PeerAddr => $addr,
%opt,
- ) or Test::More::BAIL_OUT("failed to connect to $addr: $!");
+ ) or BAIL_OUT "failed to connect to $addr: $!";
$s->autoflush(1);
$s;
}
@@ -64,8 +70,8 @@ sub require_git ($;$) {
my $cur_int = ($cur_maj << 24) | ($cur_min << 16) | ($cur_sub // 0);
if ($cur_int < $req_int) {
return 0 if $maybe;
- Test::More::plan(skip_all =>
- "git $req+ required, have $cur_maj.$cur_min.$cur_sub");
+ plan skip_all =>
+ "git $req+ required, have $cur_maj.$cur_min.$cur_sub";
}
1;
}
@@ -113,8 +119,8 @@ sub require_mods {
}
return unless @need;
my $m = join(', ', @need)." missing for $0";
- Test::More::skip($m, $maybe) if $maybe;
- Test::More::plan(skip_all => $m)
+ skip($m, $maybe) if $maybe;
+ plan(skip_all => $m)
}
sub key2script ($) {
@@ -323,6 +329,11 @@ sub xsys {
$? >> 8
}
+sub xsys_e { # like "/bin/sh -e"
+ xsys(@_) == 0 or
+ BAIL_OUT (ref $_[0] ? "@{$_[0]}" : "@_"). " failed \$?=$?"
+}
+
# like `backtick` or qx{} op, but uses spawn() for env/rdr + vfork
sub xqx {
my ($cmd, $env, $rdr) = @_;
diff --git a/lib/PublicInbox/V2Writable.pm b/lib/PublicInbox/V2Writable.pm
index edb8ba57..5a8caa08 100644
--- a/lib/PublicInbox/V2Writable.pm
+++ b/lib/PublicInbox/V2Writable.pm
@@ -16,7 +16,7 @@ use PublicInbox::ContentHash qw(content_hash content_digest);
use PublicInbox::InboxWritable;
use PublicInbox::OverIdx;
use PublicInbox::Msgmap;
-use PublicInbox::Spawn qw(spawn popen_rd);
+use PublicInbox::Spawn qw(spawn popen_rd run_die);
use PublicInbox::SearchIdx qw(log2stack crlf_adjust is_ancestor check_size
is_bad_blob);
use IO::Handle; # ->autoflush
@@ -745,9 +745,8 @@ sub git_init {
my ($self, $epoch) = @_;
my $git_dir = "$self->{ibx}->{inboxdir}/git/$epoch.git";
PublicInbox::Import::init_bare($git_dir);
- my @cmd = (qw/git config/, "--file=$git_dir/config",
- 'include.path', '../../all.git/config');
- PublicInbox::Import::run_die(\@cmd);
+ run_die([qw(git config), "--file=$git_dir/config",
+ qw(include.path ../../all.git/config)]);
fill_alternates($self, $epoch);
$git_dir
}
@@ -1222,8 +1221,7 @@ sub unindex_todo ($$$) {
return if $before == $after;
# ensure any blob can not longer be accessed via dumb HTTP
- PublicInbox::Import::run_die(['git',
- "--git-dir=$unit->{git}->{git_dir}",
+ run_die(['git', "--git-dir=$unit->{git}->{git_dir}",
qw(-c gc.reflogExpire=now gc --prune=all --quiet)]);
}
diff --git a/script/public-inbox-init b/script/public-inbox-init
index 7ac77830..85d14377 100755
--- a/script/public-inbox-init
+++ b/script/public-inbox-init
@@ -185,17 +185,20 @@ $ibx->init_inbox(0, $skip_epoch, $skip_artnum);
# needed for git prior to v2.1.0
umask(0077) if defined $perm;
+require PublicInbox::Spawn;
+PublicInbox::Spawn->import(qw(run_die));
+
foreach my $addr (@address) {
next if $seen{lc($addr)};
- PublicInbox::Import::run_die([@x, "--add", "$pfx.address", $addr]);
+ run_die([@x, "--add", "$pfx.address", $addr]);
}
-PublicInbox::Import::run_die([@x, "$pfx.url", $http_url]);
-PublicInbox::Import::run_die([@x, "$pfx.inboxdir", $inboxdir]);
+run_die([@x, "$pfx.url", $http_url]);
+run_die([@x, "$pfx.inboxdir", $inboxdir]);
if (defined($indexlevel)) {
- PublicInbox::Import::run_die([@x, "$pfx.indexlevel", $indexlevel]);
+ run_die([@x, "$pfx.indexlevel", $indexlevel]);
}
-PublicInbox::Import::run_die([@x, "$pfx.newsgroup", $ng]) if $ng ne '';
+run_die([@x, "$pfx.newsgroup", $ng]) if $ng ne '';
# needed for git prior to v2.1.0
if (defined $perm) {
diff --git a/t/convert-compact.t b/t/convert-compact.t
index e479476d..fe8a7ec2 100644
--- a/t/convert-compact.t
+++ b/t/convert-compact.t
@@ -21,8 +21,8 @@ my $ibx = {
PublicInbox::Import::init_bare($ibx->{inboxdir});
ok(umask(077), 'set restrictive umask');
-ok(PublicInbox::Import::run_die([qw(git) , "--git-dir=$ibx->{inboxdir}",
- qw(config core.sharedRepository 0644)]), 'set sharedRepository');
+xsys_e(qw(git) , "--git-dir=$ibx->{inboxdir}",
+ qw(config core.sharedRepository 0644));
$ibx = PublicInbox::Inbox->new($ibx);
my $im = PublicInbox::Import->new($ibx->git, undef, undef, $ibx);
my $mime = PublicInbox::Eml->new(<<'EOF');
diff --git a/t/index-git-times.t b/t/index-git-times.t
index f9869cfa..9b869f94 100644
--- a/t/index-git-times.t
+++ b/t/index-git-times.t
@@ -5,7 +5,6 @@ use strict;
use v5.10.1;
use Test::More;
use PublicInbox::TestCommon;
-use PublicInbox::Import;
use PublicInbox::Config;
use PublicInbox::Admin;
use File::Path qw(remove_tree);
@@ -48,7 +47,7 @@ EOF
print $w $data or die;
close $w or die;
my $cmd = ['git', "--git-dir=$v1dir", 'fast-import', '--quiet'];
- PublicInbox::Import::run_die($cmd, undef, { 0 => $r });
+ xsys_e($cmd, undef, { 0 => $r });
}
run_script(['-index', '--skip-docdata', $v1dir]) or die 'v1 index failed';
diff --git a/t/plack.t b/t/plack.t
index 1fedf426..5d65cd05 100644
--- a/t/plack.t
+++ b/t/plack.t
@@ -21,8 +21,8 @@ ok(-f $psgi, "psgi example file found");
my $pfx = 'http://example.com/test';
ok(run_script(['-init', 'test', $inboxdir, "$pfx/", $addr]),
'initialized repo');
-PublicInbox::Import::run_die([qw(git config -f), $pi_config,
- 'publicinbox.test.newsgroup', 'inbox.test']);
+xsys_e(qw(git config -f), $pi_config,
+ qw(publicinbox.test.newsgroup inbox.test));
open my $fh, '>', "$inboxdir/description" or die "open: $!\n";
print $fh "test for public-inbox\n";
close $fh or die "close: $!\n";
^ permalink raw reply related [relevance 6%]
* [PATCH 00/36] another round of lei stuff
@ 2020-12-31 13:51 7% Eric Wong
2020-12-31 13:51 6% ` [PATCH 24/36] spawn: move run_die here from PublicInbox::Import Eric Wong
0 siblings, 1 reply; 2+ results
From: Eric Wong @ 2020-12-31 13:51 UTC (permalink / raw)
To: meta
This is against lei branch @ commit
0c8106d44f317175e122744b43407bf067183175 in
https://public-inbox.org/public-inbox.git
Infrastructure stuff for reading + writing local Maildirs and a
bunch of mbox formats are done (including gz/bz2/xz support)
and it's usage should be familiar to mairix(1) users.
Infrastructure for deduplication + augmenting search results
in place and tested.
Going to skip MH and MMDF for now; but IMAP/JMAP might happen
sooner but deduplication needs low-latency.
"extinbox" renamed "external"
Basic infrastructure like PublicInbox::IPC and SharedKV
should've been done and in use ages ago... I look forward to
using them, at least.
Some DS safety fixes since lei will use it in stranger ways
than current.
Bad enough we have messages with duplicate Message-IDs, lei will
need to deal with Unsent/Drafts messages w/o Message-IDs at all!
Eric Wong (36):
import: respect init.defaultBranch
lei_store: use per-machine refname as git HEAD
revert "lei_store: use per-machine refname as git HEAD"
lei_to_mail: initial implementation for writing mbox formats
sharedkv: fork()-friendly key-value store
sharedkv: split out index_values
lei_to_mail: start atomic and compressed mbox writing
mboxreader: new class for reading various mbox formats
lei_to_mail: start --augment, dedupe, bz2 and xz
lei: implement various deduplication strategies
lei_to_mail: lazy-require LeiDedupe
lei_to_mail: support for non-seekable outputs
lei_to_mail: support Maildir, fix+test --augment
ipc: generic IPC dispatch based on Storable
ipc: support Sereal
lei_store: add ->set_eml, ->add_eml can return smsg
lei: rename "extinbox" => "external"
mid: use defined-or with `push' for uniqueness check
mid: hoist out mids_in sub
lei_store: handle messages without Message-ID at all
ipc: use shutdown(2), base atfork* callback
lei_to_mail: unlink mboxes if not augmenting
lei: add --mfolder as an option
spawn: move run_die here from PublicInbox::Import
init: remove embedded UnlinkMe package
t/run.perl: avoid uninitialized var on incomplete test
gcf2client: reap process on DESTROY
lei_to_mail: open FIFOs O_WRONLY so we block
searchidxshard: call DS->Reset at worker start
t/ipc.t: test for references via `die'
use PublicInbox::DS for dwaitpid
syscall: SFD_NONBLOCK can be a constant, again
lei: avoid Spawn package when starting daemon
avoid calling waitpid from children in DESTROY
ds: clobber $in_loop first at reset
on_destroy: support PID owner guard
MANIFEST | 12 +-
lib/PublicInbox/DS.pm | 42 +-
lib/PublicInbox/DSKQXS.pm | 4 +-
lib/PublicInbox/Daemon.pm | 4 +-
lib/PublicInbox/Gcf2Client.pm | 18 +-
lib/PublicInbox/Git.pm | 7 +-
lib/PublicInbox/IPC.pm | 165 ++++++++
lib/PublicInbox/Import.pm | 36 +-
lib/PublicInbox/LEI.pm | 44 +--
lib/PublicInbox/LeiDedupe.pm | 100 +++++
.../{LeiExtinbox.pm => LeiExternal.pm} | 18 +-
lib/PublicInbox/LeiStore.pm | 32 +-
lib/PublicInbox/LeiToMail.pm | 361 ++++++++++++++++++
lib/PublicInbox/LeiXSearch.pm | 2 +-
lib/PublicInbox/Lock.pm | 17 +-
lib/PublicInbox/MID.pm | 15 +-
lib/PublicInbox/MboxReader.pm | 127 ++++++
lib/PublicInbox/OnDestroy.pm | 5 +
lib/PublicInbox/OverIdx.pm | 2 +
lib/PublicInbox/ProcessPipe.pm | 34 +-
lib/PublicInbox/Qspawn.pm | 43 +--
lib/PublicInbox/SearchIdxShard.pm | 1 +
lib/PublicInbox/SharedKV.pm | 148 +++++++
lib/PublicInbox/Sigfd.pm | 4 +-
lib/PublicInbox/Smsg.pm | 6 +-
lib/PublicInbox/Spawn.pm | 9 +-
lib/PublicInbox/Syscall.pm | 4 +-
lib/PublicInbox/TestCommon.pm | 25 +-
lib/PublicInbox/V2Writable.pm | 10 +-
script/lei | 17 +-
script/public-inbox-init | 32 +-
script/public-inbox-watch | 4 +-
t/convert-compact.t | 4 +-
t/index-git-times.t | 3 +-
t/ipc.t | 80 ++++
t/lei.t | 22 +-
t/lei_dedupe.t | 59 +++
t/lei_store.t | 47 ++-
t/lei_to_mail.t | 246 ++++++++++++
t/lei_xsearch.t | 2 +-
t/mbox_reader.t | 75 ++++
t/on_destroy.t | 9 +
t/plack.t | 4 +-
t/run.perl | 3 +-
t/shared_kv.t | 58 +++
t/sigfd.t | 6 +-
46 files changed, 1755 insertions(+), 211 deletions(-)
create mode 100644 lib/PublicInbox/IPC.pm
create mode 100644 lib/PublicInbox/LeiDedupe.pm
rename lib/PublicInbox/{LeiExtinbox.pm => LeiExternal.pm} (75%)
create mode 100644 lib/PublicInbox/LeiToMail.pm
create mode 100644 lib/PublicInbox/MboxReader.pm
create mode 100644 lib/PublicInbox/SharedKV.pm
create mode 100644 t/ipc.t
create mode 100644 t/lei_dedupe.t
create mode 100644 t/lei_to_mail.t
create mode 100644 t/mbox_reader.t
create mode 100644 t/shared_kv.t
^ permalink raw reply [relevance 7%]
Results 1-2 of 2 | reverse | options above
-- pct% links below jump to the message on this page, permalinks otherwise --
2020-12-31 13:51 7% [PATCH 00/36] another round of lei stuff Eric Wong
2020-12-31 13:51 6% ` [PATCH 24/36] spawn: move run_die here from PublicInbox::Import Eric Wong
Code repositories for project(s) associated with this public inbox
https://80x24.org/public-inbox.git
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).