From: Eric Wong <e@80x24.org>
To: meta@public-inbox.org
Subject: [PATCH] initial spawn implementation using vfork
Date: Sat, 27 Feb 2016 02:14:23 +0000 [thread overview]
Message-ID: <20160227021423.618-1-e@80x24.org> (raw)
Under Linux, vfork maintains constant performance as
parent process size increases. fork needs to prepare pages
for copy-on-write, requiring a linear scan of the address
space.
---
lib/PublicInbox/Spawn.pm | 145 +++++++++++++++++++++++++++++++++++++++++++++
lib/PublicInbox/SpawnPP.pm | 33 +++++++++++
t/spawn.t | 53 +++++++++++++++++
3 files changed, 231 insertions(+)
create mode 100644 lib/PublicInbox/Spawn.pm
create mode 100644 lib/PublicInbox/SpawnPP.pm
create mode 100644 t/spawn.t
diff --git a/lib/PublicInbox/Spawn.pm b/lib/PublicInbox/Spawn.pm
new file mode 100644
index 0000000..ecc0aef
--- /dev/null
+++ b/lib/PublicInbox/Spawn.pm
@@ -0,0 +1,145 @@
+# Copyright (C) 2016 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+package PublicInbox::Spawn;
+use strict;
+use warnings;
+use base qw(Exporter);
+our @EXPORT_OK = qw/which spawn/;
+
+my $vfork_spawn = <<'VFORK_SPAWN';
+#include <sys/types.h>
+#include <sys/uio.h>
+#include <unistd.h>
+#include <alloca.h>
+
+#define AV_ALLOCA(av, max) alloca((max = (av_len((av)) + 1)) * sizeof(char *))
+
+static void av2c_copy(char **dst, AV *src, I32 max)
+{
+ I32 i;
+
+ for (i = 0; i < max; i++) {
+ SV **sv = av_fetch(src, i, 0);
+ dst[i] = sv ? SvPV_nolen(*sv) : 0;
+ }
+ dst[max] = 0;
+}
+
+static void *deconst(const char *s)
+{
+ union { const char *in; void *out; } u;
+ u.in = s;
+ return u.out;
+}
+
+/* needs to be safe inside a vfork'ed process */
+static void xerr(const char *msg)
+{
+ struct iovec iov[3];
+ const char *err = strerror(errno); /* should be safe in practice */
+
+ iov[0].iov_base = deconst(msg);
+ iov[0].iov_len = strlen(msg);
+ iov[1].iov_base = deconst(err);
+ iov[1].iov_len = strlen(err);
+ iov[2].iov_base = deconst("\n");
+ iov[2].iov_len = 1;
+ writev(2, iov, 3);
+ _exit(1);
+}
+
+#define REDIR(var,fd) do { \
+ if (var != fd && dup2(var, fd) < 0) \
+ xerr("error redirecting std"#var ": "); \
+} while (0)
+
+/*
+ * unstable internal API. This was easy to implement but does not
+ * support arbitrary redirects. It'll be updated depending on
+ * whatever we'll need in the future.
+ * Be sure to update PublicInbox::SpawnPP if this changes
+ */
+int public_inbox_fork_exec(int in, int out, int err,
+ SV *file, SV *cmdref, SV *envref)
+{
+ AV *cmd = (AV *)SvRV(cmdref);
+ AV *env = (AV *)SvRV(envref);
+ const char *filename = SvPV_nolen(file);
+ pid_t pid;
+ char **argv, **envp;
+ I32 max;
+
+ argv = AV_ALLOCA(cmd, max);
+ av2c_copy(argv, cmd, max);
+
+ envp = AV_ALLOCA(env, max);
+ av2c_copy(envp, env, max);
+
+ pid = vfork();
+ if (pid == 0) {
+ REDIR(in, 0);
+ REDIR(out, 1);
+ REDIR(err, 2);
+ execve(filename, argv, envp);
+ _exit(1);
+ }
+
+ return (int)pid;
+}
+VFORK_SPAWN
+
+my $inline_dir = $ENV{PERL_INLINE_DIRECTORY};
+unless (defined $inline_dir && -d $inline_dir && -w _) {
+ $vfork_spawn = undef;
+}
+if (defined $vfork_spawn) {
+ # need 0.64 or later for locking in multi-process env
+ eval 'use Inline 0.64 C => $vfork_spawn';
+ if ($@) {
+ warn "Inline::C failed for vfork: $@\n";
+ $vfork_spawn = undef;
+ }
+}
+
+unless (defined $vfork_spawn) {
+ require PublicInbox::SpawnPP;
+ no warnings 'once';
+ *public_inbox_fork_exec = *PublicInbox::SpawnPP::public_inbox_fork_exec
+}
+
+sub which ($) {
+ my ($file) = @_;
+ foreach my $p (split(':', $ENV{PATH})) {
+ $p .= "/$file";
+ return $p if -x $p;
+ }
+ undef;
+}
+
+sub spawn ($;$$) {
+ my ($cmd, $env, $opts) = @_;
+ my $f = which($cmd->[0]);
+ my @env;
+ $opts ||= {};
+
+ my %env = $opts->{-env} ? () : %ENV;
+ if ($env) {
+ foreach my $k (keys %$env) {
+ my $v = $env->{$k};
+ if (defined $v) {
+ $env{$k} = $v;
+ } else {
+ delete $env{$k};
+ }
+ }
+ }
+ while (my ($k, $v) = each %env) {
+ push @env, "$k=$v";
+ }
+ my $in = $opts->{0} || 0;
+ my $out = $opts->{1} || 1;
+ my $err = $opts->{2} || 2;
+ public_inbox_fork_exec($in, $out, $err, $f, $cmd, \@env);
+}
+
+1;
diff --git a/lib/PublicInbox/SpawnPP.pm b/lib/PublicInbox/SpawnPP.pm
new file mode 100644
index 0000000..ae552dd
--- /dev/null
+++ b/lib/PublicInbox/SpawnPP.pm
@@ -0,0 +1,33 @@
+# Copyright (C) 2016 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+package PublicInbox::SpawnPP;
+use strict;
+use warnings;
+use POSIX qw(dup2);
+
+# Pure Perl implementation for folks that do not use Inline::C
+sub public_inbox_fork_exec ($$$$$$) {
+ my ($in, $out, $err, $f, $cmd, $env) = @_;
+ my $pid = fork;
+ if ($pid == 0) {
+ if ($in != 0) {
+ dup2($in, 0) or die "dup2 failed for stdin: $!";
+ }
+ if ($out != 1) {
+ dup2($out, 1) or die "dup2 failed for stdout: $!";
+ }
+ if ($err != 2) {
+ dup2($err, 2) or die "dup2 failed for stderr$!";
+ }
+ %ENV = ();
+ foreach my $e (@$env) {
+ my ($k, $v) = split('=', $e, 2);
+ $ENV{$k} = $v;
+ }
+ exec @$cmd;
+ exit 1;
+ }
+ $pid;
+}
+
+1;
diff --git a/t/spawn.t b/t/spawn.t
new file mode 100644
index 0000000..ed9b5b0
--- /dev/null
+++ b/t/spawn.t
@@ -0,0 +1,53 @@
+# Copyright (C) 2015 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+use strict;
+use warnings;
+use Test::More;
+use PublicInbox::Spawn qw(which spawn);
+
+{
+ my $true = which('true');
+ ok($true, "'true' command found with which()");
+}
+
+{
+ my $pid = spawn(['true']);
+ ok($pid, 'spawned process');
+ is(waitpid($pid, 0), $pid, 'waitpid succeeds on spawned process');
+ is($?, 0, 'true exited successfully');
+}
+
+{
+ my ($r, $w);
+ pipe $r, $w or die "pipe failed: $!";
+ my $pid = spawn(['echo', 'hello world'], undef, { 1 => fileno($w) });
+ close $w or die "close pipe[1] failed: $!";
+ is(<$r>, "hello world\n", 'read stdout of spawned from pipe');
+ is(waitpid($pid, 0), $pid, 'waitpid succeeds on spawned process');
+ is($?, 0, 'true exited successfully');
+}
+
+{
+ my ($r, $w);
+ pipe $r, $w or die "pipe failed: $!";
+ my $pid = spawn(['sh', '-c', 'echo $HELLO'],
+ { 'HELLO' => 'world' }, { 1 => fileno($w) });
+ close $w or die "close pipe[1] failed: $!";
+ is(<$r>, "world\n", 'read stdout of spawned from pipe');
+ is(waitpid($pid, 0), $pid, 'waitpid succeeds on spawned process');
+ is($?, 0, 'sh exited successfully');
+}
+
+{
+ my ($r, $w);
+ pipe $r, $w or die "pipe failed: $!";
+ my $pid = spawn(['env'], {}, { -env => 1, 1 => fileno($w) });
+ close $w or die "close pipe[1] failed: $!";
+ ok(!defined(<$r>), 'read stdout of spawned from pipe');
+ is(waitpid($pid, 0), $pid, 'waitpid succeeds on spawned process');
+ is($?, 0, 'env(1) exited successfully');
+}
+
+done_testing();
+
+1;
--
EW
next reply other threads:[~2016-02-27 2:14 UTC|newest]
Thread overview: 3+ messages / expand[flat|nested] mbox.gz Atom feed top
2016-02-27 2:14 Eric Wong [this message]
2016-02-27 11:00 ` [PATCH v2] initial spawn implementation using vfork Eric Wong
2016-02-27 22:19 ` [PATCH 2/1] spawn: fail properly if Inline fails Eric Wong
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
List information: https://public-inbox.org/README
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20160227021423.618-1-e@80x24.org \
--to=e@80x24.org \
--cc=meta@public-inbox.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
Code repositories for project(s) associated with this public inbox
https://80x24.org/public-inbox.git
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).