RFC V2 [1/2] test-in-container

RFC V2 [1/2] test-in-container

[DJ Delorie]

	* Makefile (testroot.pristine): New rules to initialize the
	test-in-container "testroot".
	* Makerules (all-testsuite): Add tests-container.
	* Rules (tests): Add tests-container.
	(binaries-all-tests): Likewise.
	(tests-container): New, run these tests in the testroot container.
	* support/Makefile (others): Add test-container and links-dso-program.
	* support/links-dso-program-c.c: New.
	* support/links-dso-program.cc: New.
	* support/test-container.c: New.

diff --git a/Makefile b/Makefile
index bea4e27f8d..3fd19917df 100644
--- a/Makefile
+++ b/Makefile
@@ -297,6 +297,46 @@ define summarize-tests
 @! egrep -q -v '^(X?PASS|XFAIL|UNSUPPORTED):' $(objpfx)$1
 endef
 
+# The intention here is to do ONE install of our build into the
+# testroot.pristine/ directory, then rsync (internal to
+# support/test-container) that to testroot.root/ at the start of each
+# test.  That way we can promise each test a "clean" install, without
+# having to do the install for each test.
+#
+# In addition, we have to copy some files into this root in addition
+# to what glibc installs.  For example, many tests require /bin/sh be
+# present, and any shared objects that /bin/sh depends on.  We also
+# build a "test" program in either C or (if available) C++ just so we
+# can copy in any shared objects that GCC-compiled programs depend on.
+
+$(tests-container) $(addsuffix /tests,$(subdirs)) : $(objpfx)testroot.pristine/ready.ts
+$(objpfx)testroot.pristine/ready.ts :
+	test -d $(objpfx)testroot.pristine || \
+	  mkdir $(objpfx)testroot.pristine
+	# We need a working /bin/sh for some of the tests.
+	test -d $(objpfx)testroot.pristine/bin || \
+	  mkdir $(objpfx)testroot.pristine/bin
+	$(test-wrapper) cp /bin/sh $(objpfx)testroot.pristine/bin/sh
+	# Copy these DSOs first so we can overwrite them with our own.
+	for dso in `$(test-wrapper-env) LD_TRACE_LOADED_OBJECTS=1  $(objpfx)elf/$(rtld-installed-name) \
+			        $(objpfx)testroot.pristine/bin/sh \
+	                      | grep / | sed 's/^[^/]*//' | sed 's/ .*//'` ;\
+	  do \
+	    test -d `dirname $(objpfx)testroot.pristine$$dso` || \
+	      mkdir -p `dirname $(objpfx)testroot.pristine$$dso` ;\
+	    $(test-wrapper) cp $$dso $(objpfx)testroot.pristine$$dso ;\
+	  done
+	for dso in `$(test-wrapper-env) LD_TRACE_LOADED_OBJECTS=1  $(objpfx)elf/$(rtld-installed-name) \
+			        $(objpfx)support/links-dso-program \
+	                      | grep / | sed 's/^[^/]*//' | sed 's/ .*//'` ;\
+	  do \
+	    test -d `dirname $(objpfx)testroot.pristine$$dso` || \
+	      mkdir -p `dirname $(objpfx)testroot.pristine$$dso` ;\
+	    $(test-wrapper) cp $$dso $(objpfx)testroot.pristine$$dso ;\
+	  done
+	$(MAKE) install DESTDIR=$(objpfx)testroot.pristine
+	touch $(objpfx)testroot.pristine/ready.ts
+
 tests-special-notdir = $(patsubst $(objpfx)%, %, $(tests-special))
 tests: $(tests-special)
 	$(..)scripts/merge-test-results.sh -s $(objpfx) "" \
diff --git a/Makerules b/Makerules
index b2c2724fcb..21367503d6 100644
--- a/Makerules
+++ b/Makerules
@@ -1372,7 +1372,7 @@ xcheck: xtests
 # The only difference between MODULE_NAME=testsuite and MODULE_NAME=nonlib is
 # that almost all internal declarations from config.h, libc-symbols.h, and
 # include/*.h are not available to 'testsuite' code, but are to 'nonlib' code.
-all-testsuite := $(strip $(tests) $(xtests) $(test-srcs) $(test-extras))
+all-testsuite := $(strip $(tests) $(xtests) $(test-srcs) $(test-extras) $(tests-container))
 ifneq (,$(all-testsuite))
 cpp-srcs-left = $(all-testsuite)
 lib := testsuite
diff --git a/Rules b/Rules
index 706c8a749d..5460ab0d95 100644
--- a/Rules
+++ b/Rules
@@ -130,12 +130,12 @@ others: $(py-const)
 
 ifeq ($(run-built-tests),no)
 tests: $(addprefix $(objpfx),$(filter-out $(tests-unsupported), \
-                                          $(tests) $(tests-internal)) \
+                                          $(tests) $(tests-internal) $(tests-container)) \
 			     $(test-srcs)) $(tests-special) \
 			     $(tests-printers-programs)
 xtests: tests $(xtests-special)
 else
-tests: $(tests:%=$(objpfx)%.out) $(tests-internal:%=$(objpfx)%.out) \
+tests: $(tests:%=$(objpfx)%.out) $(tests-internal:%=$(objpfx)%.out) $(tests-container:%=$(objpfx)%.out) \
        $(tests-special) $(tests-printers-out)
 xtests: tests $(xtests:%=$(objpfx)%.out) $(xtests-special)
 endif
@@ -149,7 +149,7 @@ tests-expected = $(tests) $(tests-internal) $(tests-printers)
 endif
 tests:
 	$(..)scripts/merge-test-results.sh -s $(objpfx) $(subdir) \
-	  $(sort $(tests-expected) $(tests-special-notdir:.out=)) \
+	  $(sort $(tests-expected) $(tests-special-notdir:.out=) $(tests-container)) \
 	  > $(objpfx)subdir-tests.sum
 xtests:
 	$(..)scripts/merge-test-results.sh -s $(objpfx) $(subdir) \
@@ -158,7 +158,7 @@ xtests:
 
 ifeq ($(build-programs),yes)
 binaries-all-notests = $(others) $(sysdep-others)
-binaries-all-tests = $(tests) $(tests-internal) $(xtests) $(test-srcs)
+binaries-all-tests = $(tests) $(tests-internal) $(xtests) $(test-srcs) $(tests-container)
 binaries-all = $(binaries-all-notests) $(binaries-all-tests)
 binaries-static-notests = $(others-static)
 binaries-static-tests = $(tests-static) $(xtests-static)
@@ -248,6 +248,15 @@ $(objpfx)%.out: /dev/null $(objpfx)%	# Make it 2nd arg for canned sequence.
 	$(make-test-out) > $@; \
 	$(evaluate-test)
 
+
+# Any tests that require an isolated container (chroot) in which to
+# run, should be added to tests-container.
+$(tests-container:%=$(objpfx)%.out): $(objpfx)%.out : $(if $(wildcard $(objpfx)%.files),$(objpfx)%.files,/dev/null) $(objpfx)%
+	$(test-wrapper) $(common-objpfx)support/test-container env $(run-program-env) \
+	  $(host-test-program-cmd) $($*-ARGS) > $@; \
+	$(evaluate-test)
+
+
 # tests-unsupported lists tests that we will not try to build at all in
 # this configuration.  Note this runs every time because it does not
 # actually create its target.  The dependency on Makefile is meant to
diff --git a/support/Makefile b/support/Makefile
index 1bda81e55e..49ac4ab409 100644
--- a/support/Makefile
+++ b/support/Makefile
@@ -145,6 +145,26 @@ ifeq ($(build-shared),yes)
 libsupport-inhibit-o += .o
 endif
 
+others: $(objpfx)test-container $(objpfx)links-dso-program
+
+CFLAGS-test-container.c = \
+		-DSRCDIR_PATH=\"`cd .. ; pwd`\" \
+		-DOBJDIR_PATH=\"`cd $(objpfx)/..; pwd`\" \
+		-DINSTDIR_PATH=\"${prefix}\" \
+		-DLIBDIR_PATH=\"${libdir}\"
+
+others += test-container
+
+# This exists only so we can guess which OS DSOs we need to copy into
+# the testing container.
+ifeq (,$(CXX))
+$(objpfx)links-dso-program : $(objpfx)links-dso-program-c.o
+	$(LINK.o) -o $@ $^
+else
+$(objpfx)links-dso-program : $(objpfx)links-dso-program.o
+	$(LINK.o) -o $@ $^ -lstdc++
+endif
+
 tests = \
   README-testing \
   tst-support-namespace \
diff --git a/support/links-dso-program-c.c b/support/links-dso-program-c.c
new file mode 100644
index 0000000000..c1f64fbbac
--- /dev/null
+++ b/support/links-dso-program-c.c
@@ -0,0 +1,4 @@
+int
+main() {
+  return 0;
+}
diff --git a/support/links-dso-program.cc b/support/links-dso-program.cc
new file mode 100644
index 0000000000..c1f64fbbac
--- /dev/null
+++ b/support/links-dso-program.cc
@@ -0,0 +1,4 @@
+int
+main() {
+  return 0;
+}
diff --git a/support/test-container.c b/support/test-container.c
new file mode 100644
index 0000000000..2fb7c2bdd0
--- /dev/null
+++ b/support/test-container.c
@@ -0,0 +1,1070 @@
+/* Run a test case in an isolated namespace.
+   Copyright (C) 2017-2018 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+#define __USE_LARGEFILE64
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sched.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include <dirent.h>
+#include <string.h>
+/*#include <sys/capability.h>*/
+#include <sys/mount.h>
+#include <sys/stat.h>
+#include <sys/fcntl.h>
+#include <sys/file.h>
+#include <sys/wait.h>
+#include <stdarg.h>
+#include <sys/sysmacros.h>
+#include <ctype.h>
+#include <utime.h>
+#include <errno.h>
+
+#ifdef SRCDIR_PATH
+const char *srcdir = SRCDIR_PATH;
+#else
+# error please -DSRCDIR_PATH=something in the Makefile
+#endif
+
+#ifdef OBJDIR_PATH
+const char *objdir = OBJDIR_PATH;
+#else
+# error please -DOBJDIR_PATH=something in the Makefile
+#endif
+
+#ifdef INSTDIR_PATH
+const char *instdir = INSTDIR_PATH;
+#else
+# error please -DINSTDIR_PATH=something in the Makefile
+#endif
+
+#ifdef LIBDIR_PATH
+const char *libdir = LIBDIR_PATH;
+#else
+# error please -DLIBDIR_PATH=something in the Makefile
+#endif
+
+int verbose = 0;
+
+/* Running a test in a container is tricky.  There are two main
+   categories of things to do:
+
+   1. "Once" actions, like setting up the container and doing an
+      install into it.
+
+   2. "Per-test" actions, like copying in support files and
+      configuring the container.
+
+
+   "Once" actions:
+
+   * mkdir $buildroot/testroot.pristine/
+   * install into it
+   * rsync to $buildroot/testroot.root/
+
+   "Per-test" actions:
+   * maybe rsync to $buildroot/testroot.root/
+   * copy support files and test binary
+   * chroot/unshare
+   * set up any mounts (like /proc)
+
+   Magic files:
+
+   For test $srcdir/foo/mytest.c we look for $srcdir/foo/mytest.root and, if found...
+
+   * mytest.root/ is rsync'd into container
+   * mytest.root/preclean.txt causes fresh rsync (with delete) before test if present
+   * mytest.root/files.txt has a list of files to copy - TBD
+     ($B/ or $S/ for build and source directories)
+       syntax:
+         # comment
+         mv FILE FILE
+	 cp FILE FILE
+	 rm FILE
+	 FILE must start with $B/, $S/, $I/, $L/, or /
+	  (expands to build dir, source dir, install dir, library dir
+	   (in container), or container's root)
+   * mytest.root/postclean.txt causes fresh rsync (with delete) after test if present
+
+   Note that $srcdir/foo/mytest.files may be used instead of a
+   files.txt in the sysroot, if there is no other reason for a sysroot.
+
+   Design goals:
+
+   * independent of other packages which may not be installed (like
+     rsync or Docker, or even "cp")
+
+   * Simple, easy to review code (i.e. prefer simple naive code over
+     complex efficient code)
+
+   TBD:
+
+   * The current implementation is not parallel-make-safe, as one test
+     could be modifying the chroot while another is running against
+     it.
+
+*/
+
+/*--------------------------------------------------*/
+/* Utility Functions */
+
+/* Temporarily concatenate multiple strings into one.  Allows up to 10
+   temporary results; use strdup () if you need them to be
+   permanent.  */
+
+static char *
+concat (const char *str, ...)
+{
+  /* Assume initialized to NULL/zero.  */
+  static char *bufs[10];
+  static size_t buflens[10];
+  static int bufn = 0;
+  int n;
+  size_t len;
+  va_list ap, ap2;
+  char *cp;
+  char *next;
+
+  va_start (ap, str);
+  va_copy (ap2, ap);
+
+  n = bufn;
+  bufn = (bufn + 1) % 10;
+  len = strlen (str);
+
+  while ((next = va_arg (ap, char *)) != NULL)
+    len = len + strlen (next);
+
+  va_end (ap);
+
+  if (bufs[n] == NULL)
+    {
+      bufs[n] = malloc (len + 1); /* NUL */
+      buflens[n] = len + 1;
+    }
+  else if (buflens[n] < len + 1)
+    {
+      bufs[n] = realloc (bufs[n], len + 1); /* NUL */
+      buflens[n] = len + 1;
+    }
+
+  strcpy (bufs[n], str);
+  cp = strchr (bufs[n], '\0');
+  while ((next = va_arg (ap2, char *)) != NULL)
+    {
+      strcpy (cp, next);
+      cp = strchr (cp, '\0');
+    }
+  *cp = 0;
+  va_end (ap2);
+
+  return bufs[n];
+}
+
+/* Equivalent of "mkdir -p".  */
+
+static int
+mkdir_p (char *path)
+{
+  struct stat s;
+  char *slash_p;
+  int rv;
+
+  if (path[0] == 0)
+    return 0;
+
+  if (stat (path, &s) == 0)
+    {
+      if (S_ISDIR (s.st_mode))
+	return 0;
+      printf ("mkdir_p: %s exists but isn't a directory\n", path);
+      perror ("The error was");
+      exit (1);
+    }
+
+  slash_p = strrchr (path, '/');
+  if (slash_p)
+    {
+      while (slash_p > path && slash_p[-1] == '/')
+	--slash_p;
+      // Hack to allow top-level paths to be read-only and still work
+      if (slash_p > path)
+	{
+	  *slash_p = '\0';
+	  rv = mkdir_p (path);
+	  if (rv != 0)
+	    return rv;
+	  *slash_p = '/';
+	}
+    }
+
+  rv = mkdir (path, 0755);
+  if (rv != 0)
+    {
+      printf ("mkdir_p: can't create directory %s\n", path);
+      perror ("The error was");
+      return 1;
+    }
+
+  return 0;
+}
+
+/* Try to mount SRC onto DEST.  */
+
+static void
+trymount (const char *src, const char *dest)
+{
+  if (mount (src, dest, "", MS_BIND, NULL) < 0)
+    {
+      printf ("can't mount %s onto %s\n", src, dest);
+      perror ("the error was");
+      exit (1);
+    }
+}
+
+/* Special case of above for devices like /dev/zero where we have to
+   mount a device over a device, not a directory over a directory.  */
+
+static void
+devmount (const char *new_root_path, const char *which)
+{
+  int fd;
+  fd = open (concat (new_root_path, "/dev/", which, NULL),
+	     O_CREAT | O_TRUNC | O_RDWR, 0777);
+  close (fd);
+
+  trymount (concat ("/dev/", which, NULL),
+	    concat (new_root_path, "/dev/", which, NULL));
+}
+
+/* Returns true if the string "looks like" an environement variable
+   being set.  */
+
+static int
+is_env_setting (const char *a)
+{
+  int count_name = 0;
+
+  while (*a)
+    {
+      if (isalnum (*a) || *a == '_')
+	++count_name;
+      else if (*a == '=' && count_name > 0)
+	return 1;
+      else
+	return 0;
+      ++a;
+    }
+  return 0;
+}
+
+/* Break the_line into words and store in the_words.  Max nwords,
+   returns actual count.  */
+static int
+tokenize (char *the_line, char **the_words, int nwords)
+{
+  int rv = 0;
+
+  while (nwords > 0)
+    {
+      /* Skip leading whitespace, if any.  */
+      while (*the_line && isspace (*the_line))
+	++the_line;
+
+      /* End of line?  */
+      if (*the_line == 0)
+	return rv;
+
+      /* THE_LINE points to a non-whitespace character, so we have a
+	 word.  */
+      *the_words = the_line;
+      ++the_words;
+      nwords--;
+      ++rv;
+
+      /* Skip leading whitespace, if any.  */
+      while (*the_line && ! isspace (*the_line))
+	++the_line;
+
+      /* We now point at the trailing NUL *or* some whitespace.  */
+      if (*the_line == 0)
+	return rv;
+
+      /* It was whitespace, skip and keep tokenizing.  */
+      *the_line++ = 0;
+    }
+
+  /* We get here if we filled the words buffer.  */
+  return rv;
+}
+
+/*--------------------------------------------------*/
+/* mini-RSYNC implementation.  Optimize later.      */
+
+/* Set this to 1 if you need to debug the rsync function.  */
+#define RTRACE 0
+
+/* A few routines for an "rsync buffer" which stores the paths we're
+   working on.  We continuously grow and shrink the paths in each
+   buffer so there's lot of re-use.  */
+
+/* We rely on "initialized to zero" to set these up.  */
+typedef struct
+{
+  char *buf;
+  size_t len;
+  size_t size;
+} path_buf;
+
+static path_buf spath, dpath;
+
+static void
+r_setup (char *path, path_buf * pb)
+{
+  size_t len = strlen (path);
+  if (pb->buf == NULL || pb->size < len + 1)
+    {
+      /* Round up */
+      size_t sz = ALIGN_UP (len + 1, 512);
+      if (pb->buf == NULL)
+	pb->buf = (char *) malloc (sz);
+      else
+	pb->buf = (char *) realloc (pb->buf, sz);
+      if (pb->buf == NULL)
+	{
+	  printf ("Out of memory while rsyncing\n");
+	  exit (1);
+	}
+      pb->size = sz;
+    }
+  strcpy (pb->buf, path);
+  pb->len = len;
+}
+
+static void
+r_append (const char *path, path_buf * pb)
+{
+  size_t len = strlen (path) + pb->len;
+  if (pb->size < len + 1)
+    {
+      /* Round up */
+      size_t sz = ALIGN_UP (len + 1, 512);
+      pb->buf = (char *) realloc (pb->buf, sz);
+      if (pb->buf == NULL)
+	{
+	  printf ("Out of memory while rsyncing\n");
+	  exit (1);
+	}
+      pb->size = sz;
+    }
+  strcpy (pb->buf + pb->len, path);
+  pb->len = len;
+}
+
+static int
+file_exists (char *path)
+{
+  struct stat st;
+  if (lstat (path, &st) == 0)
+    return 1;
+  return 0;
+}
+
+static void
+recursive_remove (char *path)
+{
+  pid_t child;
+  int status;
+  /* FIXME: re-implement without external dependencies at some point.
+     Fortunately, this runs outside the container.  */
+
+  child = fork();
+
+  switch (child) {
+  case -1:
+    perror ("fork");
+    exit (1);
+  case 0:
+    /* Child.  */
+    execlp ("rm", "rm", "-rf", path, NULL);
+  default:
+    /* Parent.  */
+    waitpid (child, &status, 0);
+    break;
+  }
+}
+
+/* Used for both rsync and the files.txt "cp" command.  */
+
+static void
+copy_one_file (const char *sname, const char *dname)
+{
+  int sfd, dfd;
+  char buf[512];
+  size_t rsz;
+  struct stat st;
+  struct utimbuf times;
+
+  sfd = open (sname, O_RDONLY);
+  if (sfd < 0)
+    {
+      printf ("unable to open %s for reading\n", sname);
+      perror ("the error was");
+      exit (1);
+    }
+  dfd = open (dname, O_WRONLY | O_TRUNC | O_CREAT, 0600);
+  if (dfd < 0)
+    {
+      printf ("unable to open %s for writing\n", dname);
+      perror ("the error was");
+      exit (1);
+    }
+
+  while ((rsz = read (sfd, buf, 512)) > 0)
+    write (dfd, buf, rsz);
+
+  if (rsz < 0)
+    {
+      printf ("error reading from %s\n", sname);
+      perror ("the error was");
+      exit (1);
+    }
+
+  close (sfd);
+  close (dfd);
+
+  stat (sname, &st);
+  chmod (dname, st.st_mode & 0777);
+
+  times.actime = st.st_atime;
+  times.modtime = st.st_mtime;
+  utime (dname, &times);
+}
+
+/* We don't check *everything* about the two files to see if a copy is
+   needed, just the minimum to make sure we get the latest copy. */
+static int
+need_sync (char *ap, char *bp, struct stat *a, struct stat *b)
+{
+  if ((a->st_mode & S_IFMT) != (b->st_mode & S_IFMT))
+    return 1;
+
+  if (S_ISLNK (a->st_mode))
+    {
+      int rv;
+      char *al, *bl;
+
+      if (a->st_size != b->st_size)
+	return 1;
+
+      al = (char *) malloc (a->st_size + 1);
+      bl = (char *) malloc (b->st_size + 1);
+      readlink (ap, al, a->st_size + 1);
+      readlink (bp, bl, b->st_size + 1);
+      al[a->st_size] = 0;
+      bl[b->st_size] = 0;
+      rv = strcmp (al, bl);
+      free (al);
+      free (bl);
+      if (rv == 0)
+	return 0; /* links are same */
+      return 1; /* links differ */
+    }
+
+#if RTRACE
+  if (a->st_size != b->st_size)
+    printf ("SIZE\n");
+  if ((a->st_mode & 0777) != (b->st_mode & 0777))
+    printf ("MODE\n");
+  if (a->st_mtime != b->st_mtime)
+    printf ("TIME\n");
+#endif
+
+  if (a->st_size == b->st_size
+      && ((a->st_mode & 0777) == (b->st_mode & 0777))
+      && a->st_mtime == b->st_mtime)
+    return 0;
+
+  return 1;
+}
+
+static void
+rsync_1 (path_buf * src, path_buf * dest, int and_delete)
+{
+  DIR *dir;
+  struct dirent *de;
+  struct stat s, d;
+
+  r_append ("/", src);
+  r_append ("/", dest);
+#if RTRACE
+  printf ("sync %s to %s %s\n", src->buf, dest->buf,
+	  and_delete ? "and delete" : "");
+#endif
+
+  size_t staillen = src->len;
+
+  size_t dtaillen = dest->len;
+
+  dir = opendir (src->buf);
+
+  while ((de = readdir (dir)) != NULL)
+    {
+      if (strcmp (de->d_name, ".") == 0
+	  || strcmp (de->d_name, "..") == 0)
+	continue;
+
+      src->len = staillen;
+      r_append (de->d_name, src);
+      dest->len = dtaillen;
+      r_append (de->d_name, dest);
+
+      s.st_mode = ~0;
+      d.st_mode = ~0;
+
+      lstat (src->buf, &s);
+      lstat (dest->buf, &d);
+
+      if (s.st_mode == ~0)
+	{
+	  printf ("Error: %s obtained by readdir, but stat failed.\n",
+		  src->buf);
+	  exit (1);
+	}
+
+      if (! need_sync (src->buf, dest->buf, &s, &d))
+	{
+	  if (S_ISDIR (s.st_mode))
+	    rsync_1 (src, dest, and_delete);
+	  continue;
+	}
+
+      if (d.st_mode != ~0)
+	switch (d.st_mode & S_IFMT)
+	  {
+	  case S_IFDIR:
+	    if (!S_ISDIR (s.st_mode))
+	      {
+#if RTRACE
+		printf ("-D %s\n", dest->buf);
+#endif
+		recursive_remove (dest->buf);
+	      }
+	    break;
+
+	  default:
+#if RTRACE
+	    printf ("-F %s\n", dest->buf);
+#endif
+	    unlink (dest->buf);
+	    break;
+	  }
+
+      switch (s.st_mode & S_IFMT)
+	{
+	case S_IFREG:
+#if RTRACE
+	  printf ("+F %s\n", dest->buf);
+#endif
+	  copy_one_file (src->buf, dest->buf);
+	  break;
+
+	case S_IFDIR:
+#if RTRACE
+	  printf ("+D %s\n", dest->buf);
+#endif
+	  mkdir (dest->buf, (s.st_mode & 0777) | 0700);
+	  rsync_1 (src, dest, and_delete);
+	  break;
+
+	case S_IFLNK:
+	  {
+	    char *lp = (char *) malloc (s.st_size + 1);
+#if RTRACE
+	    printf ("+L %s\n", dest->buf);
+#endif
+	    readlink (src->buf, lp, s.st_size + 1);
+	    lp[s.st_size] = 0;
+	    symlink (lp, dest->buf);
+	    free (lp);
+	    break;
+	  }
+
+	default:
+	  break;
+	}
+    }
+
+  closedir (dir);
+  src->len = staillen;
+  src->buf[staillen] = 0;
+  dest->len = dtaillen;
+  dest->buf[dtaillen] = 0;
+
+  if (!and_delete)
+    return;
+
+  /* The rest of this function removes any files/directories in DEST
+     that do not exist in SRC.  This is triggered as part of a
+     preclean or postsclean step.  */
+
+  dir = opendir (dest->buf);
+
+  while ((de = readdir (dir)) != NULL)
+    {
+      if (strcmp (de->d_name, ".") == 0
+	  || strcmp (de->d_name, "..") == 0)
+	continue;
+
+      src->len = staillen;
+      r_append (de->d_name, src);
+      dest->len = dtaillen;
+      r_append (de->d_name, dest);
+
+      s.st_mode = ~0;
+      d.st_mode = ~0;
+
+      lstat (src->buf, &s);
+      lstat (dest->buf, &d);
+
+      if (d.st_mode == ~0)
+	{
+	  printf ("Error: %s obtained by readdir, but stat failed.\n",
+		  dest->buf);
+	  exit (1);
+	}
+      if (s.st_mode == ~0)
+	{
+	  /* dest exists and src doesn't, clean it.  */
+	  switch (d.st_mode & S_IFMT)
+	    {
+	    case S_IFDIR:
+	      if (!S_ISDIR (s.st_mode))
+		{
+#if RTRACE
+		  printf ("-D %s\n", dest->buf);
+#endif
+		  recursive_remove (dest->buf);
+		}
+	      break;
+
+	    default:
+#if RTRACE
+	      printf ("-F %s\n", dest->buf);
+#endif
+	      unlink (dest->buf);
+	      break;
+	    }
+	}
+    }
+
+  closedir (dir);
+}
+
+static void
+rsync (char *src, char *dest, int and_delete)
+{
+  r_setup (src, &spath);
+  r_setup (dest, &dpath);
+
+  rsync_1 (&spath, &dpath, and_delete);
+}
+
+/*--------------------------------------------------*/
+/* Main */
+
+int
+main (int argc, char **argv)
+{
+  pid_t child;
+  char *pristine_root_path;
+  char *new_root_path;
+  char *new_cwd_path;
+  char *new_objdir_path;
+  char *new_srcdir_path;
+  char **new_child_proc;
+  char *command_root;
+  char *command_base;
+  char *so_base;
+  int do_postclean = 0;
+
+  uid_t original_uid;
+  gid_t original_gid;
+  int UMAP;
+  int GMAP;
+  char tmp[100];
+  struct stat st;
+  int lock_fd;
+
+  setbuf (stdout, NULL);
+
+  // The command line we're expecting looks like this:
+  // env <set some vars> ld.so <library path> test-binary
+
+  // We need to peel off any "env" or "ld.so" portion of the command
+  // line, and keep track of which env vars we should preserve and
+  // which we drop.
+
+  if (argc < 2)
+    {
+      fprintf (stderr, "Usage: containerize <program to run> <args...>\n");
+      exit (1);
+    }
+
+  if (strcmp (argv[1], "-v") == 0)
+    {
+      verbose = 1;
+      ++argv;
+      --argc;
+    }
+
+  if (strcmp (argv[1], "env") == 0)
+    {
+      ++argv;
+      --argc;
+      while (is_env_setting (argv[1]))
+	{
+	  // List variables we do NOT want to propogate.
+#if 0
+	  // until we discover why locale/iconv tests don't
+	  // work against an installed tree...
+	  if (memcmp (argv[1], "GCONV_PATH=", 11)
+	      && memcmp (argv[1], "LOCPATH=", 8))
+#endif
+	    {
+	      // Need to keep these.  Note that putenv stores a
+	      // pointer to our argv.
+	      putenv (argv[1]);
+	    }
+	  ++argv;
+	  --argc;
+	}
+    }
+
+  if (strncmp (argv[1], concat (objdir, "/elf/ld-linux-", NULL),
+	       strlen (objdir) + 14) == 0)
+    {
+      ++argv;
+      --argc;
+      while (argv[1][0] == '-')
+	{
+	  if (strcmp (argv[1], "--library-path") == 0)
+	    {
+	      ++argv;
+	      --argc;
+	    }
+	  ++argv;
+	  --argc;
+	}
+    }
+
+  pristine_root_path = strdup (concat (objdir, "/testroot.pristine", NULL));
+  new_root_path = strdup (concat (objdir, "/testroot.root", NULL));
+  new_cwd_path = get_current_dir_name ();
+  new_child_proc = argv + 1;
+
+  lock_fd = open(concat (pristine_root_path, "/lock.fd", NULL), O_CREAT | O_TRUNC | O_RDWR, 0666);
+  if (lock_fd < 0)
+    {
+      printf ("Cannot create testroot lock.\n");
+      perror("The error was");
+      exit (77);
+    }
+  while (flock (lock_fd, LOCK_EX) != 0)
+    {
+      if (errno != EINTR)
+	{
+	  printf ("Cannot lock testroot.\n");
+	  perror("The error was");
+	  exit (77);
+	}
+    }
+
+  mkdir_p (new_root_path);
+
+  /* We look for extra setup info in a subdir in the same spot as the
+     test, with the same name but a ".root" extension.  This is that
+     directory.  We try to look in the source tree if the path we're
+     given refers to the build tree, but we rely on the path to be
+     absolute.  This is what the glibc makefiles do.  */
+  command_root = concat (argv[1], ".root", NULL);
+  if (strncmp (command_root, objdir, strlen (objdir)) == 0
+      && command_root[strlen (objdir)] == '/')
+    command_root = concat (srcdir, argv[1] + strlen (objdir), ".root", NULL);
+  command_root = strdup (command_root);
+
+  /* This cuts off the ".root" we appended above.  */
+  command_base = strdup (command_root);
+  command_base[strlen (command_base) - 5] = 0;
+
+  /* Shared object base directory.  */
+  so_base = strdup (argv[1]);
+  if (strrchr (so_base, '/') != NULL)
+    strrchr (so_base, '/')[1] = 0;
+
+  if (file_exists (concat (command_root, "/postclean.txt", NULL)))
+    do_postclean = 1;
+
+  rsync (pristine_root_path, new_root_path,
+	 1 || file_exists (concat (command_root, "/preclean.txt", NULL)));
+
+  if (stat (command_root, &st) >= 0
+      && S_ISDIR (st.st_mode))
+    rsync (command_root, new_root_path, 0);
+
+  new_objdir_path = strdup (concat (new_root_path, objdir, NULL));
+  new_srcdir_path = strdup (concat (new_root_path, srcdir, NULL));
+
+  /* new_cwd_path starts with '/' so no "/" needed between the two.  */
+  mkdir_p (concat (new_root_path, new_cwd_path, NULL));
+  mkdir_p (new_srcdir_path);
+  mkdir_p (new_objdir_path);
+
+  original_uid = getuid ();
+  original_gid = getgid ();
+
+  /* Handle the cp/mv/rm "script" here.  */
+  {
+    char *the_line = NULL;
+    size_t line_len = 0;
+    char *fname = concat (command_root, "/files.txt", NULL);
+    char *the_words[3];
+    FILE *f = fopen (fname, "r");
+
+    if (verbose && f)
+      fprintf (stderr, "reading %s\n", fname);
+
+    if (f == NULL)
+      {
+	/* Try foo.files instead of foo.root/files.txt, as a shortcut.  */
+	fname = concat (command_base, ".files", NULL);
+	f = fopen (fname, "r");
+	if (verbose && f)
+	  fprintf (stderr, "reading %s\n", fname);
+      }
+
+#if 0
+    /* I don't want to add this until we know we need it, but here it
+       is...  */
+    if (f == NULL)
+      {
+	/* Look for a Makefile-generated one also.  */
+	fname = concat (argv[1], ".files", NULL);
+	f = fopen (fname, "r");
+      }
+#endif
+
+    /* This is where we "interpret" the mini-script which is <test>.files.  */
+    if (f != NULL)
+      {
+	while (getline (&the_line, &line_len, f) > 0)
+	  {
+	    int nt = tokenize (the_line, the_words, 3);
+	    int i;
+
+	    for (i = 1; i < nt; ++i)
+	      {
+		if (memcmp (the_words[i], "$B/", 3) == 0)
+		  the_words[i] = concat (objdir, the_words[i] + 2, NULL);
+		else if (memcmp (the_words[i], "$S/", 3) == 0)
+		  the_words[i] = concat (srcdir, the_words[i] + 2, NULL);
+		else if (memcmp (the_words[i], "$I/", 3) == 0)
+		  the_words[i] = concat (new_root_path, instdir, the_words[i] + 2, NULL);
+		else if (memcmp (the_words[i], "$L/", 3) == 0)
+		  the_words[i] = concat (new_root_path, libdir, the_words[i] + 2, NULL);
+		else if (the_words[i][0] == '/')
+		  the_words[i] = concat (new_root_path, the_words[i], NULL);
+	      }
+
+	    if (nt == 3 && the_words[2][strlen (the_words[2]) - 1] == '/')
+	      {
+		char *r = strrchr (the_words[1], '/');
+		if (r)
+		  the_words[2] = concat (the_words[2], r + 1, NULL);
+		else
+		  the_words[2] = concat (the_words[2], the_words[1], NULL);
+	      }
+
+	    if (nt == 2 && strcmp (the_words[0], "so") == 0)
+	      {
+		the_words[2] = concat (new_root_path, libdir, "/", the_words[1], NULL);
+		the_words[1] = concat (so_base, the_words[1], NULL);
+		copy_one_file (the_words[1], the_words[2]);
+	      }
+	    else if (nt == 3 && strcmp (the_words[0], "cp") == 0)
+	      {
+		copy_one_file (the_words[1], the_words[2]);
+	      }
+	    else if (nt == 3 && strcmp (the_words[0], "mv") == 0)
+	      {
+		rename (the_words[1], the_words[2]);
+	      }
+	    else if (nt == 3 && strcmp (the_words[0], "chmod") == 0)
+	      {
+		long int m;
+		m = strtol (the_words[1], NULL, 0);
+		chmod (the_words[2], m);
+	      }
+	    else if (nt == 2 && strcmp (the_words[0], "rm") == 0)
+	      {
+		unlink (the_words[1]);
+	      }
+	    else if (nt > 0 && the_words[0][0] != '#')
+	      {
+		printf ("\033[31minvalid [%s]\033[0m\n", the_words[0]);
+	      }
+	  }
+	fclose (f);
+      }
+  }
+
+  // The unshare here gives us our own spaces and capabilities.
+  if (unshare (CLONE_NEWUSER | CLONE_NEWPID | CLONE_NEWNS) < 0)
+    {
+      printf ("unable to unshare user/fs, ");
+      perror ("the error was");
+      exit (1);
+    }
+
+  /* Some systems, by default, all mounts leak out of the namespace.  */
+  if (mount ("none", "/", NULL, MS_REC | MS_PRIVATE, NULL) != 0)
+    {
+      printf ("warning: could not create a private mount namespace: %m\n");
+      exit (1);
+    }
+
+  trymount (srcdir, new_srcdir_path);
+  trymount (objdir, new_objdir_path);
+
+  mkdir_p (concat (new_root_path, "/dev", NULL));
+  devmount (new_root_path, "null");
+  devmount (new_root_path, "zero");
+  devmount (new_root_path, "urandom");
+
+  // We're done with the "old" root, switch to the new one.
+  if (chroot (new_root_path) < 0)
+    {
+      printf ("Can't chroot to %s - ", new_root_path);
+      perror ("the error was");
+      exit (1);
+    }
+
+  if (chdir (new_cwd_path) < 0)
+    {
+      printf ("Can't cd to new %s - ", new_cwd_path);
+      perror ("the error was");
+      exit (1);
+    }
+
+  /* To complete the containerization, we need to fork () at least
+     once.  We can't exec, nor can we somehow link the new child to
+     our parent.  So we run the child and propogate it's exit status
+     up. */
+  child = fork ();
+  if (child < 0)
+    {
+      perror ("fork");
+      exit (1);
+    }
+  else if (child > 0)
+    {
+      /* Parent.  */
+      int status;
+      waitpid (child, &status, 0);
+
+      /* There's a bit of magic here, since the buildroot is mounted
+	 in our space, the paths are still valid, and since the mounts
+	 aren't recursive, it sees *only* the built root, not anything
+	 we would normally se if we rsync'd to "/" like mounted /dev
+	 files.  */
+      if (do_postclean)
+	  rsync (pristine_root_path, new_root_path, 1);
+
+      if (WIFEXITED (status))
+	exit (WEXITSTATUS (status));
+
+      if (WIFSIGNALED (status))
+	{
+	  printf ("%%SIGNALLED%%\n");
+	  exit (77);
+	}
+
+      printf ("%%EXITERROR%%\n");
+      exit (78);
+    }
+
+  /* The rest is the child process, which is now PID 1 and "in" the
+     new root.  */
+
+  mkdir ("/tmp", 0755);
+
+  // Now that we're pid 1 (effectively "root") we can mount /proc
+  mkdir ("/proc", 0777);
+  if (mount ("proc", "/proc", "proc", 0, NULL) < 0)
+    {
+      printf ("Unable to mount /proc: ");
+      perror ("the error was");
+    }
+
+  // We map our original UID to the same UID in the container so we
+  // can own our own files normally
+  UMAP = open ("/proc/self/uid_map", O_WRONLY);
+  if (UMAP < 0)
+    {
+      printf ("can't write to /proc/self/uid_map\n");
+      perror ("The error was");
+      exit (1);
+    }
+  sprintf (tmp, "%lld %lld 1\n", (long long) original_uid, (long long) original_uid);
+  write (UMAP, tmp, strlen (tmp));
+  close (UMAP);
+
+  // We must disable setgroups () before we can map our groups, else we
+  // get EPERM.
+  GMAP = open ("/proc/self/setgroups", O_WRONLY);
+  if (GMAP >= 0)
+    {
+      /* We support kernels old enough to not have this.  */
+      write (GMAP, "deny\n", 5);
+      close (GMAP);
+    }
+
+  // We map our original GID to the same GID in the container so we
+  // can own our own files normally
+  GMAP = open ("/proc/self/gid_map", O_WRONLY);
+  if (GMAP < 0)
+    {
+      printf ("can't write to /proc/self/gid_map\n");
+      perror ("The error was");
+      exit (1);
+    }
+  sprintf (tmp, "%lld %lld 1\n", (long long) original_gid, (long long) original_gid);
+  write (GMAP, tmp, strlen (tmp));
+  close (GMAP);
+
+  // Now run the child
+  execvp (new_child_proc[0], new_child_proc);
+
+  // Or don't run the child?
+  printf ("Unable to exec %s\n", new_child_proc[0]);
+  perror ("The error was");
+  exit (77);
+}

Re: RFC V2 [1/2] test-in-container

[Andreas Schwab]

On Feb 27 2018, DJ Delorie <dj@redhat.com> wrote:

> +static void
> +copy_one_file (const char *sname, const char *dname)
> +{
> +  int sfd, dfd;
> +  char buf[512];
> +  size_t rsz;
> +  struct stat st;
> +  struct utimbuf times;
> +
> +  sfd = open (sname, O_RDONLY);
> +  if (sfd < 0)
> +    {
> +      printf ("unable to open %s for reading\n", sname);
> +      perror ("the error was");

That doesn't work, the printf call can clobber errno.  Use error instead.

Andreas.

-- 
Andreas Schwab, schwab@linux-m68k.org
GPG Key fingerprint = 58CA 54C7 6D53 942B 1756  01D3 44D5 214B 8276 4ED5
"And now for something completely different."

Re: RFC V2 [1/2] test-in-container

[DJ Delorie]

Andreas Schwab <schwab@linux-m68k.org> writes:
>> +      printf ("unable to open %s for reading\n", sname);
>> +      perror ("the error was");
>
> That doesn't work, the printf call can clobber errno.  Use error instead.

Weird, I've been using that idiom for decades and never had that happen.
But, I'll put it on the list of things to change for V3.

Thanks!

Re: RFC V2 [1/2] test-in-container

[Florian Weimer]

On 02/27/2018 10:09 PM, Andreas Schwab wrote:
>> +static void
>> +copy_one_file (const char *sname, const char *dname)
>> +{
>> +  int sfd, dfd;
>> +  char buf[512];
>> +  size_t rsz;
>> +  struct stat st;
>> +  struct utimbuf times;
>> +
>> +  sfd = open (sname, O_RDONLY);
>> +  if (sfd < 0)
>> +    {
>> +      printf ("unable to open %s for reading\n", sname);
>> +      perror ("the error was");

> That doesn't work, the printf call can clobber errno.  Use error instead.

Doesn't error print to standard error?

%m is another option.

Thanks,
Florian

Re: RFC V2 [1/2] test-in-container

[Florian Weimer]

On 02/27/2018 09:33 PM, DJ Delorie wrote:
> +  while ((rsz = read (sfd, buf, 512)) > 0)
> +    write (dfd, buf, rsz);

This needs error checking, and you can use copy_file_range.

Thanks,
Florian

Re: RFC V2 [1/2] test-in-container

[Florian Weimer]

On 02/27/2018 09:33 PM, DJ Delorie wrote:
> +/* Equivalent of "mkdir -p".  */
> +
> +static int
> +mkdir_p (char *path)

This could go into support/, as xmkdirp.  But please use a const char * 
argument and make a copy for truncation purposes.

Thanks,
Florian

Re: RFC V2 [1/2] test-in-container

[Andreas Schwab]

On Feb 28 2018, Florian Weimer <fweimer@redhat.com> wrote:

> On 02/27/2018 10:09 PM, Andreas Schwab wrote:
>>> +static void
>>> +copy_one_file (const char *sname, const char *dname)
>>> +{
>>> +  int sfd, dfd;
>>> +  char buf[512];
>>> +  size_t rsz;
>>> +  struct stat st;
>>> +  struct utimbuf times;
>>> +
>>> +  sfd = open (sname, O_RDONLY);
>>> +  if (sfd < 0)
>>> +    {
>>> +      printf ("unable to open %s for reading\n", sname);
>>> +      perror ("the error was");
>
>> That doesn't work, the printf call can clobber errno.  Use error instead.
>
> Doesn't error print to standard error?

perror does, too.

> %m is another option.

Or strerror.

Andreas.

-- 
Andreas Schwab, schwab@linux-m68k.org
GPG Key fingerprint = 58CA 54C7 6D53 942B 1756  01D3 44D5 214B 8276 4ED5
"And now for something completely different."