From e0a1ea5e1d20918f6850a2ab8cb86c691cd8b46c Mon Sep 17 00:00:00 2001 From: Simon Josefsson Date: Fri, 14 May 2021 14:17:20 +0200 Subject: [PATCH] valgrind-tests: Better option handling. * m4/valgrind-tests.m4: Support new variables VALGRINDFLAGS and DEFAULT_VALGRINDFLAGS. * doc/valgrind-tests.texi (Running self-tests under valgrind): Improve. --- ChangeLog | 8 +++ doc/valgrind-tests.texi | 105 +++++++++++++++++++++++++++++++++++----- m4/valgrind-tests.m4 | 67 +++++++++++++++++-------- 3 files changed, 148 insertions(+), 32 deletions(-) diff --git a/ChangeLog b/ChangeLog index 4716f78a8..daeb44ab4 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,11 @@ +2021-05-14 Simon Josefsson + + valgrind-tests: Better option handling. + * m4/valgrind-tests.m4: Support new variables VALGRINDFLAGS and + DEFAULT_VALGRINDFLAGS. + * doc/valgrind-tests.texi (Running self-tests under valgrind): + Improve. + 2021-05-14 Markus Mützel (tiny change) windows-spawn: Don't assume that UNICODE is not defined. diff --git a/doc/valgrind-tests.texi b/doc/valgrind-tests.texi index fef910edf..9f3b30968 100644 --- a/doc/valgrind-tests.texi +++ b/doc/valgrind-tests.texi @@ -11,6 +11,7 @@ at the discretion of the developer. @menu * Using valgrind automatically:: +* Valgrind options:: * Using valgrind manually:: * Valgrind and shell scripts:: @end menu @@ -19,19 +20,93 @@ at the discretion of the developer. @subsection Using valgrind without developer intervention The @code{valgrind-tests} module searches for Valgrind at configure time -and declares the @code{VALGRIND} automake variable for use with automake's -@code{TESTS_ENVIRONMENT}. +and declares the @code{LOG_VALGRIND} automake variable for use with +automake's @code{LOG_COMPILER}. After importing the @code{valgrind-tests} module to your project, you use it by adding the following to the @code{Makefile.am} that runs the self-tests: @smallexample -TESTS_ENVIRONMENT = $(VALGRIND) +LOG_COMPILER = $(LOG_VALGRIND) @end smallexample This will run all self-checks under valgrind. +Replace @code{LOG_COMPILER} with @code{TESTS_ENVIRONMENT} if you are +using the old serial test harness. + +If you desire a project-wide decision that valgrind is not enabled by +default, but still allow users to enable it with +@code{--enable-valgrind-tests} you may put the following in configure.ac +before gl_INIT. + +@smallexample +gl_VALGRIND_TESTS_DEFAULT_NO +@end smallexample + +@node Valgrind options +@subsection Valgrind options + +The @code{VALGRIND} variable holds the name of the valgrind binary and +some options passed to valgrind. You may provide additional options +that are passed to valgrind using the @samp{VALGRINDFLAGS} variable, for +example: + +@smallexample +./configure VALGRINDFLAGS="--suppressions=/your/local/valgrind/suppressions/file.txt" +@end smallexample + +Alternatively during build phase: + +@smallexample +make check VALGRINDFLAGS="--suppressions=/your/local/valgrind/suppressions/file.txt" +@end smallexample + +This is useful if you have a valgrind suppression files that are needed +to avoid triggering errors for known errors, typically in system +libraries. + +The @code{VALGRIND} variable include options that are useful when +valgrind is run non-interactively through the test harness. The default +parameters are @code{-q} to silence the output, +@code{--error-exitcode=1} to cause valgrind errors to be treated as +fatal errors, and @code{--leak-check=full} to check for memory leaks. + +These options can be controlled through the @code{DEFAULT_VALGRINDFLAGS} +variable. For example, when configuring the package: + +@smallexample +./configure DEFAULT_VALGRINDFLAGS="--quiet" +@end smallexample + +Alternatively, during the build phase: + +@smallexample +make check DEFAULT_VALGRINDFLAGS="--quiet" +@end smallexample + +That would have the effect of removing @code{--error-exitcode=1} and +@code{--leak-check=full} from the default options, thus causing any +valgrind errors to be silently ignored, instead of causing fatal test +failures. + +As a developer you may use the variables in @code{configure.ac} before +calling @code{gl_INIT}, like this if your program has deeply-nested call +chains: + +@smallexample +gl_EARLY +... +VALGRINDFLAGS="$VALGRINDFLAGS --num-callers=42" +... +gl_INIT +@end smallexample + +Note that any user-supplied @code{VALGRINDFLAGS} value is preserved, +which is usually what you want. + + @node Using valgrind manually @subsection Using valgrind at the developer's discretion @@ -58,24 +133,28 @@ There are two ways to avoid this: @itemize @bullet @item -You can make use of the @code{build-aux/run-test} script from Gnulib. -Add these lines to your @code{Makefile.am}: +Using the Automake parallel-tests feature, you can use the following instead: @smallexample -# This must be the last thing that gets added to TESTS_ENVIRONMENT. -TESTS_ENVIRONMENT += $(SHELL) $(top_srcdir)/build-aux/run-test '$(VALGRIND)' +TEST_EXTENSIONS = .pl .sh +LOG_COMPILER = $(LOG_VALGRIND) @end smallexample +Then valgrind will only be used for the non-.sh and non-.pl tests. + +For old automake, you will need @code{AUTOMAKE_OPTIONS = parallel-tests} +to enable the parallel test harness. + @item -Using the Automake parallel-tests feature, you can use the following instead: +You can make use of the @code{build-aux/run-test} script from Gnulib. +Add these lines to your @code{Makefile.am}: @smallexample -AUTOMAKE_OPTIONS = parallel-tests -TEST_EXTENSIONS = .pl .sh -LOG_COMPILER = $(VALGRIND) +LOG_COMPILER += $(SHELL) $(top_srcdir)/build-aux/run-test '$(LOG_VALGRIND)' @end smallexample -Then valgrind will only be used for the non-.sh and non-.pl tests. +Replace @code{LOG_COMPILER} with @code{TESTS_ENVIRONMENT} if you use the +old serial test harness. @end itemize However, with this measure in place, binaries invoked through scripts will @@ -84,7 +163,7 @@ variables in the @code{TESTS_ENVIRONMENT} variable that are then used by the shell scripts. For example, add the following: @smallexample -TESTS_ENVIRONMENT = VALGRIND='$(VALGRIND)' +TESTS_ENVIRONMENT = VALGRIND='$(LOG_VALGRIND)' @end smallexample And then modify the shell scripts to invoke the binary prefixed with diff --git a/m4/valgrind-tests.m4 b/m4/valgrind-tests.m4 index 50d90e145..9fc9d5b37 100644 --- a/m4/valgrind-tests.m4 +++ b/m4/valgrind-tests.m4 @@ -1,4 +1,4 @@ -# valgrind-tests.m4 serial 6 +# valgrind-tests.m4 serial 7 dnl Copyright (C) 2008-2021 Free Software Foundation, Inc. dnl This file is free software; the Free Software Foundation dnl gives unlimited permission to copy and/or distribute it, @@ -8,18 +8,60 @@ dnl From Simon Josefsson # gl_VALGRIND_TESTS() # ------------------- -# Check if valgrind is available, and set VALGRIND to it if available. -AC_DEFUN([gl_VALGRIND_TESTS], +# Check if valgrind is available. + +# Sets VALGRIND to command line (including options) to invoke valgrind +# with, may be used directly in autoconf, makefiles or shell scripts. + +# Sets LOG_VALGRIND, suitable for use with LOG_COMPILER, that in a +# makefile will expand to command line to invoke self-tests with, +# i.e., LOG_VALGRIND = $(VALGRIND) $(DEFAULT_VALGRINDFLAGS) +# $(VALGRINDFLAGS). + +# Whether to look for valgrind and set the variables can be influenced +# by calling gl_VALGRIND_TESTS_DEFAULT_NO in configure.ac. +# Regardless, the user can change the choice through the options +# --enable-valgrind-tests or --disable-valgrind-tests. + +# You may modify the VALGRIND, DEFAULT_VALGRINDFLAGS and VALGRINDFLAGS +# variables before calling this function to override defaults. Either +# as developer from configure.ac or user on the ./configure command +# line. + +AC_DEFUN([gl_VALGRIND_TESTS_DEFAULT_NO], +[ + gl_valgrind_tests_default=no +]) + +AC_DEFUN_ONCE([gl_VALGRIND_TESTS], [ AC_ARG_ENABLE([valgrind-tests], AS_HELP_STRING([--disable-valgrind-tests], [don't try to run self tests under valgrind]), - [opt_valgrind_tests=$enableval], [opt_valgrind_tests=yes]) + [opt_valgrind_tests=$enableval], [opt_valgrind_tests=${gl_valgrind_tests_default:-yes}]) # Run self-tests under valgrind? if test "$opt_valgrind_tests" = "yes" && test "$cross_compiling" = no; then AC_CHECK_PROGS([VALGRIND], [valgrind]) + AC_SUBST([DEFAULT_VALGRINDFLAGS]) + if test -z "$DEFAULT_VALGRINDFLAGS"; then + DEFAULT_VALGRINDFLAGS="-q --error-exitcode=1 --leak-check=full" + fi + AC_ARG_VAR([VALGRINDFLAGS], [Additional flags for Valgrind]) + + if test -n "$VALGRIND"; then + AC_CACHE_CHECK([for valgrind options for tests], + [gl_cv_opt_valgrind_tests], + [AS_IF([$VALGRIND $DEFAULT_VALGRINDFLAGS $VALGRINDFLAGS true], + [gl_cv_opt_valgrind_tests="$DEFAULT_VALGRINDFLAGS $VALGRINDFLAGS"], + [gl_cv_opt_valgrind_tests=no]) + ]) + if test "$gl_cv_opt_valgrind_tests" != no; then + VALGRIND="$VALGRIND $gl_cv_opt_valgrind_tests" + fi + fi + if test -n "$VALGRIND"; then dnl On Ubuntu 16.04, /usr/bin/valgrind works only on 64-bit executables dnl but fails on 32-bit executables (with exit code 1) and on x86_64-x32 @@ -28,7 +70,7 @@ AC_DEFUN([gl_VALGRIND_TESTS], [gl_cv_prog_valgrind_works], [AC_RUN_IFELSE( [AC_LANG_SOURCE([[int main () { return 0; }]])], - [$VALGRIND ./conftest$ac_exeext 2>/dev/null + [$VALGRIND $gl_cv_opt_valgrind_tests ./conftest$ac_exeext 2>/dev/null if test $? = 0; then gl_cv_prog_valgrind_works=yes else @@ -42,19 +84,6 @@ AC_DEFUN([gl_VALGRIND_TESTS], fi fi - if test -n "$VALGRIND"; then - AC_CACHE_CHECK([for valgrind options for tests], - [gl_cv_opt_valgrind_tests], - [gl_valgrind_opts='-q --error-exitcode=1 --leak-check=full' - if $VALGRIND $gl_valgrind_opts ls > /dev/null 2>&1; then - gl_cv_opt_valgrind_tests="$gl_valgrind_opts" - else - gl_cv_opt_valgrind_tests=no - fi - ]) - if test "$gl_cv_opt_valgrind_tests" != no; then - VALGRIND="$VALGRIND $gl_cv_opt_valgrind_tests" - fi - fi + AC_SUBST([LOG_VALGRIND], ["\$(VALGRIND) \$(DEFAULT_VALGRINDFLAGS) \$(VALGRINDFLAGS)"]) fi ]) -- 2.20.1