From ff49cfa697466da3bbce6b740c228fe84831c174 Mon Sep 17 00:00:00 2001 From: Zack Weinberg Date: Wed, 28 Oct 2020 17:04:22 -0400 Subject: [PATCH] Autotest: add official way to execute code before all/each test. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Currently, there isn’t any documented way for an Autotest testsuite to add custom code to be run either right before the main driver loop, or at the point of each AT_SETUP. For instance, there’s no good place to put environment variable sanitization that should apply to the entire testsuite (but isn’t universally relevant), or shell function definitions to be used by custom test macros. Autoconf’s test suite is poking shell functions directly into the PREPARE_TESTS diversion, and doing environment variable sanitization in each individual test. Both of these are obviously undesirable. This patch adds three new AT_* macros that can be used to do these things in an officially-supported way: AT_PREPARE_TESTS adds code to be run right before the main driver loop, AT_PREPARE_EACH_TEST adds code to be run at the beginning of each test, and AT_TEST_HELPER_FN defines a shell function that will be available to each test. In Autoconf’s test suite, I use AT_PREPARE_TESTS to factor out environment variable sanitization that *ought* to apply across the board, and AT_TEST_HELPER_FN for the helper function used by AT_CHECK_ENV. (This fixes the testsuite bug reported by Jannick at https://lists.gnu.org/archive/html/autoconf/2020-10/msg00052.html .) Comments requested particularly on the names and implementations of the new macros. Also, it would be nice to have an example of when AT_PREPARE_EACH_TEST is useful, but I didn’t find one in the autoconf test suite. * lib/autotest/general.m4 (AT_PREPARE_TESTS, AT_PREPARE_EACH_TEST) (AT_TEST_HELPER_FN): New macros. (AT_INIT, AT_TESTED): Emit the code to report tested programs only if it’s needed, and make sure it’s after any code added by AT_PREPARE_TESTS. * tests/local.at: Add AT_PREPARE_TESTS block that ensures $MAKE is set sensibly and $MAKEFLAGS and $CONFIG_SITE are unset. Use AT_TEST_HELPER_FN for the helper function needed by AT_CHECK_ENV. (AT_CHECK_MAKE): No need to sanitize $MAKE or $MAKEFLAGS here. * tests/base.at, tests/compile.at, tests/m4sh.at, tests/torture.at: No need to unset or neutralize $CONFIG_SITE in individual tests. * doc/autoconf.texi, NEWS: Document new macros. --- NEWS | 5 ++ doc/autoconf.texi | 35 +++++++++++++ lib/autotest/general.m4 | 109 ++++++++++++++++++++++++++++------------ tests/base.at | 13 ++--- tests/compile.at | 10 ---- tests/local.at | 42 ++++++++++------ tests/m4sh.at | 4 +- tests/torture.at | 5 -- 8 files changed, 149 insertions(+), 74 deletions(-) diff --git a/NEWS b/NEWS index 42095344..a0258956 100644 --- a/NEWS +++ b/NEWS @@ -363,6 +363,11 @@ GNU Autoconf NEWS - User visible changes. *** AT_TESTED arguments can use variable or command substitutions, including in particular $EXEEXT +*** New macros AT_PREPARE_TESTS, AT_PREPARE_EACH_TEST, and AT_TEST_HELPER_FN. + + These provide an official way to define testsuite-specific + initialization code and shell functions. + * Noteworthy changes in release 2.69 (2012-04-24) [stable] ** Autoconf now requires perl 5.6 or better (but generated configure diff --git a/doc/autoconf.texi b/doc/autoconf.texi index acc290b0..f1eff146 100644 --- a/doc/autoconf.texi +++ b/doc/autoconf.texi @@ -25463,6 +25463,41 @@ Writing Testsuites via @file{atlocal} or @file{atconfig}. @end defmac +@defmac AT_PREPARE_TESTS (@var{shell-code}) +@atindex{PREPARE_TESTS} +Execute @var{shell-code} in the main testsuite process, +after initializing the test suite and processing command-line options, +but before running any tests. If this macro is used several times, +all of the @var{shell-code}s will be executed, +in the order they appeared in @file{testsuite.at}. + +One reason to use @code{AT_PREPARE_TESTS} is when the programs under +test are sensitive to environment variables: you can unset all these +variables or reset them to safe values in @var{shell-code}. + +@var{shell-code} is only executed if at least one test is going to be +run. In particular, it will not be executed if any of the @option{--help}, +@option{--version}, @option{--list}, or @option{--clean} options are +given to @command{testsuite} (@pxref{testsuite Invocation}). +@end defmac + +@defmac AT_PREPARE_EACH_TEST (@var{shell-code}) +@atindex{AT_PREPARE_EACH_TEST} +Execute @var{shell-code} in each test group's subshell, at the point of +the @code{AT_SETUP} that starts the test group. +@end defmac + +@defmac AT_TEST_HELPER_FN (@var{name}, @var{args}, @var{description}, @var{code}) +Define a shell function that will be available to the code for each test +group. Its name will be @code{ath_fn_@var{name}}, and its body will be +@var{code}. (The prefix prevents name conflicts with shell functions +defined by M4sh and Autotest.) + +@var{args} should describe the function's arguments and @var{description} +what it does; these are used only for documentation comments in the +generated testsuite script. +@end defmac + @sp 1 @defmac AT_BANNER (@var{test-category-name}) diff --git a/lib/autotest/general.m4 b/lib/autotest/general.m4 index f29ce490..ad3308dc 100644 --- a/lib/autotest/general.m4 +++ b/lib/autotest/general.m4 @@ -429,9 +429,6 @@ m4_divert_text([DEFAULTS], [ # Whether to enable colored test results. at_color=m4_ifdef([AT_color], [AT_color], [no]) -# List of the tested programs. -at_tested='m4_ifdef([AT_tested], - [m4_translit(m4_dquote(m4_defn([AT_tested])), [ ], m4_newline)])' # As many question marks as there are digits in the last test group number. # Used to normalize the test group numbers so that `ls' lists them in # numerical order. @@ -992,33 +989,7 @@ AS_BOX(m4_defn([AT_TESTSUITE_NAME])[.]) } >&AS_MESSAGE_LOG_FD m4_divert_pop([TESTS_BEGIN])dnl -m4_divert_push([PREPARE_TESTS])dnl -{ - AS_BOX([Tested programs.]) - echo -} >&AS_MESSAGE_LOG_FD - -# Report what programs are being tested. -for at_program in : `eval echo $at_tested` -do - test "$at_program" = : && continue - case $at_program in - [[\\/]* | ?:[\\/]* ) $at_program_=$at_program ;;] - * ) - _AS_PATH_WALK([$PATH], [test -f "$as_dir$at_program" && break]) - at_program_=$as_dir$at_program ;; - esac - if test -f "$at_program_"; then - { - AS_ECHO(["$at_srcdir/AT_LINE: $at_program_ --version"]) - "$at_program_" --version &AS_MESSAGE_LOG_FD 2>&1 - else - AS_ERROR([cannot find $at_program]) - fi -done - +m4_divert_push([TESTS])dnl { AS_BOX([Running the tests.]) } >&AS_MESSAGE_LOG_FD @@ -1026,8 +997,6 @@ done at_start_date=`date` at_start_time=`date +%s 2>/dev/null` AS_ECHO(["$as_me: starting at: $at_start_date"]) >&AS_MESSAGE_LOG_FD -m4_divert_pop([PREPARE_TESTS])dnl -m4_divert_push([TESTS])dnl # Create the master directory if it doesn't already exist. AS_MKDIR_P(["$at_suite_dir"]) || @@ -1794,10 +1763,83 @@ m4_defun([AT_ARG_OPTION_ARG],[_AT_ARG_OPTION([$1],[$2],1,[$3],[$4])]) # versions are logged, and in the case of embedded test suite, they # must correspond to the version of the package. PATH should be # already preset so the proper executable will be selected. -m4_define([AT_TESTED], +m4_defun([AT_TESTED], +[m4_require([_AT_TESTED])]dnl [m4_foreach_w([AT_test], [$1], [m4_append_uniq([AT_tested], "m4_defn([AT_test])", [ ])])]) +m4_defun([_AT_TESTED], +[m4_wrap([m4_divert_text([DEFAULTS], +[# List of the tested programs. +at_tested='m4_translit(m4_dquote(m4_defn([AT_tested])), [ ], m4_newline)' +])]dnl +[m4_divert_text([PREPARE_TESTS], +[{ + AS_BOX([Tested programs.]) + echo +} >&AS_MESSAGE_LOG_FD + +# Report what programs are being tested. +for at_program in : `eval echo $at_tested` +do + AS_CASE([$at_program], + [:], [continue], + [[[\\/]* | ?:[\\/]*]], [$at_program_=$at_program], + [_AS_PATH_WALK([$PATH], [test -f "$as_dir$at_program" && break]) + at_program_=$as_dir$at_program]) + + if test -f "$at_program_"; then + { + AS_ECHO(["$at_srcdir/AT_LINE: $at_program_ --version"]) + "$at_program_" --version &AS_MESSAGE_LOG_FD 2>&1 + else + AS_ERROR([cannot find $at_program]) + fi +done +])])]) + + +# AT_PREPARE_TESTS(SHELL-CODE) +# ---------------------------- +# Execute @var{shell-code} in the main testsuite process, +# after initializing the test suite and processing command-line options, +# but before running any tests. +m4_define([AT_PREPARE_TESTS], +[m4_divert_once([PREPARE_TESTS], +[m4_text_box([Prepare for this testsuite.]) +])]dnl +[m4_divert_text([PREPARE_TESTS], [$1])]) + + +# AT_PREPARE_EACH_TEST([SHELL-CODE]) +# ---------------------------------- +# Execute @var{shell-code} in each test group's subshell, +# at the point of the AT_SETUP that starts each test group. +m4_define([AT_PREPARE_EACH_TEST], +[m4_append([AT_prepare_each_test], [$1], [ +])]) + + +# AT_TEST_HELPER_FN(NAME, ARGS, DESCRIPTION, CODE) +# ------------------------------------------------ +# Define a shell function that will be available to the code for each test +# group. Its name will be ath_fn_NAME, and its body will be CODE. +# +# Implementation note: you might think this would use AT_PREPARE_EACH_TEST, +# but shell functions defined in AT_PREPARE_TESTS *are* (currently) available +# to test group subshells, and this way the code is only emitted once, not +# once for each test group. +m4_define([AT_TEST_HELPER_FN], +[AT_PREPARE_TESTS([ +AS_FUNCTION_DESCRIBE([ath_fn_$1], [$2], [$3]) +ath_fn_$1 () +{ + $4 +} +])]) + # AT_COPYRIGHT(TEXT, [FILTER = m4_newline]) # ----------------------------------------- @@ -1837,6 +1879,7 @@ at_fn_group_banner AT_ordinal 'm4_defn([AT_line])' \ "AS_ESCAPE(m4_dquote(m4_defn([AT_description])))" m4_format(["%*s"], m4_max(0, m4_eval(47 - m4_qlen(m4_defn([AT_description])))), [])m4_if( AT_banner_ordinal, [0], [], [ AT_banner_ordinal]) +m4_ifdef([AT_prepare_each_test], [m4_ifvaln([AT_prepare_each_test])])dnl m4_divert_push([TEST_SCRIPT])dnl ]) diff --git a/tests/base.at b/tests/base.at index 6a1d9742..48df3597 100644 --- a/tests/base.at +++ b/tests/base.at @@ -393,9 +393,6 @@ AT_CLEANUP AT_SETUP([AC_CACHE_CHECK]) AT_KEYWORDS([CONFIG_SITE]) -# Don't let a config.site file affect this test. -AS_UNSET([CONFIG_SITE]) - AT_DATA([configure.ac], [[AC_INIT # m4_define([ac_nothing], [ac_cv_absolutely_nothing]) @@ -669,9 +666,12 @@ cat <&AS@&t@_ORIGINAL_STDIN_FD >&AS@&t@_MESSAGE_FD AC_OUTPUT ]]) AT_CHECK_AUTOCONF -AT_CHECK([echo Hello | CONFIG_SITE=/dev/null ./configure $configure_options | grep -v '^configure: '],, [Hello +AT_CHECK([echo Hello | + ./configure $configure_options | + grep -v '^configure: '],, +[Hello ]) -AT_CHECK([echo Hello | CONFIG_SITE=/dev/null ./configure $configure_options --silent]) +AT_CHECK([echo Hello | ./configure $configure_options --silent]) AT_CLEANUP @@ -758,9 +758,6 @@ AT_CLEANUP AT_SETUP([configure directories]) -CONFIG_SITE=/dev/null -export CONFIG_SITE - AT_DATA([foo.in], [[prefix=@prefix@ exec_prefix=@exec_prefix@ diff --git a/tests/compile.at b/tests/compile.at index c93722b9..9ebe782e 100644 --- a/tests/compile.at +++ b/tests/compile.at @@ -175,11 +175,6 @@ AT_CLEANUP AT_SETUP([AC_LANG_SOURCE example]) -# Set CONFIG_SITE to a nonexistent file, so that there are -# no worries about configure output caused by sourcing a config.site. -CONFIG_SITE=no-such-file -export CONFIG_SITE - AT_DATA([configure.ac], [[# Taken from autoconf.texi:Generating Sources. # The only change is to not fail if gcc doesn't work. @@ -221,11 +216,6 @@ AT_CLEANUP AT_SETUP([AC_LANG_PROGRAM example]) -# Set CONFIG_SITE to a nonexistent file, so that there are -# no worries about configure output caused by sourcing a config.site. -CONFIG_SITE=no-such-file -export CONFIG_SITE - AT_DATA([configure.ac], [[# Taken from autoconf.texi:Generating Sources. # The only change is to not fail if gcc doesn't work. diff --git a/tests/local.at b/tests/local.at index 374e530c..a7fdbb16 100644 --- a/tests/local.at +++ b/tests/local.at @@ -27,6 +27,22 @@ AT_TESTED([autom4te autoconf autoheader autoupdate autoreconf ifnames]) # Enable colored test output. AT_COLOR_TESTS +# Sanitize the environment used for tests. +AT_PREPARE_TESTS( +[# These variables should not be inherited from the +# parent environment. +AS_UNSET([CONFIG_SITE]) +AS_UNSET([MAKEFLAGS]) + +# Ensure MAKE is set to a useful value. Unlike the above, we *do* +# want to inherit this variable from the parent environment and/or +# our command line. +: "${MAKE=make}" +export MAKE +]) + + + ## ---------------- ## ## Utility macros. ## ## ---------------- ## @@ -334,16 +350,15 @@ m4_define([AT_CHECK_CONFIGURE], # | '#'=0 # | '$'=6908 # -m4_define([AT_CHECK_ENV], -[m4_divert_once([PREPARE_TESTS], [_AT_CHECK_ENV])dnl -AT_CHECK([at_check_env])]) -m4_define([_AT_CHECK_ENV], -[AS_FUNCTION_DESCRIBE([at_check_env], [], +m4_defun([AT_CHECK_ENV], +[m4_require([_AT_CHECK_ENV])]dnl +[AT_CHECK([ath_fn_check_env])]) + +m4_defun([_AT_CHECK_ENV], +[AT_TEST_HELPER_FN([check_env], [], [Compare the directory and environment state both before and after a run, -and return non-zero status if they differ inappropriately.]) -at_check_env () -{ -# Compare directory listings. +and return non-zero status if they differ inappropriately.], +[# Compare directory listings. test -f state-ls.before || AS_ERROR([state-ls.before not present]) test -f state-ls.after \ @@ -377,7 +392,7 @@ if test -f state-env.before && test -f state-env.after; then [ALLOCA|GETLOADAVG_LIBS|KMEM_GROUP|NEED_SETGID|POW_LIB], [AWK|LEX|LEXLIB|LEX_OUTPUT_ROOT|LN_S|M4|MKDIR_P|RANLIB|SET_MAKE|YACC], [GREP|[EF]GREP|SED], - [[_@]|.[*#?$].], + [[_@]|.[*@%:@?$].], [argv|ARGC|LINENO|BASH_ARGC|BASH_ARGV|OLDPWD|PIPESTATUS|RANDOM], [SECONDS|START_TIME|ToD|_AST_FEATURES]))=' \ $act_file || @@ -391,7 +406,7 @@ if test -f state-env.before && test -f state-env.after; then $at_traceon $grep_failed || $at_diff clean-state-env.before clean-state-env.after fi -} [#]at_check_env]) +])]) # AT_CONFIG_CMP(VAR-FILE-A, VAR-FILE-B) @@ -496,10 +511,7 @@ m4_define([AT_CHECK_AUTOUPDATE], # an acceptable result, because there are situations where BSD make will # exit with status 1 but GNU make will instead exit with status 2. m4_define([AT_CHECK_MAKE], -[: "${MAKE=make}" -export MAKE -unset MAKEFLAGS -AT_CHECK( +[AT_CHECK( m4_if(m4_default([$2], [.]), [.], [], [cd "$2" && ])[$][MAKE]m4_ifnblank([$1],[ $1])[]m4_if([$3], [1], [[ dnl pacify editors that don't understand sh case: (( diff --git a/tests/m4sh.at b/tests/m4sh.at index 30e3fa58..111c27fe 100644 --- a/tests/m4sh.at +++ b/tests/m4sh.at @@ -125,9 +125,7 @@ exec sh "@S|@@" chmod a+x cfg-sh AT_CAPTURE_FILE([config.log]) -# Export CONFIG_SITE to /dev/null to avoid spurious diffs in expected -# stdout/stderr. -AT_CHECK([env CONFIG_SITE=/dev/null CONFIG_SHELL=./cfg-sh ./configure], +AT_CHECK([env CONFIG_SHELL=./cfg-sh ./configure], [0], [[configure: creating ./config.status ]], []) diff --git a/tests/torture.at b/tests/torture.at index 6ae6bfdf..002b75d5 100644 --- a/tests/torture.at +++ b/tests/torture.at @@ -1463,11 +1463,6 @@ AT_CHECK([[grep '[1-9]\.[0-9]' stdout || exit 77]], [], [ignore]) # It should understand configure.ac. AT_CHECK([[grep '[^0-9]1\.[01234][^0-9]' stdout && exit 77]], [1], [ignore]) -# Set CONFIG_SITE to a nonexistent file, so that there are -# no worries about nonstandard values for 'prefix'. -CONFIG_SITE=no-such-file -export CONFIG_SITE - # The contents of `inner/', and `inner/innermost/'. AS_MKDIR_P([inner/innermost]) -- 2.29.1