autotest annoyances

[Eric Blake]

While testing my previous patch for const/volatile vs. -Werror, I encountered
two autotest anomalies.  I first enhanced the testsuite to detect both
problems, as well as my earlier patch this week for when removing
the test group directory fails.

The first bug was that when using the -d option to testsuite (which is done by
all the testsuite.dir/###/run scripts), the run script was only regenerated
if the test failed (it was missing after a success or skip).  I also managed
to come up with a patch for this.

The second bug is that environment variables passed as options to ./testsuite
are copied directly into the run scripts, but without regards for shell
metacharacters.  That means that an environment variable that contains
spaces, such as "./run CFLAGS='-std=c89 -Werror'", will regenerate a broken
run script that tries to invoke 'testsuite CFLAGS=-std=c89 -Werror'.  However,
fixing this quickly and safely is beyond my abilities, with a pending alpha
release, so the test is just xfailed for now.

2006-04-07  Eric Blake  <address@hidden>

        * tests/autotest.at (AT_CHECK_AT_START): New macro, split out
        from...
        (AT_CHECK_AT): ...here, for use in...
        (Debugging a successful test, Debugging a failed test): ...these
        new tests.  The first of these is fixed by...
        * lib/autotest/general.m4 (_AT_CREATE_DEBUGGING_SCRIPT): New
        macro, split out from...
        (AT_INIT): ...here, so that using -d also generates a run script.

Index: lib/autotest/general.m4
===================================================================
RCS file: /sources/autoconf/autoconf/lib/autotest/general.m4,v
retrieving revision 1.206
diff -u -p -r1.206 general.m4
--- lib/autotest/general.m4     7 Apr 2006 18:31:09 -0000       1.206
+++ lib/autotest/general.m4     8 Apr 2006 00:57:29 -0000
@@ -162,6 +162,22 @@ m4_define([_AT_NORMALIZE_TEST_GROUP_NUMB
   done
 ])
 
+# _AT_CREATE_DEBUGGING_SCRIPT
+# ---------------------------
+# Create the debugging script $at_group_dir/run which will reproduce the
+# current test group.
+m4_define([_AT_CREATE_DEBUGGING_SCRIPT],
+[        {
+           echo "#! /bin/sh"
+           echo 'test "${ZSH_VERSION+set}" = set && alias -g 
'\''${1+"address@hidden"}'\''='\''"address@hidden"'\'''
+           echo "cd '$at_dir'"
+           echo 'exec ${CONFIG_SHELL-'"$SHELL"'}' "$[0]" \
+                '-v -d' "$at_debug_args" "$at_group" '${1+"address@hidden"}'
+           echo 'exit 1'
+         } >$at_group_dir/run
+         chmod +x $at_group_dir/run
+])# _AT_CREATE_DEBUGGING_SCRIPT
+
 
 # AT_INIT([TESTSUITE-NAME])
 # -------------------------
@@ -793,11 +809,12 @@ _ATEOF
          echo "$at_log_msg" >&AS_MESSAGE_LOG_FD
 
          # Cleanup the group directory, unless the user wants the files.
-         $at_debug_p ||
-           if test -d $at_group_dir; then
-             find $at_group_dir -type d ! -perm -700 -exec chmod u+rwx \{\} \;
-             rm -fr $at_group_dir
-           fi
+         if $at_debug_p ; then
+           _AT_CREATE_DEBUGGING_SCRIPT
+         elif test -d $at_group_dir; then
+           find $at_group_dir -type d ! -perm -700 -exec chmod u+rwx \{\} \;
+           rm -fr $at_group_dir
+         fi
          ;;
        *)
          # Upon failure, include the log into the testsuite's global
@@ -807,15 +824,7 @@ _ATEOF
 
          # Upon failure, keep the group directory for autopsy, and
          # create the debugging script.
-         {
-           echo "#! /bin/sh"
-           echo 'test "${ZSH_VERSION+set}" = set && alias -g 
'\''${1+"address@hidden"}'\''='\''"address@hidden"'\'''
-           echo "cd '$at_dir'"
-           echo 'exec ${CONFIG_SHELL-'"$SHELL"'}' "$[0]" \
-                '-v -d' "$at_debug_args" "$at_group" '${1+"address@hidden"}'
-           echo 'exit 1'
-         } >$at_group_dir/run
-         chmod +x $at_group_dir/run
+         _AT_CREATE_DEBUGGING_SCRIPT
          $at_errexit && break
          ;;
       esac
Index: tests/autotest.at
===================================================================
RCS file: /sources/autoconf/autoconf/tests/autotest.at,v
retrieving revision 1.11
diff -u -p -r1.11 autotest.at
--- tests/autotest.at   5 Apr 2006 16:04:22 -0000       1.11
+++ tests/autotest.at   8 Apr 2006 00:57:29 -0000
@@ -19,14 +19,15 @@ AT_BANNER([Autotest.])
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
 # 02110-1301, USA.
 
-# AT_CHECK_AT(TITLE, SUITE-CODE, [XFAIL-CONDITION], [STATUS = 0],
-#             [STDOUT := ignore], STDERR)
-# ---------------------------------------------------------------
+# AT_CHECK_AT_START(TITLE, SUITE-CODE, [XFAIL-CONDITION], [STATUS = 0],
+#                   [STDOUT := ignore], STDERR)
+# ---------------------------------------------------------------------
 # Create a new test named TITLE that runs a minimal Autotest test suite,
 # SUITE-CODE.  Call AT_XFAIL_IF with XFAIL-CONDITION.  STATUS and STDERR pass
 # directly to the AT_CHECK that call the minimal test suite.  STDOUT is not
-# used, but it is reserved for future use.
-m4_define([AT_CHECK_AT],
+# used, but it is reserved for future use.  The test group remains open
+# for further testing before AT_CLEANUP.
+m4_define([AT_CHECK_AT_START],
 [
 AT_SETUP([$1])
 AT_KEYWORDS([autotest])
@@ -49,6 +50,17 @@ AT_DATA([mysuite.at], [$2])
 AT_CHECK_AUTOM4TE([--language=autotest -o micro-suite mysuite.at])
 AT_CHECK([$CONFIG_SHELL ./micro-suite],       m4_default([$4], 0), [ignore], 
[$6])
 AT_CHECK([$CONFIG_SHELL ./micro-suite -v -x], m4_default([$4], 0), [ignore], 
[$6])
+])# AT_CHECK_AT_START
+
+# AT_CHECK_AT(TITLE, SUITE-CODE, [XFAIL-CONDITION], [STATUS = 0],
+#             [STDOUT := ignore], STDERR)
+# ---------------------------------------------------------------
+# Create a new test named TITLE that runs a minimal Autotest test suite,
+# SUITE-CODE.  Call AT_XFAIL_IF with XFAIL-CONDITION.  STATUS and STDERR pass
+# directly to the AT_CHECK that call the minimal test suite.  STDOUT is not
+# used, but it is reserved for future use.
+m4_define([AT_CHECK_AT],
+[AT_CHECK_AT_START($@)
 AT_CLEANUP
 ])
 
@@ -251,6 +263,41 @@ AT_CHECK_AT_TITLE_CHAR([Double-quote], [
 AT_CHECK_AT_TITLE_CHAR([Backslash],    [\])
 
 
+## ----------------- ##
+## Debugging a test. ##
+## ----------------- ##
+
+# At one point, using the -d switch failed to rebuild the run script.
+# Also, running from in the group dir locks the directory in some platforms.
+AT_CHECK_AT_START([Debugging a successful test],
+[[
+AT_INIT([artificial test suite])
+AT_SETUP([Environment check])
+AT_CHECK([test "$MY_VAR" = "two  spaces"])
+AT_CLEANUP
+]], [], [1], [], [ignore])
+AT_CHECK([(cd micro-suite.dir/1; ./run MY_VAR='two  spaces')],
+         [0], [ignore], [ignore])
+AT_CHECK([(cd micro-suite.dir/1; ./run MY_VAR='one space')],
+         [1], [ignore], [ignore])
+AT_CLEANUP
+
+# The run script is currently invalid when shell metacharacters are passed
+# in via an environment option.
+AT_CHECK_AT_START([Debugging a failed test],
+[[
+AT_INIT([artificial test suite])
+AT_SETUP([Environment check])
+AT_CHECK([test "$MY_VAR" = "one space"])
+AT_CLEANUP
+]], [:], [1], [], [ignore])
+AT_CHECK([(cd micro-suite.dir/1; ./run MY_VAR='two  spaces')],
+         [1], [ignore], [ignore])
+AT_CHECK([(cd micro-suite.dir/1; ./run MY_VAR='one space')],
+         [0], [ignore], [ignore])
+AT_CLEANUP
+
+
 ## --------- ##
 ## Keywords. ##
 ## --------- ##