[PATCH] {test-protocols} simple tests: support developer-defined fd redirections

[Stefano Lattarini]

Motivated by coreutils bug#8846, and related discussions:
 <http://debbugs.gnu.org/cgi/bugreport.cgi?bug=8846>
 <http://lists.gnu.org/archive/html/bug-autoconf/2011-06/msg00002.html>
In those threads it has been shown how problematic it can be to do
portable file descriptor redirections in the testsuite when using
the Automake testsuite harness.  This change should remedy to that
situation.

* lib/am/check2.am (?GENERIC?%EXT%$(EXEEXT).log,
?GENERIC?%EXT%.log, ?!GENERIC?%OBJ%): Append redirection defined
in $(AM_TESTS_FD_REDIRECT) to the command-line invocations of the
test scripts.
* lib/am/check.am [!%?PARALLEL_TESTS%] $(check-TESTS): Likewise.
* NEWS: Update.
* tests/check-fd-redirect.test: New test.
* tests/parallel-tests-fd-redirect.test: Likewise.
* tests/Makefile.am (TESTS): Update.
---
 ChangeLog                             |   20 ++++++
 NEWS                                  |    3 +
 doc/automake.texi                     |   18 +++++
 lib/am/check.am                       |    2 +-
 lib/am/check2.am                      |    4 +-
 tests/Makefile.am                     |    2 +
 tests/Makefile.in                     |   18 +++--
 tests/check-fd-redirect.test          |  110 +++++++++++++++++++++++++++++++++
 tests/parallel-tests-fd-redirect.test |  109 ++++++++++++++++++++++++++++++++
 9 files changed, 276 insertions(+), 10 deletions(-)
 create mode 100755 tests/check-fd-redirect.test
 create mode 100755 tests/parallel-tests-fd-redirect.test

diff --git a/ChangeLog b/ChangeLog
index 04ccd4a..34fea0b 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,23 @@
+2011-07-24  Stefano Lattarini  <address@hidden>
+
+       simple tests: support developer-defined fd redirections
+       Motivated by coreutils bug#8846, and related discussions:
+        <http://debbugs.gnu.org/cgi/bugreport.cgi?bug=8846>
+        <http://lists.gnu.org/archive/html/bug-autoconf/2011-06/msg00002.html>
+       In those threads it has been shown how problematic it can be to do
+       portable file descriptor redirections in the testsuite when using
+       the Automake testsuite harness.  This change should remedy to that
+       situation.
+       * lib/am/check2.am (?GENERIC?%EXT%$(EXEEXT).log,
+       ?GENERIC?%EXT%.log, ?!GENERIC?%OBJ%): Append redirection defined
+       in $(AM_TESTS_FD_REDIRECT) to the command-line invocations of the
+       test scripts.
+       * lib/am/check.am [!%?PARALLEL_TESTS%] $(check-TESTS): Likewise.
+       * NEWS: Update.
+       * tests/check-fd-redirect.test: New test.
+       * tests/parallel-tests-fd-redirect.test: Likewise.
+       * tests/Makefile.am (TESTS): Update.
+
 2011-07-22  Stefano Lattarini  <address@hidden>
 
        docs: rework and extend documentation on testsuites support
diff --git a/NEWS b/NEWS
index 4614297..620fe2e 100644
--- a/NEWS
+++ b/NEWS
@@ -72,6 +72,9 @@ New in 1.11a:
     Consistently with the existing syntax, this can be done by defining
     special makefile variables `LOG_DRIVER' and `<ext>_LOG_DRIVER'.
 
+  - A new developer-reserved variable `AM_TESTS_FD_REDIRECT' can be used
+    to redirect/define file descriptors used by the test scripts.
+
 * WARNING: Future backward-incompatibilities!
 
   - The Automake support for automatic de-ANSI-fication will be removed in
diff --git a/doc/automake.texi b/doc/automake.texi
index a47e46d..7d4700d 100644
--- a/doc/automake.texi
+++ b/doc/automake.texi
@@ -8868,6 +8868,19 @@ user, which can employ it to extend or override the 
settings in the
 former; for this to work portably, however, the contents of a non-empty
 @code{AM_TESTS_ENVIRONMENT} @emph{must} be terminated by a semicolon.
 
address@hidden AM_TESTS_FD_REDIRECT
+The @code{AM_TESTS_FD_REDIRECT} variable can be used to define file
+descriptor redirections for the test scripts.  One might think that
address@hidden could be used for this purpose, but experience
+has shown that doing so portably is practically impossible.  The main
+hurdle is constituted by Korn shells, which usually set the close-on-exec
+flag on file descriptors opened with the @command{exec} builtin, thus
+rendering an idiom like @code{AM_TESTS_ENVIRONMENT = exec 9>&2;}
+ineffectual.  This issue also affects some Bourne shells, such as the
+HP-UX's @command{/bin/sh},
address@hidden FIXME: should we offer a link to the relevant discussions on the
address@hidden bug-autoconf list?
+
 @example
 AM_TESTS_ENVIRONMENT = \
 ## Some environment initializations are kept in a separate shell file
@@ -8880,6 +8893,11 @@ AM_TESTS_ENVIRONMENT = \
     PATH=/usr/xpg4/bin:$$PATH; export PATH; \
   fi;
 @c $$ restore font-lock
+## With this, the test scripts will be able to print diagnostic messages
+## to the original standard error stream, even if the test driver
+## redirects the stderr of the test scripts to a log file before executing
+## them.
+AM_TESTS_FD_REDIRECT = 9>&2
 @end example
 
 @noindent
diff --git a/lib/am/check.am b/lib/am/check.am
index 73306bc..ffb70ba 100644
--- a/lib/am/check.am
+++ b/lib/am/check.am
@@ -367,7 +367,7 @@ check-TESTS: $(TESTS)
 ## why we also try `dir='
            elif test -f $$tst; then dir=; \
            else dir="$(srcdir)/"; fi; \
-           if $(TESTS_ENVIRONMENT) $${dir}$$tst; then \
+           if $(TESTS_ENVIRONMENT) $${dir}$$tst $(AM_TESTS_FD_REDIRECT); then \
 ## Success
              all=`expr $$all + 1`; \
              case " $(XFAIL_TESTS) " in \
diff --git a/lib/am/check2.am b/lib/am/check2.am
index b7ed0ee..f4fb3c9 100644
--- a/lib/am/check2.am
+++ b/lib/am/check2.am
@@ -19,7 +19,7 @@
 ?!GENERIC?%OBJ%: %SOURCE%
        @p='%SOURCE%'; $(am__check_pre) \
        %DRIVER% $(am__test_driver_flags) %DRIVER_FLAGS% -- \
-       %COMPILE% "$$tst"
+       %COMPILE% "$$tst" $(AM_TESTS_FD_REDIRECT)
 
 ## If no programs are built in this package, then this rule is removed
 ## at automake time.  Otherwise, %am__EXEEXT% expands to a configure time
@@ -29,5 +29,5 @@ if %am__EXEEXT%
 ?GENERIC?%EXT%$(EXEEXT).log:
        @p='%SOURCE%'; $(am__check_pre) \
        %DRIVER% $(am__test_driver_flags) %DRIVER_FLAGS% -- \
-       %COMPILE% "$$tst"
+       %COMPILE% "$$tst" $(AM_TESTS_FD_REDIRECT)
 endif %am__EXEEXT%
diff --git a/tests/Makefile.am b/tests/Makefile.am
index dea5da8..f322054 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -238,6 +238,7 @@ check10.test \
 check11.test \
 check12.test \
 check-exported-srcdir.test \
+check-fd-redirect.test \
 check-tests-in-builddir.test \
 check-tests_environment.test \
 check-no-test-driver.test \
@@ -723,6 +724,7 @@ parallel-tests7.test \
 parallel-tests8.test \
 parallel-tests9.test \
 parallel-tests10.test \
+parallel-tests-fd-redirect.test \
 parallel-tests-am_tests_environment.test \
 parallel-tests-unreadable-log.test \
 parallel-tests-subdir.test \
diff --git a/tests/Makefile.in b/tests/Makefile.in
index 3f7a585..0ce3dd9 100644
--- a/tests/Makefile.in
+++ b/tests/Makefile.in
@@ -277,12 +277,13 @@ XFAIL_TESTS = all.test auxdir2.test cond17.test gcj6.test 
\
        tap-plan-corner2.test tap-message-0.test tap-signal.test \
        txinfo5.test $(instspc_xfail_tests)
 parallel_tests = backcompat5-p.test check-exported-srcdir-p.test \
-       check-tests-in-builddir-p.test check-tests_environment-p.test \
-       check-p.test check11-p.test check12-p.test check2-p.test \
-       check3-p.test check4-p.test check5-p.test check6-p.test \
-       check7-p.test check8-p.test check9-p.test color-p.test \
-       color2-p.test comment9-p.test dejagnu-p.test exeext4-p.test \
-       maken3-p.test maken4-p.test posixsubst-tests-p.test
+       check-fd-redirect-p.test check-tests-in-builddir-p.test \
+       check-tests_environment-p.test check-p.test check11-p.test \
+       check12-p.test check2-p.test check3-p.test check4-p.test \
+       check5-p.test check6-p.test check7-p.test check8-p.test \
+       check9-p.test color-p.test color2-p.test comment9-p.test \
+       dejagnu-p.test exeext4-p.test maken3-p.test maken4-p.test \
+       posixsubst-tests-p.test
 instspc_tests = instspc-squote-build.test instspc-squote-install.test \
        instspc-dquote-build.test instspc-dquote-install.test \
        instspc-bquote-build.test instspc-bquote-install.test \
@@ -482,6 +483,7 @@ check10.test \
 check11.test \
 check12.test \
 check-exported-srcdir.test \
+check-fd-redirect.test \
 check-tests-in-builddir.test \
 check-tests_environment.test \
 check-no-test-driver.test \
@@ -967,6 +969,7 @@ parallel-tests7.test \
 parallel-tests8.test \
 parallel-tests9.test \
 parallel-tests10.test \
+parallel-tests-fd-redirect.test \
 parallel-tests-am_tests_environment.test \
 parallel-tests-unreadable-log.test \
 parallel-tests-subdir.test \
@@ -1582,7 +1585,7 @@ recheck recheck-html:
 .test.log:
        @p='$<'; $(am__check_pre) \
        $(TEST_LOG_DRIVER) $(am__test_driver_flags) $(AM_TEST_LOG_DRIVER_FLAGS) 
$(TEST_LOG_DRIVER_FLAGS) -- \
-       $(TEST_LOG_COMPILE) "$$tst"
+       $(TEST_LOG_COMPILE) "$$tst" $(AM_TESTS_FD_REDIRECT)
 
 distdir: $(DISTFILES)
        @srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \
@@ -1736,6 +1739,7 @@ uninstall-am:
 
 backcompat5-p.log: backcompat5.test
 check-exported-srcdir-p.log: check-exported-srcdir.test
+check-fd-redirect-p.log: check-fd-redirect.test
 check-tests-in-builddir-p.log: check-tests-in-builddir.test
 check-tests_environment-p.log: check-tests_environment.test
 check-p.log: check.test
diff --git a/tests/check-fd-redirect.test b/tests/check-fd-redirect.test
new file mode 100755
index 0000000..6da8704
--- /dev/null
+++ b/tests/check-fd-redirect.test
@@ -0,0 +1,110 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# Simple Tests support: redirection of file descriptors with
+# AM_TESTS_FD_REDIRECT.
+# See also related test 'parallel-tests-fd-redirect.test'.
+
+. ./defs || Exit 1
+
+cat >> configure.in << 'END'
+AC_OUTPUT
+END
+
+cat > Makefile.am <<'END'
+TESTS = foo.test
+AM_TESTS_FD_REDIRECT = 3<three 4>four 5>>five 7<&0 8>&1 9>&2
+END
+
+echo '3333' > three
+chmod a-w three
+
+: > foo.test
+chmod a+x foo.test
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE -a
+./configure
+
+do_check ()
+{
+  cat foo.test # For debugging.
+  echo 'this line will be removed' > four
+  echo 'this line will not be removed' > five
+  st=0
+  echo 'ok ok ok' | $MAKE check >stdout 2>stderr || st=1
+  cat stdout
+  cat stderr >&2
+  cat four
+  test x"$parallel_tests" != x"yes" || cat foo.log
+  test $st -eq 0
+  grep '[ /]foo\.test: foofoofoo$' stdout
+  grep '[ /]foo\.test: barbarbar$' stderr
+  grep 'this line' four && Exit 1
+  grep '^3333$' four
+  grep '^this line will not be removed$' five
+  grep '^ok ok ok$' five
+  $EGREP '(foofoofoo|barbarbar|3333|ok ok ok|this line)' foo.log && Exit 1
+  :
+}
+
+# Try using both shell script and a perl script as the test, for
+# better coverage.
+
+cat > foo.test <<'END'
+#! /bin/sh
+set -e
+
+read FOO <&3
+test 3333 -eq "$FOO"
+echo "$FOO" >&4
+
+grep '^ok ok ok$' <&7 >&5
+
+echo " " $0: foofoofoo >&8
+echo " " $0: barbarbar >&9
+END
+
+do_check
+
+echo "#! $PERL -w" > foo.test
+cat >> foo.test <<'END'
+use warnings FATAL => 'all';
+use strict;
+
+open (FD3, "<&=3") or die "opening FD3: $!";
+open (FD4, ">&=4") or die "opening FD4: $!";
+open (FD5, ">&=5") or die "opening FD5: $!";
+open (FD7, "<&=7") or die "opening FD7: $!";
+open (FD8, ">&=8") or die "opening FD8: $!";
+open (FD9, ">&=9") or die "opening FD9: $!";
+
+chomp (my $FOO = <FD3>);
+die "$FOO != 3333" if not $FOO eq "3333";
+print FD4 "$FOO\n";
+
+chomp ($_ = <FD7>);
+die "$_ != 'ok ok ok'" if not $_ eq 'ok ok ok';
+print FD5 "$_\n";
+
+print FD8 "  $0: foofoofoo\n";
+print FD9 "  $0: barbarbar\n";
+END
+
+do_check
+
+:
diff --git a/tests/parallel-tests-fd-redirect.test 
b/tests/parallel-tests-fd-redirect.test
new file mode 100755
index 0000000..e00581c
--- /dev/null
+++ b/tests/parallel-tests-fd-redirect.test
@@ -0,0 +1,109 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# parallel-tests support: redirection of file descriptors with
+# AM_TESTS_FD_REDIRECT, even when using tests without suffix, or
+# which are binary executables.
+# We use some tricks to ensure that all code paths in `lib/am/check2.am'
+# are covered, even on platforms where $(EXEEXT) would be naturally empty.
+# See also the more generic test 'check-fd-redirect.test'.
+
+required='cc native'
+parallel_tests=yes
+. ./defs || Exit 1
+
+cat >> configure.in << 'END'
+AC_PROG_CC
+AM_CONDITIONAL([real_EXEEXT], [test -n "$EXEEXT"])
+test -n "$EXEEXT" || EXEEXT=.bin
+AC_OUTPUT
+END
+
+cat > Makefile.am << 'END'
+AM_TESTS_FD_REDIRECT = 9>&1
+TEST_EXTENSIONS = .test .sh
+TESTS = foo.sh bar $(check_PROGRAMS)
+check_PROGRAMS = baz qux.test
+qux_test_SOURCES = zardoz.c
+
+## Sanity check.
+if !real_EXEEXT
+check-local:
+       test -f baz.bin
+       test -f qux.test.bin
+endif
+END
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE -a
+
+cat > foo.sh <<'END'
+#!/bin/sh
+echo " foofoofoo" >&9
+END
+chmod a+x foo.sh
+
+cat > bar <<'END'
+#!/bin/sh
+echo " barbarbar" >&9
+END
+
+chmod a+x foo.sh bar
+
+cat > baz.c <<'END'
+#include <unistd.h>
+int main (void)
+{
+  write (9, " bazbazbaz\n", 11);
+  return 0;
+}
+END
+
+cat > zardoz.c <<'END'
+#include <unistd.h>
+int main (void)
+{
+  write (9, " quxquxqux\n", 11);
+  return 0;
+}
+END
+
+./configure
+
+# Sanity checks.
+st=0
+grep '^bar\.log:.*bar' Makefile || st=1
+grep '^baz\.log:.*baz\$(EXEEXT)' Makefile || st=1
+grep '^\.test\$(EXEEXT)\.log:' Makefile || st=1
+$EGREP '^(foo|qux)\.log:' Makefile && st=1
+test $st -eq 0 || fatal_ "doesn't cover expected code paths"
+
+st=0
+$MAKE check >stdout || st=1
+cat stdout
+cat foo.log
+cat bar.log
+cat baz.log
+cat qux.log
+test $st -eq 0
+grep "^ foofoofoo$" stdout
+grep "^ barbarbar$" stdout
+grep "^ bazbazbaz$" stdout
+grep "^ quxquxqux$" stdout
+$EGREP '(foofoofoo|barbarbar|bazbazbaz|quxquxqux)' *.log && Exit 1
+
+:
-- 
1.7.2.3