[PATCH 03] New tests for `parallel-tests'.

[Ralf Wildenhues]

Compared to
<http://article.gmane.org/gmane.comp.sysutils.automake.patches/3228>,
This patch has

- LAZY_TEST_SUITE testing is moved to a later patch in the series.
- DISABLE_HARD_ERRORS adjusted to Benoit's semantics,
- tests/*-p.test listed in MAINTAINERCLEANFILES.

Cheers,
Ralf

    New tests for `parallel-tests'.
    
    * tests/parallel-tests.test: New file, to expose a bunch of
    features of `parallel-tests': VERBOSE, clean, TEST_SUITE_LOG,
    test dependencies, DISABLE_HARD_ERRORS.
    * tests/parallel-tests2.test: New file, test check-html.
    * tests/parallel-tests3.test: New file, test concurrency.
    * tests/parallel-tests4.test: New file, test suffix rules.
    * tests/parallel-tests5.test: New file, demonstrate compile/test
    concurrency.
    * tests/defs.in: Add a `required' check for rst2html.
    * tests/Makefile.am: Update.
    
    Signed-off-by: Ralf Wildenhues <address@hidden>

diff --git a/ChangeLog b/ChangeLog
index 265a261..61be223 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,17 @@
 2009-03-08  Ralf Wildenhues  <address@hidden>
 
+       New tests for `parallel-tests'.
+       * tests/parallel-tests.test: New file, to expose a bunch of
+       features of `parallel-tests': VERBOSE, clean, TEST_SUITE_LOG,
+       test dependencies, DISABLE_HARD_ERRORS.
+       * tests/parallel-tests2.test: New file, test check-html.
+       * tests/parallel-tests3.test: New file, test concurrency.
+       * tests/parallel-tests4.test: New file, test suffix rules.
+       * tests/parallel-tests5.test: New file, demonstrate compile/test
+       concurrency.
+       * tests/defs.in: Add a `required' check for rst2html.
+       * tests/Makefile.am: Update.
+
        parallel-tests: Ensure backward-compatible semantics.
        For each test in Automake's test suite that uses TESTS, generate
        an identical one that uses the `parallel-tests' option, for
diff --git a/tests/Makefile.am b/tests/Makefile.am
index fbb6118..2504941 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -480,6 +480,11 @@ overrid.test \
 parallel-am.test \
 parallel-am2.test \
 parallel-am3.test \
+parallel-tests.test \
+parallel-tests2.test \
+parallel-tests3.test \
+parallel-tests4.test \
+parallel-tests5.test \
 parse.test \
 percent.test \
 percent2.test \
diff --git a/tests/defs.in b/tests/defs.in
index 8a08998..3b8fe96 100644
--- a/tests/defs.in
+++ b/tests/defs.in
@@ -186,6 +186,16 @@ do
       rm -rf $ro_dir_temp
       test $create_status = 0 && exit 77
       ;;
+    rst2html)
+      # Try the variants that are tried in check.am.
+      while :; do
+       for r2h in $RST2HTML rst2html rst2html.py; do
+         echo "$me: running $r2h --version"
+         $r2h --version && break 2
+       done
+       exit 77
+      done
+      ;;
     runtest)
       # DejaGnu's runtest program. We rely on being able to specify
       # the program on the runtest command-line. This requires
diff --git a/tests/parallel-tests.test b/tests/parallel-tests.test
new file mode 100755
index 0000000..6e4b878
--- /dev/null
+++ b/tests/parallel-tests.test
@@ -0,0 +1,108 @@
+#! /bin/sh
+# Copyright (C) 2009  Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# Check parallel-tests features:
+# - VERBOSE
+# - clean
+# - TEST_SUITE_LOG
+# - dependencies between tests
+# - DISABLE_HARD_ERRORS
+
+. ./defs-p || Exit 1
+
+set -e
+
+cat >> configure.in << 'END'
+AC_OUTPUT
+END
+
+cat > Makefile.am << 'END'
+TEST_SUITE_LOG = mylog.log
+TESTS = foo.test bar.test baz.test
+XFAIL_TESTS = bar.test
+foo.log: bar.log
+bar.log: baz.log
+END
+
+cat >>foo.test <<'END'
+#! /bin/sh
+echo "this is $0"
+exit 0
+END
+cat >>bar.test <<'END'
+#! /bin/sh
+echo "this is $0"
+exit 99
+END
+cat >>baz.test <<'END'
+#! /bin/sh
+echo "this is $0"
+exit 1
+END
+chmod a+x foo.test bar.test baz.test
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE -a
+
+unset TESTS || :
+
+./configure
+# No hard errors: all tests should be run, there should be one failure.
+env DISABLE_HARD_ERRORS=yes $MAKE -e check >stdout && { cat stdout; Exit 1; }
+cat stdout
+test `grep -c '^FAIL' stdout` -eq 1
+test -f mylog.log
+test `grep -c '^FAIL' mylog.log` -eq 1
+test -f baz.log
+test -f bar.log
+test -f foo.log
+
+# clean should remove all log files (but not more).
+: > unrelated.log
+$MAKE clean
+test ! -f baz.log
+test ! -f bar.log
+test ! -f foo.log
+test ! -f mylog.log
+test -f unrelated.log
+
+$MAKE clean
+$MAKE check >stdout && { cat stdout; Exit 1; }
+cat stdout
+# Now, there should be two errors: bar.test is a hard error.
+test `grep -c '^FAIL' stdout` -eq 2
+test `grep -c '^FAIL' mylog.log` -eq 2
+
+# Check dependencies: baz.test needs to run before bar.test,
+# but foo.test is not needed.
+# Note that this usage has a problem: the summary will only
+# take bar.log into account, because the $(TEST_SUITE_LOG) rule
+# does not "see" baz.log.  Hmm.
+$MAKE clean
+env TESTS='bar.test' $MAKE -e check && Exit 1
+test -f baz.log
+test -f bar.log
+test ! -f foo.log
+test -f mylog.log
+
+# Test VERBOSE.
+env VERBOSE=yes $MAKE -e check > stdout && { cat stdout; Exit 1; }
+cat stdout
+grep 'this is.*bar.test' stdout
+grep 'this is.*baz.test' stdout
+
+:
diff --git a/tests/parallel-tests2.test b/tests/parallel-tests2.test
new file mode 100755
index 0000000..17a5108
--- /dev/null
+++ b/tests/parallel-tests2.test
@@ -0,0 +1,59 @@
+#! /bin/sh
+# Copyright (C) 2009  Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# Check parallel-tests features:
+# - check-html
+
+required=rst2html
+. ./defs-p || Exit 1
+
+set -e
+
+cat >> configure.in << 'END'
+AC_OUTPUT
+END
+
+cat > Makefile.am << 'END'
+TEST_SUITE_LOG = mylog.log
+TESTS = foo.test bar.test baz.test
+END
+
+cat >>foo.test <<'END'
+#! /bin/sh
+echo "this is $0"
+exit 0
+END
+cat >>bar.test <<'END'
+#! /bin/sh
+echo "this is $0"
+exit 99
+END
+cat >>baz.test <<'END'
+#! /bin/sh
+echo "this is $0"
+exit 1
+END
+chmod a+x foo.test bar.test baz.test
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE -a
+
+./configure
+$MAKE check-html >stdout && { cat stdout; Exit 1; }
+cat stdout
+test -f mylog.html
+:
diff --git a/tests/parallel-tests3.test b/tests/parallel-tests3.test
new file mode 100755
index 0000000..c53edee
--- /dev/null
+++ b/tests/parallel-tests3.test
@@ -0,0 +1,69 @@
+#! /bin/sh
+# Copyright (C) 2009  Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# Check parallel-tests features:
+# - concurrent parallel execution
+
+required=GNUmake
+. ./defs-p || Exit 1
+
+set -e
+
+cat >> configure.in << 'END'
+AC_OUTPUT
+END
+
+cat > Makefile.am << 'END'
+TESTS =
+END
+
+for i in 1 2 3 4 5 6 7 8; do
+  echo "TESTS += foo$i.test" >> Makefile.am
+  cat >foo$i.test <<'END'
+#! /bin/sh
+echo "this is $0"
+# hack around maintainer-check check:
+sleep='sleep '1
+$sleep
+exit 0
+END
+  chmod a+x foo$i.test
+done
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE -a
+
+unset TESTS || :
+
+for build in serial parallel; do
+  mkdir $build
+  cd $build
+  ../configure
+  $MAKE
+  cd ..
+done
+
+cd serial
+$MAKE -j1 check &
+cd ../parallel
+$sleep
+$MAKE -j4 check > stdout
+cd ..
+kill $!
+cat parallel/stdout
+test `grep -c PASS parallel/stdout` -eq 8
+:
diff --git a/tests/parallel-tests4.test b/tests/parallel-tests4.test
new file mode 100755
index 0000000..ca3e18b
--- /dev/null
+++ b/tests/parallel-tests4.test
@@ -0,0 +1,70 @@
+#! /bin/sh
+# Copyright (C) 2009  Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# Check parallel-tests features:
+# - suffix rules
+
+. ./defs-p || Exit 1
+set -e
+
+cat >> configure.in << 'END'
+AC_PROG_CC
+AC_OUTPUT
+END
+
+cat > Makefile.am << 'END'
+## Note that automake should not match the '/test' part
+## of 'sub/test' as '.test' suffix.
+TESTS = foo.chk bar.test $(check_PROGRAMS) sub/test
+check_PROGRAMS = baz bla.test bli.suff
+TEST_EXTENSIONS = .chk .test
+END
+
+mkdir sub
+
+cat >foo.chk << 'END'
+#! /bin/sh
+exit 0
+END
+chmod a+x foo.chk
+cp foo.chk bar.test
+cp foo.chk sub/test
+
+cat >baz.c << 'END'
+int main (void)
+{
+  return 0;
+}
+END
+cp baz.c bla.c
+cp baz.c bli.c
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE -a
+
+unset TESTS || :
+
+./configure
+$MAKE
+$MAKE check
+test -f foo.log
+test -f bar.log
+test -f baz.log
+test -f bla.log
+test -f bli.suff.log
+test -f sub/test.log
+:
diff --git a/tests/parallel-tests5.test b/tests/parallel-tests5.test
new file mode 100755
index 0000000..f109cc2
--- /dev/null
+++ b/tests/parallel-tests5.test
@@ -0,0 +1,68 @@
+#! /bin/sh
+# Copyright (C) 2009  Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# Check parallel-tests features:
+# - concurrent program compilation and testing (with EXTRA_PROGRAMS)
+#
+# Actually, this test doesn't ensure that things happen concurrently.
+# It merely serves as demonstration.  :-)
+
+required=GNUmake
+. ./defs-p || Exit 1
+
+set -e
+
+cat >> configure.in << 'END'
+AC_PROG_CC
+AC_OUTPUT
+END
+
+cat > Makefile.am << 'END'
+TESTS = $(programs) $(scripts)
+EXTRA_PROGRAMS = $(programs)
+CLEANFILES = $(EXTRA_PROGRAMS)
+dist_noinst_SCRIPTS = $(scripts)
+programs =
+scripts =
+END
+
+for i in 1 2 3 4 5 6 7 8; do
+  echo "scripts += foo$i.test" >> Makefile.am
+  echo "programs += foo$i.prog" >> Makefile.am
+  cat >foo$i.test <<'END'
+#! /bin/sh
+echo "this is $0"
+exit 0
+END
+  cat >foo$i.c <<'END'
+int main()
+{
+  return 0;
+}
+END
+  chmod a+x foo$i.test
+done
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE -a
+
+unset TESTS || :
+
+./configure
+$MAKE -j4 check
+$MAKE distcheck
+: