Re: Automake testsuite misuses DejaGnu [PATCH v0]

[Jacob Bachmeyer]

Jim Meyering wrote:
> [...]
> Even a sample fix for one of the currently-failing tests would be helpful.
>   

This is the first draft; this patch breaks 1.6.1 because versions of 
DejaGnu prior to 1.6.3 require srcdir to point exactly to the testsuite, 
while 1.6.3 allows the testsuite to be in ${srcdir}/testsuite.

8<------
diff -urN -x '*~' automake-1.16.3-original/t/check12.sh 
automake-1.16.3/t/check12.sh
--- automake-1.16.3-original/t/check12.sh       2020-11-18 19:21:03.000000000 
-0600
+++ automake-1.16.3/t/check12.sh        2021-06-29 01:47:21.669276386 -0500
@@ -60,8 +60,8 @@
DEJATOOL = hammer spanner
AM_RUNTESTFLAGS = HAMMER=$(srcdir)/hammer SPANNER=$(srcdir)/spanner
EXTRA_DIST += $(DEJATOOL)
-EXTRA_DIST += hammer.test/hammer.exp
-EXTRA_DIST += spanner.test/spanner.exp
+EXTRA_DIST += testsuite/hammer.test/hammer.exp
+EXTRA_DIST += testsuite/spanner.test/spanner.exp
END

cat > hammer << 'END'
@@ -77,9 +77,10 @@
END
chmod +x hammer spanner

-mkdir hammer.test spanner.test
+mkdir testsuite
+mkdir testsuite/hammer.test testsuite/spanner.test

-cat > hammer.test/hammer.exp << 'END'
+cat > testsuite/hammer.test/hammer.exp << 'END'
set test test_hammer
spawn $HAMMER
expect {
@@ -88,7 +89,7 @@
}
END

-cat > spanner.test/spanner.exp << 'END'
+cat > testsuite/spanner.test/spanner.exp << 'END'
set test test_spanner
spawn $SPANNER
expect {
diff -urN -x '*~' automake-1.16.3-original/t/dejagnu3.sh 
automake-1.16.3/t/dejagnu3.sh
--- automake-1.16.3-original/t/dejagnu3.sh      2020-11-18 19:21:03.000000000 
-0600
+++ automake-1.16.3/t/dejagnu3.sh       2021-06-29 01:19:19.161147525 -0500
@@ -34,12 +34,13 @@
AUTOMAKE_OPTIONS = dejagnu
DEJATOOL = hammer
AM_RUNTESTFLAGS = HAMMER=$(srcdir)/hammer
-EXTRA_DIST = hammer hammer.test/hammer.exp
+EXTRA_DIST = hammer testsuite/hammer.test/hammer.exp
END

-mkdir hammer.test
+mkdir testsuite
+mkdir testsuite/hammer.test

-cat > hammer.test/hammer.exp << 'END'
+cat > testsuite/hammer.test/hammer.exp << 'END'
set test test
spawn $HAMMER
expect {
diff -urN -x '*~' automake-1.16.3-original/t/dejagnu4.sh 
automake-1.16.3/t/dejagnu4.sh
--- automake-1.16.3-original/t/dejagnu4.sh      2020-11-18 19:21:03.000000000 
-0600
+++ automake-1.16.3/t/dejagnu4.sh       2021-06-29 01:25:08.309780437 -0500
@@ -49,13 +49,14 @@

AM_RUNTESTFLAGS = HAMMER=$(srcdir)/hammer SPANNER=$(srcdir)/spanner

-EXTRA_DIST  = hammer  hammer.test/hammer.exp
-EXTRA_DIST += spanner spanner.test/spanner.exp
+EXTRA_DIST  = hammer  testsuite/hammer.test/hammer.exp
+EXTRA_DIST += spanner testsuite/spanner.test/spanner.exp
END

-mkdir hammer.test spanner.test
+mkdir testsuite
+mkdir testsuite/hammer.test testsuite/spanner.test

-cat > hammer.test/hammer.exp << 'END'
+cat > testsuite/hammer.test/hammer.exp << 'END'
set test test
spawn $HAMMER
expect {
@@ -64,7 +65,7 @@
}
END

-cat > spanner.test/spanner.exp << 'END'
+cat > testsuite/spanner.test/spanner.exp << 'END'
set test test
spawn $SPANNER
expect {
diff -urN -x '*~' automake-1.16.3-original/t/dejagnu5.sh 
automake-1.16.3/t/dejagnu5.sh
--- automake-1.16.3-original/t/dejagnu5.sh      2020-11-18 19:21:03.000000000 
-0600
+++ automake-1.16.3/t/dejagnu5.sh       2021-06-29 01:26:36.511645792 -0500
@@ -34,12 +34,13 @@

cat > Makefile.am << END
AUTOMAKE_OPTIONS = dejagnu
-EXTRA_DIST = $package $package.test/$package.exp
+EXTRA_DIST = $package testsuite/$package.test/$package.exp
AM_RUNTESTFLAGS = PACKAGE=\$(srcdir)/$package
END

-mkdir $package.test
-cat > $package.test/$package.exp << 'END'
+mkdir testsuite
+mkdir testsuite/$package.test
+cat > testsuite/$package.test/$package.exp << 'END'
set test "a_dejagnu_test"
spawn $PACKAGE
expect {
diff -urN -x '*~' automake-1.16.3-original/t/dejagnu6.sh 
automake-1.16.3/t/dejagnu6.sh
--- automake-1.16.3-original/t/dejagnu6.sh      2020-11-18 19:21:03.000000000 
-0600
+++ automake-1.16.3/t/dejagnu6.sh       2021-06-29 01:28:07.151396859 -0500
@@ -35,8 +35,9 @@
AM_RUNTESTFLAGS = FAILDEJA=$(srcdir)/faildeja
END

-mkdir faildeja.test
-cat > faildeja.test/faildeja.exp << 'END'
+mkdir testsuite
+mkdir testsuite/faildeja.test
+cat > testsuite/faildeja.test/faildeja.exp << 'END'
set test failing_deja_test
spawn $FAILDEJA
expect {
diff -urN -x '*~' automake-1.16.3-original/t/dejagnu7.sh 
automake-1.16.3/t/dejagnu7.sh
--- automake-1.16.3-original/t/dejagnu7.sh      2020-11-18 19:21:03.000000000 
-0600
+++ automake-1.16.3/t/dejagnu7.sh       2021-06-29 01:29:38.877097021 -0500
@@ -39,8 +39,9 @@
AM_RUNTESTFLAGS = --status FAILTCL=$(srcdir)/failtcl
END

-mkdir failtcl.test
-cat > failtcl.test/failtcl.exp << 'END'
+mkdir testsuite
+mkdir testsuite/failtcl.test
+cat > testsuite/failtcl.test/failtcl.exp << 'END'
set test test
spawn $FAILTCL
expect {
diff -urN -x '*~' automake-1.16.3-original/t/dejagnu-absolute-builddir.sh 
automake-1.16.3/t/dejagnu-absolute-builddir.sh
--- automake-1.16.3-original/t/dejagnu-absolute-builddir.sh     2020-11-18 
19:21:03.000000000 -0600
+++ automake-1.16.3/t/dejagnu-absolute-builddir.sh      2021-06-29 
01:36:15.647497570 -0500
@@ -27,12 +27,13 @@
cat > Makefile.am << 'END'
AUTOMAKE_OPTIONS = dejagnu
DEJATOOL = tcl env
-EXTRA_DIST = tcl.test/tcl.exp
+EXTRA_DIST = testsuite/tcl.test/tcl.exp
END

-mkdir tcl.test
+mkdir testsuite
+mkdir testsuite/tcl.test

-cat > tcl.test/tcl.exp << 'END'
+cat > testsuite/tcl.test/tcl.exp << 'END'
send_user "tcl_objdir: $objdir\n"
if { [ regexp "^/" $objdir ] } {
    pass "test_tcl_objdir"
diff -urN -x '*~' automake-1.16.3-original/t/dejagnu-relative-srcdir.sh 
automake-1.16.3/t/dejagnu-relative-srcdir.sh
--- automake-1.16.3-original/t/dejagnu-relative-srcdir.sh       2020-11-18 
19:21:03.000000000 -0600
+++ automake-1.16.3/t/dejagnu-relative-srcdir.sh        2021-06-29 
01:38:43.275577188 -0500
@@ -27,20 +27,22 @@
cat > Makefile.am << 'END'
AUTOMAKE_OPTIONS = dejagnu
DEJATOOL = tcl env
-EXTRA_DIST = env.test/env.exp tcl.test/tcl.exp lib/tcl.exp
+EXTRA_DIST  = testsuite/env.test/env.exp testsuite/tcl.test/tcl.exp
+EXTRA_DIST += testsuite/lib/tcl.exp
END

-mkdir env.test tcl.test lib
+mkdir testsuite
+mkdir testsuite/env.test testsuite/tcl.test testsuite/lib

# DejaGnu can change $srcdir behind our backs, so we have to
# save its original value.  Thanks to Ian Lance Taylor for the
# suggestion.
-cat > lib/tcl.exp << 'END'
+cat > testsuite/lib/tcl.exp << 'END'
send_user "tcl_lib_srcdir: $srcdir\n"
set orig_srcdir $srcdir
END

-cat > env.test/env.exp << 'END'
+cat > testsuite/env.test/env.exp << 'END'
set env_srcdir $env(srcdir)
send_user "env_srcdir: $env_srcdir\n"
if { [ regexp "^\.(\./\.\.)?$" $env_srcdir ] } {
@@ -50,7 +52,7 @@
}
END

-cat > tcl.test/tcl.exp << 'END'
+cat > testsuite/tcl.test/tcl.exp << 'END'
send_user "tcl_srcdir: $srcdir\n"
if { [ regexp "^\.(\./\.\.)?$" $srcdir ] } {
    pass "test_tcl_src"
diff -urN -x '*~' automake-1.16.3-original/t/dejagnu-siteexp-extend.sh 
automake-1.16.3/t/dejagnu-siteexp-extend.sh
--- automake-1.16.3-original/t/dejagnu-siteexp-extend.sh        2020-11-18 
19:21:03.000000000 -0600
+++ automake-1.16.3/t/dejagnu-siteexp-extend.sh 2021-06-29 01:43:25.922327525 
-0500
@@ -40,7 +40,7 @@
AUTOMAKE_OPTIONS = dejagnu
DEJATOOL = tool

-EXTRA_DIST = tool.test/tool.exp
+EXTRA_DIST = testsuite/tool.test/tool.exp

EXTRA_DEJAGNU_SITE_CONFIG = foo.exp
EXTRA_DIST += foo.exp
@@ -48,9 +48,10 @@

echo 'set foo "/foo/"' > foo.exp

-mkdir tool.test
-write_check_for foo > tool.test/tool.exp
-cat tool.test/tool.exp
+mkdir testsuite
+mkdir testsuite/tool.test
+write_check_for foo > testsuite/tool.test/tool.exp
+cat testsuite/tool.test/tool.exp

$ACLOCAL
$AUTOCONF
@@ -63,9 +64,9 @@
cat site.exp
grep 'PASS: test_foo' tool.sum

-write_check_for bar >> tool.test/tool.exp
-write_check_for baz >> tool.test/tool.exp
-cat tool.test/tool.exp
+write_check_for bar >> testsuite/tool.test/tool.exp
+write_check_for baz >> testsuite/tool.test/tool.exp
+cat testsuite/tool.test/tool.exp

# Ensure that foo.exp will be newer than site.exp, which will
# thus have to be remade.
@@ -118,8 +119,8 @@

# Check that the user can edit the site.exp file, and that his edits
# are retained.
-write_check_for zardoz >> tool.test/tool.exp
-cat tool.test/tool.exp
+write_check_for zardoz >> testsuite/tool.test/tool.exp
+cat testsuite/tool.test/tool.exp
echo 'set zardoz "/zardoz/"' >> site.exp

$MAKE check
diff -urN -x '*~' automake-1.16.3-original/t/dejagnu-siteexp-useredit.sh 
automake-1.16.3/t/dejagnu-siteexp-useredit.sh
--- automake-1.16.3-original/t/dejagnu-siteexp-useredit.sh      2020-11-18 
19:21:03.000000000 -0600
+++ automake-1.16.3/t/dejagnu-siteexp-useredit.sh       2021-06-29 
01:44:58.285997782 -0500
@@ -30,8 +30,9 @@

# Deliberately select a variable defined automatically by
# the Makefile-generated site.exp.
-mkdir foo.test
-cat > foo.test/foo.exp << 'END'
+mkdir testsuite
+mkdir testsuite/foo.test
+cat > testsuite/foo.test/foo.exp << 'END'
send_user "objdir: $objdir\n"
set pipe "|"
if { $objdir == "${pipe}objdir${pipe}" } {
8<------


-- Jacob