From 42a465249b6d3425efbf23df8f2b62fc0e088cfb Mon Sep 17 00:00:00 2001 From: Stefano Lattarini Date: Thu, 21 Apr 2011 10:25:01 +0200 Subject: [PATCH] check: document and test $(TEST_SUITE_LOG) overriding * doc/automake.texi (Simple Tests using parallel-tests): Explain how and why TEST_SUITE_LOG can be overridden at runtime. * tests/parallel-tests-log-override-1.test: New test, check that the newly documented idiom and similar usages are truly supported. * tests/parallel-tests-log-override-2.test: Likewise. * tests/parallel-tests-log-override-recheck.test: Likewise. * tests/Makefile.am (TESTS): Update. --- ChangeLog | 11 +++ doc/automake.texi | 17 ++++ tests/Makefile.am | 3 + tests/Makefile.in | 3 + tests/parallel-tests-log-override-1.test | 112 +++++++++++++++++++++++++ tests/parallel-tests-log-override-2.test | 88 +++++++++++++++++++ tests/parallel-tests-log-override-recheck.test | 92 ++++++++++++++++++++ 7 files changed, 326 insertions(+) create mode 100755 tests/parallel-tests-log-override-1.test create mode 100755 tests/parallel-tests-log-override-2.test create mode 100755 tests/parallel-tests-log-override-recheck.test diff --git a/ChangeLog b/ChangeLog index b9c71f3..06b9788 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,14 @@ +2011-05-11 Stefano Lattarini + + check: document and test $(TEST_SUITE_LOG) overriding + * doc/automake.texi (Simple Tests using parallel-tests): Explain + how and why TEST_SUITE_LOG can be overridden at runtime. + * tests/parallel-tests-log-override-1.test: New test, check that + the newly documented idiom and similar usages are truly supported. + * tests/parallel-tests-log-override-2.test: Likewise. + * tests/parallel-tests-log-override-recheck.test: Likewise. + * tests/Makefile.am (TESTS): Update. + 2011-05-07 Stefano Lattarini tests: fix spurious failure of txinfo21.test on FreeBSD diff --git a/doc/automake.texi b/doc/automake.texi index 3c4bddd..3d5dcc5 100644 --- a/doc/automake.texi +++ b/doc/automake.texi @@ -8772,6 +8772,20 @@ use a command like this to run only a subset of the tests: env TESTS="foo.test bar.test" make -e check @end example +Note however that the command above will unconditionally overwrite the +@file{test-suite.log} file, thus clobbering the recorded results +of any previous testsuite run. This might be undesirable for packages +whose testsuite takes long time to execute. Luckily, this problem can +easily be avoided by overriding also @code{TEST_SUITE_LOG} at runtime; +for example, + +@example +env TEST_SUITE_LOG=partial.log TESTS="..." make -e check +@end example + +will write the result of the partial testsuite runs to the +@file{partial.log}, without touching @file{test-suite.log}. + @item You can set the @code{TEST_LOGS} variable. By default, this variable is computed at @command{make} run time from the value of @code{TESTS} as @@ -8782,6 +8796,9 @@ set x subset*.log; shift env TEST_LOGS="foo.log $*" make -e check @end example +The comments made above about @code{TEST_SUITE_LOG} overriding applies +here too. + @item @vindex RECHECK_LOGS @cindex lazy test execution diff --git a/tests/Makefile.am b/tests/Makefile.am index 49d8942..33599ad 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -590,6 +590,9 @@ parallel-tests9.test \ parallel-tests10.test \ parallel-tests-unreadable-log.test \ parallel-tests-subdir.test \ +parallel-tests-log-override-1.test \ +parallel-tests-log-override-2.test \ +parallel-tests-log-override-recheck.test \ parse.test \ percent.test \ percent2.test \ diff --git a/tests/Makefile.in b/tests/Makefile.in index 78d723c..44aac47 100644 --- a/tests/Makefile.in +++ b/tests/Makefile.in @@ -861,6 +861,9 @@ parallel-tests9.test \ parallel-tests10.test \ parallel-tests-unreadable-log.test \ parallel-tests-subdir.test \ +parallel-tests-log-override-1.test \ +parallel-tests-log-override-2.test \ +parallel-tests-log-override-recheck.test \ parse.test \ percent.test \ percent2.test \ diff --git a/tests/parallel-tests-log-override-1.test b/tests/parallel-tests-log-override-1.test new file mode 100755 index 0000000..6e98380 --- /dev/null +++ b/tests/parallel-tests-log-override-1.test @@ -0,0 +1,112 @@ +#! /bin/sh +# Copyright (C) 2011 Free Software Foundation, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# Check parallel-tests features: runtime redefinition of $(TEST_SUITE_LOG). + +parallel_tests=yes +. ./defs || Exit 1 + +set -e + +cat >> configure.in <<'END' +AC_OUTPUT +END + +cat > Makefile.am <<'END' +TESTS = pass.test skip.test xfail.test +XFAIL_TESTS = xfail.test +EXTRA_DIST = $(TESTS) +END + +cat > pass.test <<'END' +#! /bin/sh +exit 0 +END + +cat > skip.test <<'END' +#! /bin/sh +echo "% test skipped %" +exit 77 +END + +cat > xfail.test <<'END' +#! /bin/sh +echo "# expected failure #" +exit 1 +END + +chmod a+x *.test + +test_log_edit () +{ + sed -e "s|^ *$me 1\.0:.*$| $me 1.0: ???|" \ + -e "s|^=====*|=======================|" $* +} + +test_log_expected () +{ + test_log_edit orig > exp + test_log_edit $1 > got + diff exp got || Exit 1 + rm -f exp got +} + +$ACLOCAL +$AUTOCONF +$AUTOMAKE -a + +./configure + +$MAKE check +ls -l +cat test-suite.log +cp test-suite.log orig + +$MAKE clean +test -f test-suite.log && Exit 99 # Sanity check. + +# Check that we can override the testsuite log file at runtime. +TEST_SUITE_LOG=zardoz.log $MAKE -e check +ls -l +test ! -f test-suite.log +cat zardoz.log +test_log_expected zardoz.log +# Sanity check the distribution too (this also does minimal checks on +# VPATH support). +TEST_SUITE_LOG=zardoz.log $MAKE -e distcheck + +# Check that cleanup rules remove the correct file even when +# user overrides are in place. +cp orig test-suite.log +TEST_SUITE_LOG=zardoz.log $MAKE -e clean +ls -l +test ! -f zardoz.log +diff orig test-suite.log + +# Check that the default testsuite log doesn't get unduly modified. +# Also check that the testsuite log file doesn't need to be named +# accordingly to the `*.log' pattern. +chmod a-w test-suite.log +TEST_SUITE_LOG=TheLogFile $MAKE -e check +ls -l +diff orig test-suite.log +test_log_expected TheLogFile +TEST_SUITE_LOG=TheLogFile $MAKE -e clean +ls -l +test ! -f TheLogFile +diff orig test-suite.log + +: diff --git a/tests/parallel-tests-log-override-2.test b/tests/parallel-tests-log-override-2.test new file mode 100755 index 0000000..649360c --- /dev/null +++ b/tests/parallel-tests-log-override-2.test @@ -0,0 +1,88 @@ +#! /bin/sh +# Copyright (C) 2011 Free Software Foundation, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# Check parallel-tests features: runtime redefinition of: +# - $(TEST_SUITE_LOG) and $(TESTS) +# - $(TEST_SUITE_LOG) and $(TEST_LOGS) + +parallel_tests=yes +. ./defs || Exit 1 + +set -e + +cat >> configure.in <<'END' +AC_OUTPUT +END + +cat > Makefile.am <<'END' +TESTS = pass.test pass2.test skip.test skip2.test fail.test +END + +cat > pass.test <<'END' +#! /bin/sh +exit 0 +END + +cp pass.test pass2.test + +cat > skip.test <<'END' +#! /bin/sh +echo "% skipped test %" +exit 77 +END + +cp skip.test skip2.test + +cat > fail.test <<'END' +#! /bin/sh +exit 1 +END + +chmod a+x *.test + +$ACLOCAL +$AUTOCONF +$AUTOMAKE -a + +./configure + +for test_list_override in \ + 'TESTS=pass.test skip.test' \ + 'TEST_LOGS=pass.log skip.log' +do + env TEST_SUITE_LOG=partial.log "$test_list_override" \ + $MAKE -e check >stdout || { cat stdout; Exit 1; } + cat stdout + ls -l + cat pass.log + cat skip.log + cat partial.log + test ! -f test-suite.log + test ! -f pass2.log + test ! -f skip2.log + test ! -f fail.log + grep '^PASS: .*pass\.test' stdout + grep '^SKIP: .*skip\.test' stdout + $FGREP 'skip.test' partial.log + $FGREP '% skipped test %' partial.log + for t in pass2 skip2 fail; do + $FGREP "$t.test" stdout && Exit 1 + $FGREP "$t.test" partial.log && Exit 1 + done + rm -f *.log +done + +: diff --git a/tests/parallel-tests-log-override-recheck.test b/tests/parallel-tests-log-override-recheck.test new file mode 100755 index 0000000..8a50d29 --- /dev/null +++ b/tests/parallel-tests-log-override-recheck.test @@ -0,0 +1,92 @@ +#! /bin/sh +# Copyright (C) 2011 Free Software Foundation, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# Check parallel-tests features: runtime redefinition of $(TEST_SUITE_LOG) +# for the recheck target. + +parallel_tests=yes +. ./defs || Exit 1 + +set -e + +cat >> configure.in <<'END' +AC_OUTPUT +END + +cat > Makefile.am <<'END' +TESTS = foo.test bar.test baz.test +END + +cat > foo.test <<'END' +#! /bin/sh +echo "this is $0" +exit 0 +END + +cat > bar.test <<'END' +#! /bin/sh +echo "this is $0" +exit 99 +END + +cat > baz.test <<'END' +#! /bin/sh +echo "this is $0" +exit ${BAZ_EXIT_STATUS-1} +END + +chmod a+x *.test + +unset BAZ_EXIT_STATUS || : + +$ACLOCAL +$AUTOCONF +$AUTOMAKE -a + +./configure +$MAKE check >stdout && { cat stdout; Exit 1; } +cat stdout + +chmod a-rw test-suite.log +TEST_SUITE_LOG=my.log $MAKE -e recheck >stdout \ + && { cat stdout; Exit 1; } +cat stdout +ls -l +grep '^2 of 2 .*failed' stdout +for x in stdout my.log; do + $FGREP foo.test $x && Exit 1 + $FGREP bar.test $x + $FGREP baz.test $x +done + +chmod a-rw my.log +BAZ_EXIT_STATUS=0 TEST_SUITE_LOG=my2.log $MAKE -e recheck >stdout \ + && { cat stdout; Exit 1; } +cat stdout +ls -l +grep '^1 of 2 .*failed' stdout +$FGREP foo.test stdout && Exit 1 +$FGREP bar.test stdout +$FGREP baz.test stdout +$FGREP foo.test my2.log && Exit 1 +$FGREP bar.test my2.log +$FGREP baz.test my2.log && Exit 1 + +chmod u+r test-suite.log my.log +$FGREP baz.test test-suite.log +$FGREP baz.test my.log + +: -- 2.7.4