From f31a38aed9c8ffbd33d797aa2df5b5864860bb17 Mon Sep 17 00:00:00 2001 From: Stefano Lattarini Date: Sun, 22 May 2011 09:35:17 +0200 Subject: [PATCH] parallel-tests: stricter checks on DISABLE_HARD_ERRORS support * tests/parallel-tests-harderror.test: New test, doing more in-depth checks on DISABLE_HARD_ERRORS. * tests/parallel-tests.test: Remove tests on DISABLE_HARD_ERRORS, now redundant. * tests/Makefile.am (TESTS): Update. --- ChangeLog | 9 ++++ tests/Makefile.am | 1 + tests/Makefile.in | 1 + tests/parallel-tests-harderror.test | 98 +++++++++++++++++++++++++++++++++++++ tests/parallel-tests.test | 18 ++----- 5 files changed, 114 insertions(+), 13 deletions(-) create mode 100755 tests/parallel-tests-harderror.test diff --git a/ChangeLog b/ChangeLog index 6eb09d7..6ec063d 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,12 @@ +2011-05-22 Stefano Lattarini + + parallel-tests: stricter checks on DISABLE_HARD_ERRORS support + * tests/parallel-tests-harderror.test: New test, doing more + in-depth checks on DISABLE_HARD_ERRORS. + * tests/parallel-tests.test: Remove tests on DISABLE_HARD_ERRORS, + now redundant. + * tests/Makefile.am (TESTS): Update. + 2011-05-20 Stefano Lattarini testsuite: avoid re-running few tests with 'parallel-tests' option diff --git a/tests/Makefile.am b/tests/Makefile.am index 33599ad..86a49c0 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -588,6 +588,7 @@ parallel-tests7.test \ parallel-tests8.test \ parallel-tests9.test \ parallel-tests10.test \ +parallel-tests-harderror.test \ parallel-tests-unreadable-log.test \ parallel-tests-subdir.test \ parallel-tests-log-override-1.test \ diff --git a/tests/Makefile.in b/tests/Makefile.in index 80f6b15..42272c7 100644 --- a/tests/Makefile.in +++ b/tests/Makefile.in @@ -856,6 +856,7 @@ parallel-tests7.test \ parallel-tests8.test \ parallel-tests9.test \ parallel-tests10.test \ +parallel-tests-harderror.test \ parallel-tests-unreadable-log.test \ parallel-tests-subdir.test \ parallel-tests-log-override-1.test \ diff --git a/tests/parallel-tests-harderror.test b/tests/parallel-tests-harderror.test new file mode 100755 index 0000000..8680e29 --- /dev/null +++ b/tests/parallel-tests-harderror.test @@ -0,0 +1,98 @@ +#! /bin/sh +# Copyright (C) 2011 Free Software Foundation, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# Check parallel-tests features: DISABLE_HARD_ERRORS + +parallel_tests=yes +. ./defs || Exit 1 + +set -e + +cat >> configure.in << 'END' +AC_CONFIG_FILES([sub/Makefile]) +AC_OUTPUT +END + +mkdir sub + +cat > Makefile.am << 'END' +SUBDIRS = . sub +TESTS = foo.test +XFAIL_TESTS = foo.test +EXTRA_DIST = $(TESTS) +END + +cat > sub/Makefile.am << 'END' +TESTS = bar.test +XFAIL_TESTS = $(TESTS) +EXTRA_DIST = $(TESTS) +END + +cat > foo.test <<'END' +#! /bin/sh +exit 99 +END +chmod a+x foo.test +cp foo.test sub/bar.test + +$ACLOCAL +$AUTOCONF +$AUTOMAKE -a + +./configure + +# DISABLE_HARD_ERRORS is not defined as a make variable, so that it +# should be possible to define either from the environment or from +# command-line, also when recursive make invocations are involved, +# and also without using the `-e' make flag; this should work also +# for non-GNU make. Moreover, it shouldn't be necessary to define +# DISABLE_HARD_ERRORS to "yes" to really disable hard errors: any +# non-empty value should do. +$MAKE check DISABLE_HARD_ERRORS=yes +DISABLE_HARD_ERRORS=x $MAKE check + +# But an empty values for DISABLE_HARD_ERRORS means that hard errors +# are not to be counted like normal failures. + +$MAKE check DISABLE_HARD_ERRORS='' && Exit 1 +cat test-suite.log +grep '^FAIL: foo\.test .*exit.*99' test-suite.log + +cd sub +# The `-e' is wanted here. +DISABLE_HARD_ERRORS='' $MAKE -e check && Exit 1 +cat test-suite.log +grep '^FAIL: bar\.test .*exit.*99' test-suite.log +cd .. + +# Check the distributions. +$MAKE DISABLE_HARD_ERRORS=y distcheck + +# Finally, DISABLE_HARD_ERRORS should work also when the developer +# sets it directly in Makefile.am or Makefile. And its effects +# should remain local to that specific Makefile, obviously. + +echo 'DISABLE_HARD_ERRORS = yes' >> Makefile.am +$AUTOMAKE Makefile +./config.status Makefile +VERBOSE=yes $MAKE check && Exit 1 +grep '^FAIL' test-suite.log && Exit 1 +grep '^FAIL: bar\.test .*exit.*99' sub/test-suite.log + +echo 'DISABLE_HARD_ERRORS = zardoz' >> sub/Makefile +VERBOSE=yes $MAKE check + +: diff --git a/tests/parallel-tests.test b/tests/parallel-tests.test index 399b23b..54f0cd5 100755 --- a/tests/parallel-tests.test +++ b/tests/parallel-tests.test @@ -19,7 +19,6 @@ # - clean # - TEST_SUITE_LOG # - dependencies between tests -# - DISABLE_HARD_ERRORS # - TESTS # - TEST_LOGS # - RECHECK_LOGS @@ -68,12 +67,13 @@ $AUTOCONF $AUTOMAKE -a ./configure -# No hard errors: all tests should be run, there should be one failure. -env DISABLE_HARD_ERRORS=yes $MAKE -e check >stdout && { cat stdout; Exit 1; } + +$MAKE check >stdout && { cat stdout; Exit 1; } cat stdout -test `grep -c '^FAIL' stdout` -eq 1 +# There should be two errors: bar.test is a hard error. +test `grep -c '^FAIL' stdout` -eq 2 test -f mylog.log -test `grep -c '^FAIL' mylog.log` -eq 1 +test `grep -c '^FAIL' mylog.log` -eq 2 test -f baz.log test -f bar.log test -f foo.log @@ -89,19 +89,11 @@ test ! -f foo.log test ! -f mylog.log test -f unrelated.log -$MAKE clean -$MAKE check >stdout && { cat stdout; Exit 1; } -cat stdout -# Now, there should be two errors: bar.test is a hard error. -test `grep -c '^FAIL' stdout` -eq 2 -test `grep -c '^FAIL' mylog.log` -eq 2 - # Check dependencies: baz.test needs to run before bar.test, # but foo.test is not needed. # Note that this usage has a problem: the summary will only # take bar.log into account, because the $(TEST_SUITE_LOG) rule # does not "see" baz.log. Hmm. -$MAKE clean env TESTS='bar.test' $MAKE -e check && Exit 1 test -f baz.log test -f bar.log -- 2.7.4