parallel-tests: no more spurious successes for FreeBSD make

Work around a bug of FreeBSD make bug that was causing the
automake-generated "check" target to complete with success
even if some tests failed; this happened only when FreeBSD
make was run in concurrent mode (as in, e.g., "make -j2
check").  The bug is not present in NetBSD make.

This change fixes automake bug#9245:
 <http://debbugs.gnu.org/cgi/bugreport.cgi?bug=9245>

See also FreeBSD PR bin/159730:
 <http://www.freebsd.org/cgi/query-pr.cgi?pr=159730>

* lib/am/check.am [%?PARALLEL_TESTS%] $(TEST_SUITE_LOG): Use a
more "safe" (and apparently redundant) idiom to exit with error,
so that the non-zero exit status is picked up also by FreeBSD
make when it's running in concurrent mode.
* NEWS: Update.
* tests/check-concurrency-bug9245.test: New test.
* tests/Makefile.am (TESTS): Update.

diff --git a/ChangeLog b/ChangeLog
index 7c85938..8de3ebc 100644 (file)
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,23 @@
+2011-08-16  Stefano Lattarini  <stefano.lattarini@gmail.com>
+
+       parallel-tests: no more spurious successes for FreeBSD make
+       Work around a bug of FreeBSD make bug that was causing the
+       automake-generated "check" target to complete with success
+       even if some tests failed; this happened only when FreeBSD
+       make was run in concurrent mode (as in, e.g., "make -j2
+       check").  The bug is not present in NetBSD make.
+       This change fixes automake bug#9245:
+        <http://debbugs.gnu.org/cgi/bugreport.cgi?bug=9245>
+       See also FreeBSD PR bin/159730:
+        <http://www.freebsd.org/cgi/query-pr.cgi?pr=159730>
+       * lib/am/check.am [%?PARALLEL_TESTS%] $(TEST_SUITE_LOG): Use a
+       more "safe" (and apparently redundant) idiom to exit with error,
+       so that the non-zero exit status is picked up also by FreeBSD
+       make when it's running in concurrent mode.
+       * NEWS: Update.
+       * tests/check-concurrency-bug9245.test: New test.
+       * tests/Makefile.am (TESTS): Update.
+
 2011-08-11  Stefano Lattarini  <stefano.lattarini@gmail.com>
 
        hacking: we now require autoconf 2.68

diff --git a/NEWS b/NEWS
index 29c552b..5086f75 100644 (file)
--- a/NEWS
+++ b/NEWS
@@ -23,7 +23,9 @@ Bugs fixed in 1.11.0a:
 * Bugs introduced by 1.11:
 
   - The `parallel-tests' test driver works around a GNU make 3.80 bug with
-    trailing white space in the test list (`TESTS = foo $(EMPTY)').
+    trailing white space in the test list (`TESTS = foo $(EMPTY)'), and
+    does not report spurious successes when used with concurrent FreeBSD
+    make (e.g., "make check -j3").
 
   - The `silent-rules' option now also silences all compile rules if dependency
     tracking is disabled.  Also, when `silent-rules' is not used, the output from

diff --git a/lib/Automake/tests/Makefile.in b/lib/Automake/tests/Makefile.in
index 2f553ed..17a2002 100644 (file)
--- a/lib/Automake/tests/Makefile.in
+++ b/lib/Automake/tests/Makefile.in
@@ -405,7 +405,7 @@ $(TEST_SUITE_LOG): $(TEST_LOGS)
          col="$$red";                                                  \
        fi;                                                             \
        echo "$$msg" | $(am__text_box) "col=$$col" "std=$$std";         \
-       $$exit
+       $$exit || exit 1
 
 # Run all the tests.
 check-TESTS:

diff --git a/lib/am/check.am b/lib/am/check.am
index 0b54312..3d0188d 100644 (file)
--- a/lib/am/check.am
+++ b/lib/am/check.am
@@ -224,7 +224,12 @@ $(TEST_SUITE_LOG): $(TEST_LOGS)
          col="$$red";                                                  \
        fi;                                                             \
        echo "$$msg" | $(am__text_box) "col=$$col" "std=$$std";         \
-       $$exit
+## The use of "exit 1" below is required to work around a FreeBSD make bug
+## (present only when running in concurrent mode).  See automake bug#9245:
+##  <http://debbugs.gnu.org/cgi/bugreport.cgi?bug=9245>
+## and FreeBSD PR bin/159730:
+##  <http://www.freebsd.org/cgi/query-pr.cgi?pr=159730>.
+       $$exit || exit 1
 
 RECHECK_LOGS = $(TEST_LOGS)
 

diff --git a/tests/Makefile.am b/tests/Makefile.am
index b545dda..c2a55fd 100644 (file)
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -160,6 +160,7 @@ check11.test \
 check-exported-srcdir.test \
 check-tests-in-builddir.test \
 check-tests_environment.test \
+check-concurrency-bug9245.test \
 tests-environment-backcompat.test \
 checkall.test \
 clean.test \

diff --git a/tests/Makefile.in b/tests/Makefile.in
index fe44a40..5a3259e 100644 (file)
--- a/tests/Makefile.in
+++ b/tests/Makefile.in
@@ -291,6 +291,7 @@ yacc-dist-nobuild-subdir.test \
 txinfo5.test
 
 parallel_tests = \
+check-concurrency-bug9245-p.test \
 check-exported-srcdir-p.test \
 check-tests-in-builddir-p.test \
 check-tests_environment-p.test \
@@ -437,6 +438,7 @@ check11.test \
 check-exported-srcdir.test \
 check-tests-in-builddir.test \
 check-tests_environment.test \
+check-concurrency-bug9245.test \
 tests-environment-backcompat.test \
 checkall.test \
 clean.test \
@@ -1274,7 +1276,7 @@ $(TEST_SUITE_LOG): $(TEST_LOGS)
          col="$$red";                                                  \
        fi;                                                             \
        echo "$$msg" | $(am__text_box) "col=$$col" "std=$$std";         \
-       $$exit
+       $$exit || exit 1
 
 # Run all the tests.
 check-TESTS:

diff --git a/tests/check-concurrency-bug9245.test b/tests/check-concurrency-bug9245.test
new file mode 100755 (executable)
index 0000000..e5dd153
--- /dev/null
+++ b/tests/check-concurrency-bug9245.test
@@ -0,0 +1,62 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# Look for a bug where FreeBSD make in concurrent mode reported success
+# even when the Automake-generated parallel testsuite harness failed.
+# See automake bug#9245.
+
+. ./defs || Exit 1
+
+cat >> configure.in << 'END'
+AC_OUTPUT
+END
+
+cat > Makefile.am << 'END'
+TESTS = foo.test bar.test
+END
+
+cat > foo.test <<'END'
+#!/bin/sh
+exit 1
+END
+chmod a+x foo.test
+
+cp foo.test bar.test
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE -a
+
+./configure
+
+# Some make implementations don't grok the `-j' option.
+$MAKE -j1 || Exit 77
+
+for j in '' -j1 -j2; do
+  $MAKE $j check && Exit 1
+  TESTS=foo.test $MAKE $j -e check && Exit 1
+  if test x"$parallel_tests" = x"yes"; then
+    $MAKE $j recheck && Exit 1
+    TEST_LOGS=foo.log $MAKE $j -e check && Exit 1
+    rm -f test-suite.log
+    $MAKE $j test-suite.log && Exit 1
+    test -f test-suite.log || Exit 1
+  else
+    : # For shells with buggy 'set -e'.
+  fi
+done
+
+: