tests: ignore minor 'recheck' regression for BSD make

It turns out that, with NetBSD 5.1 make and FreeBSD 9 make, running
"make recheck" two times in a row quickly fails to correctly re-run
the failed tests in the second run.

That issue has been introduced likely introduced in commit
'v1.12.1-95-gd5443e4' of 20102-07-01, "parallel-tests: reimplement
fix for bug#11791".

Anyway, the use case that has been broken is not realistic (who is
going to run "make recheck" two times in one second, without modifying
any of the tests or the tested programs in the meantime?), so we believe
the best fix is to simply work around the issue in the affected test
cases, rather than risking to slow down or uglify the 'recheck' rule.

* t/parallel-tests9.sh: Enhance a little.
* t/tap-recheck.sh: Adjust adding proper '$sleep' calls were required.
* t/parallel-tests-log-override-recheck.sh: Likewise.
* t/test-driver-custom-multitest-recheck.sh: Likewise.
* t/test-driver-custom-multitest-recheck2.sh: Likewise.

Signed-off-by: Stefano Lattarini <stefano.lattarini@gmail.com>

diff --git a/t/parallel-tests-log-override-recheck.sh b/t/parallel-tests-log-override-recheck.sh
index 493fbfea0ee03d4732c746bbc611c286ea1f5823..79c16fc65e4c61b1138829dd188dc11bd6c5abd6 100755 (executable)
--- a/t/parallel-tests-log-override-recheck.sh
+++ b/t/parallel-tests-log-override-recheck.sh
@@ -77,6 +77,8 @@ for x in stdout my.log; do
   $FGREP bazbaz $x
 done
 
+using_gmake || $sleep # Required by BSD make.
+
 chmod a-rw my.log
 BAZ_EXIT_STATUS=0 TEST_SUITE_LOG=my2.log $MAKE -e recheck >stdout \
   && { cat stdout; exit 1; }

diff --git a/t/parallel-tests9.sh b/t/parallel-tests9.sh
index 583c674b55b2cf1904f1428d1619e3b8624e558f..ea9dba665dd68cf74d9e766e925280ece986baf8 100755 (executable)
--- a/t/parallel-tests9.sh
+++ b/t/parallel-tests9.sh
@@ -59,12 +59,17 @@ $MAKE check >stdout && { cat stdout; exit 1; }
 cat stdout
 count_test_results total=3 pass=1 fail=1 skip=0 xfail=0 xpass=0 error=1
 
-$MAKE recheck >stdout && { cat stdout; exit 1; }
-cat stdout
-count_test_results total=2 pass=0 fail=1 skip=0 xfail=0 xpass=0 error=1
-grep 'foo\.test' stdout && exit 1
-grep '^ERROR: bar\.test$' stdout
-grep '^FAIL: baz\.test$' stdout
+# Running this two times in a row should produce the same results the
+# second time.
+for i in 1 2; do
+  using_gmake || $sleep # Required by BSD make.
+  $MAKE recheck >stdout && { cat stdout; exit 1; }
+  cat stdout
+  count_test_results total=2 pass=0 fail=1 skip=0 xfail=0 xpass=0 error=1
+  grep 'foo\.test' stdout && exit 1
+  grep '^ERROR: bar\.test$' stdout
+  grep '^FAIL: baz\.test$' stdout
+done
 
 # Ensure that recheck builds check_SCRIPTS, and that
 # recheck reruns nothing if check has not been run.

diff --git a/t/tap-recheck.sh b/t/tap-recheck.sh
index 877f70e75b0da539c53981d739bafcdee6bc9b86..3c542ba8ba10d6f7ff19452ab830a3a7bb2ea9a1 100755 (executable)
--- a/t/tap-recheck.sh
+++ b/t/tap-recheck.sh
@@ -130,6 +130,7 @@ for vpath in : false; do
   count_test_results total=9 pass=3 fail=2 xpass=1 xfail=1 skip=1 error=1
 
   : Let us make b.test pass.
+  using_gmake || $sleep # Required by BSD make.
   echo OK > b.ok
   do_recheck --fail
   # a.test has been successful the first time, so no need to re-run it.
@@ -141,6 +142,7 @@ for vpath in : false; do
   count_test_results total=7 pass=2 fail=2 xpass=1 xfail=1 skip=1 error=0
 
   : Let us make the first part of c.test pass.
+  using_gmake || $sleep # Required by BSD make.
   echo OK > c.pass
   do_recheck --fail
   test ! -e a.run
@@ -150,6 +152,7 @@ for vpath in : false; do
   count_test_results total=5 pass=1 fail=1 xpass=1 xfail=1 skip=1 error=0
 
   : Let us make also the second part of c.test pass.
+  using_gmake || $sleep # Required by BSD make.
   echo KO > c.xfail
   do_recheck --fail
   test ! -e a.run
@@ -160,6 +163,7 @@ for vpath in : false; do
 
   : Nothing changed, so only d.test should be run.
   for i in 1 2; do
+    using_gmake || $sleep # Required by BSD make.
     do_recheck --fail
     test ! -e a.run
     test ! -e b.run
@@ -169,6 +173,7 @@ for vpath in : false; do
   done
 
   : Let us make d.test run more testcases, and experience _more_ failures.
+  using_gmake || $sleep # Required by BSD make.
   echo 'test_count=9' > d.count
   unindent > d.extra <<'END'
     echo 'ok # SKIP s'
@@ -190,6 +195,7 @@ END
   count_test_results total=11 pass=2 fail=4 xpass=1 xfail=0 skip=2 error=2
 
   : Let us finally make d.test pass.
+  using_gmake || $sleep # Required by BSD make.
   echo 'test_count=1' > d.count
   echo : > d.extra
   do_recheck --pass
@@ -200,6 +206,7 @@ END
   count_test_results total=1 pass=0 fail=0 xpass=0 xfail=0 skip=1 error=0
 
   : All tests have been successful or skipped, nothing should be re-run.
+  using_gmake || $sleep # Required by BSD make.
   do_recheck --pass
   test ! -e a.run
   test ! -e b.run

diff --git a/t/test-driver-custom-multitest-recheck.sh b/t/test-driver-custom-multitest-recheck.sh
index ae313979cafa3f66ee7fad5b241d4bce4c985db1..362abbff088d678b65acc9448dd60c23d7ef3edd 100755 (executable)
--- a/t/test-driver-custom-multitest-recheck.sh
+++ b/t/test-driver-custom-multitest-recheck.sh
@@ -107,6 +107,7 @@ for vpath in : false; do
   $srcdir/configure
 
   : A "make recheck" in a clean tree should run no tests.
+  using_gmake || $sleep # Required by BSD make.
   do_recheck --pass
   cat test-suite.log
   test ! -e a.run
@@ -131,6 +132,7 @@ for vpath in : false; do
   count_test_results total=9 pass=3 fail=2 xpass=1 xfail=1 skip=1 error=1
 
   : Let us make b.test pass.
+  using_gmake || $sleep # Required by BSD make.
   echo OK > b.ok
   do_recheck --fail
   # a.test has been successful the first time, so no need to re-run it.
@@ -142,6 +144,7 @@ for vpath in : false; do
   count_test_results total=7 pass=2 fail=2 xpass=1 xfail=1 skip=1 error=0
 
   : Let us make the first part of c.test pass.
+  using_gmake || $sleep # Required by BSD make.
   echo OK > c.pass
   do_recheck --fail
   test ! -e a.run
@@ -151,6 +154,7 @@ for vpath in : false; do
   count_test_results total=5 pass=1 fail=1 xpass=1 xfail=1 skip=1 error=0
 
   : Let us make also the second part of c.test pass.
+  using_gmake || $sleep # Required by BSD make.
   echo KO > c.xfail
   do_recheck --fail
   test ! -e a.run
@@ -161,6 +165,7 @@ for vpath in : false; do
 
   : Nothing changed, so only d.test should be run.
   for i in 1 2; do
+    using_gmake || $sleep # Required by BSD make.
     do_recheck --fail
     test ! -e a.run
     test ! -e b.run
@@ -170,6 +175,7 @@ for vpath in : false; do
   done
 
   : Let us make d.test run more testcases, and experience _more_ failures.
+  using_gmake || $sleep # Required by BSD make.
   unindent > d.extra <<'END'
     echo SKIP: s
     echo FAIL: f 1
@@ -190,6 +196,7 @@ END
   count_test_results total=11 pass=2 fail=4 xpass=1 xfail=0 skip=2 error=2
 
   : Let us finally make d.test pass.
+  using_gmake || $sleep # Required by BSD make.
   echo : > d.extra
   do_recheck --pass
   test ! -e a.run
@@ -199,6 +206,7 @@ END
   count_test_results total=1 pass=0 fail=0 xpass=0 xfail=0 skip=1 error=0
 
   : All tests have been successful or skipped, nothing should be re-run.
+  using_gmake || $sleep # Required by BSD make.
   do_recheck --pass
   test ! -e a.run
   test ! -e b.run

diff --git a/t/test-driver-custom-multitest-recheck2.sh b/t/test-driver-custom-multitest-recheck2.sh
index 623cbad4b71f864cf02be1f87f1de23cbf004381..b6f543aee98767637cffe7972365a9b3a0a986e0 100755 (executable)
--- a/t/test-driver-custom-multitest-recheck2.sh
+++ b/t/test-driver-custom-multitest-recheck2.sh
@@ -105,6 +105,7 @@ for vpath in : false; do
   unset var
 
   : a.test was successful the first time, no need to re-run it.
+  using_gmake || $sleep # Required by BSD make.
   env TESTS=a.test $MAKE -e recheck >stdout \
     || { cat stdout; exit 1; }
   cat stdout
@@ -114,6 +115,7 @@ for vpath in : false; do
   test ! -e c.run
 
   : b.test failed, it should be re-run.  And make it pass this time.
+  using_gmake || $sleep # Required by BSD make.
   echo OK > b.ok
   TEST_LOGS=b.log $MAKE -e recheck >stdout \
     || { cat stdout; exit 1; }
@@ -126,6 +128,7 @@ for vpath in : false; do
   rm -f *.run
 
   : No need to re-run a.test or b.test anymore.
+  using_gmake || $sleep # Required by BSD make.
   TEST_LOGS=b.log $MAKE -e recheck >stdout \
     || { cat stdout; exit 1; }
   cat stdout
@@ -133,6 +136,7 @@ for vpath in : false; do
   test ! -e a.run
   test ! -e b.run
   test ! -e c.run
+  using_gmake || $sleep # Required by BSD make.
   TESTS='a.test b.test' $MAKE -e recheck >stdout \
     || { cat stdout; exit 1; }
   cat stdout
@@ -158,6 +162,7 @@ for vpath in : false; do
   : c.test contained and hard error the last time, so it should be re-run.
   : This time, make it pass
   # Use 'echo', not ':'; see comments above for why.
+  using_gmake || $sleep # Required by BSD make.
   echo dummy > c.ok
   env TESTS='c.test a.test' $MAKE -e recheck >stdout \
     || { cat stdout; exit 1; }
@@ -171,6 +176,7 @@ for vpath in : false; do
 
   : Nothing should be rerun anymore, as all tests have been eventually
   : successful.
+  using_gmake || $sleep # Required by BSD make.
   $MAKE recheck >stdout || { cat stdout; exit 1; }
   cat stdout
   count_test_results total=0 pass=0 fail=0 xpass=0 xfail=0 skip=0 error=0