2 # Copyright (C) 2011 Free Software Foundation, Inc.
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 2, or (at your option)
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
14 # You should have received a copy of the GNU General Public License
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
17 # Parallel testsuite harness: check APIs for the registering of test
18 # results in `*.trs' files, as documented in the automake manual.
23 cat >> configure.in << 'END'
27 cat > Makefile.am << 'END'
28 TEST_LOG_DRIVER = ./dummy-driver
29 TESTS = foo.test bar.test
32 cat > dummy-driver <<'END'
35 while test $# -gt 0; do
37 --log-file) log_file=$2; shift;;
38 --trs-file) trs_file=$2; shift;;
39 --test-name) test_name=$2; shift;;
40 --expect-failure|--color-tests|--enable-hard-errors) shift;;
42 *) echo "$0: invalid option/argument: '$1'" >&2; exit 2;;
49 chmod a+x dummy-driver
54 $MAKE check >stdout || st=$?
56 # Our dummy driver make no testsuite progress report.
57 grep ': .*\.test' stdout && Exit 1
58 # Nor it writes to the log files.
59 test -s foo.log && Exit 1
60 test -s bar.log && Exit 1
64 # This must be different from the one defined in `test/defs', as that
65 # assumes that the driver does proper testsuite progress reporting.
68 total=ERR pass=ERR fail=ERR xpass=ERR xfail=ERR skip=ERR error=ERR
71 grep "^# TOTAL: *$total$" stdout || rc=1
72 grep "^# PASS: *$pass$" stdout || rc=1
73 grep "^# XFAIL: *$xfail$" stdout || rc=1
74 grep "^# SKIP: *$skip$" stdout || rc=1
75 grep "^# FAIL: *$fail$" stdout || rc=1
76 grep "^# XPASS: *$xpass$" stdout || rc=1
77 grep "^# ERROR: *$error$" stdout || rc=1
78 test $st -eq 0 || Exit 1
87 # Basic checks. Also that that `:global-test-result:' fields and
88 # "old-style" directives with format "RESULT: test-name" are now ignored.
91 echo blah blah blah > bar.test
93 count_test_results total=0 pass=0 fail=0 xpass=0 xfail=0 skip=0 error=0
96 :test-global-result: PASS
101 :test-global-result: ERROR
104 count_test_results total=2 pass=0 fail=1 xpass=0 xfail=0 skip=1 error=0
109 :test-global-result: XPASS
111 echo ERROR: bar.test > bar.test
113 count_test_results total=1 pass=1 fail=0 xpass=0 xfail=0 skip=0 error=0
116 :test-global-result: SKIP
120 :test-global-result: PASS
123 count_test_results total=1 pass=0 fail=1 xpass=0 xfail=0 skip=0 error=0
138 count_test_results total=8 pass=4 fail=0 xpass=0 xfail=1 skip=3 error=0
140 # Check that all results expected to be supported are *really* supported.
152 count_test_results total=6 pass=1 fail=1 xpass=1 xfail=1 skip=1 error=1
156 count_test_results total=12 pass=2 fail=2 xpass=2 xfail=2 skip=2 error=2
158 # Check that we are liberal w.r.t. whitespace use.
162 for RESULT in PASS FAIL XPASS XFAIL SKIP ERROR; do
163 sed -e 's/^ *//' -e 's/|//g' >> foo.test <<END
164 |:test-result:$RESULT|
165 |:test-result: $tab $RESULT|
166 |:test-result:$RESULT $tab|
167 |:test-result:$tab$tab $RESULT$tab $tab |
169 echo " $tab $tab$tab :test-result: $RESULT" >> bar.test
171 cat foo.test # For debugging.
172 cat bar.test # Likewise.
174 count_test_results total=30 pass=5 fail=5 xpass=5 xfail=5 skip=5 error=5