2 # Copyright (C) 2011-2012 Free Software Foundation, Inc.
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 2, or (at your option)
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
14 # You should have received a copy of the GNU General Public License
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
17 # Auxiliary script for tests on TAP support: checking testsuite summary.
22 br='============================================================================'
26 *) fatal_ "invalid \$use_colors value '$use_colors'"
31 cat > configure.ac <<END
32 AC_INIT([GNU AutoTAP], [5.12], [bug-automake@gnu.org])
33 AM_INIT_AUTOMAKE([parallel-tests])
34 AC_CONFIG_FILES([Makefile])
38 cat > Makefile.am << 'END'
39 TEST_LOG_DRIVER = $(srcdir)/tap-driver
40 TEST_LOG_COMPILER = cat
44 # The following shell variables are influential for this function:
50 1,--pass) expect_failure=no;;
51 1,--fail) expect_failure=yes;;
52 *) fatal_ "invalid usage of 'do_check'";;
58 if test $use_colors = yes; then
59 # Forced colorization should take place also with non-ANSI terminals;
60 # hence the "TERM=dumb" definition.
61 make_cmd="env TERM=dumb AM_COLOR_TESTS=always $MAKE -e"
65 $make_cmd check > stdout || st=$?
67 if test $expect_failure = yes; then
68 test $st -gt 0 || Exit 1
70 test $st -eq 0 || Exit 1
72 $PERL "$am_testauxdir"/extract-testsuite-summary.pl stdout >summary.got \
73 || fatal_ "cannot extract testsuite summary"
76 if test $use_colors = yes; then
77 # Use cmp, not diff, because the files might contain binary data.
82 $compare summary.exp summary.got || Exit 1
85 if test $use_colors = yes; then
93 echo AUTOMAKE_OPTIONS = color-tests >> Makefile.am
95 red= grn= lgn= blu= mgn= brg= std=
100 ${grn}Testsuite summary for GNU AutoTAP 5.12${std}
103 success_footer=${grn}${br}${std}
107 ${red}Testsuite summary for GNU AutoTAP 5.12${std}
112 ${red}See ./test-suite.log${std}
113 ${red}Please report to bug-automake@gnu.org${std}
118 $AUTOMAKE --add-missing
123 { echo 1..1 && echo ok; } > all.test
124 do_check --pass <<END
126 ${brg}# TOTAL: 1${std}
127 ${grn}# PASS: 1${std}
137 { echo 1..1 && echo 'ok # SKIP'; } > all.test
138 do_check --pass <<END
140 ${brg}# TOTAL: 1${std}
142 ${blu}# SKIP: 1${std}
151 { echo 1..1 && echo 'not ok # TODO'; } > all.test
152 do_check --pass <<END
154 ${brg}# TOTAL: 1${std}
157 ${lgn}# XFAIL: 1${std}
165 { echo 1..1 && echo not ok; } > all.test
166 do_check --fail <<END
168 ${brg}# TOTAL: 1${std}
172 ${red}# FAIL: 1${std}
179 { echo 1..1 && echo 'ok # TODO'; } > all.test
180 do_check --fail <<END
182 ${brg}# TOTAL: 1${std}
187 ${red}# XPASS: 1${std}
193 { echo 1..1 && echo 'Bail out!'; } > all.test
194 do_check --fail <<END
196 ${brg}# TOTAL: 1${std}
202 ${mgn}# ERROR: 1${std}
206 # 3 non-failing results.
213 do_check --pass <<END
215 ${brg}# TOTAL: 3${std}
216 ${grn}# PASS: 1${std}
217 ${blu}# SKIP: 1${std}
218 ${lgn}# XFAIL: 1${std}
225 # 1 pass, 1 skip, 1 fail.
232 do_check --fail <<END
234 ${brg}# TOTAL: 3${std}
235 ${grn}# PASS: 1${std}
236 ${blu}# SKIP: 1${std}
238 ${red}# FAIL: 1${std}
244 # 1 pass, 1 xfail, 1 xpass.
251 do_check --fail <<END
253 ${brg}# TOTAL: 3${std}
254 ${grn}# PASS: 1${std}
256 ${lgn}# XFAIL: 1${std}
258 ${red}# XPASS: 1${std}
263 # 1 skip, 1 xfail, 1 error.
270 do_check --fail <<END
272 ${brg}# TOTAL: 3${std}
274 ${blu}# SKIP: 1${std}
275 ${lgn}# XFAIL: 1${std}
278 ${mgn}# ERROR: 1${std}
292 do_check --fail <<END
294 ${brg}# TOTAL: 6${std}
295 ${grn}# PASS: 1${std}
296 ${blu}# SKIP: 1${std}
297 ${lgn}# XFAIL: 1${std}
298 ${red}# FAIL: 1${std}
299 ${red}# XPASS: 1${std}
300 ${mgn}# ERROR: 1${std}
304 # Prepare some common data for later.
305 for i in 0 1 2 3 4 5 6 7 8 9; do
306 for j in 0 1 2 3 4 5 6 7 8 9; do
313 # Lots of non-failures (300 per kind).
314 (cat tap && cat tap && cat tap) > all.test
315 test `wc -l <all.test` -eq 900 || Exit 99 # Sanity check.
316 echo 1..900 >> all.test # Test plan.
317 do_check --pass <<END
319 ${brg}# TOTAL: 900${std}
320 ${grn}# PASS: 300${std}
321 ${blu}# SKIP: 300${std}
322 ${lgn}# XFAIL: 300${std}
329 # 1 failure and lots of non-failures means failure.
330 (cat tap && echo "not ok" && cat tap) > all.test
331 test `wc -l <all.test` -eq 601 || Exit 99 # Sanity check.
332 echo 1..601 >> all.test # Test plan.
333 do_check --fail <<END
335 ${brg}# TOTAL: 601${std}
336 ${grn}# PASS: 200${std}
337 ${blu}# SKIP: 200${std}
338 ${lgn}# XFAIL: 200${std}
339 ${red}# FAIL: 1${std}
345 # 1 error and lots of non-failures means failure.
346 (cat tap && sed 30q tap && echo 'Bail out!') > all.test
347 test `wc -l <all.test` -eq 331 || Exit 99 # Sanity check.
348 echo 1..331 >> all.test # Test plan.
349 do_check --fail <<END
351 ${brg}# TOTAL: 331${std}
352 ${grn}# PASS: 110${std}
353 ${blu}# SKIP: 110${std}
354 ${lgn}# XFAIL: 110${std}
357 ${mgn}# ERROR: 1${std}