2 # Copyright (C) 2011-2012 Free Software Foundation, Inc.
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 2, or (at your option)
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
14 # You should have received a copy of the GNU General Public License
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
18 # - unplanned tests are properly reported as errors
22 . "$am_testauxdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
29 $MAKE check >stdout && { cat stdout; Exit 1; }
31 count_test_results total=3 pass=1 fail=0 xpass=0 xfail=0 skip=0 error=2
32 grep '^ERROR: all\.test - too many tests run (expected 1, got 2)$' stdout
33 grep '^ERROR: all\.test 2 # UNPLANNED$' stdout
41 $MAKE check >stdout && { cat stdout; Exit 1; }
43 count_test_results total=4 pass=2 fail=0 xpass=0 xfail=0 skip=0 error=2
44 grep '^ERROR: all\.test - too many tests run (expected 2, got 3)$' stdout
45 grep '^ERROR: all\.test 3 # UNPLANNED$' stdout
47 # Interaction with XFAIL_TESTS.
56 XFAIL_TESTS=all.test $MAKE -e check >stdout && { cat stdout; Exit 1; }
58 count_test_results total=6 pass=0 fail=0 xpass=0 xfail=1 skip=1 error=4
59 grep '^ERROR: all\.test - too many tests run (expected 2, got 5)$' stdout
60 grep '^ERROR: all\.test 3 # UNPLANNED$' stdout
61 grep '^ERROR: all\.test 4 # UNPLANNED$' stdout
62 grep '^ERROR: all\.test 5 # UNPLANNED$' stdout
81 ok 13 - bar bar # TODO
86 not ok 17 - bar bar # TODO
91 ok 21 - bar bar # SKIP
123 $MAKE check >stdout && { cat stdout; Exit 1; }
125 count_test_results total=22 pass=1 fail=0 xpass=0 xfail=0 skip=0 error=21
127 echo 'PASS: all.test 1' > exp
128 sed -e '/^$/d' -e 's/.*/ERROR: all.test & # UNPLANNED/' t >> exp
129 echo 'ERROR: all.test - too many tests run (expected 1, got 21)' >> exp
131 $FGREP ': all.test' stdout > got
137 # Note that, if the TAP input has a trailing plan, it is not possible
138 # to flag unplanned tests as such, since we do not know they're unplanned
139 # until the plan is reached; still, we should give at least an error
140 # message about the unmatched number of tests once we've got the plan.
142 for x in 'ok' 'ok 3' 'not ok' 'not ok # TODO' 'ok # TODO' 'ok # SKIP'; do
143 unindent > all.test <<END
149 $MAKE check >stdout && { cat stdout; Exit 1; }
151 test $($FGREP -c ': all.test' stdout) -eq 4
152 $EGREP '^PASS: all\.test 1($| )' stdout
153 $EGREP '^SKIP: all\.test 2($| )' stdout
154 $EGREP ': all\.test 3($| )' stdout
155 grep '^ERROR: all\.test - too many tests run (expected 2, got 3)$' stdout