From 40705c5cc5898ff924fea0c6f54f8953205a630d Mon Sep 17 00:00:00 2001 From: Stefano Lattarini Date: Fri, 19 Aug 2011 16:47:12 +0200 Subject: [PATCH] coverage: some more tests on corner cases of TAP support * tests/tap-no-spurious.test: Extend checks verifying that a line matching, say, the regex "^ok[a-zA-Z0-9_]" is *not* considered a TAP result. * tests/tap-no-spurious-numbers.test: New test, checking that our TAP driver doesn't spuriously recognize as TAP result numbers what is not (even if it seems pretty close). * tests/tap-negative-numbers.test: New test, checking that our TAP driver doesn't spuriously recognize negative TAP result numbers, but correctly interprets them as test descriptions instead. * tests/tap-plan-leading-zero.test, tap-numbers-leading-zero.test: New tests, checking how our driver fares in recognizing numbers with leading zeroes in TAP results or TAP plans. * tests/tap-planskip-malformed.test: New test, checking that a malformed TAP plan is not recognized as a valid plan. * tests/tap-plan-whitespace.test: New test, checking that a TAP plan line with trailing whitespace is recognized and handled correctly. * tests/Makefile.am (tap_with_common_setup_tests): Update. --- ChangeLog | 23 ++++++ tests/Makefile.am | 6 ++ tests/Makefile.in | 5 ++ tests/tap-negative-numbers.test | 58 +++++++++++++++ tests/tap-no-spurious-numbers.test | 99 +++++++++++++++++++++++++ tests/tap-no-spurious.test | 25 ++++++- tests/tap-numbers-leading-zero.test | 109 ++++++++++++++++++++++++++++ tests/tap-plan-leading-zero.test | 57 +++++++++++++++ tests/tap-plan-malformed.test | 60 +++++++++++++++ tests/tap-plan-whitespace.test | 41 +++++++++++ 10 files changed, 479 insertions(+), 4 deletions(-) create mode 100755 tests/tap-negative-numbers.test create mode 100755 tests/tap-no-spurious-numbers.test create mode 100755 tests/tap-numbers-leading-zero.test create mode 100755 tests/tap-plan-leading-zero.test create mode 100755 tests/tap-plan-malformed.test create mode 100755 tests/tap-plan-whitespace.test diff --git a/ChangeLog b/ChangeLog index f86d1cdfc..bdccc31f9 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,26 @@ +2011-08-20 Stefano Lattarini + + coverage: some more tests on corner cases of TAP support + * tests/tap-no-spurious.test: Extend checks verifying that a line + matching, say, the regex "^ok[a-zA-Z0-9_]" is *not* considered a + TAP result. + * tests/tap-no-spurious-numbers.test: New test, checking that our + TAP driver doesn't spuriously recognize as TAP result numbers what + is not (even if it seems pretty close). + * tests/tap-negative-numbers.test: New test, checking that our + TAP driver doesn't spuriously recognize negative TAP result + numbers, but correctly interprets them as test descriptions + instead. + * tests/tap-plan-leading-zero.test, tap-numbers-leading-zero.test: + New tests, checking how our driver fares in recognizing numbers + with leading zeroes in TAP results or TAP plans. + * tests/tap-planskip-malformed.test: New test, checking that a + malformed TAP plan is not recognized as a valid plan. + * tests/tap-plan-whitespace.test: New test, checking that a TAP + plan line with trailing whitespace is recognized and handled + correctly. + * tests/Makefile.am (tap_with_common_setup_tests): Update. + 2011-08-19 Stefano Lattarini testsuite: fix some redundant autotools calls in tests on TAP diff --git a/tests/Makefile.am b/tests/Makefile.am index 2d630a9c3..e7f8570d4 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -1171,9 +1171,12 @@ tap-merge-stdout-stderr.test \ tap-no-merge-stdout-stderr.test \ tap-no-disable-hard-error.test \ tap-no-spurious-summary.test \ +tap-no-spurious-numbers.test \ tap-no-spurious.test \ tap-not-ok-skip.test \ tap-numeric-description.test \ +tap-negative-numbers.test \ +tap-numbers-leading-zero.test \ tap-out-of-order.test \ tap-passthrough.test \ tap-passthrough-exit.test \ @@ -1181,6 +1184,9 @@ tap-plan.test \ tap-plan-corner.test \ tap-plan-errors.test \ tap-plan-middle.test \ +tap-plan-whitespace.test \ +tap-plan-leading-zero.test \ +tap-plan-malformed.test \ tap-missing-plan-and-bad-exit.test \ tap-planskip.test \ tap-planskip-late.test \ diff --git a/tests/Makefile.in b/tests/Makefile.in index dc7cb80b8..96edc94ce 100644 --- a/tests/Makefile.in +++ b/tests/Makefile.in @@ -1412,9 +1412,12 @@ tap-merge-stdout-stderr.test \ tap-no-merge-stdout-stderr.test \ tap-no-disable-hard-error.test \ tap-no-spurious-summary.test \ +tap-no-spurious-numbers.test \ tap-no-spurious.test \ tap-not-ok-skip.test \ tap-numeric-description.test \ +tap-negative-numbers.test \ +tap-numbers-leading-zero.test \ tap-out-of-order.test \ tap-passthrough.test \ tap-passthrough-exit.test \ @@ -1422,6 +1425,7 @@ tap-plan.test \ tap-plan-corner.test \ tap-plan-errors.test \ tap-plan-middle.test \ +tap-plan-leading-zero.test \ tap-missing-plan-and-bad-exit.test \ tap-planskip.test \ tap-planskip-late.test \ @@ -1432,6 +1436,7 @@ tap-planskip-whitespace.test \ tap-planskip-badexit.test \ tap-planskip-bailout.test \ tap-planskip-later-errors.test \ +tap-planskip-malformed.test \ tap-realtime.test \ tap-recheck-logs.test \ tap-todo-skip-together.test \ diff --git a/tests/tap-negative-numbers.test b/tests/tap-negative-numbers.test new file mode 100755 index 000000000..9fdee871a --- /dev/null +++ b/tests/tap-negative-numbers.test @@ -0,0 +1,58 @@ +#! /bin/sh +# Copyright (C) 2011 Free Software Foundation, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# TAP support: +# - don't spuriously recognize negative TAP result numbers, but correctly +# interpret them as test descriptions instead + +parallel_tests=yes +. ./defs || Exit 1 + +. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh" + +cat > all.test <<'END' +1..7 +ok -1 +not ok -3 +ok -2 # SKIP +not ok -5 # TODO +ok -04 # TODO +ok -121 +not ok -50000 +END + +$MAKE check >stdout && { cat stdout; Exit 1; } +cat stdout + +count_test_results total=7 pass=2 fail=2 xpass=1 xfail=1 skip=1 error=0 + +cat > exp <<'END' +PASS: all.test 1 -1 +FAIL: all.test 2 -3 +SKIP: all.test 3 -2 # SKIP +XFAIL: all.test 4 -5 # TODO +XPASS: all.test 5 -04 # TODO +PASS: all.test 6 -121 +FAIL: all.test 7 -50000 +END + +$FGREP ': all.test' stdout > got + +cat exp +cat got +diff exp got + +: diff --git a/tests/tap-no-spurious-numbers.test b/tests/tap-no-spurious-numbers.test new file mode 100755 index 000000000..e3ee0c818 --- /dev/null +++ b/tests/tap-no-spurious-numbers.test @@ -0,0 +1,99 @@ +#! /bin/sh +# Copyright (C) 2011 Free Software Foundation, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# TAP support: +# - we shouldn't spuriously recognize as TAP result numbers what it +# not, even if it seems pretty close + +parallel_tests=yes +. ./defs || Exit 1 + +. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh" + +cat > prefixes <<'END' +A +a +_ ++ +- += +/ +* +. +: +, +; +$ +@ +% +& +# +? +! +| +\ +" +` +' +( +) +[ +] +{ +} +< +> +END + +n=`wc -l all.test + +highno=1000 + +for result in 'ok' 'not ok'; do + for spacing in "" " " "$tab"; do + subst="$result &$spacing$higno" + sed -e "s|.*|$subst|" prefixes + for directive in TODO SKIP; do + test "$result $directive" != "not ok SKIP" || continue + sed -e '/^#$/d' -e "s|.*|$subst # $directive|" prefixes + done + done +done >> all.test + +cat all.test # For debugging. + +# Sanity checks. +grep '#.*#' all.test && framework_failure_ "creating all.test" +test `wc -l stdout || : +cat stdout + +count_test_results total=$total pass=$pass fail=$fail skip=$skip \ + xpass=$xpass xfail=$xfail error=$error + +: diff --git a/tests/tap-no-spurious.test b/tests/tap-no-spurious.test index 11e2d1a79..3e2afac10 100755 --- a/tests/tap-no-spurious.test +++ b/tests/tap-no-spurious.test @@ -46,14 +46,10 @@ cat >> all.test < \\; do + printf '%s\n' "$r$s1$s2" + done + done +done >> all.test +set -x # Reset shell xtraces. + # The prove(1) utility doesn't bail out on these, so our driver # shouldn't either. cat >> all.test <<'END' @@ -88,6 +98,13 @@ Bail out # Bail out! END +cat all.test # For debugging. + +# Minor sanity check. +test `grep -c '^ok1$' all.test` -eq 1 \ + && test `grep -c '^not ok1$' all.test` -eq 1 \ + || framework_failure_ "creating all.test" + $MAKE check >stdout || { cat stdout; Exit 1; } cat stdout diff --git a/tests/tap-numbers-leading-zero.test b/tests/tap-numbers-leading-zero.test new file mode 100755 index 000000000..81c979737 --- /dev/null +++ b/tests/tap-numbers-leading-zero.test @@ -0,0 +1,109 @@ +#! /bin/sh +# Copyright (C) 2011 Free Software Foundation, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# TAP support: +# - how does TAP result numbers with leading zero fares? + +parallel_tests=yes +. ./defs || Exit 1 + +. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh" + +do_checks () +{ + $MAKE check >stdout && { cat stdout; Exit 1; } + cat stdout + count_test_results "$@" + # Allow some normalization of leading zeros, without forcing it. + LC_ALL=C sed -n 's/: all\.test 0*\([0-9]\)/: all.test \1/p' stdout > got + cat exp + cat got + diff exp got +} + +cat > all.test < exp < all.test < exp <. + +# TAP support: +# - TAP plans with numbers having leading zeroes, as in "1..01" +# - TAP "SKIP" plans with multiple zeroes, as in "1..00 # SKIP" +# This is consistent with the behaviour of the `prove' utility. + +parallel_tests=yes +. ./defs || Exit 1 + +. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh" + +cat > a.test < b.test < c.test +for i in 1 2 3 4 5 6 7 8 9 10; do echo ok $i; done >> c.test + +echo 1..00100 > d.test +for i in 1 2 3 4 5 6 7 8 9 10; do + for j in 1 2 3 4 5 6 7 8 9 10; do + echo ok + done +done >> d.test + +echo 1..00 > e.test +echo '1..000 # SKIP' > f.test + +env TESTS='a.test b.test c.test d.test e.test f.test' \ + $MAKE -e check >stdout || { cat stdout; Exit 1; } +cat stdout + +count_test_results total=115 pass=113 xfail=0 skip=2 fail=0 xpass=0 error=0 + +: diff --git a/tests/tap-plan-malformed.test b/tests/tap-plan-malformed.test new file mode 100755 index 000000000..9eb8698e6 --- /dev/null +++ b/tests/tap-plan-malformed.test @@ -0,0 +1,60 @@ +#! /bin/sh +# Copyright (C) 2011 Free Software Foundation, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# TAP support: a malformed TAP plan is not recognized. The checks in +# here should be consistent with the behaviour of the `prove' utility. + +parallel_tests=yes +. ./defs || Exit 1 + +. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh" + +cat > a.test < b.test < c.test < d.test < e.test <stdout && { cat stdout; Exit 1; } +cat stdout + +count_test_results total=9 pass=2 fail=0 xpass=0 xfail=2 skip=0 error=5 + +for tst in $tests_list; do + grep "^ERROR: $tst - missing test plan$" stdout +done + +: diff --git a/tests/tap-plan-whitespace.test b/tests/tap-plan-whitespace.test new file mode 100755 index 000000000..3cdd79de2 --- /dev/null +++ b/tests/tap-plan-whitespace.test @@ -0,0 +1,41 @@ +#! /bin/sh +# Copyright (C) 2011 Free Software Foundation, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# TAP support: +# - plan line with trailing whitespace is recognized and handled correctly + +parallel_tests=yes +. ./defs || Exit 1 + +. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh" + +sed 's/\$$//' > foo.test < bar.test <stdout || { cat stdout; Exit 1; } +cat stdout + +count_test_results total=3 pass=3 fail=0 error=0 xpass=0 xfail=0 skip=0 + +: -- 2.34.1