From e61b581a99e7a0af8f8dbb793a707f94b0336395 Mon Sep 17 00:00:00 2001 From: Stefano Lattarini Date: Tue, 9 Aug 2011 22:30:26 +0200 Subject: [PATCH] testsuite: improvements to our shell TAP library and APIs These improvements impact only the Automake's own testsuite. * tests/tap-functions.sh (plan_): Allow new special arguments "unknown" a.k.a. "later" and "lazy" a.k.a. "now". Perform sanity check the given arguments. Adjust incorrect comments. (lazy_plan_): Removed, superseded by the new version of `plan_'. (not): New function, run a command and inverts its exit status. (ok_, not_ok_, skip_): These are just shorthands for common `result_' usages. (xfail_, xpass_, reset_test_count_): Removed, they are not used, nor does a legitimate use for them seem in sight. (not): New function, to run a command inverting its exit status. (command_not_ok_): Removed, superseded by `command_ok_' + `not'. (command_ok_): Extended to accept TAP directives. * tests/defs (trap): Don't use `late_plan_' anymore, use `plan_ "now"' instead. * tests/ac-output-old.tap: Update to take advantage of the new APIs. * tests/self-check-env-sanitize.tap: Likewise. * tests/suffix-chain.tap: Likewise. * tests/suffix10.tap: Likewise. * tests/suffix11.tap: Likewise. * tests/suffix3.tap: Likewise. * tests/suffix8.tap: Likewise. * tests/tap-bad-prog.tap: Likewise. * tests/self-check-exit.tap: Likewise, plus a small improvement in the name of a variable. --- ChangeLog | 30 ++++++++ tests/ac-output-old.tap | 2 +- tests/defs | 2 +- tests/self-check-env-sanitize.tap | 2 +- tests/self-check-exit.tap | 15 ++-- tests/suffix-chain.tap | 6 +- tests/suffix10.tap | 4 +- tests/suffix11.tap | 6 +- tests/suffix3.tap | 10 +-- tests/suffix8.tap | 6 +- tests/tap-bad-prog.tap | 10 +-- tests/tap-functions.sh | 152 ++++++++++++++------------------------ 12 files changed, 111 insertions(+), 134 deletions(-) diff --git a/ChangeLog b/ChangeLog index 394af5f..c34bd51 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,35 @@ 2011-08-09 Stefano Lattarini + testsuite: improvements to our shell TAP library and APIs + These improvements impact only the Automake's own testsuite. + * tests/tap-functions.sh (plan_): Allow new special arguments + "unknown" a.k.a. "later" and "lazy" a.k.a. "now". Perform + sanity check the given arguments. Adjust incorrect comments. + (lazy_plan_): Removed, superseded by the new version of `plan_'. + (not): New function, run a command and inverts its exit status. + (ok_, not_ok_, skip_): These are just shorthands for common + `result_' usages. + (xfail_, xpass_, reset_test_count_): Removed, they are not used, + nor does a legitimate use for them seem in sight. + (not): New function, to run a command inverting its exit status. + (command_not_ok_): Removed, superseded by `command_ok_' + `not'. + (command_ok_): Extended to accept TAP directives. + * tests/defs (trap): Don't use `late_plan_' anymore, use + `plan_ "now"' instead. + * tests/ac-output-old.tap: Update to take advantage of the new + APIs. + * tests/self-check-env-sanitize.tap: Likewise. + * tests/suffix-chain.tap: Likewise. + * tests/suffix10.tap: Likewise. + * tests/suffix11.tap: Likewise. + * tests/suffix3.tap: Likewise. + * tests/suffix8.tap: Likewise. + * tests/tap-bad-prog.tap: Likewise. + * tests/self-check-exit.tap: Likewise, plus a small improvement + in the name of a variable. + +2011-08-09 Stefano Lattarini + testsuite: convert some `suffix*' tests to TAP This will allow us to explicitly declare some expected failures involving Solaris make as such, instead of partly papering over diff --git a/tests/ac-output-old.tap b/tests/ac-output-old.tap index ee2d150..5eeb672 100755 --- a/tests/ac-output-old.tap +++ b/tests/ac-output-old.tap @@ -48,7 +48,7 @@ acoutbs_check () command_ok_ "can ./configure in $dir" ./configure command_ok_ "zot created in $dir" test -f zot ls -a > lst || bailout_ "cannot get file listing in $dir" - command_not_ok_ "'\\' not leaked in filenames in $dir" grep '\\' lst + command_ok_ "'\\' not leaked in filenames in $dir" not grep '\\' lst cd .. } diff --git a/tests/defs b/tests/defs index affb1c1..047e22f 100644 --- a/tests/defs +++ b/tests/defs @@ -869,7 +869,7 @@ if test "$sh_errexit_works" = yes; then fi if test $using_tap = yes; then if test $have_tap_plan_ != yes; then - late_plan_ + plan_ "now" fi test $exit_status -eq 0 \ && test $tap_xfail_count_ -eq 0 \ diff --git a/tests/self-check-env-sanitize.tap b/tests/self-check-env-sanitize.tap index 16e3a99..4556e47 100755 --- a/tests/self-check-env-sanitize.tap +++ b/tests/self-check-env-sanitize.tap @@ -49,7 +49,7 @@ do_grep () } for var in $vars; do - command_not_ok_ "$var [err status]" do_run $var + command_ok_ "$var [err status]" not do_run $var command_ok_ "$var [err message]" do_grep $var done diff --git a/tests/self-check-exit.tap b/tests/self-check-exit.tap index ff0278a..fc3fdca 100755 --- a/tests/self-check-exit.tap +++ b/tests/self-check-exit.tap @@ -29,9 +29,9 @@ AM_TESTS_REEXEC=no; export AM_TESTS_REEXEC init='. ./defs; stderr_fileno_=2; unset am_explicit_skips' for st in 1 2 3 4 5 77 99 126 127 128 129 130 255; do - for ex in "Exit $st" "sh -c 'exit $st'"; do - rc=0; $SHELL -c "$init; $ex; :" || rc=$? - command_ok_ "$ex" test $rc -eq $st + for exit_cmd in "Exit $st" "sh -c 'exit $st'"; do + rc=0; $SHELL -c "$init; $exit_cmd; :" || rc=$? + command_ok_ "$exit_cmd" test $rc -eq $st done done @@ -74,12 +74,9 @@ if $SHELL -c 'set -e; trap "exit \$?" 0; non-existent-program'; then else maybe_todo="" reason="" fi -if $SHELL -c "$init; non-existent-program; :"; then - r='not ok' -else - r='ok' -fi -result_ "$r" -D "$maybe_todo" -r "$reason" "command not found" +rc=0; $SHELL -c "$init; non-existent-program; :" || rc=$? +command_ok_ "command not found" -D "$maybe_todo" -r "$reason" \ + -- test $rc -gt 0 : Non-executable command. test -f Makefile && test ! -x Makefile || \ diff --git a/tests/suffix-chain.tap b/tests/suffix-chain.tap index 0ea685a..974c3f7 100755 --- a/tests/suffix-chain.tap +++ b/tests/suffix-chain.tap @@ -48,9 +48,9 @@ command_ok_ "configure" ./configure directive=''; make_can_chain_suffix_rules || directive=TODO for target in all distcheck; do - if $MAKE $target; then r='ok'; else r='not ok'; fi - result_ "$r" -D "$directive" -r "make doesn't chain suffix rules" \ - -- "make $target" + command_ok_ "make $target" \ + -D "$directive" -r "suffix rules not chained" \ + $MAKE $target done command_ok_ "clean" $MAKE clean diff --git a/tests/suffix10.tap b/tests/suffix10.tap index 95c4533..d71450d 100755 --- a/tests/suffix10.tap +++ b/tests/suffix10.tap @@ -62,7 +62,7 @@ command_ok_ "configure" ./configure command_ok_ "make test" $MAKE test directive=''; make_can_chain_suffix_rules || directive=TODO -if $MAKE all; then r='ok'; else r='not ok'; fi -result_ "$r" -D "$directive" -r "make doesn't chain suffix rules" "make all" +command_ok_ "make all" -D "$directive" -r "suffix rules not chained" \ + $MAKE all : diff --git a/tests/suffix11.tap b/tests/suffix11.tap index 30fa874..e740f6c 100755 --- a/tests/suffix11.tap +++ b/tests/suffix11.tap @@ -74,9 +74,9 @@ command_ok_ "make test-real" $MAKE test-real directive=''; make_can_chain_suffix_rules || directive=TODO for target in '' distcheck; do - if $MAKE $target; then r='ok'; else r='not ok'; fi - result_ "$r" -D "$directive" -r "make doesn't chain suffix rules" \ - -- "make $target" + command_ok_ "make $target" \ + -D "$directive" -r "suffix rules not chained" \ + $MAKE $target done : diff --git a/tests/suffix3.tap b/tests/suffix3.tap index a4908d4..f1596ce 100755 --- a/tests/suffix3.tap +++ b/tests/suffix3.tap @@ -45,8 +45,8 @@ command_ok_ "automake" $AUTOMAKE # The foo.cc intermediate step is implicit, it's a mistake if # Automake requires this file somewhere. Also, Automake should # not require the file `foo.c' anywhere. -command_not_ok_ "intermediate files not mentioned" \ - $FGREP foo.c Makefile.in +command_ok_ "intermediate files not mentioned" \ + not $FGREP foo.c Makefile.in # However Automake must figure that foo.zoo is eventually # transformed into foo.o, and use this latter file (to link foo). command_ok_ "final object file figured out" \ @@ -68,9 +68,9 @@ END directive=''; make_can_chain_suffix_rules || directive=TODO for target in all distcheck; do - if $MAKE $target; then r='ok'; else r='not ok'; fi - result_ "$r" -D "$directive" -r "make doesn't chain suffix rules" \ - -- "make $target" + command_ok_ "make $target" \ + -D "$directive" -r "suffix rules not chained" \ + $MAKE $target done # FIXME: should we check that intermediate file `foo.cc' has diff --git a/tests/suffix8.tap b/tests/suffix8.tap index 50d0967..7b07496 100755 --- a/tests/suffix8.tap +++ b/tests/suffix8.tap @@ -83,9 +83,9 @@ command_ok_ "make test1" $MAKE test1 directive=''; make_can_chain_suffix_rules || directive=TODO for target in test2 all distcheck; do - if $MAKE $target; then r='ok'; else r='not ok'; fi - result_ "$r" -D "$directive" -r "make doesn't chain suffix rules" \ - -- "make $target" + command_ok_ "make $target" \ + -D "$directive" -r "suffix rules not chained" \ + $MAKE $target done : diff --git a/tests/tap-bad-prog.tap b/tests/tap-bad-prog.tap index cf475a1..8ba153f 100755 --- a/tests/tap-bad-prog.tap +++ b/tests/tap-bad-prog.tap @@ -77,13 +77,7 @@ fi # Check that no spurious test results is reported. This is lower-priority # (and in fact the check currently fails. -if count_test_results total=3 pass=0 fail=0 xpass=0 xfail=0 skip=0 error=3 -then - r='ok' -else - r='not ok' -fi -result_ "$r" --directive TODO --reason 'still get "missing plan"' \ - 'no spurious results' +command_ok_ 'no spurious results' -D TODO -r 'still get "missing plan"' \ + count_test_results total=3 pass=0 fail=0 xpass=0 xfail=0 skip=0 error=3 : diff --git a/tests/tap-functions.sh b/tests/tap-functions.sh index 5cd6c1a..603b53c 100644 --- a/tests/tap-functions.sh +++ b/tests/tap-functions.sh @@ -42,31 +42,41 @@ else incr_ () { eval "$1=\`expr \${$1} + 1\`"; } fi -# plan_ NUMBER-OF-PLANNED-TESTS -# ----------------------------- -# Print a TAP plan for the given number of tests. This must be called -# before reporting any test result; in fact, it must be called before -# emitting anything on standard output. -plan_ () +# not COMMAND [ARGS...] +# --------------------- +# Run the given command and invert its exit status. +not () { - echo "1..$1" - have_tap_plan_=yes + if "$@"; then return 1; else return 0; fi } -# late_plan_ -# ---------- -# Print a TAP plan that accounts for the number of tests seen so far. -# This must be called after all the tests result have been reported; -# in fact, after this has been called, nothing more can be print on -# standard output. -late_plan_ () +# plan_ [unknown|later|lazy|now|NUMBER-OF-PLANNED-TESTS] +# ------------------------------------------------------ +# Print a TAP plan for the given number of tests. This must be called +# before reporting any test result. If called with the special argument +# "unknown" or "later", it will do nothing, expecting the calling script +# to declare the plan later. If called with the special argument "lazy" +# or "now", it will print a TAP plan that accounts for the number of tests +# seen so far. +plan_ () { - echo "1..$tap_count_" - have_tap_plan_=yes + if test $# -eq 0; then + bailout_ "plan_: missing argument" + elif test $# -ge 2; then + bailout_ "plan_: too many arguments" + elif test x"$1" = x"unknown" || test x"$1" = x"later"; then + : No-op. + elif test x"$1" = x"lazy" || test x"$1" = x"now"; then + echo "1..$tap_count_" # Number of test results seen so far. + have_tap_plan_=yes + elif test $1 -ge 0; then + echo "1..$1" + have_tap_plan_=yes + else + bailout_ "plan_: invalid argument '$1'" + fi } - -# Initialize it to avoid interferences from the environment. -have_tap_plan_=no +have_tap_plan_=no # Avoid interferences from the environment. # diag_ [EXPLANATION] # ------------------ @@ -139,30 +149,10 @@ result_ () set -x # Restore shell xtraces. } -# ok_ [DESCRIPTION...] -# -------------------- -# Report a successful test. -ok_ () -{ - result_ 'ok' -- ${1+"$@"} -} - -# not_ok_ [DESCRIPTION...] -# ------------------------ -# Report a failed test. -not_ok_ () -{ - result_ 'not ok' -- ${1+"$@"} -} - -# skip_ [-r REASON] [--] [DESCRIPTION...] -# --------------------------------------- -# Report a skipped test. If the `-r' option is present, its argument is -# give as the reason of the skip. -skip_ () -{ - result_ 'ok' -D SKIP ${1+"$@"} -} +# Shorthands for common usages of `result_'. +ok_ () { result_ 'ok' ${1+"$@"}; } +not_ok_ () { result_ 'not ok' ${1+"$@"}; } +skip_ () { result_ 'ok' -D SKIP ${1+"$@"}; } # skip_row_ COUNT [-r REASON] [--] [DESCRIPTION...] # ------------------------------------------------- @@ -175,24 +165,6 @@ skip_row_ () for i_ in `seq $skip_count_`; do skip_ ${1+"$@"}; done } -# xfail_ [-r REASON] [DESCRIPTION...] -# ---------------------------------- -# Report a test that failed expectedly. If the `-r' option is present, its -# argument is give as the reason why the failure is expected. -xfail_ () -{ - result_ 'not ok' -D TODO ${1+"$@"} -} - -# xpass_ [-r REASON] [DESCRIPTION...] -# ----------------------------------- -# Report a test that failed unexpectedly. If the `-r' option is present, its -# argument is give as the reason why the failure is expected. -xpass_ () -{ - result_ 'ok' -D TODO ${1+"$@"} -} - # skip_all_ [REASON ...] # ---------------------- # Skip all the tests in a test script. Must be used before calling `plan_' @@ -233,44 +205,28 @@ framework_failure_ () bailout_ "set-up failure"${1+": $*"} } -# command_ok_ TEST-DESCRIPTION [--] CMD [ARGS...] -# ----------------------------------------------- -# Report a passed test if the given command returns with success, -# a failed test otherwise. +# command_ok_ TEST-DESCRIPTION [OPTIONS..] [--] CMD [ARGS...] +# ----------------------------------------------------------- +# Helper subroutine for when a TAP result must be determined by the +# outcome of a command. command_ok_ () { - tap_desc_=$1; shift - test x"$1" != x"--" || shift - if "$@"; then - ok_ "$tap_desc_" - else - not_ok_ "$tap_desc_" - fi -} - -# command_not_ok_ TEST-DESCRIPTION [--] CMD [ARGS...] -# --------------------------------------------------- -# Report a failed test if the given command returns with success, -# a failed test otherwise. -command_not_ok_ () -{ - tap_desc_=$1; shift - test x"$1" != x"--" || shift - if "$@"; then - not_ok_ "$tap_desc_" - else - ok_ "$tap_desc_" - fi -} - -# reset_test_count_ COUNT -# ----------------------- -# Reset the count of the TAP test results seen so far to COUNT. -# This function is for use in corner cases only (e.g., when `ok_' and -# `not_ok_' must be used inside a subshell). Be careful when using it! -reset_test_count_ () -{ - tap_count_=$1 + tap_directive_= tap_reason_= + test $# -gt 0 || bailout_ "command_ok_: missing argument" + tap_description_=$1; shift + while test $# -gt 0; do + case $1 in + -D|--directive) tap_directive_=$2; shift;; + -r|--reason) tap_reason_=$2; shift;; + --) shift; break;; + -*) bailout_ "command_ok_: invalid option '$1'";; + *) break;; + esac + shift + done + tap_result_="ok"; "$@" || tap_result_="not ok" + result_ "$tap_result_" -D "$tap_directive_" -r "$tap_reason_" \ + -- "$tap_description_" } : -- 2.7.4