+2011-08-04 Stefano Lattarini <stefano.lattarini@gmail.com>
+
+ test harness: be aware of more metadata, simplify test drivers
+ * lib/am/check.am ($(TEST_SUITE_LOG)): When producing the
+ global test log, take into account the new metadata field
+ `:global-test-result:, to write nicely formatted RST section
+ titles, instead of leaving this chore to the individual test
+ drivers.
+ (am_rst_section): Re-introduce this variable, as removed in
+ commit `v1.11-870-ga27c9c4'.
+ * lib/test-driver, lib/tap-driver: Write the global test result
+ as metadata in the `.trs' file, not as (part of) a formatted RST
+ subsection title in the `.log' file. Related simplifications;
+ in particular, get rid of temporary files usage.
+ * doc/automake.texi (Log files generation and test results
+ recording): Document the new metadata.
+ * tests/check12.test: Update.
+ * tests/parallel-tests-harderror.test: Likewise.
+ * tests/parallel-tests-interrupt.test: Likewise.
+ * tests/parallel-tests-log-override-2.test: Likewise.
+ * tests/parallel-tests-log-override-recheck.test: Likewise.
+ * tests/parallel-tests-unreadable.test: Likewise.
+ * tests/tap-global-result.test: Likewise.
+ * tests/test-metadata-results.test: Likewise.
+ * tests/test-log.test: Likewise, and another minor unrelated fix.
+ * tests/test-metadata-global-result.test: New test.
+ * tests/Makefile.am (TESTS): Update.
+
2011-08-03 Stefano Lattarini <stefano.lattarini@gmail.com>
tap: check that also a trailing TAP plan can hold a skip directive
successful. What happens when two or more @code{:copy-in-global-log:}
fields are present in the same @file{.trs} file is undefined behaviour.
+@c Keep in sync with 'test-metadata-global-result.test'.
+@item @code{:test-global-result:}
+@cindex :test-global-result:
+@cindex reStructuredText field, @code{:test-global-result:}
+This is used to declare the "global result" of the script. Currently,
+the value of this field is needed only to be reported (more or less
+verbatim) in the generated global log file @code{$(TEST_SUITE_LOG)},
+so it's quite free-form. For example, a test script which run 10 test
+cases, 6 of which pass and 4 of which are skipped, could reasonably have
+a @code{PASS/SKIP} value for this field, while a test script which run
+19 successful tests and one failed test could have an @code{ALMOST
+PASSED} value. What happens when two or more @code{:test-global-result:}
+fields are present in the same @file{.trs} file is undefined behaviour.
@end table
@noindent
am__base_list = \
sed '$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;s/\n/ /g' | \
sed '$$!N;$$!N;$$!N;$$!N;s/\n/ /g'
-# Restructured Text title.
+# Restructured Text title and section.
am__rst_title = sed 's/.*/ & /;h;s/./=/g;p;x;p;g;p;s/.*//'
+am__rst_section = sed 'p;s/./=/g;p;g'
# Solaris 10 'make', and several other traditional 'make' implementations,
# pass "-e" to $(SHELL), and POSIX 2008 even requires this. Work around it
# by disabling -e (using the XSI extension "set +e") if it's set.
echo ".. contents:: :depth: 2"; \
echo; \
for b in $$bases; do \
- if grep "^$$ws*:copy-in-global-log:$$ws*no$$ws*$$" $$b.trs \
- >/dev/null; then :; \
- elif test ! -r $$b.log; then \
- echo "ERROR: cannot read $$b.log" >&2; \
- echo && echo "WARNING: could not read $$b.log!"; \
- else \
- echo && cat $$b.log; \
- fi; \
- done; \
+ if grep "^$$ws*:copy-in-global-log:$$ws*no$$ws*$$" $$b.trs \
+ >/dev/null; then continue; \
+ fi; \
+ glob_res=`sed -n -e "s/$$ws*$$//" \
+ -e "s/^$$ws*:global-test-result:$$ws*//p" \
+ $$b.trs`; \
+ test -n "$$glob_res" || glob_res=RUN; \
+ echo "$$glob_res: $$b" | $(am__rst_section); \
+ if test ! -r $$b.log; then \
+ echo "fatal: making $@: $$b.log is unreadable" >&2; \
+ exit 1; \
+ fi; \
+ cat $$b.log; echo; \
+ done; \
} >$(TEST_SUITE_LOG).tmp; \
mv $(TEST_SUITE_LOG).tmp $(TEST_SUITE_LOG); \
if $$success; then \
## test is XFAIL or not. You can disable this feature by setting the
## variable DISABLE_HARD_ERRORS to a nonempty value.
-# Restructured Text title.
+# Restructured Text title and section.
am__rst_title = sed 's/.*/ & /;h;s/./=/g;p;x;p;g;p;s/.*//'
+am__rst_section = sed 'p;s/./=/g;p;g'
# Solaris 10 'make', and several other traditional 'make' implementations,
# pass "-e" to $(SHELL), and POSIX 2008 even requires this. Work around it
echo; \
for b in $$bases; do \
## FIXME: one fork per test -- this is horrendously inefficient!
- if grep "^$$ws*:copy-in-global-log:$$ws*no$$ws*$$" $$b.trs \
- >/dev/null; then :; \
-## If we cannot read the .log file of a test ...
- elif test ! -r $$b.log; then \
-## ... print an error message to the console ...
- echo "ERROR: cannot read $$b.log" >&2; \
-## ... and a warning in the `test-suite.log' file.
- echo && echo "WARNING: could not read $$b.log!"; \
- else \
- echo && cat $$b.log; \
- fi; \
- done; \
+ if grep "^$$ws*:copy-in-global-log:$$ws*no$$ws*$$" $$b.trs \
+ >/dev/null; then continue; \
+ fi; \
+## Get the declared "global result" of the test.
+## FIXME: yet another one fork per test here!
+ glob_res=`sed -n -e "s/$$ws*$$//" \
+ -e "s/^$$ws*:global-test-result:$$ws*//p" \
+ $$b.trs`; \
+## If no global result is explicitly declared, we'll merely mark the
+## test as "RUN" in the global test log.
+ test -n "$$glob_res" || glob_res=RUN; \
+## Write the name and result of the test as an RST section title.
+ echo "$$glob_res: $$b" | $(am__rst_section); \
+## If we should have remade any unreadable `.log', above.
+ if test ! -r $$b.log; then \
+ echo "fatal: making $@: $$b.log is unreadable" >&2; \
+ exit 1; \
+ fi; \
+ cat $$b.log; echo; \
+ done; \
} >$(TEST_SUITE_LOG).tmp; \
mv $(TEST_SUITE_LOG).tmp $(TEST_SUITE_LOG); \
## Emit the test summary on the console.
sub write_test_results ()
{
open RES, ">", $trs_file or die "opening $trs_file: $!\n";
+ print RES ":global-test-result: " .
+ get_global_test_result . "\n";
print RES ":recheck: " .
(must_recheck ? "yes" : "no") . "\n";
print RES ":copy-in-global-log: " .
# Redirect stderr and stdout to a temporary log file. Save the
# original stdout stream, since we need it to print testsuite
# progress output.
+ open LOG, ">", $log_file or die "opening $log_file: $!\n";
open OLDOUT, ">&STDOUT" or die "duplicating stdout: $!\n";
- open TMP, ">$log_file-t" or die "opening $log_file-t: $!\n";
- open STDOUT, ">&TMP" or die "redirecting stdout: $!\n";
- open STDERR, ">&TMP" or die "redirecting stderr: $!\n";
+ open STDOUT, ">&LOG" or die "redirecting stdout: $!\n";
+ open STDERR, ">&LOG" or die "redirecting stderr: $!\n";
$parser = TAP::Parser->new ({ exec => \@_, merge => $cfg{merge} });
$parser->ignore_exit(1) if $cfg{"ignore-exit"};
}
sub finish ()
{
write_test_results;
-
- open LOG, ">", $log_file or die "opening $log_file: $!\n";
- my $global_result = get_global_test_result;
- my $global_result_line = "$global_result: $test_script_name";
- print LOG "$global_result_line\n";
- print LOG "=" x length ($global_result_line) . "\n";
- print LOG "\n";
-
- close TMP or die "closing $log_file-t: $!\n";
- # FIXME: remove this hack!
- my $test_output = `cat $log_file-t && rm -f $log_file-t`;
- print LOG $test_output;
close LOG or die "closing $log_file: $!\n";
-
exit 0;
}
#! /bin/sh
# test-driver - basic driver script for the `parallel-tests' mode.
-scriptversion=2011-08-01.21; # UTC
+scriptversion=2011-08-04.22; # UTC
# Copyright (C) 2011 Free Software Foundation, Inc.
#
END
}
-# Restructured Text section.
-rst_section () { sed 'p;s/./=/g;p;g'; }
-
# TODO: better error handling in option parsing (in particular, ensure
# TODO: $logfile, $trsfile and $test_name are defined).
test_name= # Used for reporting.
red= grn= lgn= blu= mgn= std=
fi
-tmpfile=$logfile-t
-do_exit='rm -f $tmpfile; (exit $st); exit $st'
+do_exit='rm -f $logfile; (exit $st); exit $st'
trap "st=129; $do_exit" 1
trap "st=130; $do_exit" 2
trap "st=141; $do_exit" 13
trap "st=143; $do_exit" 15
-rm -f $tmpfile
# Test script is run here.
-"$@" >$tmpfile 2>&1
+"$@" >$logfile 2>&1
estatus=$?
if test $enable_hard_errors = no && test $estatus -eq 99; then
estatus=1
echo "${col}${res}${std}: $test_name"
# Register the test result, and other relevant metadata.
-echo ":test-result: $res (exit status: $estatus)" > $trsfile
+echo ":test-result: $res" > $trsfile
+echo ":global-test-result: $res" >> $trsfile
echo ":recheck: $recheck" >> $trsfile
echo ":copy-in-global-log: $gcopy" >> $trsfile
-# Now write log file.
-{
- echo "$res: $test_name (exit: $estatus)" | rst_section
- echo
- cat $tmpfile
-} > $logfile
-rm -f $tmpfile
-
# Local Variables:
# mode: shell-script
# sh-indentation: 2
test-driver-fail.test \
test-log.test \
test-metadata-global-log.test \
+test-metadata-global-result.test \
test-metadata-recheck.test \
test-metadata-results.test \
test-missing.test \
am__base_list = \
sed '$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;s/\n/ /g' | \
sed '$$!N;$$!N;$$!N;$$!N;s/\n/ /g'
-# Restructured Text title.
+# Restructured Text title and section.
am__rst_title = sed 's/.*/ & /;h;s/./=/g;p;x;p;g;p;s/.*//'
+am__rst_section = sed 'p;s/./=/g;p;g'
# Solaris 10 'make', and several other traditional 'make' implementations,
# pass "-e" to $(SHELL), and POSIX 2008 even requires this. Work around it
# by disabling -e (using the XSI extension "set +e") if it's set.
test-driver-fail.test \
test-log.test \
test-metadata-global-log.test \
+test-metadata-global-result.test \
test-metadata-recheck.test \
test-metadata-results.test \
test-missing.test \
echo ".. contents:: :depth: 2"; \
echo; \
for b in $$bases; do \
- if grep "^$$ws*:copy-in-global-log:$$ws*no$$ws*$$" $$b.trs \
- >/dev/null; then :; \
- elif test ! -r $$b.log; then \
- echo "ERROR: cannot read $$b.log" >&2; \
- echo && echo "WARNING: could not read $$b.log!"; \
- else \
- echo && cat $$b.log; \
- fi; \
- done; \
+ if grep "^$$ws*:copy-in-global-log:$$ws*no$$ws*$$" $$b.trs \
+ >/dev/null; then continue; \
+ fi; \
+ glob_res=`sed -n -e "s/$$ws*$$//" \
+ -e "s/^$$ws*:global-test-result:$$ws*//p" \
+ $$b.trs`; \
+ test -n "$$glob_res" || glob_res=RUN; \
+ echo "$$glob_res: $$b" | $(am__rst_section); \
+ if test ! -r $$b.log; then \
+ echo "fatal: making $@: $$b.log is unreadable" >&2; \
+ exit 1; \
+ fi; \
+ cat $$b.log; echo; \
+ done; \
} >$(TEST_SUITE_LOG).tmp; \
mv $(TEST_SUITE_LOG).tmp $(TEST_SUITE_LOG); \
if $$success; then \
cat > a.test << 'END'
#!/bin/sh
+echo a.test: exit ${A_EXIT_STATUS-0}
exit ${A_EXIT_STATUS-0}
END
cat > b.test << 'END'
#!/bin/sh
+echo b.test: exit ${B_EXIT_STATUS-0}
exit ${B_EXIT_STATUS-0}
END
chmod +x a.test b.test
test -f spanner.sum
# This checks will be run only by the autogenerated `check12-p.test'.
if test x"$parallel_tests" = x"yes"; then
- cat test-suite.log
- grep '^PASS: a\.test (exit: 0)' a.log
- grep '^PASS: b\.test (exit: 0)' b.log
+ test -f test-suite.log
+ test -f a.log
+ test -f b.log
else
:
fi
# This checks will be run only by the autogenerated `check12-p.test'.
if test x"$parallel_tests" = x"yes"; then
cat test-suite.log
- grep '^PASS: a\.test (exit: 0)' a.log
- grep '^FAIL: b\.test (exit: 1)' b.log
- grep '^FAIL: b\.test (exit: 1)' test-suite.log
+ cat a.log
+ cat b.log
+ grep '^a\.test: exit 0$' a.log
+ grep '^b\.test: exit 1$' b.log
+ grep '^FAIL: b$' test-suite.log
+ grep '^b\.test: exit 1$' test-suite.log
+ grep '^a\.test' test-suite.log && Exit 1
else :; fi
CHECKLOCAL_EXIT_STATUS=1 $MAKE check && Exit 1
# This checks will be run only by the autogenerated `check12-p.test'.
if test x"$parallel_tests" = x"yes"; then
cat test-suite.log
- grep '^PASS: a\.test (exit: 0)' a.log
- grep '^FAIL: b\.test (exit: 23)' b.log
- grep '^FAIL: b\.test (exit: 23)' test-suite.log
+ cat a.log
+ cat b.log
+ grep '^a\.test: exit 0$' a.log
+ grep '^b\.test: exit 23$' b.log
+ grep '^FAIL: b$' test-suite.log
+ grep '^b\.test: exit 23$' test-suite.log
+ grep '^a\.test' test-suite.log && Exit 1
else :; fi
grep 'check-local failed :-(' local.log
$MAKE check DISABLE_HARD_ERRORS='' && Exit 1
cat test-suite.log
-grep '^ERROR: foo\.test .*exit.*99' test-suite.log
+grep '^ERROR: foo$' test-suite.log
cd sub
# The `-e' is wanted here.
DISABLE_HARD_ERRORS='' $MAKE -e check && Exit 1
cat test-suite.log
-grep '^ERROR: bar\.test .*exit.*99' test-suite.log
+grep '^ERROR: bar$' test-suite.log
cd ..
# Check the distributions.
./config.status Makefile
VERBOSE=yes $MAKE check && Exit 1
grep '^FAIL' test-suite.log && Exit 1
-grep '^ERROR: bar\.test .*exit.*99' sub/test-suite.log
+grep '^ERROR: bar$' sub/test-suite.log
echo 'DISABLE_HARD_ERRORS = zardoz' >> sub/Makefile
VERBOSE=yes $MAKE check
exec 2>&9
echo "foo is starting to run"
ls -l >&2
-cat foo.log-t >&2 || : > fail
-grep '^foo is starting to run$' foo.log-t >&2 || : > fail
+cat foo.log >&2 || : > fail
+grep '^foo is starting to run$' foo.log >&2 || : > fail
cat pid >&2 || : > fail
kill -$signum `cat pid` || : > fail
END
trapped_signals='1 2 13 15'
for signum in $trapped_signals; do
- rm -f pid fail *.log *.log-t
+ rm -f pid fail *.log
env signum=$signum $MAKE check && { ls -l; Exit 1; }
ls -l
# These files shouldn't exist, but in case they do, their content might
# provide helpful information about the causes of the failure(s).
- cat foo.log-t || :
cat foo.log || :
cat test-suite.log || :
test -f fail && Exit 1
- test -f foo.log-t && Exit 1
- test -f foo.log && Exit 1
+ ls | $EGREP 'foo.*\.(log|tmp)' && Exit 1
done
:
test ! -f fail.log
grep '^PASS: pass\.test$' stdout
grep '^SKIP: skip\.test$' stdout
- $FGREP 'skip.test' partial.log
+ $FGREP 'SKIP: skip' partial.log
$FGREP '% skipped test %' partial.log
- $EGREP '(pass2|skip2|fail)\.test' stdout partial.log && Exit 1
+ $EGREP '(pass2|skip2|fail)\.test' stdout && Exit 1
+ $EGREP '(pass2|skip2|fail)' partial.log && Exit 1
rm -f *.log
done
END
cat > Makefile.am <<'END'
-TESTS = foo.test bar.test baz.test
+TESTS = foofoo.test barbar.test bazbaz.test
END
-cat > foo.test <<'END'
+cat > foofoo.test <<'END'
#! /bin/sh
echo "this is $0"
exit 0
END
-cat > bar.test <<'END'
+cat > barbar.test <<'END'
#! /bin/sh
echo "this is $0"
exit 99
END
-cat > baz.test <<'END'
+cat > bazbaz.test <<'END'
#! /bin/sh
echo "this is $0"
exit ${BAZ_EXIT_STATUS-1}
ls -l
count_test_results total=2 pass=0 fail=1 skip=0 xfail=0 xpass=0 error=1
for x in stdout my.log; do
- $FGREP foo.test $x && Exit 1
- $FGREP bar.test $x
- $FGREP baz.test $x
+ $FGREP foofoo $x && Exit 1
+ $FGREP barbar $x
+ $FGREP bazbaz $x
done
chmod a-rw my.log
cat stdout
ls -l
count_test_results total=2 pass=1 fail=0 skip=0 xfail=0 xpass=0 error=1
-$FGREP foo.test stdout && Exit 1
-$FGREP bar.test stdout
-$FGREP baz.test stdout
-$FGREP foo.test my2.log && Exit 1
-$FGREP bar.test my2.log
-$FGREP baz.test my2.log && Exit 1
+$FGREP foofoo stdout && Exit 1
+$FGREP barbar stdout
+$FGREP bazbaz stdout
+$FGREP foofoo my2.log && Exit 1
+$FGREP barbar my2.log
+$FGREP bazbaz my2.log && Exit 1
chmod u+r test-suite.log my.log
-$FGREP baz.test test-suite.log
-$FGREP baz.test my.log
+$FGREP bazbaz test-suite.log
+$FGREP bazbaz my.log
:
grep '^:test-result: PASS' foo.trs
grep '^barbarbar$' bar.log
grep '^:test-result: SKIP' bar.trs
- grep '^SKIP: bar.test' test-suite.log
+ grep '^SKIP: bar' test-suite.log
grep '^barbarbar$' test-suite.log
- $EGREP 'foo\.test|foofoofoo' test-suite.log && Exit 1
+ $EGREP ':.*foo|foofoofoo' test-suite.log && Exit 1
done
:
TESTS="`echo *.test`" $MAKE -e check || :
cat test-suite.log
-grep 'ok\.test|not seen' test-suite.log && Exit 1
+grep ':.*ok|not seen' test-suite.log && Exit 1
for s in skip todo fail xpass bail error; do
$FGREP "::$s::" test-suite.log
# TODO: add scripts with TAP errors (multiple plans, out-of-order
# tests, etc).
-TESTS="`echo *.test`" $MAKE -e check >stdout && { cat stdout; Exit 1; }
+tests=`echo *.test` # Also required later.
+
+TESTS="$tests" $MAKE -e check >stdout && { cat stdout; Exit 1; }
cat stdout
+# Dirty tricks required here.
for tst in ok skip skipall fail fail2 xpass xpass2 error error2 \
hodgepodge hodgepodge-all; do
- sed -e 2q $tst.log > $tst.res
+ echo :copy-in-global-log: yes >> $tst.trs
done
-cat *.res # For debugging.
-
-grep '^PASS:' ok.res
-grep '^SKIP:' skip.res
-grep '^SKIP:' skipall.res
-grep '^FAIL:' fail.res
-grep '^FAIL:' fail2.res
-grep '^FAIL:' xpass.res
-grep '^FAIL:' xpass2.res
-grep '^ERROR:' error.res
-grep '^ERROR:' error2.res
-grep '^ERROR:' hodgepodge.res
-grep '^ERROR:' hodgepodge-all.res
+rm -f test-suite.log
+TESTS="$tests" $MAKE -e test-suite.log && Exit 1
+cat test-suite.log
+
+have_rst_section ()
+{
+ eqeq=`echo "$1" | sed 's/./=/g'`
+ # Assume $1 contains no RE metacharacters.
+ sed -n "/^$1$/,/^$eqeq$/p" test-suite.log > got
+ (echo "$1" && echo "$eqeq") > exp
+ cat exp
+ cat got
+ diff exp got
+}
+
+have_rst_section 'PASS: ok'
+have_rst_section 'SKIP: skip'
+have_rst_section 'SKIP: skipall'
+have_rst_section 'FAIL: fail'
+have_rst_section 'FAIL: fail2'
+have_rst_section 'FAIL: xpass'
+have_rst_section 'FAIL: xpass2'
+have_rst_section 'ERROR: error'
+have_rst_section 'ERROR: error2'
+have_rst_section 'ERROR: hodgepodge'
+have_rst_section 'ERROR: hodgepodge-all'
:
#! /bin/sh
echo "$pmarker skip $pmarker"
echo "# $cmarker skip $cmarker"
-exit 1
+exit 77
END
cat > xfail.test <<END
test `$FGREP -c "$pmarker" my.log` -eq 5
test `$FGREP -c "$cmarker" my.log` -eq 5
+have_rst_section ()
+{
+ eqeq=`echo "$1" | sed 's/./=/g'`
+ # Assume $1 contains no RE metacharacters.
+ sed -n "/^$1$/,/^$eqeq$/p" $2 > got
+ (echo "$1" && echo "$eqeq") > exp
+ cat exp
+ cat got
+ diff exp got
+}
+
# Passed test scripts shouldn't be mentioned in the global log.
-$EGREP '(^pass|[^x]pass)\.test' my.log && Exit 1
+$EGREP ':.*[^x]pass' my.log && Exit 1
# But failing (expectedly or not) and skipped ones should.
-$FGREP 'xfail.test' my.log
-$FGREP 'skip.test' my.log
-$FGREP 'fail.test' my.log
-$FGREP 'xpass.test' my.log
-$FGREP 'error.test' my.log
+have_rst_section 'SKIP: skip' my.log
+have_rst_section 'FAIL: fail' my.log
+have_rst_section 'XFAIL: xfail' my.log
+have_rst_section 'XPASS: xpass' my.log
+have_rst_section 'ERROR: error' my.log
touch error2.log test-suite.log global.log
TEST_SUITE_LOG=my.log $MAKE -e mostlyclean
--- /dev/null
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Parallel testsuite harness: check APIs for the registering the
+# "global test result" in `*.trs' files, as documented in the automake
+# manual.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+cat >> configure.in << 'END'
+AC_OUTPUT
+END
+
+cat > Makefile.am << 'END'
+TEST_EXTENSIONS = .test .x
+TEST_LOG_DRIVER = ./dummy-driver
+X_LOG_DRIVER = ./dummy-driver
+TESTS = foo.test zar-doz.test
+END
+
+cat > dummy-driver <<'END'
+#! /bin/sh
+set -e; set -u
+while test $# -gt 0; do
+ case $1 in
+ --log-file) log_file=$2; shift;;
+ --trs-file) trs_file=$2; shift;;
+ --test-name) test_name=$2; shift;;
+ --expect-failure|--color-tests|--enable-hard-errors) shift;;
+ --) shift; break;;
+ *) echo "$0: invalid option/argument: '$1'" >&2; exit 2;;
+ esac
+ shift
+done
+echo logloglog > $log_file
+cp $1 $trs_file
+END
+chmod a+x dummy-driver
+
+# Do this in a subroutine to avoid quoting problem in the backticked
+# command substitution below.
+get_escaped_line()
+{
+ sed -e 's,[$^/\\\.],\\&,g' -e 1q "$@"
+}
+
+have_result ()
+{
+ cat > exp; echo >> exp; echo logloglog >> exp
+ eline=`get_escaped_line exp`
+ sed -n -e "/^$eline$/,/^logloglog$/p" test-suite.log > got
+ cat exp; cat got
+ diff exp got
+}
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE
+
+./configure
+
+: Basic checks.
+
+echo :global-test-result: PASS > foo.test
+echo :global-test-result: ERROR > zar-doz.x
+
+$MAKE check
+cat test-suite.log
+
+have_result <<END
+PASS: foo
+=========
+END
+
+have_result <<END
+ERROR: zar-doz
+==============
+END
+
+: Try usage documented in the manual.
+
+echo :global-test-result: PASS/SKIP > foo.test
+echo :global-test-result: ALMOST PASSED > zar-doz.x
+
+$MAKE check
+cat test-suite.log
+
+have_result <<END
+PASS/SKIP: foo
+==============
+END
+
+have_result <<END
+ALMOST PASSED: zar-doz
+======================
+END
+
+: Fields ':test-result:' does not interfere with the global test result.
+
+cat > foo.test << 'END'
+:test-result: FAIL
+:global-test-result: PASS
+:test-result: ERROR
+END
+
+cat > zar-doz.x << 'END'
+:global-test-result: FAIL
+:test-result: SKIP
+:test-result: XFAIL
+END
+
+$MAKE check && Exit 1
+cat test-suite.log
+
+have_result <<END
+PASS: foo
+=========
+END
+
+have_result <<END
+FAIL: zar-doz
+=============
+END
+
+: What happens when ':global-test-result:' is absent.
+
+cat > foo.test << 'END'
+:test-result: PASS
+:test-result: ERROR
+END
+: > zar-doz.x
+
+$MAKE check && Exit 1
+cat test-suite.log
+
+have_result <<END
+RUN: foo
+========
+END
+
+have_result <<END
+RUN: zar-doz
+============
+END
+
+# Leading and trailing whitespace gets eaten/normalized.
+
+echo ":global-test-result:SKIP${tab} ${tab}${tab}" > foo.test
+echo ":global-test-result:${tab} ${tab}XFAIL ${tab} " > zar-doz.x
+
+$MAKE check
+cat test-suite.log
+
+have_result <<END
+SKIP: foo
+=========
+END
+
+have_result <<END
+XFAIL: zar-doz
+==============
+END
+
+# Whitespaces before and after `:global-test-result:' are handled OK.
+
+echo " $tab:global-test-result:PASS" > foo.test
+echo "${tab}${tab}:global-test-result:${tab} ${tab}SKIP" > zar-doz.x
+
+$MAKE check
+cat test-suite.log
+
+have_result <<END
+PASS: foo
+=========
+END
+
+have_result <<END
+SKIP: zar-doz
+=============
+END
+
+:
./configure
-# Basic checks. Also that that "old-style" directives with format
-# "RESULT: test-name" are be ignored now.
+# Basic checks. Also that that `:global-test-result:' fields and
+# "old-style" directives with format "RESULT: test-name" are now ignored.
+
+: > foo.test
+echo blah blah blah > bar.test
+mk_check
+count_test_results total=0 pass=0 fail=0 xpass=0 xfail=0 skip=0 error=0
+
+cat > foo.test <<END
+:test-global-result: PASS
+:test-result: FAIL
+END
+cat > bar.test <<END
+:test-result: SKIP
+:test-global-result: ERROR
+END
+mk_check && Exit 1
+count_test_results total=2 pass=0 fail=1 xpass=0 xfail=0 skip=1 error=0
cat > foo.test <<END
FAIL: foo.test
:test-result: PASS
+:test-global-result: XPASS
END
echo ERROR: bar.test > bar.test
mk_check
count_test_results total=1 pass=1 fail=0 xpass=0 xfail=0 skip=0 error=0
cat > foo.test <<END
-PASS: foo.test
-:test-result: FAIL
-END
-echo SKIP: bar.test > bar.test
-: > bar.test
-mk_check && Exit 1
-count_test_results total=1 pass=0 fail=1 xpass=0 xfail=0 skip=0 error=0
-
-cat > foo.test <<END
-PASS: foo.test
+:test-global-result: SKIP
:test-result: FAIL
END
cat > bar.test <<END
-ERROR: foo.test
-:test-result: SKIP
+:test-global-result: PASS
END
mk_check && Exit 1
-count_test_results total=2 pass=0 fail=1 xpass=0 xfail=0 skip=1 error=0
+count_test_results total=1 pass=0 fail=1 xpass=0 xfail=0 skip=0 error=0
cat > foo.test <<END
:test-result: XFAIL
projects / platform / upstream / automake.git / commitdiff