From b9e9d546e8ab45139be6d8e939602cd712c36c8c Mon Sep 17 00:00:00 2001 From: Stefano Lattarini Date: Sun, 24 Jul 2011 16:47:24 +0200 Subject: [PATCH] simple tests: support developer-defined fd redirections Motivated by coreutils bug#8846, and related discussions: In those threads it has been shown how problematic it can be to do portable file descriptor redirections in the testsuite when using the Automake testsuite harness. This change should remedy to that situation. * lib/am/check2.am (?GENERIC?%EXT%$(EXEEXT).log, ?GENERIC?%EXT%.log, ?!GENERIC?%OBJ%): Append redirection defined in $(AM_TESTS_FD_REDIRECT) to the command-line invocations of the test scripts. * lib/am/check.am [!%?PARALLEL_TESTS%] $(check-TESTS): Likewise. * NEWS: Update. * doc/automake.texi (Script-based Testsuites): Document the new feature. * tests/check-fd-redirect.test: New test. * tests/parallel-tests-fd-redirect.test: Likewise. * tests/parallel-tests-am_tests_environment.test: Remove checks about the use of redirections in AM_TESTS_ENVIRONMENT: they would check deprecated (if not undefined) behaviour now. Strengthen a couple of still valid checks, to keep the test more in sync with the documentation. Improve debugging information. * tests/Makefile.am (TESTS): Update. --- ChangeLog | 27 ++++++ NEWS | 3 + doc/automake.texi | 23 +++++- lib/Automake/tests/Makefile.in | 2 +- lib/am/check.am | 2 +- lib/am/check2.am | 4 +- tests/Makefile.am | 2 + tests/Makefile.in | 18 ++-- tests/check-fd-redirect.test | 110 +++++++++++++++++++++++++ tests/parallel-tests-am_tests_environment.test | 48 +++++------ tests/parallel-tests-fd-redirect.test | 109 ++++++++++++++++++++++++ 11 files changed, 309 insertions(+), 39 deletions(-) create mode 100755 tests/check-fd-redirect.test create mode 100755 tests/parallel-tests-fd-redirect.test diff --git a/ChangeLog b/ChangeLog index 4904bf4..694cd0b 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,30 @@ +2011-07-31 Stefano Lattarini + + simple tests: support developer-defined fd redirections + Motivated by coreutils bug#8846, and related discussions: + + + In those threads it has been shown how problematic it can be to do + portable file descriptor redirections in the testsuite when using + the Automake testsuite harness. This change should remedy to that + situation. + * lib/am/check2.am (?GENERIC?%EXT%$(EXEEXT).log, + ?GENERIC?%EXT%.log, ?!GENERIC?%OBJ%): Append redirection defined + in $(AM_TESTS_FD_REDIRECT) to the command-line invocations of the + test scripts. + * lib/am/check.am [!%?PARALLEL_TESTS%] $(check-TESTS): Likewise. + * NEWS: Update. + * doc/automake.texi (Script-based Testsuites): Document the new + feature. + * tests/check-fd-redirect.test: New test. + * tests/parallel-tests-fd-redirect.test: Likewise. + * tests/parallel-tests-am_tests_environment.test: Remove checks + about the use of redirections in AM_TESTS_ENVIRONMENT: they would + check deprecated (if not undefined) behaviour now. Strengthen a + couple of still valid checks, to keep the test more in sync with + the documentation. Improve debugging information. + * tests/Makefile.am (TESTS): Update. + 2011-07-22 Stefano Lattarini docs: rework and extend documentation on testsuites support diff --git a/NEWS b/NEWS index 4614297..620fe2e 100644 --- a/NEWS +++ b/NEWS @@ -72,6 +72,9 @@ New in 1.11a: Consistently with the existing syntax, this can be done by defining special makefile variables `LOG_DRIVER' and `_LOG_DRIVER'. + - A new developer-reserved variable `AM_TESTS_FD_REDIRECT' can be used + to redirect/define file descriptors used by the test scripts. + * WARNING: Future backward-incompatibilities! - The Automake support for automatic de-ANSI-fication will be removed in diff --git a/doc/automake.texi b/doc/automake.texi index a47e46d..e1a1ca3 100644 --- a/doc/automake.texi +++ b/doc/automake.texi @@ -8868,6 +8868,19 @@ user, which can employ it to extend or override the settings in the former; for this to work portably, however, the contents of a non-empty @code{AM_TESTS_ENVIRONMENT} @emph{must} be terminated by a semicolon. +@vindex AM_TESTS_FD_REDIRECT +The @code{AM_TESTS_FD_REDIRECT} variable can be used to define file +descriptor redirections for the test scripts. One might think that +@code{AM_TESTS_ENVIRONMENT} could be used for this purpose, but experience +has shown that doing so portably is practically impossible. The main +hurdle is constituted by Korn shells, which usually set the close-on-exec +flag on file descriptors opened with the @command{exec} builtin, thus +rendering an idiom like @code{AM_TESTS_ENVIRONMENT = exec 9>&2;} +ineffectual. This issue also affects some Bourne shells, such as the +HP-UX's @command{/bin/sh}, +@c FIXME: should we offer a link to the relevant discussions on the +@c bug-autoconf list? + @example AM_TESTS_ENVIRONMENT = \ ## Some environment initializations are kept in a separate shell file @@ -8880,6 +8893,11 @@ AM_TESTS_ENVIRONMENT = \ PATH=/usr/xpg4/bin:$$PATH; export PATH; \ fi; @c $$ restore font-lock +## With this, the test scripts will be able to print diagnostic messages +## to the original standard error stream, even if the test driver +## redirects the stderr of the test scripts to a log file before executing +## them. +AM_TESTS_FD_REDIRECT = 9>&2 @end example @noindent @@ -9229,8 +9247,9 @@ support for @code{check_*} variables (@code{check_PROGRAMS}, use of @code{VERBOSE} environment variable to get verbose output on testsuite failures; @item -definition and honoring of @code{TESTS_ENVIRONMENT} and -@code{AM_TESTS_ENVIRONMENT} variables; +definition and honoring of @code{TESTS_ENVIRONMENT}, +@code{AM_TESTS_ENVIRONMENT} and @code{AM_TESTS_FD_REDIRECT} +variables; @item definition of generic and extension-specific @code{LOG_COMPILER} and @code{LOG_FLAGS} variables. diff --git a/lib/Automake/tests/Makefile.in b/lib/Automake/tests/Makefile.in index 6139c8b..3777b5d 100644 --- a/lib/Automake/tests/Makefile.in +++ b/lib/Automake/tests/Makefile.in @@ -475,7 +475,7 @@ recheck recheck-html: .pl.log: @p='$<'; $(am__check_pre) \ $(PL_LOG_DRIVER) $(am__test_driver_flags) $(AM_PL_LOG_DRIVER_FLAGS) $(PL_LOG_DRIVER_FLAGS) -- \ - $(PL_LOG_COMPILE) "$$tst" + $(PL_LOG_COMPILE) "$$tst" $(AM_TESTS_FD_REDIRECT) distdir: $(DISTFILES) @srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ diff --git a/lib/am/check.am b/lib/am/check.am index 73306bc..ffb70ba 100644 --- a/lib/am/check.am +++ b/lib/am/check.am @@ -367,7 +367,7 @@ check-TESTS: $(TESTS) ## why we also try `dir=' elif test -f $$tst; then dir=; \ else dir="$(srcdir)/"; fi; \ - if $(TESTS_ENVIRONMENT) $${dir}$$tst; then \ + if $(TESTS_ENVIRONMENT) $${dir}$$tst $(AM_TESTS_FD_REDIRECT); then \ ## Success all=`expr $$all + 1`; \ case " $(XFAIL_TESTS) " in \ diff --git a/lib/am/check2.am b/lib/am/check2.am index b7ed0ee..f4fb3c9 100644 --- a/lib/am/check2.am +++ b/lib/am/check2.am @@ -19,7 +19,7 @@ ?!GENERIC?%OBJ%: %SOURCE% @p='%SOURCE%'; $(am__check_pre) \ %DRIVER% $(am__test_driver_flags) %DRIVER_FLAGS% -- \ - %COMPILE% "$$tst" + %COMPILE% "$$tst" $(AM_TESTS_FD_REDIRECT) ## If no programs are built in this package, then this rule is removed ## at automake time. Otherwise, %am__EXEEXT% expands to a configure time @@ -29,5 +29,5 @@ if %am__EXEEXT% ?GENERIC?%EXT%$(EXEEXT).log: @p='%SOURCE%'; $(am__check_pre) \ %DRIVER% $(am__test_driver_flags) %DRIVER_FLAGS% -- \ - %COMPILE% "$$tst" + %COMPILE% "$$tst" $(AM_TESTS_FD_REDIRECT) endif %am__EXEEXT% diff --git a/tests/Makefile.am b/tests/Makefile.am index dea5da8..f322054 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -238,6 +238,7 @@ check10.test \ check11.test \ check12.test \ check-exported-srcdir.test \ +check-fd-redirect.test \ check-tests-in-builddir.test \ check-tests_environment.test \ check-no-test-driver.test \ @@ -723,6 +724,7 @@ parallel-tests7.test \ parallel-tests8.test \ parallel-tests9.test \ parallel-tests10.test \ +parallel-tests-fd-redirect.test \ parallel-tests-am_tests_environment.test \ parallel-tests-unreadable-log.test \ parallel-tests-subdir.test \ diff --git a/tests/Makefile.in b/tests/Makefile.in index 3f7a585..0ce3dd9 100644 --- a/tests/Makefile.in +++ b/tests/Makefile.in @@ -277,12 +277,13 @@ XFAIL_TESTS = all.test auxdir2.test cond17.test gcj6.test \ tap-plan-corner2.test tap-message-0.test tap-signal.test \ txinfo5.test $(instspc_xfail_tests) parallel_tests = backcompat5-p.test check-exported-srcdir-p.test \ - check-tests-in-builddir-p.test check-tests_environment-p.test \ - check-p.test check11-p.test check12-p.test check2-p.test \ - check3-p.test check4-p.test check5-p.test check6-p.test \ - check7-p.test check8-p.test check9-p.test color-p.test \ - color2-p.test comment9-p.test dejagnu-p.test exeext4-p.test \ - maken3-p.test maken4-p.test posixsubst-tests-p.test + check-fd-redirect-p.test check-tests-in-builddir-p.test \ + check-tests_environment-p.test check-p.test check11-p.test \ + check12-p.test check2-p.test check3-p.test check4-p.test \ + check5-p.test check6-p.test check7-p.test check8-p.test \ + check9-p.test color-p.test color2-p.test comment9-p.test \ + dejagnu-p.test exeext4-p.test maken3-p.test maken4-p.test \ + posixsubst-tests-p.test instspc_tests = instspc-squote-build.test instspc-squote-install.test \ instspc-dquote-build.test instspc-dquote-install.test \ instspc-bquote-build.test instspc-bquote-install.test \ @@ -482,6 +483,7 @@ check10.test \ check11.test \ check12.test \ check-exported-srcdir.test \ +check-fd-redirect.test \ check-tests-in-builddir.test \ check-tests_environment.test \ check-no-test-driver.test \ @@ -967,6 +969,7 @@ parallel-tests7.test \ parallel-tests8.test \ parallel-tests9.test \ parallel-tests10.test \ +parallel-tests-fd-redirect.test \ parallel-tests-am_tests_environment.test \ parallel-tests-unreadable-log.test \ parallel-tests-subdir.test \ @@ -1582,7 +1585,7 @@ recheck recheck-html: .test.log: @p='$<'; $(am__check_pre) \ $(TEST_LOG_DRIVER) $(am__test_driver_flags) $(AM_TEST_LOG_DRIVER_FLAGS) $(TEST_LOG_DRIVER_FLAGS) -- \ - $(TEST_LOG_COMPILE) "$$tst" + $(TEST_LOG_COMPILE) "$$tst" $(AM_TESTS_FD_REDIRECT) distdir: $(DISTFILES) @srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ @@ -1736,6 +1739,7 @@ uninstall-am: backcompat5-p.log: backcompat5.test check-exported-srcdir-p.log: check-exported-srcdir.test +check-fd-redirect-p.log: check-fd-redirect.test check-tests-in-builddir-p.log: check-tests-in-builddir.test check-tests_environment-p.log: check-tests_environment.test check-p.log: check.test diff --git a/tests/check-fd-redirect.test b/tests/check-fd-redirect.test new file mode 100755 index 0000000..6da8704 --- /dev/null +++ b/tests/check-fd-redirect.test @@ -0,0 +1,110 @@ +#! /bin/sh +# Copyright (C) 2011 Free Software Foundation, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# Simple Tests support: redirection of file descriptors with +# AM_TESTS_FD_REDIRECT. +# See also related test 'parallel-tests-fd-redirect.test'. + +. ./defs || Exit 1 + +cat >> configure.in << 'END' +AC_OUTPUT +END + +cat > Makefile.am <<'END' +TESTS = foo.test +AM_TESTS_FD_REDIRECT = 3four 5>>five 7<&0 8>&1 9>&2 +END + +echo '3333' > three +chmod a-w three + +: > foo.test +chmod a+x foo.test + +$ACLOCAL +$AUTOCONF +$AUTOMAKE -a +./configure + +do_check () +{ + cat foo.test # For debugging. + echo 'this line will be removed' > four + echo 'this line will not be removed' > five + st=0 + echo 'ok ok ok' | $MAKE check >stdout 2>stderr || st=1 + cat stdout + cat stderr >&2 + cat four + test x"$parallel_tests" != x"yes" || cat foo.log + test $st -eq 0 + grep '[ /]foo\.test: foofoofoo$' stdout + grep '[ /]foo\.test: barbarbar$' stderr + grep 'this line' four && Exit 1 + grep '^3333$' four + grep '^this line will not be removed$' five + grep '^ok ok ok$' five + $EGREP '(foofoofoo|barbarbar|3333|ok ok ok|this line)' foo.log && Exit 1 + : +} + +# Try using both shell script and a perl script as the test, for +# better coverage. + +cat > foo.test <<'END' +#! /bin/sh +set -e + +read FOO <&3 +test 3333 -eq "$FOO" +echo "$FOO" >&4 + +grep '^ok ok ok$' <&7 >&5 + +echo " " $0: foofoofoo >&8 +echo " " $0: barbarbar >&9 +END + +do_check + +echo "#! $PERL -w" > foo.test +cat >> foo.test <<'END' +use warnings FATAL => 'all'; +use strict; + +open (FD3, "<&=3") or die "opening FD3: $!"; +open (FD4, ">&=4") or die "opening FD4: $!"; +open (FD5, ">&=5") or die "opening FD5: $!"; +open (FD7, "<&=7") or die "opening FD7: $!"; +open (FD8, ">&=8") or die "opening FD8: $!"; +open (FD9, ">&=9") or die "opening FD9: $!"; + +chomp (my $FOO = ); +die "$FOO != 3333" if not $FOO eq "3333"; +print FD4 "$FOO\n"; + +chomp ($_ = ); +die "$_ != 'ok ok ok'" if not $_ eq 'ok ok ok'; +print FD5 "$_\n"; + +print FD8 " $0: foofoofoo\n"; +print FD9 " $0: barbarbar\n"; +END + +do_check + +: diff --git a/tests/parallel-tests-am_tests_environment.test b/tests/parallel-tests-am_tests_environment.test index 072e614..436b6df 100755 --- a/tests/parallel-tests-am_tests_environment.test +++ b/tests/parallel-tests-am_tests_environment.test @@ -31,27 +31,37 @@ cat > Makefile.am << 'END' TEST_EXTENSIONS = .sh .test TESTS = foo.test bar.sh SH_LOG_COMPILER = sh -AM_TESTS_ENVIRONMENT = $(MKDIR_P) quux.dir; exec 9>&2; fd=9; export fd; -EXTRA_DIST = $(TESTS) +AM_TESTS_ENVIRONMENT = \ + $(MKDIR_P) quux.dir; \ + if test -f $(srcdir)/test-env.sh; then \ + . $(srcdir)/test-env.sh; \ + fi; \ + FOO=1; export FOO; +EXTRA_DIST = $(TESTS) test-env.sh END cat > foo.test << 'END' #! /bin/sh -echo "FOO='$FOO'" -echo "BAR='$BAR'" -test x"$FOO" = x"$BAR" +ls -l && test -d quux.dir END chmod a+x foo.test cat > bar.sh << 'END' -: ${fd=2} -eval "echo 'diagnostic message from test bar' >&${fd}" +echo "FOO='$FOO'" +echo "BAR='$BAR'" +test x"$FOO" = x"$BAR" END -minicheck () +debug_info () { + cat test-suite.log cat foo.log cat bar.log +} + +minicheck () +{ + debug_info test -d quux.dir } @@ -67,31 +77,17 @@ $AUTOMAKE -a ./configure -FOO=1; export FOO - -TESTS_ENVIRONMENT='BAR=1' $MAKE check || { cat test-suite.log; Exit 1; } -minicheck -miniclean -TESTS_ENVIRONMENT='BAR=2' $MAKE check && { cat test-suite.log; Exit 1; } +TESTS_ENVIRONMENT='BAR=1' $MAKE check || { debug_info; Exit 1; } minicheck miniclean -BAR=$FOO; export BAR - -$MAKE check 2>stderr || { cat stderr >&2; Exit1; } -cat stderr >&2 +TESTS_ENVIRONMENT='BAR=2' $MAKE check && { debug_info; Exit 1; } minicheck -grep 'diagnostic message from test bar' stderr -grep 'diagnostic message from test bar' bar.log && Exit 1 miniclean -TESTS_ENVIRONMENT='fd=2' $MAKE check 2>stderr || { cat stderr >&2; Exit1; } -cat stderr >&2 +echo 'BAR=1 && export BAR' > test-env.sh +$MAKE check || { debug_info; Exit 1; } minicheck -grep 'diagnostic message from test bar' stderr && Exit 1 -grep 'diagnostic message from test bar' bar.log -miniclean - $MAKE distcheck : diff --git a/tests/parallel-tests-fd-redirect.test b/tests/parallel-tests-fd-redirect.test new file mode 100755 index 0000000..e00581c --- /dev/null +++ b/tests/parallel-tests-fd-redirect.test @@ -0,0 +1,109 @@ +#! /bin/sh +# Copyright (C) 2011 Free Software Foundation, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# parallel-tests support: redirection of file descriptors with +# AM_TESTS_FD_REDIRECT, even when using tests without suffix, or +# which are binary executables. +# We use some tricks to ensure that all code paths in `lib/am/check2.am' +# are covered, even on platforms where $(EXEEXT) would be naturally empty. +# See also the more generic test 'check-fd-redirect.test'. + +required='cc native' +parallel_tests=yes +. ./defs || Exit 1 + +cat >> configure.in << 'END' +AC_PROG_CC +AM_CONDITIONAL([real_EXEEXT], [test -n "$EXEEXT"]) +test -n "$EXEEXT" || EXEEXT=.bin +AC_OUTPUT +END + +cat > Makefile.am << 'END' +AM_TESTS_FD_REDIRECT = 9>&1 +TEST_EXTENSIONS = .test .sh +TESTS = foo.sh bar $(check_PROGRAMS) +check_PROGRAMS = baz qux.test +qux_test_SOURCES = zardoz.c + +## Sanity check. +if !real_EXEEXT +check-local: + test -f baz.bin + test -f qux.test.bin +endif +END + +$ACLOCAL +$AUTOCONF +$AUTOMAKE -a + +cat > foo.sh <<'END' +#!/bin/sh +echo " foofoofoo" >&9 +END +chmod a+x foo.sh + +cat > bar <<'END' +#!/bin/sh +echo " barbarbar" >&9 +END + +chmod a+x foo.sh bar + +cat > baz.c <<'END' +#include +int main (void) +{ + write (9, " bazbazbaz\n", 11); + return 0; +} +END + +cat > zardoz.c <<'END' +#include +int main (void) +{ + write (9, " quxquxqux\n", 11); + return 0; +} +END + +./configure + +# Sanity checks. +st=0 +grep '^bar\.log:.*bar' Makefile || st=1 +grep '^baz\.log:.*baz\$(EXEEXT)' Makefile || st=1 +grep '^\.test\$(EXEEXT)\.log:' Makefile || st=1 +$EGREP '^(foo|qux)\.log:' Makefile && st=1 +test $st -eq 0 || fatal_ "doesn't cover expected code paths" + +st=0 +$MAKE check >stdout || st=1 +cat stdout +cat foo.log +cat bar.log +cat baz.log +cat qux.log +test $st -eq 0 +grep "^ foofoofoo$" stdout +grep "^ barbarbar$" stdout +grep "^ bazbazbaz$" stdout +grep "^ quxquxqux$" stdout +$EGREP '(foofoofoo|barbarbar|bazbazbaz|quxquxqux)' *.log && Exit 1 + +: -- 2.7.4