lib/am/check.am

   1 ## automake - create Makefile.in from Makefile.am
   2 ## Copyright (C) 2001, 2003, 2006, 2007, 2008, 2009, 2010, 2011 Free
   3 ## Software Foundation, Inc.
   4
   5 ## This program is free software; you can redistribute it and/or modify
   6 ## it under the terms of the GNU General Public License as published by
   7 ## the Free Software Foundation; either version 2, or (at your option)
   8 ## any later version.
   9
  10 ## This program is distributed in the hope that it will be useful,
  11 ## but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 ## GNU General Public License for more details.
  14
  15 ## You should have received a copy of the GNU General Public License
  16 ## along with this program.  If not, see <http://www.gnu.org/licenses/>.
  17
  18 if %?COLOR%
  19 # If stdout is a non-dumb tty, use colors.  If test -t is not supported,
  20 # then this fails; a conservative approach.  Of course do not redirect
  21 # stdout here, just stderr.
  22 am__tty_colors = \
  23 red=; grn=; lgn=; blu=; std=; \
  24 test "X$(AM_COLOR_TESTS)" != Xno \
  25 && test "X$$TERM" != Xdumb \
  26 && { test "X$(AM_COLOR_TESTS)" = Xalways || test -t 1 2>/dev/null; } \
  27 && { \
  28   red='\e[0;31m'; \
  29   grn='\e[0;32m'; \
  30   lgn='\e[1;32m'; \
  31   blu='\e[1;34m'; \
  32   std='\e[m'; \
  33 }
  34 else !%?COLOR%
  35 am__tty_colors = \
  36 red=; grn=; lgn=; blu=; std=
  37 endif !%?COLOR%
  38
  39 .PHONY: check-TESTS
  40
  41 if %?PARALLEL_TESTS%
  42
  43 include inst-vars.am
  44
  45 ## New parallel test driver.
  46 ##
  47 ## This code is adapted from check.mk which was originally
  48 ## written at EPITA/LRDE, further developed at Gostai, then made
  49 ## its way from GNU coreutils to end up, largely rewritten, in
  50 ## Automake.
  51 ##
  52 ## It provides special support for "unit tests", that is to say,
  53 ## tests that (once run) no longer need to be re-compiled and
  54 ## re-run at each "make check", unless their sources changed.  To
  55 ## enable unit-test supports, set RECHECK_LOGS to empty.  In such a
  56 ## setting, that heavily relies on correct dependencies, its users may
  57 ## prefer to define EXTRA_PROGRAMS instead of check_PROGRAMS, because
  58 ## it allows intertwined compilation and execution of the tests.
  59 ## Sometimes this helps catching errors earlier (you don't have to
  60 ## wait for all the tests to be compiled).
  61 ##
  62 ## Define TEST_SUITE_LOG to be the name of the global log to create.
  63 ## Define TEST_LOGS to the set of logs to include in it.  It defaults
  64 ## to $(TESTS), with `.test' and `@EXEEXT@' removed, and `'.log'
  65 ## appended.
  66 ##
  67 ## In addition to the magic "exit 77 means SKIP" feature (which was
  68 ## imported from automake), there is a magic "exit 99 means FAIL" feature
  69 ## which is useful if you need to issue a hard error no matter whether the
  70 ## test is XFAIL or not.  You can disable this feature by setting the
  71 ## variable DISABLE_HARD_ERRORS to a nonempty value.
  72
  73 # Restructured Text title and section.
  74 am__rst_title   = sed 's/.*/   &   /;h;s/./=/g;p;x;p;g;p;s/.*//'
  75 am__rst_section = sed 'p;s/./=/g;p;g'
  76
  77 # Put stdin (possibly several lines separated by ".  ") in a box.
  78 # Prefix each line by 'col' and terminate each with 'std', for coloring.
  79 # Multi line coloring is problematic with "less -R", so we really need
  80 # to color each line individually.
  81 am__text_box = $(AWK) '{                        \
  82   n = split($$0, lines, "\\.  "); max = 0;      \
  83   for (i = 1; i <= n; ++i)                      \
  84     if (max < length(lines[i]))                 \
  85       max = length(lines[i]);                   \
  86   for (i = 0; i < max; ++i)                     \
  87     line = line "=";                            \
  88   print col line std;                           \
  89   for (i = 1; i <= n; ++i)                      \
  90     if (lines[i])                               \
  91       print col lines[i] std;                   \
  92   print col line std;                           \
  93 }'
  94
  95 # Solaris 10 'make', and several other traditional 'make' implementations,
  96 # pass "-e" to $(SHELL), and POSIX 2008 even requires this.  Work around it
  97 # by disabling -e (using the XSI extension "set +e") if it's set.
  98 am__sh_e_setup = case $$- in *e*) set +e;; esac
  99
 100 # To be inserted before the command running the test.  Creates the
 101 # directory for the log if needed.  Stores in $dir the directory
 102 # containing $f, in $tst the test, in $log the log.  Executes the
 103 # developer- defined test setup AM_TESTS_ENVIRONMENT (if any), and
 104 # passes TESTS_ENVIRONMENT.  Saves and restores TERM around uses of
 105 # TESTS_ENVIRONMENT and AM_TESTS_ENVIRONMENT, in case any of them
 106 # unsets it.
 107 am__check_pre =                                         \
 108 $(am__sh_e_setup);                                      \
 109 $(am__vpath_adj_setup) $(am__vpath_adj)                 \
 110 srcdir=$(srcdir); export srcdir;                        \
 111 rm -f $@-t;                                             \
 112 am__trap='rm -f '\''$(abs_builddir)/$@-t'\''; (exit $$st); exit $$st'; \
 113 trap "st=129; $$am__trap" 1; trap "st=130; $$am__trap" 2;       \
 114 trap "st=141; $$am__trap" 13; trap "st=143; $$am__trap" 15; \
 115 am__odir=`echo "./$@" | sed 's|/[^/]*$$||'`;            \
 116 test "x$$am__odir" = x. || $(MKDIR_P) "$$am__odir" || exit $$?; \
 117 if test -f "./$$f"; then dir=./;                        \
 118 elif test -f "$$f"; then dir=;                          \
 119 else dir="$(srcdir)/"; fi;                              \
 120 tst=$$dir$$f; log='$@'; __SAVED_TERM=$$TERM;            \
 121 $(AM_TESTS_ENVIRONMENT)                                 \
 122 $(TESTS_ENVIRONMENT)
 123
 124 # To be appended to the command running the test.  Handle the stdout
 125 # and stderr redirection, and catch the exit status.
 126 am__check_post =                                        \
 127 >$@-t 2>&1;                                             \
 128 estatus=$$?;                                            \
 129 if test -n '$(DISABLE_HARD_ERRORS)'                     \
 130    && test $$estatus -eq 99; then                       \
 131   estatus=1;                                            \
 132 fi;                                                     \
 133 TERM=$$__SAVED_TERM; export TERM;                       \
 134 $(am__tty_colors);                                      \
 135 xfailed=PASS;                                           \
 136 case " $(XFAIL_TESTS) " in                              \
 137   *[\ \ ]$$f[\ \        ]* | *[\ \      ]$$dir$$f[\ \   ]*) \
 138     xfailed=XFAIL;;                                     \
 139 esac;                                                   \
 140 case $$estatus.$$xfailed in                             \
 141     0.XFAIL) col=$$red; res=XPASS;;                     \
 142     0.*)     col=$$grn; res=PASS ;;                     \
 143     77.*)    col=$$blu; res=SKIP ;;                     \
 144     99.*)    col=$$red; res=FAIL ;;                     \
 145     *.XFAIL) col=$$lgn; res=XFAIL;;                     \
 146     *.*)     col=$$red; res=FAIL ;;                     \
 147 esac;                                                   \
 148 echo "$${col}$$res$${std}: $$f";                        \
 149 echo "$$res: $$f (exit: $$estatus)" |                   \
 150   $(am__rst_section) >$@;                               \
 151 cat $@-t >>$@;                                          \
 152 rm -f $@-t
 153
 154 $(TEST_SUITE_LOG): $(TEST_LOGS)
 155         @$(am__sh_e_setup);                                             \
 156         list='$(TEST_LOGS)';                                            \
 157         results=`for f in $$list; do                                    \
 158                    test -r $$f && read line < $$f && echo "$$line"      \
 159                      || echo FAIL;                                      \
 160                  done`;                                                 \
 161         all=`echo "$$results" | sed '/^$$/d' | wc -l | sed -e 's/^[      ]*//'`; \
 162         fail=`echo "$$results" | grep -c '^FAIL'`;                      \
 163         pass=`echo "$$results" | grep -c '^PASS'`;                      \
 164         skip=`echo "$$results" | grep -c '^SKIP'`;                      \
 165         xfail=`echo "$$results" | grep -c '^XFAIL'`;                    \
 166         xpass=`echo "$$results" | grep -c '^XPASS'`;                    \
 167         failures=`expr $$fail + $$xpass`;                               \
 168         all=`expr $$all - $$skip`;                                      \
 169         if test "$$all" -eq 1; then tests=test; All=;                   \
 170         else tests=tests; All="All "; fi;                               \
 171         case fail=$$fail:xpass=$$xpass:xfail=$$xfail in                 \
 172           fail=0:xpass=0:xfail=0)                                       \
 173             msg="$$All$$all $$tests passed.  ";                         \
 174             exit=true;;                                                 \
 175           fail=0:xpass=0:xfail=*)                                       \
 176             msg="$$All$$all $$tests behaved as expected";               \
 177             if test "$$xfail" -eq 1; then xfailures=failure;            \
 178             else xfailures=failures; fi;                                \
 179             msg="$$msg ($$xfail expected $$xfailures).  ";              \
 180             exit=true;;                                                 \
 181           fail=*:xpass=0:xfail=*)                                       \
 182             msg="$$fail of $$all $$tests failed.  ";                    \
 183             exit=false;;                                                \
 184           fail=*:xpass=*:xfail=*)                                       \
 185             msg="$$failures of $$all $$tests did not behave as expected"; \
 186             if test "$$xpass" -eq 1; then xpasses=pass;                 \
 187             else xpasses=passes; fi;                                    \
 188             msg="$$msg ($$xpass unexpected $$xpasses).  ";              \
 189             exit=false;;                                                \
 190           *)                                                            \
 191             echo >&2 "incorrect case"; exit 4;;                         \
 192         esac;                                                           \
 193         if test "$$skip" -ne 0; then                                    \
 194           if test "$$skip" -eq 1; then                                  \
 195             msg="$$msg($$skip test was not run).  ";                    \
 196           else                                                          \
 197             msg="$$msg($$skip tests were not run).  ";                  \
 198           fi;                                                           \
 199         fi;                                                             \
 200         {                                                               \
 201           echo "$(PACKAGE_STRING): $(subdir)/$(TEST_SUITE_LOG)" |       \
 202             $(am__rst_title);                                           \
 203           echo "$$msg";                                                 \
 204           echo;                                                         \
 205           echo ".. contents:: :depth: 2";                               \
 206           echo;                                                         \
 207           for f in $$list; do                                           \
 208             test -r $$f && read line < $$f || line=;                    \
 209             case $$line in                                              \
 210               PASS:*|XFAIL:*);;                                         \
 211               *) echo; cat $$f;;                                        \
 212             esac;                                                       \
 213           done;                                                         \
 214         } >$(TEST_SUITE_LOG).tmp;                                       \
 215         mv $(TEST_SUITE_LOG).tmp $(TEST_SUITE_LOG);                     \
 216         if test "$$failures" -ne 0; then                                \
 217           msg="$${msg}See $(subdir)/$(TEST_SUITE_LOG).  ";              \
 218           if test -n "$(PACKAGE_BUGREPORT)"; then                       \
 219             msg="$${msg}Please report to $(PACKAGE_BUGREPORT).  ";      \
 220           fi;                                                           \
 221         fi;                                                             \
 222         test x"$$VERBOSE" = x || $$exit || cat $(TEST_SUITE_LOG);       \
 223         $(am__tty_colors);                                              \
 224         if $$exit; then                                                 \
 225           col="$$grn";                                                  \
 226          else                                                           \
 227           col="$$red";                                                  \
 228         fi;                                                             \
 229         echo "$$msg" | $(am__text_box) "col=$$col" "std=$$std";         \
 230 ## The use of "exit 1" below is required to work around a FreeBSD make bug
 231 ## (present only when running in concurrent mode).  See automake bug#9245:
 232 ##  <http://debbugs.gnu.org/cgi/bugreport.cgi?bug=9245>
 233 ## and FreeBSD PR bin/159730:
 234 ##  <http://www.freebsd.org/cgi/query-pr.cgi?pr=159730>.
 235         $$exit || exit 1
 236
 237 RECHECK_LOGS = $(TEST_LOGS)
 238
 239 # Run all the tests.
 240 check-TESTS:
 241 ## Expand $(RECHECK_LOGS) only once, to avoid exceeding line length limits.
 242         @list='$(RECHECK_LOGS)'; test -z "$$list" || rm -f $$list
 243 ## We always have to remove TEST_SUITE_LOG, to ensure its rule is run
 244 ## in any case even in lazy mode: otherwise, if no test needs rerunning,
 245 ## or a prior run plus reruns all happen within the same timestamp
 246 ## (can happen with a prior `make TESTS=<subset>'),
 247 ## then we get no log output.
 248 ## OTOH, this means that, in the rule for `$(TEST_SUITE_LOG)', we
 249 ## cannot use `$?' to compute the set of lazily rerun tests, lest
 250 ## we rely on .PHONY to work portably.
 251 ##
 252 ## Trailing whitespace in `TESTS = foo.test $(empty)' causes GNU make
 253 ## 3.80 to erroneously expand $(TESTS_LOGS) to `foo.log .log'.
 254 ## Work around this bug.
 255         @test -z "$(TEST_SUITE_LOG)" || rm -f $(TEST_SUITE_LOG)
 256         @list='$(TEST_LOGS)';                                           \
 257         list=`for f in $$list; do                                       \
 258           test .log = $$f || echo $$f;                                  \
 259         done | tr '\012\015' '  '`;                                     \
 260         $(MAKE) $(AM_MAKEFLAGS) $(TEST_SUITE_LOG) TEST_LOGS="$$list"
 261
 262 AM_RECURSIVE_TARGETS += check
 263
 264 ## -------------- ##
 265 ## Produce HTML.  ##
 266 ## -------------- ##
 267
 268 .log.html:
 269         @list='$(RST2HTML) $$RST2HTML rst2html rst2html.py';            \
 270         for r2h in $$list; do                                           \
 271           if ($$r2h --version) >/dev/null 2>&1; then                    \
 272             R2H=$$r2h;                                                  \
 273           fi;                                                           \
 274         done;                                                           \
 275         if test -z "$$R2H"; then                                        \
 276           echo >&2 "cannot find rst2html, cannot create $@";            \
 277           exit 2;                                                       \
 278         fi;                                                             \
 279         $$R2H $< >$@.tmp
 280         @mv $@.tmp $@
 281
 282 # Be sure to run check first, and then to convert the result.
 283 # Beware of concurrent executions.  Run "check" not "check-TESTS", as
 284 # check-SCRIPTS and other dependencies are rebuilt by the former only.
 285 # And expect check to fail.
 286 check-html:
 287         @if $(MAKE) $(AM_MAKEFLAGS) check; then                 \
 288           rv=0; else rv=$$?;                                    \
 289         fi;                                                     \
 290         $(MAKE) $(AM_MAKEFLAGS) $(TEST_SUITE_HTML) || exit 4;   \
 291         exit $$rv
 292
 293 .PHONY: check-html
 294 .MAKE: check-html
 295
 296 AM_RECURSIVE_TARGETS += check-html
 297
 298 ## -------------------- ##
 299 ## Rechecking failures. ##
 300 ## -------------------- ##
 301
 302 ## Rerun all FAILed or XPASSed tests.
 303 recheck recheck-html:
 304         @target=`echo $@ | sed 's,^re,,'`;                              \
 305         list='$(TEST_LOGS)';                                            \
 306         list=`for f in $$list; do                                       \
 307                 test -f $$f || continue;                                \
 308                 if test -r $$f && read line < $$f; then                 \
 309                   case $$line in FAIL*|XPASS*) echo $$f;; esac;         \
 310                 else echo $$f; fi;                                      \
 311               done | tr '\012\015' '  '`;                               \
 312 ## This apparently useless munging helps to avoid a nasty bug (a
 313 ## segmentation fault!) on Solaris XPG4 make.
 314         list=`echo "$$list" | sed 's/ *$$//'`;                          \
 315         $(MAKE) $(AM_MAKEFLAGS) $$target AM_MAKEFLAGS='$(AM_MAKEFLAGS) TEST_LOGS="'"$$list"'"'
 316
 317 .PHONY: recheck recheck-html
 318 .MAKE: recheck recheck-html
 319
 320 AM_RECURSIVE_TARGETS += recheck recheck-html
 321
 322 else !%?PARALLEL_TESTS%
 323
 324 check-TESTS: $(TESTS)
 325         @failed=0; all=0; xfail=0; xpass=0; skip=0; \
 326         srcdir=$(srcdir); export srcdir; \
 327 ## Make sure Solaris VPATH-expands all members of this list, even
 328 ## the first and the last one; thus the spaces around $(TESTS)
 329         list=' $(TESTS) '; \
 330         $(am__tty_colors); \
 331         if test -n "$$list"; then \
 332           for tst in $$list; do \
 333             if test -f ./$$tst; then dir=./; \
 334 ## Note: Solaris 2.7 seems to expand TESTS using VPATH.  That's
 335 ## why we also try `dir='
 336             elif test -f $$tst; then dir=; \
 337             else dir="$(srcdir)/"; fi; \
 338             if $(TESTS_ENVIRONMENT) $${dir}$$tst; then \
 339 ## Success
 340               all=`expr $$all + 1`; \
 341               case " $(XFAIL_TESTS) " in \
 342               *[\ \     ]$$tst[\ \      ]*) \
 343                 xpass=`expr $$xpass + 1`; \
 344                 failed=`expr $$failed + 1`; \
 345                 col=$$red; res=XPASS; \
 346               ;; \
 347               *) \
 348                 col=$$grn; res=PASS; \
 349               ;; \
 350               esac; \
 351             elif test $$? -ne 77; then \
 352 ## Failure
 353               all=`expr $$all + 1`; \
 354               case " $(XFAIL_TESTS) " in \
 355               *[\ \     ]$$tst[\ \      ]*) \
 356                 xfail=`expr $$xfail + 1`; \
 357                 col=$$lgn; res=XFAIL; \
 358               ;; \
 359               *) \
 360                 failed=`expr $$failed + 1`; \
 361                 col=$$red; res=FAIL; \
 362               ;; \
 363               esac; \
 364             else \
 365 ## Skipped
 366               skip=`expr $$skip + 1`; \
 367               col=$$blu; res=SKIP; \
 368             fi; \
 369             echo "$${col}$$res$${std}: $$tst"; \
 370           done; \
 371 ## Prepare the banner
 372           if test "$$all" -eq 1; then \
 373             tests="test"; \
 374             All=""; \
 375           else \
 376             tests="tests"; \
 377             All="All "; \
 378           fi; \
 379           if test "$$failed" -eq 0; then \
 380             if test "$$xfail" -eq 0; then \
 381               banner="$$All$$all $$tests passed"; \
 382             else \
 383               if test "$$xfail" -eq 1; then failures=failure; else failures=failures; fi; \
 384               banner="$$All$$all $$tests behaved as expected ($$xfail expected $$failures)"; \
 385             fi; \
 386           else \
 387             if test "$$xpass" -eq 0; then \
 388               banner="$$failed of $$all $$tests failed"; \
 389             else \
 390               if test "$$xpass" -eq 1; then passes=pass; else passes=passes; fi; \
 391               banner="$$failed of $$all $$tests did not behave as expected ($$xpass unexpected $$passes)"; \
 392             fi; \
 393           fi; \
 394 ## DASHES should contain the largest line of the banner.
 395           dashes="$$banner"; \
 396           skipped=""; \
 397           if test "$$skip" -ne 0; then \
 398             if test "$$skip" -eq 1; then \
 399               skipped="($$skip test was not run)"; \
 400             else \
 401               skipped="($$skip tests were not run)"; \
 402             fi; \
 403             test `echo "$$skipped" | wc -c` -le `echo "$$banner" | wc -c` || \
 404               dashes="$$skipped"; \
 405           fi; \
 406           report=""; \
 407           if test "$$failed" -ne 0 && test -n "$(PACKAGE_BUGREPORT)"; then \
 408             report="Please report to $(PACKAGE_BUGREPORT)"; \
 409             test `echo "$$report" | wc -c` -le `echo "$$banner" | wc -c` || \
 410               dashes="$$report"; \
 411           fi; \
 412           dashes=`echo "$$dashes" | sed s/./=/g`; \
 413           if test "$$failed" -eq 0; then \
 414             col="$$grn"; \
 415           else \
 416             col="$$red"; \
 417           fi; \
 418 ## Multi line coloring is problematic with "less -R", so we really need
 419 ## to color each line individually.
 420           echo "$${col}$$dashes$${std}"; \
 421           echo "$${col}$$banner$${std}"; \
 422           test -z "$$skipped" || echo "$${col}$$skipped$${std}"; \
 423           test -z "$$report" || echo "$${col}$$report$${std}"; \
 424           echo "$${col}$$dashes$${std}"; \
 425           test "$$failed" -eq 0; \
 426         else :; fi
 427
 428 endif !%?PARALLEL_TESTS%