From 5eeb36603d197ed73a031cea70843e9b43d312b9 Mon Sep 17 00:00:00 2001 From: Stefano Lattarini Date: Sun, 26 Feb 2012 00:30:18 +0100 Subject: [PATCH] parallel-tests: optimize global log creation With this change, the time required to execute the test case 'testsuite-summary-speed.sh' has dropped as follows: + Slow Debian i686 system, 1 core x 1.5 GHz, 768 MB of RAM, GNU make 3.81: - 1 run: 4 minutes => 30/40 seconds + Fast Solaris 10 i686 system, 4 cores x 3 GHz, 20 GB of RAM, Solaris CCS make: - 2 runs: 6 minutes => 30 seconds + Very fast Fedora ppc64 system, 64 cores x 3.5 GHz, 64 GB of RAM, GNU make 3.82: - 5 runs: 7 minutes => 1 minute and 10 seconds * lib/am/check.am ($(TEST_SUITE_LOG)): Optimize for speed, by avoiding lots of forks with the help of ... (am__create_global_log): ... this new internal variable, basically defining a smart awk program, and ... (am__global_test_result_rx, am__copy_in_global_log_rx): ... these new internal variables, used by the one above. (am__rst_section): Remove as obsolete * t/rst-formatting.trs: Remove. * t/list-of-tests.mk: Update. * t/testsuite-summary-count-many.sh: Improve its stress testing by using 1 million tests rather than just 5 thousands: we can afford this with our new optimization, and still have the test completed in a reasonable time. Signed-off-by: Stefano Lattarini --- lib/am/check.am | 82 +++++++++++++++++++++++----------- t/list-of-tests.mk | 1 - t/rst-formatting.sh | 93 --------------------------------------- t/testsuite-summary-count-many.sh | 4 +- 4 files changed, 58 insertions(+), 122 deletions(-) delete mode 100755 t/rst-formatting.sh diff --git a/lib/am/check.am b/lib/am/check.am index f2bfb6a..a65fef9 100644 --- a/lib/am/check.am +++ b/lib/am/check.am @@ -56,6 +56,8 @@ include inst-vars.am ## (among them, TAP). am__recheck_rx = ^[ ]*:recheck:[ ]* +am__global_test_result_rx = ^[ ]*:global-test-result:[ ]* +am__copy_in_global_log_rx = ^[ ]*:copy-in-global-log:[ ]* # A command that, given a newline-separated list of test names on the # standard input, print the name of the tests that are to be re-run @@ -106,9 +108,59 @@ am__list_recheck_tests = $(AWK) '{ \ close ($$0 ".log"); \ }' -# Restructured Text title and section. +# A command that, given a newline-separated list of test names on the +# standard input, create the global log from their .trs and .log files. +am__create_global_log = $(AWK) ' \ +function fatal(msg) \ +{ \ + print "fatal: making $@: " msg | "cat >&2"; \ + exit 1; \ +} \ +function rst_section(header) \ +{ \ + print header; \ + len = length(header); \ + for (i = 1; i <= len; i = i + 1) \ + printf "="; \ + printf "\n\n"; \ +} \ +{ \ +## By default, we assume the test log is to be copied in the global log, +## and that its result is simply "RUN" (i.e., we still don't know what +## it outcome was, but we know that at least it has run). + copy_in_global_log = 1; \ + global_test_result = "RUN"; \ + while ((rc = (getline line < ($$0 ".trs"))) != 0) \ + { \ + if (rc < 0) \ + fatal("failed to read from " $$0 ".trs"); \ + if (line ~ /$(am__global_test_result_rx)/) \ + { \ + sub("$(am__global_test_result_rx)", "", line); \ + sub("[ ]*$$", "", line); \ + global_test_result = line; \ + } \ + else if (line ~ /$(am__copy_in_global_log_rx)[nN][oO]/) \ + copy_in_global_log = 0; \ + }; \ + if (copy_in_global_log) \ + { \ + rst_section(global_test_result ": " $$0); \ + while ((rc = (getline line < ($$0 ".log"))) != 0) \ + { \ + if (rc < 0) \ + fatal("failed to read from " $$0 ".log"); \ + print line; \ + }; \ + }; \ +## Don't leak open file descriptors, as this could cause serious +## problems when there are many tests (yes, even on Linux). + close ($$0 ".trs"); \ + close ($$0 ".log"); \ +}' + +# Restructured Text title. am__rst_title = { sed 's/.*/ & /;h;s/./=/g;p;x;s/ *$$//;p;g' && echo; } -am__rst_section = { sed 'p;s/./=/g;' && echo; } # Solaris 10 'make', and several other traditional 'make' implementations, # pass "-e" to $(SHELL), and POSIX 2008 even requires this. Work around it @@ -295,29 +347,9 @@ $(TEST_SUITE_LOG): $(TEST_LOGS) echo; \ echo ".. contents:: :depth: 2"; \ echo; \ - for i in $$bases; do \ -## FIXME: one fork per test -- this is horrendously inefficient! - if grep "^$$ws*:copy-in-global-log:$$ws*no$$ws*$$" $$i.trs \ - >/dev/null; then continue; \ - fi; \ -## Get the declared "global result" of the test. -## FIXME: yet another one fork per test here! - glob_res=`sed -n -e "s/$$ws*$$//" \ - -e "s/^$$ws*:global-test-result:$$ws*//p" \ - $$i.trs`; \ -## If no global result is explicitly declared, we'll merely mark the -## test as "RUN" in the global test log. - test -n "$$glob_res" || glob_res=RUN; \ -## Write the name and result of the test as an RST section title. - echo "$$glob_res: $$i" | $(am__rst_section); \ -## If we should have remade any unreadable '.log', above. - if test ! -r $$i.log; then \ - echo "fatal: making $@: $$i.log is unreadable" >&2; \ - exit 1; \ - fi; \ - cat $$i.log; echo; \ - done; \ - } >$(TEST_SUITE_LOG).tmp; \ + for b in $$bases; do echo $$b; done \ + | $(am__create_global_log); \ + } >$(TEST_SUITE_LOG).tmp || exit 1; \ mv $(TEST_SUITE_LOG).tmp $(TEST_SUITE_LOG); \ ## Emit the test summary on the console. if $$success; then \ diff --git a/t/list-of-tests.mk b/t/list-of-tests.mk index 2830073..ff6a668 100644 --- a/t/list-of-tests.mk +++ b/t/list-of-tests.mk @@ -934,7 +934,6 @@ t/req.sh \ t/reqd.sh \ t/reqd2.sh \ t/repeated-options.sh \ -t/rst-formatting.sh \ t/rulepat.sh \ t/self-check-cleanup.tap \ t/self-check-configure-help.sh \ diff --git a/t/rst-formatting.sh b/t/rst-formatting.sh deleted file mode 100755 index f23c28c..0000000 --- a/t/rst-formatting.sh +++ /dev/null @@ -1,93 +0,0 @@ -#! /bin/sh -# Copyright (C) 2012 Free Software Foundation, Inc. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2, or (at your option) -# any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -# Parallel testsuite harness: check APIs for the registering the -# "global test result" in '*.trs' files, as documented in the automake -# manual. - -am_create_testdir=empty -. ./defs || Exit 1 - -sed -n '/^am__rst_[a-z_][a-z_]* =/p' "$am_amdir"/check.am > Makefile \ - || framework_failure_ "fetching definitions from check.am" - -cat >> Makefile << 'END' -test: - printf '%s\n' "$$in" | $(am__rst_title) > title-got - printf '%s\n' "$$in" | $(am__rst_section) > section-got - cat title-exp - cat title-got - diff title-exp title-got - cat section-exp - cat section-got - diff section-exp section-got -END - -# ------------------------------------------------------------------------- - -cat > title-exp <<'END' -============== - ab cd ef -============== - -END - -cat > section-exp <<'END' -ab cd ef -======== - -END - -env in='ab cd ef' $MAKE test - -# ------------------------------------------------------------------------- - -cat > title-exp <<'END' -============================================================================ - 0123456789012345678901234567890123456789012345678901234567890123456789 -============================================================================ - -END - -cat > section-exp <<'END' -0123456789012345678901234567890123456789012345678901234567890123456789 -====================================================================== - -END - -in=0123456789012345678901234567890123456789012345678901234567890123456789 -env in=$in $MAKE test - -# ------------------------------------------------------------------------- - -cat > title-exp <<'END' -======= - x -======= - -END - -cat > section-exp <<'END' -x -= - -END - -env in=x $MAKE test - -# ------------------------------------------------------------------------- - -: diff --git a/t/testsuite-summary-count-many.sh b/t/testsuite-summary-count-many.sh index 58157ad..ff38097 100755 --- a/t/testsuite-summary-count-many.sh +++ b/t/testsuite-summary-count-many.sh @@ -56,9 +56,7 @@ $PERL -w -e ' use warnings FATAL => "all"; use strict; - # FIXME: we would like this to be 1000 or even 10000, but the current - # implementation is too slow to handle that :-( - my $base = 5; + my $base = 1000; my %count = ( TOTAL => $base * 1000, PASS => $base * 700, -- 2.7.4