From 88785561eaf67250a97fc85a6dab15b6ceb83b41 Mon Sep 17 00:00:00 2001 From: David Steele Date: Mon, 1 Feb 2016 16:26:31 +0000 Subject: [PATCH] Upgraded test harness A new Serial mode has been added to the test harness that will run test cases in serial without using Testkit Lite. This executes all the dali test cases in ~35 seconds, and dumps the test case output to stdout/stderr. It can be run using "execute.sh -S" Have changed the Parallel mode to always re-run failing test cases, causing it to display the failing checks. All types of execution now write to summary.xml, and display a human readable version of it before exiting. execute.sh's exit status has been changed - 0 means no failures, 1 means there was a test case failure. Ensured TestGLAbstraction initialization covers all member variables. Change-Id: I14c46b190f0ab4c832746971f39a4ecce0eee03d Signed-off-by: David Steele --- automated-tests/README.md | 23 +++--- automated-tests/build.sh | 3 + automated-tests/execute.sh | 84 +++++++++++++++----- automated-tests/scripts/output_summary.pl | 78 +++++++++++++++++++ automated-tests/scripts/summarize.pl | 2 + .../src/dali-devel/tct-dali-devel-core.cpp | 17 +++- .../src/dali-devel/utc-Dali-Context.cpp | 2 +- .../src/dali-internal/tct-dali-internal-core.cpp | 17 +++- .../dali-test-suite-utils/test-gl-abstraction.cpp | 20 +++-- .../dali/dali-test-suite-utils/test-harness.cpp | 91 +++++++++++++++------- .../src/dali/dali-test-suite-utils/test-harness.h | 3 +- automated-tests/src/dali/tct-dali-core.cpp | 17 +++- 12 files changed, 280 insertions(+), 77 deletions(-) create mode 100755 automated-tests/scripts/output_summary.pl diff --git a/automated-tests/README.md b/automated-tests/README.md index 65876e8..8beb8b0 100644 --- a/automated-tests/README.md +++ b/automated-tests/README.md @@ -80,29 +80,34 @@ Or without cleaning down the build area (Useful for fast build/run/debug cycles) Executing the tests ------------------- +To see a list of all of the options: + + ./execute.sh -h + To execute tests, cd into automated-tests and run ./execute.sh This will execute dali and dali-internal test sets. Note that the output summary for the first will be printed before running the second. -By default the tests execute in parallel, which is faster but does not produce any test case output files. Use this to execute the tests in series: +By default the tests execute in parallel, which is faster but does not produce any test case output files. Use this to execute the tests in series and log test output to stdout/err - ./execute.sh -s + ./execute.sh -S -To see the results, copy the style folder from web-tct_2.2.1_r1/tools/tct-mgr/style into automated-tests and run +To use test kit lite, (which is very slow), - firefox --new-window summary.xml + ./execute.sh -s -To see a list of all of the options: +To see the test kit lite results, copy the style folder from web-tct_2.2.1_r1/tools/tct-mgr/style into automated-tests and run - ./execute.sh -h + firefox --new-window summary.xml To execute a subset of tests, you can run individual test sets, e.g. ./execute.sh dali -To get coverage output, run +To get coverage output (you need to first build dali libraries with +--coverage), run ./coverage.sh @@ -193,10 +198,10 @@ On desktop, you can debug the tests by running gdb on the test program: replace `` with the name of the failing testcase. -For example, using testcase UtcDaliNinePatch01 from the dali-core test suite: +For example, using testcase UtcDaliActorAddP from the dali-core test suite: $ gdb build/src/dali/tct-dali-core - gdb> r UtcDaliNinePatch01 + gdb> r UtcDaliActorAddP On target, you can re-install the test RPM and associated debug RPMs manually using diff --git a/automated-tests/build.sh b/automated-tests/build.sh index 7c7584b..2f92135 100755 --- a/automated-tests/build.sh +++ b/automated-tests/build.sh @@ -48,3 +48,6 @@ else fi done fi + +echo "Build succeeded" +exit 0 diff --git a/automated-tests/execute.sh b/automated-tests/execute.sh index 04dbd1f..7d430c9 100755 --- a/automated-tests/execute.sh +++ b/automated-tests/execute.sh @@ -1,6 +1,6 @@ #!/bin/bash -TEMP=`getopt -o hsr --long help,serial,rerun -n 'execute.sh' -- "$@"` +TEMP=`getopt -o hsSm --long help,serial,tct,modules -n 'execute.sh' -- "$@"` if [ $? != 0 ] ; then echo "Terminating..." >&2 ; exit 1 ; fi @@ -9,34 +9,63 @@ eval set -- "$TEMP" function usage { - echo -e "Usage: execute.sh\t\tExecute test cases from all modules in parallel" - echo -e " execute.sh \tExecute test cases from the given module in parallel" - echo -e " execute.sh -s\t\tExecute test cases in serial using Testkit-Lite" - echo -e " execute.sh -r\t\tExecute test cases in parallel, re-running failed test cases in serial afterwards" + echo -e "Usage: execute.sh [-s|-S|-r] [module|testcase]" + echo -e " execute.sh\t\tExecute test cases from all modules in parallel" + echo -e " execute.sh [module]\tExecute test cases from the given module in parallel" + echo -e " execute.sh -s [module]\t\tExecute test cases in serial using Testkit-Lite" + echo -e " execute.sh -S [module]\t\tExecute test cases in serial" echo -e " execute.sh \tFind and execute the given test case" exit 2 } -opt_serial=0 -opt_rerun="" +opt_tct=0 +opt_serial="" +opt_modules=0 while true ; do case "$1" in -h|--help) usage ;; - -s|--serial) opt_serial=1 ; shift ;; - -r|--rerun) opt_rerun="-r" ; shift ;; + -s|--tct) opt_tct=1 ; shift ;; + -S|--serial) opt_serial="-s" ; shift ;; + -m|--modules) opt_modules=1 ; shift ;; --) shift; break;; *) echo "Internal error $1!" ; exit 1 ;; esac done -function execute +function execute_tct { scripts/tctestsgen.sh $1 `pwd` desktop $2 testkit-lite -f `pwd`/tests.xml -o tct-${1}-core-tests.xml -A --comm localhost scripts/add_style.pl $1 } +function summary_start +{ + start=`date +"%Y-%m-%d_%H_%M_%S"` + cat > summary.xml < + + + + + $start + $start + +EOF +} + +function summary_end +{ + cat >> summary.xml < +EOF +} +if [ $opt_modules == 1 ] ; then + modules= get_modules + echo $modules + exit 0 +fi # Clean up old test results rm -f tct*core-tests.xml @@ -51,47 +80,56 @@ find build \( -name "*.gcda" \) -exec rm '{}' \; ASCII_BOLD="\e[1m" ASCII_RESET="\e[0m" -if [ $opt_serial = 1 ] ; then +modules=`ls -1 src/ | grep -v CMakeList | grep -v common | grep -v manual` +if [ -f summary.xml ] ; then unlink summary.xml ; fi + +if [ $opt_tct == 1 ] ; then + # Use Test-kit lite # Run all test case executables serially, create XML output if [ -n "$1" ] ; then - execute $1 $* + execute_tct $1 $* else - for mod in `ls -1 src/ | grep -v CMakeList ` + for mod in $modules do if [ $mod != 'common' ] && [ $mod != 'manual' ]; then echo -ne "$ASCII_BOLD" echo -e "Executing $mod$ASCII_RESET" - execute $mod $* + execute_tct $mod $* fi done fi scripts/summarize.pl + else - # if $1 is an executable filename, execute it· + # Execute test cases using own test harness if [ -z "$1" ] ; then # No arguments: - # Execute each test executable in turn, using parallel execution - for mod in `ls -1 src/ | grep -v CMakeList | grep -v common | grep -v manual` + # Execute each test executable in turn (by default, runs tests in parallel) + summary_start + for mod in $modules do echo -e "$ASCII_BOLD" echo -e "Executing $mod$ASCII_RESET" - build/src/$mod/tct-$mod-core $opt_rerun + build/src/$mod/tct-$mod-core -r $opt_serial done + summary_end elif [ -f "build/src/$1/tct-$1-core" ] ; then # First argument is an executable filename - execute only that with any # remaining arguments + summary_start module=$1 shift; - build/src/$module/tct-$module-core $opt_rerun $* + build/src/$module/tct-$module-core -r $opt_serial $* + summary_end else # First argument is not an executable. Is it a test case name? # Try executing each executable with the test case name until success/known failure - for mod in `ls -1 src/ | grep -v CMakeList | grep -v common | grep -v manual` + for mod in $modules do output=`build/src/$mod/tct-$mod-core $1` ret=$? @@ -104,3 +142,9 @@ else echo $1 not found fi fi + +if [ -f summary.xml ] ; then + scripts/output_summary.pl +fi + +exit $? diff --git a/automated-tests/scripts/output_summary.pl b/automated-tests/scripts/output_summary.pl new file mode 100755 index 0000000..dec392d --- /dev/null +++ b/automated-tests/scripts/output_summary.pl @@ -0,0 +1,78 @@ +#!/usr/bin/perl + +# Reads summary.xml and produces human readable output + +use strict; +use XML::Parser; +use Encode; +use Getopt::Long; +use Cwd; + +my $pwd = getcwd; +my $text = ""; +my $module=""; +my %modules=(); + +sub handle_start +{ + my ($p, $elt, %attrs) = @_; + + if($elt =~ /suite/) + { + $module=$attrs{"name"}; + } + if($elt =~ /_case/) + { + $text = ""; + } +} + +sub handle_end +{ + my ($p, $elt) = @_; + if($elt =~ /pass_case/) + { + $modules{$module}->{"pass"}=$text; + $text=""; + } + elsif($elt =~ /fail_case/) + { + $modules{$module}->{"fail"}=$text; + $text=""; + } +} + +sub handle_char +{ + my ($p, $str) = @_; + $text .= $str; +} + +my($parser) = new XML::Parser(Handlers => {Start => \&handle_start, + End => \&handle_end, + Char => \&handle_char}); +$parser->parsefile("summary.xml"); + +my $RED_COLOR="\e[1;31m"; +my $GREEN_COLOR="\e[1;32m"; +my $ASCII_RESET="\e[0m"; +my $ASCII_BOLD="\e[1m"; + +print "\n"; +my $totalFailures=0; +foreach $module (keys(%modules)) +{ + my $result_colour = $GREEN_COLOR; + if( $modules{$module}->{"fail"} ) + { + $result_colour = $RED_COLOR; + } + my $numPasses = $modules{$module}->{"pass"}; + my $numFailures = $modules{$module}->{"fail"}; + $totalFailures += $numFailures; + print( "$ASCII_BOLD$module results:$ASCII_RESET\n" ); + printf("Number of test passes: %s%4d (%5.2f%%)%s\n", $ASCII_BOLD, $numPasses, 100.0 * $numPasses / ($numPasses+$numFailures), $ASCII_RESET); + printf("%sNumber of test failures:%s %s%4d%s\n\n", $result_colour, $ASCII_RESET, $ASCII_BOLD, $numFailures, $ASCII_RESET); +} + +exit $totalFailures == 0; diff --git a/automated-tests/scripts/summarize.pl b/automated-tests/scripts/summarize.pl index c90eb89..8270376 100755 --- a/automated-tests/scripts/summarize.pl +++ b/automated-tests/scripts/summarize.pl @@ -1,5 +1,7 @@ #!/usr/bin/perl +# Generates an XML summary of test cases from Test-kit lite output XML. + use strict; use XML::Parser; use Encode; diff --git a/automated-tests/src/dali-devel/tct-dali-devel-core.cpp b/automated-tests/src/dali-devel/tct-dali-devel-core.cpp index 69d3233..4553f75 100644 --- a/automated-tests/src/dali-devel/tct-dali-devel-core.cpp +++ b/automated-tests/src/dali-devel/tct-dali-devel-core.cpp @@ -8,8 +8,9 @@ int main(int argc, char * const argv[]) { int result = TestHarness::EXIT_STATUS_BAD_ARGUMENT; - const char* optString = "r"; - bool optRerunFailed(false); + const char* optString = "rs"; + bool optRerunFailed(true); + bool optRunSerially(false); int nextOpt = 0; do @@ -20,6 +21,9 @@ int main(int argc, char * const argv[]) case 'r': optRerunFailed = true; break; + case 's': + optRunSerially = true; + break; case '?': TestHarness::Usage(argv[0]); exit(TestHarness::EXIT_STATUS_BAD_ARGUMENT); @@ -29,7 +33,14 @@ int main(int argc, char * const argv[]) if( optind == argc ) // no testcase name in argument list { - result = TestHarness::RunAllInParallel(argv[0], tc_array, optRerunFailed); + if( optRunSerially ) + { + result = TestHarness::RunAll( argv[0], tc_array ); + } + else + { + result = TestHarness::RunAllInParallel( argv[0], tc_array, optRerunFailed ); + } } else { diff --git a/automated-tests/src/dali-devel/utc-Dali-Context.cpp b/automated-tests/src/dali-devel/utc-Dali-Context.cpp index 09cdcd9..e3adc4b 100644 --- a/automated-tests/src/dali-devel/utc-Dali-Context.cpp +++ b/automated-tests/src/dali-devel/utc-Dali-Context.cpp @@ -27,7 +27,7 @@ using namespace Dali; namespace { // Size of the VertexAttributeArray enables -// GLES specification states that there's minimum of +// GLES specification states that there's a minimum of 8 const unsigned int TEST_MAX_ATTRIBUTE_CACHE_SIZE = 8; enum TestAttribType diff --git a/automated-tests/src/dali-internal/tct-dali-internal-core.cpp b/automated-tests/src/dali-internal/tct-dali-internal-core.cpp index 359a7d2..8c734e0 100644 --- a/automated-tests/src/dali-internal/tct-dali-internal-core.cpp +++ b/automated-tests/src/dali-internal/tct-dali-internal-core.cpp @@ -8,8 +8,9 @@ int main(int argc, char * const argv[]) { int result = TestHarness::EXIT_STATUS_BAD_ARGUMENT; - const char* optString = "r"; - bool optRerunFailed(false); + const char* optString = "rs"; + bool optRerunFailed(true); + bool optRunSerially(false); int nextOpt = 0; do @@ -20,6 +21,9 @@ int main(int argc, char * const argv[]) case 'r': optRerunFailed = true; break; + case 's': + optRunSerially = true; + break; case '?': TestHarness::Usage(argv[0]); exit(TestHarness::EXIT_STATUS_BAD_ARGUMENT); @@ -29,7 +33,14 @@ int main(int argc, char * const argv[]) if( optind == argc ) // no testcase name in argument list { - result = TestHarness::RunAllInParallel(argv[0], tc_array, optRerunFailed); + if( optRunSerially ) + { + result = TestHarness::RunAll( argv[0], tc_array ); + } + else + { + result = TestHarness::RunAllInParallel( argv[0], tc_array, optRerunFailed ); + } } else { diff --git a/automated-tests/src/dali/dali-test-suite-utils/test-gl-abstraction.cpp b/automated-tests/src/dali/dali-test-suite-utils/test-gl-abstraction.cpp index d8af042..4a12bdb 100644 --- a/automated-tests/src/dali/dali-test-suite-utils/test-gl-abstraction.cpp +++ b/automated-tests/src/dali/dali-test-suite-utils/test-gl-abstraction.cpp @@ -43,24 +43,19 @@ void TestGlAbstraction::Initialize() mIsRenderbufferResult = 0; mIsShaderResult = 0; mIsTextureResult = 0; - mVertexAttribArrayChanged = false; - + mActiveTextureUnit = 0; mCheckFramebufferStatusResult = 0; mFramebufferStatus = 0; mFramebufferColorAttached = 0; mFramebufferDepthAttached = 0; mFramebufferStencilAttached = 0; - mNumBinaryFormats = 0; mBinaryFormats = 0; mProgramBinaryLength = 0; - mGetProgramBinaryCalled = false; - mLastAutoTextureIdUsed = 0; + mVertexAttribArrayChanged = false; + mGetProgramBinaryCalled = false; - mLastShaderIdUsed = 0; - mLastProgramIdUsed = 0; - mLastUniformIdUsed = 0; mLastShaderCompiled = 0; mLastClearBitMask = 0; mClearCount = 0; @@ -71,6 +66,10 @@ void TestGlAbstraction::Initialize() mLastBlendFuncDstRgb = 0; mLastBlendFuncSrcAlpha = 0; mLastBlendFuncDstAlpha = 0; + mLastAutoTextureIdUsed = 0; + mLastShaderIdUsed = 0; + mLastProgramIdUsed = 0; + mLastUniformIdUsed = 0; mUniforms.clear(); mProgramUniforms1i.clear(); @@ -78,6 +77,11 @@ void TestGlAbstraction::Initialize() mProgramUniforms2f.clear(); mProgramUniforms3f.clear(); mProgramUniforms4f.clear(); + + for( unsigned int i=0; i RunningTestCases; -namespace +const char* basename(const char* path) { -const char* RED_COLOR="\e[1;31m"; -const char* GREEN_COLOR="\e[1;32m"; -const char* ASCII_RESET="\e[0m"; -const char* ASCII_BOLD="\e[1m"; + const char* ptr=path; + const char* slash=NULL; + for( ; *ptr != '\0' ; ++ptr ) + { + if(*ptr == '/') slash=ptr; + } + if(slash != NULL) ++slash; + return slash; } - int RunTestCase( struct ::testcase_s& testCase ) { int result = EXIT_STATUS_TESTCASE_FAILED; @@ -71,7 +74,24 @@ int RunTestCaseInChildProcess( struct ::testcase_s& testCase, bool suppressOutpu close(STDOUT_FILENO); close(STDERR_FILENO); } - exit( RunTestCase( testCase ) ); + else + { + printf("\n"); + for(int i=0; i<80; ++i) printf("#"); + printf("\nTC: %s\n", testCase.name); + fflush(stdout); + } + + int status = RunTestCase( testCase ); + + if( ! suppressOutput ) + { + fflush(stdout); + fflush(stderr); + fclose(stdout); + fclose(stderr); + } + exit( status ); } else if(pid == -1) { @@ -81,7 +101,7 @@ int RunTestCaseInChildProcess( struct ::testcase_s& testCase, bool suppressOutpu else // Parent process { int status = 0; - int childPid = waitpid(-1, &status, 0); + int childPid = waitpid(pid, &status, 0); if( childPid == -1 ) { perror("waitpid"); @@ -100,38 +120,55 @@ int RunTestCaseInChildProcess( struct ::testcase_s& testCase, bool suppressOutpu } else if(WIFSIGNALED(status) ) { + int signal = WTERMSIG(status); testResult = EXIT_STATUS_TESTCASE_ABORTED; - -#ifdef WCOREDUMP - if(WCOREDUMP(status)) + if( signal == SIGABRT ) + { + printf("Test case %s failed: test case asserted\n", testCase.name ); + } + else { - printf("Test case %s failed: due to a crash\n", testCase.name); + printf("Test case %s failed: exit with signal %s\n", testCase.name, strsignal(WTERMSIG(status))); } -#endif - printf("Test case %s failed: exit with signal %s\n", testCase.name, strsignal(WTERMSIG(status))); } else if(WIFSTOPPED(status)) { printf("Test case %s failed: stopped with signal %s\n", testCase.name, strsignal(WSTOPSIG(status))); } } + fflush(stdout); + fflush(stderr); return testResult; } -void OutputStatistics( int numPasses, int numFailures ) +void OutputStatistics( const char* processName, int numPasses, int numFailures ) { - const char* failureColor = GREEN_COLOR; - if( numFailures > 0 ) + FILE* fp=fopen("summary.xml", "a"); + if( fp != NULL ) { - failureColor = RED_COLOR; + fprintf( fp, + " \n" + " %d\n" + " %d\n" + " %5.2f\n" + " %d\n" + " %5.2f\n" + " 0\n" + " 0.00\n" + " 0\n" + " 0.00\n" + " \n", + basename(processName), + numPasses+numFailures, + numPasses, + (float)numPasses/(numPasses+numFailures), + numFailures, + (float)numFailures/(numPasses+numFailures) ); + fclose(fp); } - printf("\rNumber of test passes: %s%4d (%5.2f%%)%s\n", ASCII_BOLD, numPasses, 100.0f * (float)numPasses / (numPasses+numFailures), ASCII_RESET); - printf("%sNumber of test failures:%s %s%4d%s\n", failureColor, ASCII_RESET, ASCII_BOLD, numFailures, ASCII_RESET); - } - -int RunAll(const char* processName, ::testcase tc_array[], bool reRunFailed) +int RunAll( const char* processName, ::testcase tc_array[] ) { int numFailures = 0; int numPasses = 0; @@ -139,7 +176,7 @@ int RunAll(const char* processName, ::testcase tc_array[], bool reRunFailed) // Run test cases in child process( to kill output/handle signals ), but run serially. for( unsigned int i=0; tc_array[i].name; i++) { - int result = RunTestCaseInChildProcess( tc_array[i], true ); + int result = RunTestCaseInChildProcess( tc_array[i], false ); if( result == 0 ) { numPasses++; @@ -150,13 +187,11 @@ int RunAll(const char* processName, ::testcase tc_array[], bool reRunFailed) } } - OutputStatistics(numPasses, numFailures); + OutputStatistics( processName, numPasses, numFailures); return numFailures; } - - // Constantly runs up to MAX_NUM_CHILDREN processes int RunAllInParallel( const char* processName, ::testcase tc_array[], bool reRunFailed) { @@ -250,7 +285,7 @@ int RunAllInParallel( const char* processName, ::testcase tc_array[], bool reRu } } - OutputStatistics( numPasses, numFailures ); + OutputStatistics( processName, numPasses, numFailures ); if( reRunFailed ) { diff --git a/automated-tests/src/dali/dali-test-suite-utils/test-harness.h b/automated-tests/src/dali/dali-test-suite-utils/test-harness.h index e6dc517..53ac0b9 100644 --- a/automated-tests/src/dali/dali-test-suite-utils/test-harness.h +++ b/automated-tests/src/dali/dali-test-suite-utils/test-harness.h @@ -85,10 +85,9 @@ int RunAllInParallel(const char* processName, testcase tc_array[], bool reRunFai * Run all test cases in serial * @param[in] processName The name of this process * @param[in] tc_array The array of auto-generated testkit-lite test cases - * @param[in] reRunFailed True if failed test cases should be re-run * @return 0 on success */ -int RunAll(const char* processName, testcase tc_array[], bool reRunFailed); +int RunAll( const char* processName, testcase tc_array[] ); /** * Find the named test case in the given array, and run it diff --git a/automated-tests/src/dali/tct-dali-core.cpp b/automated-tests/src/dali/tct-dali-core.cpp index f8bd4de..d0a829d 100644 --- a/automated-tests/src/dali/tct-dali-core.cpp +++ b/automated-tests/src/dali/tct-dali-core.cpp @@ -8,8 +8,9 @@ int main(int argc, char * const argv[]) { int result = TestHarness::EXIT_STATUS_BAD_ARGUMENT; - const char* optString = "r"; - bool optRerunFailed(false); + const char* optString = "rs"; + bool optRerunFailed(true); + bool optRunSerially(false); int nextOpt = 0; do @@ -20,6 +21,9 @@ int main(int argc, char * const argv[]) case 'r': optRerunFailed = true; break; + case 's': + optRunSerially = true; + break; case '?': TestHarness::Usage(argv[0]); exit(TestHarness::EXIT_STATUS_BAD_ARGUMENT); @@ -29,7 +33,14 @@ int main(int argc, char * const argv[]) if( optind == argc ) // no testcase name in argument list { - result = TestHarness::RunAllInParallel(argv[0], tc_array, optRerunFailed); + if( optRunSerially ) + { + result = TestHarness::RunAll( argv[0], tc_array ); + } + else + { + result = TestHarness::RunAllInParallel( argv[0], tc_array, optRerunFailed ); + } } else { -- 2.7.4