From 62e611ced86d6cafa3b2a95bcd8d96a12ed4a426 Mon Sep 17 00:00:00 2001 From: David Steele Date: Fri, 28 Nov 2014 20:34:14 +0000 Subject: [PATCH] Updated test harness to allow parallel execution Also ensured build script stops on error WARNING! Executing in parallel bypasses TestKit-Lite and does not produce XML output. Change-Id: Ic756ebf6bc69db023494253d6a45b13419323451 Signed-off-by: David Steele --- automated-tests/build.sh | 1 + automated-tests/execute.sh | 56 +++++- .../src/dali-internal/tct-dali-internal-core.cpp | 223 +++++++++++++++++++-- .../src/dali-unmanaged/tct-dali-unmanaged-core.cpp | 223 +++++++++++++++++++-- automated-tests/src/dali/tct-dali-core.cpp | 223 +++++++++++++++++++-- 5 files changed, 664 insertions(+), 62 deletions(-) diff --git a/automated-tests/build.sh b/automated-tests/build.sh index fe0d2b4..6129590 100755 --- a/automated-tests/build.sh +++ b/automated-tests/build.sh @@ -44,6 +44,7 @@ else if [ $mod != 'common' ] && [ $mod != 'manual' ]; then echo BUILDING $mod build $mod + if [ $? -ne 0 ]; then echo "Build failed" ; exit 1; fi fi done fi diff --git a/automated-tests/execute.sh b/automated-tests/execute.sh index 95f57ed..1c77009 100755 --- a/automated-tests/execute.sh +++ b/automated-tests/execute.sh @@ -1,5 +1,23 @@ #!/bin/bash +TEMP=`getopt -o sr --long serial,rerun -n 'execute.sh' -- "$@"` + +if [ $? != 0 ] ; then echo "Terminating..." >&2 ; exit 1 ; fi + +# Note the quotes around `$TEMP': they are essential! +eval set -- "$TEMP" + +opt_parallel=1 +opt_rerun="" +while true ; do + case "$1" in + -s|--serial) opt_parallel=0 ; shift ;; + -r|--rerun) opt_rerun="-r" ; shift ;; + --) shift; break;; + *) echo "Internal error $1!" ; exit 1 ;; + esac +done + function execute { scripts/tctestsgen.sh $1 `pwd` desktop $2 @@ -17,18 +35,34 @@ fi find build \( -name "*.gcda" \) -exec rm '{}' \; -if [ -n "$1" ] ; then - echo EXECUTING ONLY $1 - execute $* +if [ $opt_parallel = 1 ] ; then + + if [ -n "$1" ] ; then + if [ -f "build/src/$1/tct-$1-core" ] ; then + build/src/$1/tct-$1-core -p $opt_rerun + fi + else + for mod in `ls -1 src/ | grep -v CMakeList ` + do + if [ $mod != 'common' ] && [ $mod != 'manual' ]; then + echo EXECUTING $mod + build/src/$mod/tct-$mod-core -p $opt_rerun + fi + done + fi else - for mod in `ls -1 src/ | grep -v CMakeList ` - do - if [ $mod != 'common' ] && [ $mod != 'manual' ]; then - echo EXECUTING $mod - execute $mod $* + if [ -n "$1" ] ; then + execute $1 $* + else + for mod in `ls -1 src/ | grep -v CMakeList ` + do + if [ $mod != 'common' ] && [ $mod != 'manual' ]; then + echo EXECUTING $mod + execute $mod $* + fi + done fi - done -fi -scripts/summarize.pl + scripts/summarize.pl +fi diff --git a/automated-tests/src/dali-internal/tct-dali-internal-core.cpp b/automated-tests/src/dali-internal/tct-dali-internal-core.cpp index f609ca4..ecefe36 100644 --- a/automated-tests/src/dali-internal/tct-dali-internal-core.cpp +++ b/automated-tests/src/dali-internal/tct-dali-internal-core.cpp @@ -1,31 +1,220 @@ #include #include #include "tct-dali-internal-core.h" +#include +#include +#include +#include +#include +#include +#include -int main(int argc, const char *argv[]) +int RunTestCase( struct testcase_s& testCase ) { - int result = -1; - int i; + int result = 1; + if( testCase.startup ) + { + testCase.startup(); + } + result = testCase.function(); + if( testCase.cleanup ) + { + testCase.cleanup(); + } + return result; +} - if (argc != 2) { - printf("Usage: %s \n", argv[0]); - return 2; - } +#define MAX_NUM_CHILDREN 16 + +struct TestCase +{ + int testCase; + const char* testCaseName; + + TestCase() + : testCase(0), + testCaseName(NULL) + { + } - for (i = 0; tc_array[i].name; i++) { - if (!strcmp(argv[1], tc_array[i].name)) { - if (tc_array[i].startup) - tc_array[i].startup(); + TestCase(int tc, const char* name) + : testCase(tc), + testCaseName(name) + { + } + TestCase(const TestCase& rhs) + : testCase(rhs.testCase), + testCaseName(rhs.testCaseName) + { + } + TestCase& operator=(const TestCase& rhs) + { + testCase = rhs.testCase; + testCaseName = rhs.testCaseName; + return *this; + + } +}; + + +typedef std::map RunningTestCases; + +// Constantly runs up to MAX_NUM_CHILDREN processes +int RunAllInParallel(const char* processName, bool reRunFailed) +{ + int numFailures = 0; + int numPasses = 0; + int numTestCases = sizeof(tc_array)/sizeof(struct testcase_s) - 1; - result = tc_array[i].function(); + RunningTestCases children; + std::vector failedTestCases; - if (tc_array[i].cleanup) - tc_array[i].cleanup(); + // Fork up to MAX_NUM_CHILDREN processes, then + // wait. As soon as a proc completes, fork the next. + + int nextTestCase = 0; + int numRunningChildren = 0; + while( nextTestCase < numTestCases || numRunningChildren > 0) + { + if( nextTestCase < numTestCases ) + { + while( numRunningChildren < MAX_NUM_CHILDREN ) + { + int pid = fork(); + if( pid == 0 ) // Child process + { + close(STDOUT_FILENO); + close(STDERR_FILENO); + exit( RunTestCase( tc_array[nextTestCase] ) ); + } + else if(pid == -1) + { + perror("fork"); + exit(2); + } + else // Parent process + { + TestCase tc(nextTestCase, tc_array[nextTestCase].name); + children[pid] = tc; + nextTestCase++; + numRunningChildren++; + } + } + } + + int status=0; + int childPid = waitpid(-1, &status, 0); + if( childPid == -1 ) + { + perror("waitpid"); + exit(2); + } + + if( WIFEXITED(status) ) + { + if( childPid > 0 ) + { + int testResult = WEXITSTATUS(status); + if( testResult ) + { + printf("Test case %s failed: %d\n", children[childPid].testCaseName, testResult); + failedTestCases.push_back(children[childPid].testCase); + numFailures++; + } + else + { + numPasses++; + } + numRunningChildren--; + } + } - return result; + else if( WIFSIGNALED(status) ) + { + if( childPid > 0 ) + { + RunningTestCases::iterator iter = children.find(childPid); + if( iter != children.end() ) + { + printf("Test case %s exited with signal %d\n", iter->second.testCaseName, WTERMSIG(status)); + failedTestCases.push_back(iter->second.testCase); + } + else + { + printf("Unknown child process: %d signaled %d\n", childPid, WTERMSIG(status)); } + + numFailures++; + numRunningChildren--; + } + } + } + + printf("\rNumber of test passes: %d \n", numPasses); + printf("Number of test failures: %d\n", numFailures); + + if( reRunFailed ) + { + for( unsigned int i=0; i\n", argv[0]); + return 2; + } + result = FindAndRunTestCase(argv[1]); + } + return result; } diff --git a/automated-tests/src/dali-unmanaged/tct-dali-unmanaged-core.cpp b/automated-tests/src/dali-unmanaged/tct-dali-unmanaged-core.cpp index 65fe1e4..795f044 100644 --- a/automated-tests/src/dali-unmanaged/tct-dali-unmanaged-core.cpp +++ b/automated-tests/src/dali-unmanaged/tct-dali-unmanaged-core.cpp @@ -1,31 +1,220 @@ #include #include #include "tct-dali-unmanaged-core.h" +#include +#include +#include +#include +#include +#include +#include -int main(int argc, const char *argv[]) +int RunTestCase( struct testcase_s& testCase ) { - int result = -1; - int i; + int result = 1; + if( testCase.startup ) + { + testCase.startup(); + } + result = testCase.function(); + if( testCase.cleanup ) + { + testCase.cleanup(); + } + return result; +} - if (argc != 2) { - printf("Usage: %s \n", argv[0]); - return 2; - } +#define MAX_NUM_CHILDREN 16 + +struct TestCase +{ + int testCase; + const char* testCaseName; + + TestCase() + : testCase(0), + testCaseName(NULL) + { + } - for (i = 0; tc_array[i].name; i++) { - if (!strcmp(argv[1], tc_array[i].name)) { - if (tc_array[i].startup) - tc_array[i].startup(); + TestCase(int tc, const char* name) + : testCase(tc), + testCaseName(name) + { + } + TestCase(const TestCase& rhs) + : testCase(rhs.testCase), + testCaseName(rhs.testCaseName) + { + } + TestCase& operator=(const TestCase& rhs) + { + testCase = rhs.testCase; + testCaseName = rhs.testCaseName; + return *this; + + } +}; + + +typedef std::map RunningTestCases; + +// Constantly runs up to MAX_NUM_CHILDREN processes +int RunAllInParallel(const char* processName, bool reRunFailed) +{ + int numFailures = 0; + int numPasses = 0; + int numTestCases = sizeof(tc_array)/sizeof(struct testcase_s) - 1; - result = tc_array[i].function(); + RunningTestCases children; + std::vector failedTestCases; - if (tc_array[i].cleanup) - tc_array[i].cleanup(); + // Fork up to MAX_NUM_CHILDREN processes, then + // wait. As soon as a proc completes, fork the next. + + int nextTestCase = 0; + int numRunningChildren = 0; + while( nextTestCase < numTestCases || numRunningChildren > 0) + { + if( nextTestCase < numTestCases ) + { + while( numRunningChildren < MAX_NUM_CHILDREN ) + { + int pid = fork(); + if( pid == 0 ) // Child process + { + close(STDOUT_FILENO); + close(STDERR_FILENO); + exit( RunTestCase( tc_array[nextTestCase] ) ); + } + else if(pid == -1) + { + perror("fork"); + exit(2); + } + else // Parent process + { + TestCase tc(nextTestCase, tc_array[nextTestCase].name); + children[pid] = tc; + nextTestCase++; + numRunningChildren++; + } + } + } + + int status=0; + int childPid = waitpid(-1, &status, 0); + if( childPid == -1 ) + { + perror("waitpid"); + exit(2); + } + + if( WIFEXITED(status) ) + { + if( childPid > 0 ) + { + int testResult = WEXITSTATUS(status); + if( testResult ) + { + printf("Test case %s failed: %d\n", children[childPid].testCaseName, testResult); + failedTestCases.push_back(children[childPid].testCase); + numFailures++; + } + else + { + numPasses++; + } + numRunningChildren--; + } + } - return result; + else if( WIFSIGNALED(status) ) + { + if( childPid > 0 ) + { + RunningTestCases::iterator iter = children.find(childPid); + if( iter != children.end() ) + { + printf("Test case %s exited with signal %d\n", iter->second.testCaseName, WTERMSIG(status)); + failedTestCases.push_back(iter->second.testCase); + } + else + { + printf("Unknown child process: %d signaled %d\n", childPid, WTERMSIG(status)); } + + numFailures++; + numRunningChildren--; + } + } + } + + printf("\rNumber of test passes: %d \n", numPasses); + printf("Number of test failures: %d\n", numFailures); + + if( reRunFailed ) + { + for( unsigned int i=0; i\n", argv[0]); + return 2; + } + result = FindAndRunTestCase(argv[1]); + } + return result; } diff --git a/automated-tests/src/dali/tct-dali-core.cpp b/automated-tests/src/dali/tct-dali-core.cpp index 84613cd..92548fc 100644 --- a/automated-tests/src/dali/tct-dali-core.cpp +++ b/automated-tests/src/dali/tct-dali-core.cpp @@ -1,31 +1,220 @@ #include #include #include "tct-dali-core.h" +#include +#include +#include +#include +#include +#include +#include -int main(int argc, const char *argv[]) +int RunTestCase( struct testcase_s& testCase ) { - int result = -1; - int i; + int result = 1; + if( testCase.startup ) + { + testCase.startup(); + } + result = testCase.function(); + if( testCase.cleanup ) + { + testCase.cleanup(); + } + return result; +} - if (argc != 2) { - printf("Usage: %s \n", argv[0]); - return 2; - } +#define MAX_NUM_CHILDREN 16 + +struct TestCase +{ + int testCase; + const char* testCaseName; + + TestCase() + : testCase(0), + testCaseName(NULL) + { + } - for (i = 0; tc_array[i].name; i++) { - if (!strcmp(argv[1], tc_array[i].name)) { - if (tc_array[i].startup) - tc_array[i].startup(); + TestCase(int tc, const char* name) + : testCase(tc), + testCaseName(name) + { + } + TestCase(const TestCase& rhs) + : testCase(rhs.testCase), + testCaseName(rhs.testCaseName) + { + } + TestCase& operator=(const TestCase& rhs) + { + testCase = rhs.testCase; + testCaseName = rhs.testCaseName; + return *this; + + } +}; + + +typedef std::map RunningTestCases; + +// Constantly runs up to MAX_NUM_CHILDREN processes +int RunAllInParallel(const char* processName, bool reRunFailed) +{ + int numFailures = 0; + int numPasses = 0; + int numTestCases = sizeof(tc_array)/sizeof(struct testcase_s) - 1; - result = tc_array[i].function(); + RunningTestCases children; + std::vector failedTestCases; - if (tc_array[i].cleanup) - tc_array[i].cleanup(); + // Fork up to MAX_NUM_CHILDREN processes, then + // wait. As soon as a proc completes, fork the next. + + int nextTestCase = 0; + int numRunningChildren = 0; + while( nextTestCase < numTestCases || numRunningChildren > 0) + { + if( nextTestCase < numTestCases ) + { + while( numRunningChildren < MAX_NUM_CHILDREN ) + { + int pid = fork(); + if( pid == 0 ) // Child process + { + close(STDOUT_FILENO); + close(STDERR_FILENO); + exit( RunTestCase( tc_array[nextTestCase] ) ); + } + else if(pid == -1) + { + perror("fork"); + exit(2); + } + else // Parent process + { + TestCase tc(nextTestCase, tc_array[nextTestCase].name); + children[pid] = tc; + nextTestCase++; + numRunningChildren++; + } + } + } + + int status=0; + int childPid = waitpid(-1, &status, 0); + if( childPid == -1 ) + { + perror("waitpid"); + exit(2); + } + + if( WIFEXITED(status) ) + { + if( childPid > 0 ) + { + int testResult = WEXITSTATUS(status); + if( testResult ) + { + printf("Test case %s failed: %d\n", children[childPid].testCaseName, testResult); + failedTestCases.push_back(children[childPid].testCase); + numFailures++; + } + else + { + numPasses++; + } + numRunningChildren--; + } + } - return result; + else if( WIFSIGNALED(status) ) + { + if( childPid > 0 ) + { + RunningTestCases::iterator iter = children.find(childPid); + if( iter != children.end() ) + { + printf("Test case %s exited with signal %d\n", iter->second.testCaseName, WTERMSIG(status)); + failedTestCases.push_back(iter->second.testCase); + } + else + { + printf("Unknown child process: %d signaled %d\n", childPid, WTERMSIG(status)); } + + numFailures++; + numRunningChildren--; + } + } + } + + printf("\rNumber of test passes: %d \n", numPasses); + printf("Number of test failures: %d\n", numFailures); + + if( reRunFailed ) + { + for( unsigned int i=0; i\n", argv[0]); + return 2; + } + result = FindAndRunTestCase(argv[1]); + } + return result; } -- 2.7.4