From 1a0dc2cc53708bbc32546da6031e996465544f51 Mon Sep 17 00:00:00 2001 From: David Steele Date: Fri, 18 Dec 2020 17:24:20 +0000 Subject: [PATCH] All tests now output results to xml files Change-Id: I7cbf0540d6f32bdb516f75b42384ed3f44685de2 --- automated-tests/execute.sh | 40 +- .../tct-dali-scene-loader-internal-core.cpp | 46 +-- .../tct-dali-scene-loader-core.cpp | 46 +-- .../tct-dali-toolkit-internal-core.cpp | 46 +-- .../tct-dali-toolkit-styling-core.cpp | 46 +-- .../tct-dali-toolkit-third-party-core.cpp | 46 +-- .../dali-toolkit-test-utils/test-harness.cpp | 406 +++++++++++++++++---- .../dali-toolkit-test-utils/test-harness.h | 58 ++- .../src/dali-toolkit/tct-dali-toolkit-core.cpp | 46 +-- 9 files changed, 419 insertions(+), 361 deletions(-) diff --git a/automated-tests/execute.sh b/automated-tests/execute.sh index add0e78..adee84b 100755 --- a/automated-tests/execute.sh +++ b/automated-tests/execute.sh @@ -1,6 +1,6 @@ #!/bin/bash -TEMP=`getopt -o dhsSmf --long debug,help,failnorerun,serial,tct,modules -n 'execute.sh' -- "$@"` +TEMP=`getopt -o dhsSmfq --long debug,help,failnorerun,quiet,serial,tct,modules -n 'execute.sh' -- "$@"` if [ $? != 0 ] ; then echo "Terminating..." >&2 ; exit 1 ; fi @@ -9,13 +9,14 @@ eval set -- "$TEMP" function usage { - echo -e "Usage: execute.sh [-d][-s|-S|-r] [module|testcase]" + echo -e "Usage: execute.sh [-d][-s|-S|-r][-q] [module|testcase]" echo -e " execute.sh\t\tExecute test cases from all modules in parallel" echo -e " execute.sh -f \tExecute test cases from all modules in parallel without rerunning failed test cases" echo -e " execute.sh -d \tDebug testcase" echo -e " execute.sh [module]\tExecute test cases from the given module in parallel" echo -e " execute.sh -s [module]\t\tExecute test cases in serial using Testkit-Lite" echo -e " execute.sh -S [module]\t\tExecute test cases in serial" + echo -e " execute.sh -q|--quiet ...\tExecute test cases, but don't write output" echo -e " execute.sh \tFind and execute the given test case" exit 2 } @@ -25,6 +26,7 @@ opt_serial="" opt_modules=0 opt_debug=0 opt_noFailedRerun=""; +opt_quiet=""; while true ; do case "$1" in -h|--help) usage ;; @@ -32,6 +34,7 @@ while true ; do -s|--tct) opt_tct=1 ; shift ;; -f|--nofailedrerun) opt_noFailedRerun="-f" ; shift ;; -S|--serial) opt_serial="-s" ; shift ;; + -q|--quiet) opt_quiet="-q" ; shift ;; -m|--modules) opt_modules=1 ; shift ;; --) shift; break;; *) echo "Internal error $1!" ; exit 1 ;; @@ -67,6 +70,31 @@ function summary_end EOF } +function output_start +{ + start=`date +"%Y-%m-%d_%H_%M_%S"` + cat > tct-${1}-core-tests.xml < + + + +$start$start + + +EOF +} + + +function output_end +{ + cat >> tct-${1}-core-tests.xml < + + +EOF +} + + if [ $opt_modules == 1 ] ; then modules= get_modules echo $modules @@ -118,7 +146,9 @@ else do echo -e "$ASCII_BOLD" echo -e "Executing $mod$ASCII_RESET" - dbus-launch build/src/$mod/tct-$mod-core $opt_serial $opt_noFailedRerun + output_start $mod + dbus-launch build/src/$mod/tct-$mod-core $opt_serial $opt_noFailedRerun $opt_quiet + output_end $mod done summary_end @@ -128,7 +158,9 @@ else summary_start module=$1 shift; - dbus-launch build/src/$module/tct-$module-core $opt_serial $opt_noFailedRerun $* + output_start ${module} + dbus-launch build/src/$module/tct-$module-core $opt_serial $opt_noFailedRerun $opt_quiet $* + output_end ${module} summary_end else diff --git a/automated-tests/src/dali-scene-loader-internal/tct-dali-scene-loader-internal-core.cpp b/automated-tests/src/dali-scene-loader-internal/tct-dali-scene-loader-internal-core.cpp index 9dea347..7bfea6e 100644 --- a/automated-tests/src/dali-scene-loader-internal/tct-dali-scene-loader-internal-core.cpp +++ b/automated-tests/src/dali-scene-loader-internal/tct-dali-scene-loader-internal-core.cpp @@ -1,51 +1,7 @@ -#include -#include -#include #include #include "tct-dali-scene-loader-internal-core.h" int main(int argc, char * const argv[]) { - int result = TestHarness::EXIT_STATUS_BAD_ARGUMENT; - - const char* optString = "sf"; - bool optRerunFailed(true); - bool optRunSerially(false); - - int nextOpt = 0; - do - { - nextOpt = getopt( argc, argv, optString ); - switch(nextOpt) - { - case 'f': - optRerunFailed = false; - break; - case 's': - optRunSerially = true; - break; - case '?': - TestHarness::Usage(argv[0]); - exit(TestHarness::EXIT_STATUS_BAD_ARGUMENT); - break; - } - } while( nextOpt != -1 ); - - if( optind == argc ) // no testcase name in argument list - { - if( optRunSerially ) - { - result = TestHarness::RunAll( argv[0], tc_array ); - } - else - { - result = TestHarness::RunAllInParallel( argv[0], tc_array, optRerunFailed ); - } - } - else - { - // optind is index of next argument - interpret as testcase name - result = TestHarness::FindAndRunTestCase(tc_array, argv[optind]); - } - return result; + return TestHarness::RunTests(argc, argv, tc_array); } diff --git a/automated-tests/src/dali-scene-loader/tct-dali-scene-loader-core.cpp b/automated-tests/src/dali-scene-loader/tct-dali-scene-loader-core.cpp index 90b7a2c..5b79502 100644 --- a/automated-tests/src/dali-scene-loader/tct-dali-scene-loader-core.cpp +++ b/automated-tests/src/dali-scene-loader/tct-dali-scene-loader-core.cpp @@ -1,51 +1,7 @@ -#include -#include -#include #include #include "tct-dali-scene-loader-core.h" int main(int argc, char * const argv[]) { - int result = TestHarness::EXIT_STATUS_BAD_ARGUMENT; - - const char* optString = "sf"; - bool optRerunFailed(true); - bool optRunSerially(false); - - int nextOpt = 0; - do - { - nextOpt = getopt( argc, argv, optString ); - switch(nextOpt) - { - case 'f': - optRerunFailed = false; - break; - case 's': - optRunSerially = true; - break; - case '?': - TestHarness::Usage(argv[0]); - exit(TestHarness::EXIT_STATUS_BAD_ARGUMENT); - break; - } - } while( nextOpt != -1 ); - - if( optind == argc ) // no testcase name in argument list - { - if( optRunSerially ) - { - result = TestHarness::RunAll( argv[0], tc_array ); - } - else - { - result = TestHarness::RunAllInParallel( argv[0], tc_array, optRerunFailed ); - } - } - else - { - // optind is index of next argument - interpret as testcase name - result = TestHarness::FindAndRunTestCase(tc_array, argv[optind]); - } - return result; + return TestHarness::RunTests(argc, argv, tc_array); } diff --git a/automated-tests/src/dali-toolkit-internal/tct-dali-toolkit-internal-core.cpp b/automated-tests/src/dali-toolkit-internal/tct-dali-toolkit-internal-core.cpp index 39ed044..fe0ed63 100644 --- a/automated-tests/src/dali-toolkit-internal/tct-dali-toolkit-internal-core.cpp +++ b/automated-tests/src/dali-toolkit-internal/tct-dali-toolkit-internal-core.cpp @@ -1,51 +1,7 @@ -#include -#include -#include #include #include "tct-dali-toolkit-internal-core.h" int main(int argc, char * const argv[]) { - int result = TestHarness::EXIT_STATUS_BAD_ARGUMENT; - - const char* optString = "sf"; - bool optRerunFailed(true); - bool optRunSerially(false); - - int nextOpt = 0; - do - { - nextOpt = getopt( argc, argv, optString ); - switch(nextOpt) - { - case 'f': - optRerunFailed = false; - break; - case 's': - optRunSerially = true; - break; - case '?': - TestHarness::Usage(argv[0]); - exit(TestHarness::EXIT_STATUS_BAD_ARGUMENT); - break; - } - } while( nextOpt != -1 ); - - if( optind == argc ) // no testcase name in argument list - { - if( optRunSerially ) - { - result = TestHarness::RunAll( argv[0], tc_array ); - } - else - { - result = TestHarness::RunAllInParallel( argv[0], tc_array, optRerunFailed ); - } - } - else - { - // optind is index of next argument - interpret as testcase name - result = TestHarness::FindAndRunTestCase(tc_array, argv[optind]); - } - return result; + return TestHarness::RunTests(argc, argv, tc_array); } diff --git a/automated-tests/src/dali-toolkit-styling/tct-dali-toolkit-styling-core.cpp b/automated-tests/src/dali-toolkit-styling/tct-dali-toolkit-styling-core.cpp index ef04d46..16b61c2 100644 --- a/automated-tests/src/dali-toolkit-styling/tct-dali-toolkit-styling-core.cpp +++ b/automated-tests/src/dali-toolkit-styling/tct-dali-toolkit-styling-core.cpp @@ -1,51 +1,7 @@ -#include -#include -#include #include #include "tct-dali-toolkit-styling-core.h" int main(int argc, char * const argv[]) { - int result = TestHarness::EXIT_STATUS_BAD_ARGUMENT; - - const char* optString = "sf"; - bool optRerunFailed(true); - bool optRunSerially(false); - - int nextOpt = 0; - do - { - nextOpt = getopt( argc, argv, optString ); - switch(nextOpt) - { - case 'f': - optRerunFailed = false; - break; - case 's': - optRunSerially = true; - break; - case '?': - TestHarness::Usage(argv[0]); - exit(TestHarness::EXIT_STATUS_BAD_ARGUMENT); - break; - } - } while( nextOpt != -1 ); - - if( optind == argc ) // no testcase name in argument list - { - if( optRunSerially ) - { - result = TestHarness::RunAll( argv[0], tc_array ); - } - else - { - result = TestHarness::RunAllInParallel( argv[0], tc_array, optRerunFailed ); - } - } - else - { - // optind is index of next argument - interpret as testcase name - result = TestHarness::FindAndRunTestCase(tc_array, argv[optind]); - } - return result; + return TestHarness::RunTests(argc, argv, tc_array); } diff --git a/automated-tests/src/dali-toolkit-third-party/tct-dali-toolkit-third-party-core.cpp b/automated-tests/src/dali-toolkit-third-party/tct-dali-toolkit-third-party-core.cpp index d188f08..779e56b 100644 --- a/automated-tests/src/dali-toolkit-third-party/tct-dali-toolkit-third-party-core.cpp +++ b/automated-tests/src/dali-toolkit-third-party/tct-dali-toolkit-third-party-core.cpp @@ -1,51 +1,7 @@ -#include -#include -#include #include #include "tct-dali-toolkit-third-party-core.h" int main(int argc, char * const argv[]) { - int result = TestHarness::EXIT_STATUS_BAD_ARGUMENT; - - const char* optString = "sf"; - bool optRerunFailed(true); - bool optRunSerially(false); - - int nextOpt = 0; - do - { - nextOpt = getopt( argc, argv, optString ); - switch(nextOpt) - { - case 'f': - optRerunFailed = false; - break; - case 's': - optRunSerially = true; - break; - case '?': - TestHarness::Usage(argv[0]); - exit(TestHarness::EXIT_STATUS_BAD_ARGUMENT); - break; - } - } while( nextOpt != -1 ); - - if( optind == argc ) // no testcase name in argument list - { - if( optRunSerially ) - { - result = TestHarness::RunAll( argv[0], tc_array ); - } - else - { - result = TestHarness::RunAllInParallel( argv[0], tc_array, optRerunFailed ); - } - } - else - { - // optind is index of next argument - interpret as testcase name - result = TestHarness::FindAndRunTestCase(tc_array, argv[optind]); - } - return result; + return TestHarness::RunTests(argc, argv, tc_array); } diff --git a/automated-tests/src/dali-toolkit/dali-toolkit-test-utils/test-harness.cpp b/automated-tests/src/dali-toolkit/dali-toolkit-test-utils/test-harness.cpp index b53630d..587d4a5 100644 --- a/automated-tests/src/dali-toolkit/dali-toolkit-test-utils/test-harness.cpp +++ b/automated-tests/src/dali-toolkit/dali-toolkit-test-utils/test-harness.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020 Samsung Electronics Co., Ltd. + * Copyright (c) 2021 Samsung Electronics Co., Ltd. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -22,14 +22,21 @@ #include #include -#include +#include #include - +#include #include +#include #include +#include +#include #include +#include #include +using std::chrono::steady_clock; +using std::chrono::system_clock; + namespace TestHarness { typedef std::map RunningTestCases; @@ -48,18 +55,185 @@ const char* basename(const char* path) return slash; } -void SuppressLogOutput() +std::vector Split(const std::string& aString, char delimiter) { - // Close stdout and stderr to suppress the log output - close(STDOUT_FILENO); // File descriptor number for stdout is 1 - close(STDERR_FILENO); // File descriptor number for stderr is 2 + std::vector tokens; + std::string token; + std::istringstream tokenStream(aString); + while(std::getline(tokenStream, token, delimiter)) + { + tokens.push_back(token); + } + return tokens; +} - // The POSIX specification requires that /dev/null must be provided, - // The open function always chooses the lowest unused file descriptor - // It is sufficient for stdout to be writable. - open("/dev/null", O_WRONLY); // Redirect file descriptor number 1 (i.e. stdout) to /dev/null - // When stderr is opened it must be both readable and writable. - open("/dev/null", O_RDWR); // Redirect file descriptor number 2 (i.e. stderr) to /dev/null +std::string Join(const std::vector& tokens, char delimiter) +{ + std::ostringstream oss; + + unsigned int delimiterCount = 0; + for(auto& token : tokens) + { + oss << token; + if(delimiterCount < tokens.size() - 1) + { + oss << delimiter; + } + ++delimiterCount; + } + return oss.str(); +} + +std::string ChildOutputFilename(int pid) +{ + std::ostringstream os; + os << "/tmp/tct-child." << pid; + return os.str(); +} + +std::string TestModuleFilename(const char* processName) +{ + auto pathComponents = Split(processName, '/'); + auto aModule = pathComponents.back(); + aModule += "-tests.xml"; + return aModule; +} + +std::string TestModuleName(const char* processName) +{ + auto pathComponents = Split(processName, '/'); + auto aModule = pathComponents.back(); + auto moduleComponents = Split(aModule, '-'); + + moduleComponents[1][0] = std::toupper(moduleComponents[1][0]); + moduleComponents[2][0] = std::toupper(moduleComponents[2][0]); + + std::ostringstream oss; + for(unsigned int i = 1; i < moduleComponents.size() - 1; ++i) // [0]=tct, [n-1]=core + { + oss << moduleComponents[i]; + + if(i > 1 && i < moduleComponents.size() - 2) // skip first and last delimiter + { + oss << '-'; + } + } + + return oss.str(); +} + +std::string ReadAndEscape(std::string filename) +{ + std::ostringstream os; + std::ifstream ifs; + ifs.open(filename, std::ifstream::in); + while(ifs.good()) + { + std::string line; + std::getline(ifs, line); + for(auto c : line) + { + switch(c) + { + case '<': + os << "<"; + break; + case '>': + os << ">"; + break; + case '&': + os << "&"; + break; + default: + os << c; + break; + } + } + os << "\\" + << "n"; + } + ifs.close(); + return os.str(); +} + +void OutputTestResult( + std::ofstream& ofs, + const char* pathToExecutable, + std::string testSuiteName, + TestCase& testCase, + std::string startTime, + std::string endTime) +{ + std::string outputFilename = ChildOutputFilename(testCase.childPid); + std::string testOutput = ReadAndEscape(outputFilename); + + ofs << "" << std::endl + << "" + << pathToExecutable << testCase.name << "" << std::endl + << "" + << "" << (testCase.result == 0 ? "PASS" : "FAIL") << "" << std::endl + << "" << startTime << "" + << "" << endTime << "" + << "" + << "" << std::endl; + + unlink(outputFilename.c_str()); +} + +void OutputTestResults(const char* processName, RunningTestCases& children) +{ + std::ofstream ofs; + std::string filename = TestModuleFilename(processName); + std::string moduleName = TestModuleName(processName); + ofs.open(filename, std::ofstream::out | std::ofstream::app); + + // Sort completed cases by original test case id + std::vector childTestCases; + childTestCases.reserve(children.size()); + for(auto& element : children) childTestCases.push_back(element.second); + std::sort(childTestCases.begin(), childTestCases.end(), [](const TestCase& a, const TestCase& b) { + return a.testCase < b.testCase; + }); + + const int BUFSIZE = 256; + char buffer[BUFSIZE]; + for(auto& testCase : childTestCases) + { + auto tt = system_clock::to_time_t(testCase.startSystemTime); + strftime(buffer, BUFSIZE, "%c", localtime(&tt)); + std::string startTime(buffer); + OutputTestResult(ofs, processName, moduleName, testCase, startTime, startTime); + } + + ofs.close(); +} + +void OutputStatistics(const char* processName, int32_t numPasses, int32_t numFailures) +{ + FILE* fp = fopen("summary.xml", "a"); + if(fp != NULL) + { + fprintf(fp, + " \n" + " %d\n" + " %d\n" + " %5.2f\n" + " %d\n" + " %5.2f\n" + " 0\n" + " 0.00\n" + " 0\n" + " 0.00\n" + " \n", + basename(processName), + numPasses + numFailures, + numPasses, + (float)numPasses * 100.0f / (numPasses + numFailures), + numFailures, + (float)numFailures * 100.0f / (numPasses + numFailures)); + fclose(fp); + } } int32_t RunTestCase(struct ::testcase_s& testCase) @@ -89,35 +263,55 @@ int32_t RunTestCase(struct ::testcase_s& testCase) return result; } -int32_t RunTestCaseInChildProcess(struct ::testcase_s& testCase, bool suppressOutput) +int32_t RunTestCaseRedirectOutput(TestCase& testCase, bool suppressOutput) +{ + // Executing in child process + // Close stdout and stderr to suppress the log output + close(STDOUT_FILENO); // File descriptor number for stdout is 1 + + // The POSIX specification requires that /dev/null must be provided, + // The open function always chooses the lowest unused file descriptor + // It is sufficient for stdout to be writable. + open("/dev/null", O_WRONLY); // Redirect file descriptor number 1 (i.e. stdout) to /dev/null + + fflush(stderr); + close(STDERR_FILENO); + if(suppressOutput) + { + stderr = fopen("/dev/null", "w+"); // Redirect fd 2 to /dev/null + } + else + { + // When stderr is opened it must be both readable and writable. + std::string childOutputFilename = ChildOutputFilename(getpid()); + stderr = fopen(childOutputFilename.c_str(), "w+"); + } + + int32_t status = RunTestCase(*testCase.tctPtr); + + fflush(stderr); + fclose(stderr); + + return status; +} + +int32_t RunTestCaseInChildProcess(TestCase& testCase, bool redirect) { int32_t testResult = EXIT_STATUS_TESTCASE_FAILED; int32_t pid = fork(); if(pid == 0) // Child process { - if(suppressOutput) + if(redirect) { - SuppressLogOutput(); + int status = RunTestCaseRedirectOutput(testCase, false); + exit(status); } else { - printf("\n"); - for(int32_t i = 0; i < 80; ++i) printf("#"); - printf("\nTC: %s\n", testCase.name); - fflush(stdout); + int status = RunTestCase(*testCase.tctPtr); + exit(status); } - - int32_t status = RunTestCase(testCase); - - if(!suppressOutput) - { - fflush(stdout); - fflush(stderr); - fclose(stdout); - fclose(stderr); - } - exit(status); } else if(pid == -1) { @@ -126,8 +320,9 @@ int32_t RunTestCaseInChildProcess(struct ::testcase_s& testCase, bool suppressOu } else // Parent process { - int32_t status = 0; - int32_t childPid = waitpid(pid, &status, 0); + int32_t status = 0; + int32_t childPid = waitpid(pid, &status, 0); + testCase.childPid = childPid; if(childPid == -1) { perror("waitpid"); @@ -167,43 +362,32 @@ int32_t RunTestCaseInChildProcess(struct ::testcase_s& testCase, bool suppressOu return testResult; } -void OutputStatistics(const char* processName, int32_t numPasses, int32_t numFailures) +int32_t RunAll(const char* processName, ::testcase tc_array[], bool quiet) { - FILE* fp = fopen("summary.xml", "a"); - if(fp != NULL) + int32_t numFailures = 0; + int32_t numPasses = 0; + std::ofstream ofs; + std::string filename = TestModuleFilename(processName); + std::string moduleName = TestModuleName(processName); + ofs.open(filename, std::ofstream::out | std::ofstream::app); + const int BUFSIZE = 256; + char buffer[BUFSIZE]; + + // Run test cases in child process( to handle signals ), but run serially. + for(uint32_t i = 0; tc_array[i].name; i++) { - fprintf(fp, - " \n" - " %d\n" - " %d\n" - " %5.2f\n" - " %d\n" - " %5.2f\n" - " 0\n" - " 0.00\n" - " 0\n" - " 0.00\n" - " \n", - basename(processName), - numPasses + numFailures, - numPasses, - (float)numPasses / (numPasses + numFailures), - numFailures, - (float)numFailures / (numPasses + numFailures)); - fclose(fp); - } -} + auto tt = system_clock::to_time_t(system_clock::now()); + strftime(buffer, BUFSIZE, "%c", localtime(&tt)); + std::string startTime(buffer); -int32_t RunAll(const char* processName, ::testcase tc_array[]) -{ - int32_t numFailures = 0; - int32_t numPasses = 0; + TestCase testCase(i, &tc_array[i]); + testCase.result = RunTestCaseInChildProcess(testCase, quiet); - // Run test cases in child process( to kill output/handle signals ), but run serially. - for(uint32_t i = 0; tc_array[i].name; i++) - { - int32_t result = RunTestCaseInChildProcess(tc_array[i], false); - if(result == 0) + tt = system_clock::to_time_t(system_clock::now()); + strftime(buffer, BUFSIZE, "%c", localtime(&tt)); + std::string endTime(buffer); + + if(testCase.result == 0) { numPasses++; } @@ -211,7 +395,12 @@ int32_t RunAll(const char* processName, ::testcase tc_array[]) { numFailures++; } + if(!quiet) + { + OutputTestResult(ofs, processName, moduleName, testCase, startTime, endTime); + } } + ofs.close(); OutputStatistics(processName, numPasses, numFailures); @@ -219,7 +408,7 @@ int32_t RunAll(const char* processName, ::testcase tc_array[]) } // Constantly runs up to MAX_NUM_CHILDREN processes -int32_t RunAllInParallel(const char* processName, ::testcase tc_array[], bool reRunFailed) +int32_t RunAllInParallel(const char* processName, ::testcase tc_array[], bool reRunFailed, bool quiet) { int32_t numFailures = 0; int32_t numPasses = 0; @@ -241,8 +430,9 @@ int32_t RunAllInParallel(const char* processName, ::testcase tc_array[], bool re int32_t pid = fork(); if(pid == 0) // Child process { - SuppressLogOutput(); - exit(RunTestCase(tc_array[nextTestCase])); + TestCase testCase(nextTestCase, &tc_array[nextTestCase]); + int status = RunTestCaseRedirectOutput(testCase, quiet); + exit(status); } else if(pid == -1) { @@ -252,7 +442,9 @@ int32_t RunAllInParallel(const char* processName, ::testcase tc_array[], bool re else // Parent process { TestCase tc(nextTestCase, tc_array[nextTestCase].name); - tc.startTime = std::chrono::steady_clock::now(); + tc.startTime = steady_clock::now(); + tc.startSystemTime = system_clock::now(); + tc.childPid = pid; children[pid] = tc; nextTestCase++; @@ -289,11 +481,12 @@ int32_t RunAllInParallel(const char* processName, ::testcase tc_array[], bool re { if(WIFEXITED(status)) { - int32_t testResult = WEXITSTATUS(status); - if(testResult) + auto& testCase = children[childPid]; + testCase.result = WEXITSTATUS(status); + if(testCase.result) { - printf("Test case %s failed: %d\n", children[childPid].testCaseName, testResult); - failedTestCases.push_back(children[childPid].testCase); + printf("Test case %s failed: %d\n", testCase.name, testCase.result); + failedTestCases.push_back(testCase.testCase); numFailures++; } else @@ -310,7 +503,8 @@ int32_t RunAllInParallel(const char* processName, ::testcase tc_array[], bool re RunningTestCases::iterator iter = children.find(childPid); if(iter != children.end()) { - printf("Test case %s exited with signal %s\n", iter->second.testCaseName, strsignal(status)); + printf("Test case %s exited with signal %s\n", iter->second.name, strsignal(status)); + iter->second.result = 1; failedTestCases.push_back(iter->second.testCase); } else @@ -324,6 +518,11 @@ int32_t RunAllInParallel(const char* processName, ::testcase tc_array[], bool re } } + if(!quiet) + { + OutputTestResults(processName, children); + } + OutputStatistics(processName, numPasses, numFailures); if(reRunFailed) @@ -338,7 +537,9 @@ int32_t RunAllInParallel(const char* processName, ::testcase tc_array[], bool re printf("="); } printf("\n"); - RunTestCaseInChildProcess(tc_array[failedTestCases[i]], false); + int index = failedTestCases[i]; + TestCase testCase(index, &tc_array[index]); + RunTestCaseInChildProcess(testCase, false); } } @@ -368,11 +569,62 @@ void Usage(const char* program) " %s \t\t Execute a test case\n" " %s \t\t Execute all test cases in parallel\n" " %s -r\t\t Execute all test cases in parallel, rerunning failed test cases\n" - " %s -s\t\t Execute all test cases serially\n", + " %s -s\t\t Execute all test cases serially\n" + " %s -q\t\t Run without output\n", + program, program, program, program, program); } +int RunTests(int argc, char* const argv[], ::testcase tc_array[]) +{ + int result = TestHarness::EXIT_STATUS_BAD_ARGUMENT; + const char* optString = "sfq"; + bool optRerunFailed(true); + bool optRunSerially(false); + bool optQuiet(false); + + int nextOpt = 0; + do + { + nextOpt = getopt(argc, argv, optString); + switch(nextOpt) + { + case 'f': + optRerunFailed = false; + break; + case 's': + optRunSerially = true; + break; + case 'q': + optQuiet = true; + break; + case '?': + TestHarness::Usage(argv[0]); + exit(TestHarness::EXIT_STATUS_BAD_ARGUMENT); + break; + } + } while(nextOpt != -1); + + if(optind == argc) // no testcase name in argument list + { + if(optRunSerially) + { + result = TestHarness::RunAll(argv[0], tc_array, optQuiet); + } + else + { + result = TestHarness::RunAllInParallel(argv[0], tc_array, optRerunFailed, optQuiet); + } + } + else + { + // optind is index of next argument - interpret as testcase name + result = TestHarness::FindAndRunTestCase(tc_array, argv[optind]); + } + return result; +} + } // namespace TestHarness diff --git a/automated-tests/src/dali-toolkit/dali-toolkit-test-utils/test-harness.h b/automated-tests/src/dali-toolkit/dali-toolkit-test-utils/test-harness.h index b210918..554846f 100644 --- a/automated-tests/src/dali-toolkit/dali-toolkit-test-utils/test-harness.h +++ b/automated-tests/src/dali-toolkit/dali-toolkit-test-utils/test-harness.h @@ -41,33 +41,63 @@ const int32_t MAX_NUM_CHILDREN(16); struct TestCase { int32_t testCase; - const char* testCaseName; + const char* name; std::chrono::steady_clock::time_point startTime; + std::chrono::system_clock::time_point startSystemTime; + int32_t result; + pid_t childPid{0}; + testcase* tctPtr; + TestCase(int32_t index, testcase* testCase) + : testCase(index), + name(testCase->name), + startTime(), + startSystemTime(), + result(0), + childPid(0), + tctPtr(testCase) + { + } TestCase() : testCase(0), - testCaseName(NULL), - startTime() + name(NULL), + startTime(), + startSystemTime(), + result(0), + childPid(0), + tctPtr(nullptr) { } TestCase(int32_t tc, const char* name) : testCase(tc), - testCaseName(name), - startTime() + name(name), + startTime(), + startSystemTime(), + result(0), + childPid(0), + tctPtr(nullptr) { } TestCase(const TestCase& rhs) : testCase(rhs.testCase), - testCaseName(rhs.testCaseName), - startTime(rhs.startTime) + name(rhs.name), + startTime(rhs.startTime), + startSystemTime(rhs.startSystemTime), + result(rhs.result), + childPid(rhs.childPid), + tctPtr(rhs.tctPtr) { } TestCase& operator=(const TestCase& rhs) { - testCase = rhs.testCase; - testCaseName = rhs.testCaseName; - startTime = rhs.startTime; + testCase = rhs.testCase; + name = rhs.name; + startTime = rhs.startTime; + startSystemTime = rhs.startSystemTime; + result = rhs.result; + childPid = rhs.childPid; + tctPtr = rhs.tctPtr; return *this; } }; @@ -109,6 +139,14 @@ int32_t FindAndRunTestCase(::testcase tc_array[], const char* testCaseName); */ void Usage(const char* program); +/** + * Main function. + * @param[in] argc Argument count + * @param[in] argv Argument vector + * @param[in] tc_array Array of test cases + */ +int RunTests(int argc, char* const argv[], ::testcase tc_array[]); + } // namespace TestHarness #endif diff --git a/automated-tests/src/dali-toolkit/tct-dali-toolkit-core.cpp b/automated-tests/src/dali-toolkit/tct-dali-toolkit-core.cpp index 5e64bd3..6d6077b 100644 --- a/automated-tests/src/dali-toolkit/tct-dali-toolkit-core.cpp +++ b/automated-tests/src/dali-toolkit/tct-dali-toolkit-core.cpp @@ -1,51 +1,7 @@ -#include -#include -#include #include #include "tct-dali-toolkit-core.h" int main(int argc, char * const argv[]) { - int result = TestHarness::EXIT_STATUS_BAD_ARGUMENT; - - const char* optString = "sf"; - bool optRerunFailed(true); - bool optRunSerially(false); - - int nextOpt = 0; - do - { - nextOpt = getopt( argc, argv, optString ); - switch(nextOpt) - { - case 'f': - optRerunFailed = false; - break; - case 's': - optRunSerially = true; - break; - case '?': - TestHarness::Usage(argv[0]); - exit(TestHarness::EXIT_STATUS_BAD_ARGUMENT); - break; - } - } while( nextOpt != -1 ); - - if( optind == argc ) // no testcase name in argument list - { - if( optRunSerially ) - { - result = TestHarness::RunAll( argv[0], tc_array ); - } - else - { - result = TestHarness::RunAllInParallel( argv[0], tc_array, optRerunFailed ); - } - } - else - { - // optind is index of next argument - interpret as testcase name - result = TestHarness::FindAndRunTestCase(tc_array, argv[optind]); - } - return result; + return TestHarness::RunTests(argc, argv, tc_array); } -- 2.7.4