2 * Copyright (c) 2020 Samsung Electronics Co., Ltd.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
17 #include "test-harness.h"
19 #include <sys/types.h>
31 typedef std::map<int32_t, TestCase> RunningTestCases;
33 const char* basename(const char* path)
36 const char* slash=NULL;
37 for( ; *ptr != '\0' ; ++ptr )
39 if(*ptr == '/') slash=ptr;
41 if(slash != NULL) ++slash;
45 void SuppressLogOutput()
47 // Close stdout and stderr to suppress the log output
48 close(STDOUT_FILENO); // File descriptor number for stdout is 1
49 close(STDERR_FILENO); // File descriptor number for stderr is 2
51 // The POSIX specification requires that /dev/null must be provided,
52 // The open function always chooses the lowest unused file descriptor
53 // It is sufficient for stdout to be writable.
54 open("/dev/null", O_WRONLY); // Redirect file descriptor number 1 (i.e. stdout) to /dev/null
55 // When stderr is opened it must be both readable and writable.
56 open("/dev/null", O_RDWR); // Redirect file descriptor number 2 (i.e. stderr) to /dev/null
59 int32_t RunTestCase( struct ::testcase_s& testCase )
61 int32_t result = EXIT_STATUS_TESTCASE_FAILED;
63 // dont want to catch exception as we want to be able to get
64 // gdb stack trace from the first error
65 // by default tests should all always pass with no exceptions
66 if( testCase.startup )
72 result = testCase.function();
76 // just catch test fail exception, return is already set to EXIT_STATUS_TESTCASE_FAILED
78 if( testCase.cleanup )
87 int32_t RunTestCaseInChildProcess( struct ::testcase_s& testCase, bool suppressOutput )
89 int32_t testResult = EXIT_STATUS_TESTCASE_FAILED;
92 if( pid == 0 ) // Child process
101 for(int32_t i=0; i<80; ++i) printf("#");
102 printf("\nTC: %s\n", testCase.name);
106 int32_t status = RunTestCase( testCase );
108 if( ! suppressOutput )
120 exit(EXIT_STATUS_FORK_FAILED);
122 else // Parent process
125 int32_t childPid = waitpid(pid, &status, 0);
129 exit(EXIT_STATUS_WAITPID_FAILED);
131 if( WIFEXITED(status) )
135 testResult = WEXITSTATUS(status);
138 printf("Test case %s failed: %d\n", testCase.name, testResult);
142 else if(WIFSIGNALED(status) )
144 int32_t signal = WTERMSIG(status);
145 testResult = EXIT_STATUS_TESTCASE_ABORTED;
146 if( signal == SIGABRT )
148 printf("Test case %s failed: test case asserted\n", testCase.name );
152 printf("Test case %s failed: exit with signal %s\n", testCase.name, strsignal(WTERMSIG(status)));
155 else if(WIFSTOPPED(status))
157 printf("Test case %s failed: stopped with signal %s\n", testCase.name, strsignal(WSTOPSIG(status)));
165 void OutputStatistics( const char* processName, int32_t numPasses, int32_t numFailures )
167 FILE* fp=fopen("summary.xml", "a");
171 " <suite name=\"%s\">\n"
172 " <total_case>%d</total_case>\n"
173 " <pass_case>%d</pass_case>\n"
174 " <pass_rate>%5.2f</pass_rate>\n"
175 " <fail_case>%d</fail_case>\n"
176 " <fail_rate>%5.2f</fail_rate>\n"
177 " <block_case>0</block_case>\n"
178 " <block_rate>0.00</block_rate>\n"
179 " <na_case>0</na_case>\n"
180 " <na_rate>0.00</na_rate>\n"
182 basename(processName),
183 numPasses+numFailures,
185 (float)numPasses/(numPasses+numFailures),
187 (float)numFailures/(numPasses+numFailures) );
192 int32_t RunAll( const char* processName, ::testcase tc_array[] )
194 int32_t numFailures = 0;
195 int32_t numPasses = 0;
197 // Run test cases in child process( to kill output/handle signals ), but run serially.
198 for( uint32_t i=0; tc_array[i].name; i++)
200 int32_t result = RunTestCaseInChildProcess( tc_array[i], false );
211 OutputStatistics( processName, numPasses, numFailures);
216 // Constantly runs up to MAX_NUM_CHILDREN processes
217 int32_t RunAllInParallel( const char* processName, ::testcase tc_array[], bool reRunFailed)
219 int32_t numFailures = 0;
220 int32_t numPasses = 0;
222 RunningTestCases children;
223 std::vector<int32_t> failedTestCases;
225 // Fork up to MAX_NUM_CHILDREN processes, then
226 // wait. As soon as a proc completes, fork the next.
228 int32_t nextTestCase = 0;
229 int32_t numRunningChildren = 0;
231 while( tc_array[nextTestCase].name || numRunningChildren > 0)
233 // Create more children (up to the max number or til the end of the array)
234 while( numRunningChildren < MAX_NUM_CHILDREN && tc_array[nextTestCase].name )
236 int32_t pid = fork();
237 if( pid == 0 ) // Child process
240 exit( RunTestCase( tc_array[nextTestCase] ) );
245 exit(EXIT_STATUS_FORK_FAILED);
247 else // Parent process
249 TestCase tc(nextTestCase, tc_array[nextTestCase].name);
252 numRunningChildren++;
256 // Wait for the next child to finish
259 int32_t childPid = waitpid(-1, &status, 0);
263 exit(EXIT_STATUS_WAITPID_FAILED);
266 if( WIFEXITED(status) )
270 int32_t testResult = WEXITSTATUS(status);
273 printf("Test case %s failed: %d\n", children[childPid].testCaseName, testResult);
274 failedTestCases.push_back(children[childPid].testCase);
281 numRunningChildren--;
285 else if( WIFSIGNALED(status) || WIFSTOPPED(status))
287 status = WIFSIGNALED(status)?WTERMSIG(status):WSTOPSIG(status);
291 RunningTestCases::iterator iter = children.find(childPid);
292 if( iter != children.end() )
294 printf("Test case %s exited with signal %s\n", iter->second.testCaseName, strsignal(status));
295 failedTestCases.push_back(iter->second.testCase);
299 printf("Unknown child process: %d signaled %s\n", childPid, strsignal(status));
303 numRunningChildren--;
308 OutputStatistics( processName, numPasses, numFailures );
312 for( uint32_t i=0; i<failedTestCases.size(); i++)
314 char* testCaseStrapline;
315 int32_t numChars = asprintf(&testCaseStrapline, "Test case %s", tc_array[failedTestCases[i]].name );
316 printf("\n%s\n", testCaseStrapline);
317 for(int32_t j=0; j<numChars; j++)
322 RunTestCaseInChildProcess( tc_array[failedTestCases[i] ], false );
331 int32_t FindAndRunTestCase(::testcase tc_array[], const char* testCaseName)
333 int32_t result = EXIT_STATUS_TESTCASE_NOT_FOUND;
335 for( int32_t i = 0; tc_array[i].name; i++ )
337 if( !strcmp(testCaseName, tc_array[i].name) )
339 return RunTestCase( tc_array[i] );
343 printf("Unknown testcase name: \"%s\"\n", testCaseName);
347 void Usage(const char* program)
350 " %s <testcase name>\t\t Execute a test case\n"
351 " %s \t\t Execute all test cases in parallel\n"
352 " %s -r\t\t Execute all test cases in parallel, rerunning failed test cases\n"
353 " %s -s\t\t Execute all test cases serially\n",
354 program, program, program, program);