2 * Copyright (c) 2020 Samsung Electronics Co., Ltd.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
17 #include "test-harness.h"
21 #include <sys/types.h>
35 typedef std::map<int32_t, TestCase> RunningTestCases;
37 const double MAXIMUM_CHILD_LIFETIME(60.0f); // 1 minute
39 const char* basename(const char* path)
41 const char* ptr = path;
42 const char* slash = NULL;
43 for(; *ptr != '\0'; ++ptr)
45 if(*ptr == '/') slash = ptr;
47 if(slash != NULL) ++slash;
51 void SuppressLogOutput()
53 // Close stdout and stderr to suppress the log output
54 close(STDOUT_FILENO); // File descriptor number for stdout is 1
55 close(STDERR_FILENO); // File descriptor number for stderr is 2
57 // The POSIX specification requires that /dev/null must be provided,
58 // The open function always chooses the lowest unused file descriptor
59 // It is sufficient for stdout to be writable.
60 open("/dev/null", O_WRONLY); // Redirect file descriptor number 1 (i.e. stdout) to /dev/null
61 // When stderr is opened it must be both readable and writable.
62 open("/dev/null", O_RDWR); // Redirect file descriptor number 2 (i.e. stderr) to /dev/null
65 int32_t RunTestCase(struct ::testcase_s& testCase)
67 int32_t result = EXIT_STATUS_TESTCASE_FAILED;
69 // dont want to catch exception as we want to be able to get
70 // gdb stack trace from the first error
71 // by default tests should all always pass with no exceptions
78 result = testCase.function();
82 // just catch test fail exception, return is already set to EXIT_STATUS_TESTCASE_FAILED
92 int32_t RunTestCaseInChildProcess(struct ::testcase_s& testCase, bool suppressOutput)
94 int32_t testResult = EXIT_STATUS_TESTCASE_FAILED;
97 if(pid == 0) // Child process
106 for(int32_t i = 0; i < 80; ++i) printf("#");
107 printf("\nTC: %s\n", testCase.name);
111 int32_t status = RunTestCase(testCase);
125 exit(EXIT_STATUS_FORK_FAILED);
127 else // Parent process
130 int32_t childPid = waitpid(pid, &status, 0);
134 exit(EXIT_STATUS_WAITPID_FAILED);
136 if(WIFEXITED(status))
140 testResult = WEXITSTATUS(status);
143 printf("Test case %s failed: %d\n", testCase.name, testResult);
147 else if(WIFSIGNALED(status))
149 int32_t signal = WTERMSIG(status);
150 testResult = EXIT_STATUS_TESTCASE_ABORTED;
151 if(signal == SIGABRT)
153 printf("Test case %s failed: test case asserted\n", testCase.name);
157 printf("Test case %s failed: exit with signal %s\n", testCase.name, strsignal(WTERMSIG(status)));
160 else if(WIFSTOPPED(status))
162 printf("Test case %s failed: stopped with signal %s\n", testCase.name, strsignal(WSTOPSIG(status)));
170 void OutputStatistics(const char* processName, int32_t numPasses, int32_t numFailures)
172 FILE* fp = fopen("summary.xml", "a");
176 " <suite name=\"%s\">\n"
177 " <total_case>%d</total_case>\n"
178 " <pass_case>%d</pass_case>\n"
179 " <pass_rate>%5.2f</pass_rate>\n"
180 " <fail_case>%d</fail_case>\n"
181 " <fail_rate>%5.2f</fail_rate>\n"
182 " <block_case>0</block_case>\n"
183 " <block_rate>0.00</block_rate>\n"
184 " <na_case>0</na_case>\n"
185 " <na_rate>0.00</na_rate>\n"
187 basename(processName),
188 numPasses + numFailures,
190 (float)numPasses / (numPasses + numFailures),
192 (float)numFailures / (numPasses + numFailures));
197 int32_t RunAll(const char* processName, ::testcase tc_array[])
199 int32_t numFailures = 0;
200 int32_t numPasses = 0;
202 // Run test cases in child process( to kill output/handle signals ), but run serially.
203 for(uint32_t i = 0; tc_array[i].name; i++)
205 int32_t result = RunTestCaseInChildProcess(tc_array[i], false);
216 OutputStatistics(processName, numPasses, numFailures);
221 // Constantly runs up to MAX_NUM_CHILDREN processes
222 int32_t RunAllInParallel(const char* processName, ::testcase tc_array[], bool reRunFailed)
224 int32_t numFailures = 0;
225 int32_t numPasses = 0;
227 RunningTestCases children;
228 std::vector<int32_t> failedTestCases;
230 // Fork up to MAX_NUM_CHILDREN processes, then
231 // wait. As soon as a proc completes, fork the next.
233 int32_t nextTestCase = 0;
234 int32_t numRunningChildren = 0;
236 while(tc_array[nextTestCase].name || numRunningChildren > 0)
238 // Create more children (up to the max number or til the end of the array)
239 while(numRunningChildren < MAX_NUM_CHILDREN && tc_array[nextTestCase].name)
241 int32_t pid = fork();
242 if(pid == 0) // Child process
245 exit(RunTestCase(tc_array[nextTestCase]));
250 exit(EXIT_STATUS_FORK_FAILED);
252 else // Parent process
254 TestCase tc(nextTestCase, tc_array[nextTestCase].name);
255 tc.startTime = std::chrono::steady_clock::now();
259 numRunningChildren++;
263 // Check to see if any children have finished yet
265 int32_t childPid = waitpid(-1, &status, WNOHANG);
268 // No children have finished.
269 // Check if any have exceeded execution time
270 auto endTime = std::chrono::steady_clock::now();
272 for(auto& tc : children)
274 std::chrono::steady_clock::duration timeSpan = endTime - tc.second.startTime;
275 std::chrono::duration<double> seconds = std::chrono::duration_cast<std::chrono::duration<double>>(timeSpan);
276 if(seconds.count() > MAXIMUM_CHILD_LIFETIME)
278 // Kill the child process. A subsequent call to waitpid will process signal result below.
279 kill(tc.first, SIGKILL);
283 else if(childPid == -1) // waitpid errored
286 exit(EXIT_STATUS_WAITPID_FAILED);
288 else // a child has finished
290 if(WIFEXITED(status))
292 int32_t testResult = WEXITSTATUS(status);
295 printf("Test case %s failed: %d\n", children[childPid].testCaseName, testResult);
296 failedTestCases.push_back(children[childPid].testCase);
303 numRunningChildren--;
306 else if(WIFSIGNALED(status) || WIFSTOPPED(status))
308 status = WIFSIGNALED(status) ? WTERMSIG(status) : WSTOPSIG(status);
310 RunningTestCases::iterator iter = children.find(childPid);
311 if(iter != children.end())
313 printf("Test case %s exited with signal %s\n", iter->second.testCaseName, strsignal(status));
314 failedTestCases.push_back(iter->second.testCase);
318 printf("Unknown child process: %d signaled %s\n", childPid, strsignal(status));
322 numRunningChildren--;
327 OutputStatistics(processName, numPasses, numFailures);
331 for(uint32_t i = 0; i < failedTestCases.size(); i++)
333 char* testCaseStrapline;
334 int32_t numChars = asprintf(&testCaseStrapline, "Test case %s", tc_array[failedTestCases[i]].name);
335 printf("\n%s\n", testCaseStrapline);
336 for(int32_t j = 0; j < numChars; j++)
341 RunTestCaseInChildProcess(tc_array[failedTestCases[i]], false);
348 int32_t FindAndRunTestCase(::testcase tc_array[], const char* testCaseName)
350 int32_t result = EXIT_STATUS_TESTCASE_NOT_FOUND;
352 for(int32_t i = 0; tc_array[i].name; i++)
354 if(!strcmp(testCaseName, tc_array[i].name))
356 return RunTestCase(tc_array[i]);
360 printf("Unknown testcase name: \"%s\"\n", testCaseName);
364 void Usage(const char* program)
368 " %s <testcase name>\t\t Execute a test case\n"
369 " %s \t\t Execute all test cases in parallel\n"
370 " %s -r\t\t Execute all test cases in parallel, rerunning failed test cases\n"
371 " %s -s\t\t Execute all test cases serially\n",
378 } // namespace TestHarness