2 * Copyright (c) 2023 Samsung Electronics Co., Ltd.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
17 #include "test-harness.h"
21 #include <sys/types.h>
37 using std::chrono::steady_clock;
38 using std::chrono::system_clock;
42 typedef std::map<int32_t, TestCase> RunningTestCases;
44 const double MAXIMUM_CHILD_LIFETIME(60.0f); // 1 minute
46 const char* basename(const char* path)
48 const char* ptr = path;
49 const char* slash = NULL;
50 for(; *ptr != '\0'; ++ptr)
52 if(*ptr == '/') slash = ptr;
54 if(slash != NULL) ++slash;
58 std::vector<std::string> Split(const std::string& aString, char delimiter)
60 std::vector<std::string> tokens;
62 std::istringstream tokenStream(aString);
63 while(std::getline(tokenStream, token, delimiter))
65 tokens.push_back(token);
70 std::string Join(const std::vector<std::string>& tokens, char delimiter)
72 std::ostringstream oss;
74 unsigned int delimiterCount = 0;
75 for(auto& token : tokens)
78 if(delimiterCount < tokens.size() - 1)
87 std::string ChildOutputFilename(int pid)
89 std::ostringstream os;
90 os << "/tmp/tct-child." << pid;
94 std::string TestModuleFilename(const char* processName)
96 auto pathComponents = Split(processName, '/');
97 auto aModule = pathComponents.back();
98 aModule += "-tests.xml";
102 std::string TestModuleName(const char* processName)
104 auto pathComponents = Split(processName, '/');
105 auto aModule = pathComponents.back();
106 auto moduleComponents = Split(aModule, '-');
108 moduleComponents[1][0] = std::toupper(moduleComponents[1][0]);
109 moduleComponents[2][0] = std::toupper(moduleComponents[2][0]);
111 std::ostringstream oss;
112 for(unsigned int i = 1; i < moduleComponents.size() - 1; ++i) // [0]=tct, [n-1]=core
114 oss << moduleComponents[i];
116 if(i > 1 && i < moduleComponents.size() - 2) // skip first and last delimiter
125 std::string GetWChan(int pid)
127 std::ostringstream procwchan;
128 procwchan << "/proc/" << pid << "/wchan";
130 ifs.open(procwchan.str(), std::ifstream::in);
132 std::getline(ifs, line);
137 std::string ReadAndEscape(std::string filename)
139 std::ostringstream os;
141 ifs.open(filename, std::ifstream::in);
145 std::getline(ifs, line);
171 void OutputTestResult(
173 const char* pathToExecutable,
174 std::string testSuiteName,
176 std::string startTime,
179 std::string outputFilename = ChildOutputFilename(testCase.childPid);
180 std::string testOutput = ReadAndEscape(outputFilename);
182 ofs << "<testcase component=\"CoreAPI/" << testSuiteName << "/default\" execution_type=\"auto\" id=\""
183 << testCase.name << "\" purpose=\"\" result=\"" << (testCase.result == 0 ? "PASS" : "FAIL") << "\">" << std::endl
184 << "<description><test_script_entry test_script_expected_result=\"0\">"
185 << pathToExecutable << testCase.name << "</test_script_entry>" << std::endl
187 << "<result_info><actual_result>" << (testCase.result == 0 ? "PASS" : "FAIL") << "</actual_result>" << std::endl
188 << "<start>" << startTime << "</start>"
189 << "<end>" << endTime << "</end>"
190 << "<stdout><![CDATA[]]></stdout>"
191 << "<stderr><![CDATA[" << testOutput << "]]></stderr></result_info></testcase>" << std::endl;
193 unlink(outputFilename.c_str());
196 void OutputTestResults(const char* processName, RunningTestCases& children)
199 std::string filename = TestModuleFilename(processName);
200 std::string moduleName = TestModuleName(processName);
201 ofs.open(filename, std::ofstream::out | std::ofstream::app);
203 // Sort completed cases by original test case id
204 std::vector<TestCase> childTestCases;
205 childTestCases.reserve(children.size());
206 for(auto& element : children) childTestCases.push_back(element.second);
207 std::sort(childTestCases.begin(), childTestCases.end(), [](const TestCase& a, const TestCase& b) {
208 return a.testCase < b.testCase;
211 const int BUFSIZE = 256;
212 char buffer[BUFSIZE];
213 for(auto& testCase : childTestCases)
215 auto tt = system_clock::to_time_t(testCase.startSystemTime);
216 strftime(buffer, BUFSIZE, "%c", localtime(&tt));
217 std::string startTime(buffer);
218 OutputTestResult(ofs, processName, moduleName, testCase, startTime, startTime);
224 void OutputStatistics(const char* processName, int32_t numPasses, int32_t numFailures)
226 FILE* fp = fopen("summary.xml", "a");
230 " <suite name=\"%s-tests\">\n"
231 " <total_case>%d</total_case>\n"
232 " <pass_case>%d</pass_case>\n"
233 " <pass_rate>%5.2f</pass_rate>\n"
234 " <fail_case>%d</fail_case>\n"
235 " <fail_rate>%5.2f</fail_rate>\n"
236 " <block_case>0</block_case>\n"
237 " <block_rate>0.00</block_rate>\n"
238 " <na_case>0</na_case>\n"
239 " <na_rate>0.00</na_rate>\n"
241 basename(processName),
242 numPasses + numFailures,
244 numPasses > 0 ? (float)numPasses * 100.0f / (numPasses + numFailures) : 0.0f,
246 numPasses > 0 ? (float)numFailures * 100.0f / (numPasses + numFailures) : 0.0f);
251 int32_t RunTestCase(struct ::testcase_s& testCase)
253 int32_t result = EXIT_STATUS_TESTCASE_FAILED;
255 // dont want to catch exception as we want to be able to get
256 // gdb stack trace from the first error
257 // by default tests should all always pass with no exceptions
264 result = testCase.function();
268 // just catch test fail exception, return is already set to EXIT_STATUS_TESTCASE_FAILED
278 int32_t RunTestCaseRedirectOutput(TestCase& testCase, bool suppressOutput)
280 // Executing in child process
281 // Close stdout and stderr to suppress the log output
282 close(STDOUT_FILENO); // File descriptor number for stdout is 1
284 // The POSIX specification requires that /dev/null must be provided,
285 // The open function always chooses the lowest unused file descriptor
286 // It is sufficient for stdout to be writable.
287 open("/dev/null", O_WRONLY); // Redirect file descriptor number 1 (i.e. stdout) to /dev/null
290 close(STDERR_FILENO);
293 stderr = fopen("/dev/null", "w+"); // Redirect fd 2 to /dev/null
297 // When stderr is opened it must be both readable and writable.
298 std::string childOutputFilename = ChildOutputFilename(getpid());
299 stderr = fopen(childOutputFilename.c_str(), "w+");
302 int32_t status = RunTestCase(*testCase.tctPtr);
310 int32_t RunTestCaseInChildProcess(TestCase& testCase, bool redirect)
312 int32_t testResult = EXIT_STATUS_TESTCASE_FAILED;
314 int32_t pid = fork();
315 if(pid == 0) // Child process
319 int status = RunTestCaseRedirectOutput(testCase, false);
324 int status = RunTestCase(*testCase.tctPtr);
331 exit(EXIT_STATUS_FORK_FAILED);
333 else // Parent process
336 int32_t childPid = waitpid(pid, &status, 0);
337 testCase.childPid = childPid;
341 exit(EXIT_STATUS_WAITPID_FAILED);
343 if(WIFEXITED(status))
347 testResult = WEXITSTATUS(status);
350 printf("Test case %s failed: %d\n", testCase.name, testResult);
354 else if(WIFSIGNALED(status))
356 int32_t signal = WTERMSIG(status);
357 testResult = EXIT_STATUS_TESTCASE_ABORTED;
358 if(signal == SIGABRT)
360 printf("Test case %s failed: test case asserted\n", testCase.name);
364 printf("Test case %s failed: exit with signal %s\n", testCase.name, strsignal(WTERMSIG(status)));
367 else if(WIFSTOPPED(status))
369 printf("Test case %s failed: stopped with signal %s\n", testCase.name, strsignal(WSTOPSIG(status)));
377 int32_t RunAll(const char* processName, ::testcase tc_array[], std::string match, bool quiet)
379 int32_t numFailures = 0;
380 int32_t numPasses = 0;
382 std::string filename = TestModuleFilename(processName);
383 std::string moduleName = TestModuleName(processName);
384 ofs.open(filename, std::ofstream::out | std::ofstream::app);
385 const int BUFSIZE = 256;
386 char buffer[BUFSIZE];
388 // Run test cases in child process( to handle signals ), but run serially.
389 for(uint32_t i = 0; tc_array[i].name; i++)
391 auto tt = system_clock::to_time_t(system_clock::now());
392 strftime(buffer, BUFSIZE, "%c", localtime(&tt));
393 std::string startTime(buffer);
395 TestCase testCase(i, &tc_array[i]);
399 if(match.compare(0, match.size(), testCase.name, match.size()))
407 testCase.result = RunTestCaseInChildProcess(testCase, quiet);
409 tt = system_clock::to_time_t(system_clock::now());
410 strftime(buffer, BUFSIZE, "%c", localtime(&tt));
411 std::string endTime(buffer);
413 if(testCase.result == 0)
423 OutputTestResult(ofs, processName, moduleName, testCase, startTime, endTime);
429 OutputStatistics(processName, numPasses, numFailures);
434 // Constantly runs up to MAX_NUM_CHILDREN processes
435 int32_t RunAllInParallel(const char* processName, ::testcase tc_array[], std::string match, bool reRunFailed, bool quiet)
437 int32_t numFailures = 0;
438 int32_t numPasses = 0;
440 RunningTestCases children;
441 std::vector<int32_t> failedTestCases;
443 // Fork up to MAX_NUM_CHILDREN processes, then
444 // wait. As soon as a proc completes, fork the next.
446 int32_t nextTestCase = 0;
447 int32_t numRunningChildren = 0;
449 while(tc_array[nextTestCase].name || numRunningChildren > 0)
451 // Create more children (up to the max number or til the end of the array)
452 while(numRunningChildren < MAX_NUM_CHILDREN && tc_array[nextTestCase].name)
457 if(match.compare(0, match.size(), tc_array[nextTestCase].name, match.size()))
464 int32_t pid = fork();
465 if(pid == 0) // Child process
467 TestCase testCase(nextTestCase, &tc_array[nextTestCase]);
468 int status = RunTestCaseRedirectOutput(testCase, quiet);
474 exit(EXIT_STATUS_FORK_FAILED);
476 else // Parent process
478 TestCase tc(nextTestCase, tc_array[nextTestCase].name);
479 tc.startTime = steady_clock::now();
480 tc.startSystemTime = system_clock::now();
485 numRunningChildren++;
495 int32_t childPid = 0;
496 if(numRunningChildren > 0) // Only wait if there are running children.
498 // Check to see if any children have finished yet
499 childPid = waitpid(-1, &status, WNOHANG);
504 // No children have finished.
505 // Check if any have exceeded execution time
506 auto endTime = std::chrono::steady_clock::now();
508 for(auto& tc : children)
510 std::chrono::steady_clock::duration timeSpan = endTime - tc.second.startTime;
511 double seconds = double(timeSpan.count()) * std::chrono::steady_clock::period::num / std::chrono::steady_clock::period::den;
513 if(seconds > MAXIMUM_CHILD_LIFETIME)
515 // Kill the child process. A subsequent call to waitpid will process signal result below.
516 if(!tc.second.finished)
518 printf("Child process %s WCHAN:%s\n", tc.second.name, GetWChan(tc.first).c_str());
519 kill(tc.first, SIGKILL);
520 tc.second.finished = true; // Only send kill signal once.
525 else if(childPid == -1) // waitpid errored
528 exit(EXIT_STATUS_WAITPID_FAILED);
530 else // a child has finished
532 if(WIFEXITED(status))
534 auto& testCase = children[childPid];
535 testCase.result = WEXITSTATUS(status);
536 testCase.finished = true;
539 printf("Test case %s failed: %d\n", testCase.name, testCase.result);
540 failedTestCases.push_back(testCase.testCase);
547 numRunningChildren--;
550 else if(WIFSIGNALED(status) || WIFSTOPPED(status))
552 status = WIFSIGNALED(status) ? WTERMSIG(status) : WSTOPSIG(status);
554 RunningTestCases::iterator iter = children.find(childPid);
555 if(iter != children.end())
557 iter->second.finished = true;
558 printf("Test case %s exited with signal %s\n", iter->second.name, strsignal(status));
559 iter->second.result = 1;
560 failedTestCases.push_back(iter->second.testCase);
564 printf("Unknown child process: %d signaled %s\n", childPid, strsignal(status));
568 numRunningChildren--;
575 OutputTestResults(processName, children);
578 OutputStatistics(processName, numPasses, numFailures);
582 for(uint32_t i = 0; i < failedTestCases.size(); i++)
584 char* testCaseStrapline;
585 int32_t numChars = asprintf(&testCaseStrapline, "Test case %s", tc_array[failedTestCases[i]].name);
586 printf("\n%s\n", testCaseStrapline);
587 for(int32_t j = 0; j < numChars; j++)
592 int index = failedTestCases[i];
593 TestCase testCase(index, &tc_array[index]);
594 RunTestCaseInChildProcess(testCase, false);
601 int32_t FindAndRunTestCase(::testcase tc_array[], const char* testCaseName)
603 int32_t result = EXIT_STATUS_TESTCASE_NOT_FOUND;
605 for(int32_t i = 0; tc_array[i].name; i++)
607 if(!strcmp(testCaseName, tc_array[i].name))
609 return RunTestCase(tc_array[i]);
613 printf("Unknown testcase name: \"%s\"\n", testCaseName);
617 void Usage(const char* program)
621 " %s <testcase name>\t\t Execute a test case\n"
622 " %s \t\t Execute all test cases in parallel\n"
623 " %s -r\t\t Execute all test cases in parallel, rerunning failed test cases\n"
624 " %s -s\t\t Execute all test cases serially\n"
625 " %s -q\t\t Run without output\n",
633 int RunTests(int argc, char* const argv[], ::testcase tc_array[])
635 int result = TestHarness::EXIT_STATUS_BAD_ARGUMENT;
636 const char* optString = "sfqm:";
637 bool optRerunFailed(true);
638 bool optRunSerially(false);
639 bool optQuiet(false);
640 std::string optMatch;
645 nextOpt = getopt(argc, argv, optString);
649 optRerunFailed = false;
652 optRunSerially = true;
661 TestHarness::Usage(argv[0]);
662 exit(TestHarness::EXIT_STATUS_BAD_ARGUMENT);
665 } while(nextOpt != -1);
667 if(optind == argc) // no testcase name in argument list
671 result = TestHarness::RunAll(argv[0], tc_array, optMatch, optQuiet);
675 result = TestHarness::RunAllInParallel(argv[0], tc_array, optMatch, optRerunFailed, optQuiet);
680 // optind is index of next argument - interpret as testcase name
681 result = TestHarness::FindAndRunTestCase(tc_array, argv[optind]);
686 } // namespace TestHarness