2 * Copyright (c) 2011 Samsung Electronics Co., Ltd All Rights Reserved
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
17 * @file test_runner.cpp
18 * @author Przemyslaw Dobrowolski (p.dobrowolsk@samsung.com)
19 * @author Lukasz Wrzosek (l.wrzosek@samsung.com)
21 * @brief This file is the implementation file of test runner
24 #include <dpl/test/test_runner.h>
25 #include <dpl/test/test_results_collector.h>
26 #include <dpl/exception.h>
27 #include <dpl/scoped_free.h>
28 #include <dpl/log/log.h>
29 #include <dpl/colors.h>
38 #include <libxml/xpath.h>
39 #include <libxml/xpathInternals.h>
40 #include <libxml/parser.h>
41 #include <libxml/tree.h>
43 #include <dpl/singleton_impl.h>
44 IMPLEMENT_SINGLETON(DPL::Test::TestRunner)
48 std::string getXMLNode(xmlNodePtr node)
51 xmlChar * value = xmlNodeGetContent(node);
52 ret = std::string(reinterpret_cast<char*>(value));
62 namespace // anonymous
64 std::string BaseName(std::string aPath)
66 ScopedFree<char> path(strdup(aPath.c_str()));
67 if (nullptr == path.Get()) {
68 throw std::bad_alloc();
70 char* baseName = basename(path.Get());
71 std::string retValue = baseName;
74 } // namespace anonymous
76 //! \brief Failed test message creator
78 //! \param[in] aTest string for tested expression
79 //! \param[in] aFile source file name
80 //! \param[in] aLine source file line
81 //! \param[in] aMessage error message
82 TestRunner::TestFailed::TestFailed(const char* aTest,
85 const std::string &aMessage)
87 std::ostringstream assertMsg;
88 assertMsg << "[" << BaseName(aFile) << ":" << aLine
89 << "] Assertion failed ("
90 << aTest << ") " << aMessage;
91 m_message = assertMsg.str();
94 TestRunner::TestFailed::TestFailed(const std::string &message)
99 void TestRunner::RegisterTest(const char *testName, TestCase proc)
101 m_testGroups[m_currentGroup].push_back(TestCaseStruct(testName, proc));
104 void TestRunner::InitGroup(const char* name)
106 m_currentGroup = name;
109 void TestRunner::normalizeXMLTag(std::string& str, const std::string& testcase)
111 //Add testcase if missing
112 std::string::size_type pos = str.find(testcase);
115 str = testcase + "_" + str;
118 //dpl test runner cannot have '-' character in name so it have to be replaced
119 // for TCT case to make comparision works
120 std::replace(str.begin(), str.end(), '-', '_');
123 bool TestRunner::filterGroupsByXmls(const std::vector<std::string> & files)
125 DECLARE_EXCEPTION_TYPE(DPL::Exception, XMLError)
127 const std::string idPath = "/test_definition/suite/set/testcase/@id";
130 std::map<std::string, bool> casesMap;
132 std::string testsuite;
133 if(!m_testGroups.empty())
135 for(TestCaseGroupMap::const_iterator cit = m_testGroups.begin(); cit != m_testGroups.end(); ++cit)
137 if(!cit->second.empty())
139 for(TestCaseStructList::const_iterator cj = cit->second.begin(); cj != cit->second.end(); ++cj)
141 std::string name = cj->name;
142 std::string::size_type st = name.find('_');
143 if(st != std::string::npos)
145 name = name.substr(0, st);
150 if(!testsuite.empty()) break;
161 for (const std::string &file : files)
164 xmlXPathContextPtr xpathCtx;
166 doc = xmlReadFile(file.c_str(), nullptr, 0);
167 if (doc == nullptr) {
168 ThrowMsg(XMLError, "File Problem");
171 xpathCtx = xmlXPathNewContext(doc);
172 if (xpathCtx == nullptr) {
174 "Error: unable to create new XPath context\n");
176 xpathCtx->node = xmlDocGetRootElement(doc);
180 xmlXPathObjectPtr xpathObject;
181 //get requested node's values
182 xpathObject = xmlXPathEvalExpression(BAD_CAST idPath.c_str(), xpathCtx);
183 if (xpathObject == nullptr)
185 ThrowMsg(XMLError, "XPath evaluation failure: " << idPath);
187 xmlNodeSetPtr nodes = xpathObject->nodesetval;
188 unsigned size = (nodes) ? nodes->nodeNr : 0;
189 LogDebug("Found " << size << " nodes matching xpath");
190 for(unsigned i = 0; i < size; ++i)
192 LogPedantic("Type: " << nodes->nodeTab[i]->type);
193 if (nodes->nodeTab[i]->type == XML_ATTRIBUTE_NODE) {
194 xmlNodePtr curNode = nodes->nodeTab[i];
195 result = getXMLNode(curNode);
196 LogPedantic("Result: " << result);
197 normalizeXMLTag(result, testsuite);
198 casesMap.insert(make_pair(result, false));
201 //Cleanup of XPath data
202 xmlXPathFreeObject(xpathObject);
203 xmlXPathFreeContext(xpathCtx);
209 LogError("Libxml error: " << _rethrown_exception.DumpToString());
214 if(!filterByXML(casesMap))
222 bool TestRunner::filterByXML(std::map<std::string, bool> & casesMap)
224 for (auto &group : m_testGroups) {
225 TestCaseStructList newList;
226 for (auto &tc : group.second)
228 if (casesMap.find(tc.name) != casesMap.end()) {
229 casesMap[tc.name] = true;
230 newList.push_back(tc);
233 group.second = newList;
235 for (auto &cs : casesMap)
237 if(cs.second == false)
239 LogError("Cannot find testcase from XML file: " << cs.first);
246 TestRunner::Status TestRunner::RunTestCase(const TestCaseStruct& testCase)
248 setCurrentTestCase(&(const_cast<TestCaseStruct &>(testCase)));
251 } catch (const TestFailed &e) {
252 // Simple test failure
253 CollectResult(testCase.name,
255 TestResultsCollectorBase::FailStatus::FAILED,
258 setCurrentTestCase(nullptr);
260 } catch (const Ignored &e) {
262 // Simple test have to be implemented
263 CollectResult(testCase.name,
265 TestResultsCollectorBase::FailStatus::IGNORED,
269 setCurrentTestCase(nullptr);
271 } catch (const DPL::Exception &e) {
272 // DPL exception failure
273 CollectResult(testCase.name,
275 TestResultsCollectorBase::FailStatus::INTERNAL,
276 "DPL exception:" + e.GetMessage());
278 setCurrentTestCase(nullptr);
280 } catch (const std::exception &) {
281 // std exception failure
282 CollectResult(testCase.name,
284 TestResultsCollectorBase::FailStatus::INTERNAL,
287 setCurrentTestCase(nullptr);
290 // Unknown exception failure
291 CollectResult(testCase.name,
293 TestResultsCollectorBase::FailStatus::INTERNAL,
294 "unknown exception");
296 setCurrentTestCase(nullptr);
300 CollectResult(testCase.name,
302 TestResultsCollectorBase::FailStatus::NONE,
304 testCase.m_isPerformanceTest,
305 testCase.m_performanceTestDurationTime,
306 testCase.m_performanceMaxTime);
307 setCurrentTestCase(nullptr);
313 void TestRunner::RunTests()
315 using namespace DPL::Colors::Text;
318 for (auto &collector : m_collectors) {
319 collector.second->Start();
323 for (auto &group : m_testGroups) {
324 count += group.second.size();
326 fprintf(stderr, "%sFound %d testcases...%s\n", GREEN_BEGIN, count, GREEN_END);
327 fprintf(stderr, "%s%s%s\n", GREEN_BEGIN, "Running tests...", GREEN_END);
328 for (auto &group : m_testGroups) {
329 TestCaseStructList list = group.second;
331 for (auto &collector : m_collectors) {
332 collector.second->CollectCurrentTestGroupName(group.first);
336 for (TestCaseStructList::const_iterator iterator = list.begin();
337 iterator != list.end();
340 TestCaseStruct test = *iterator;
341 if (m_startTestId == test.name) {
345 if (m_startTestId.empty()) {
348 if (m_terminate == true) {
349 // Terminate quietly without any logs
356 std::for_each(m_collectors.begin(),
358 [] (const TestResultsCollectors::value_type & collector)
360 collector.second->Finish();
364 fprintf(stderr, "%s%s%s\n\n", GREEN_BEGIN, "Finished", GREEN_END);
367 TestRunner::TestCaseStruct *TestRunner::getCurrentTestCase()
369 return m_currentTestCase;
372 void TestRunner::setCurrentTestCase(TestCaseStruct* testCase)
374 m_currentTestCase = testCase;
377 void TestRunner::beginPerformanceTestTime(std::chrono::system_clock::duration maxTimeInMicroseconds)
379 TestCaseStruct* testCase = getCurrentTestCase();
383 testCase->m_isPerformanceTest = true;
384 testCase->m_performanceMaxTime = maxTimeInMicroseconds;
385 testCase->m_performanceTestStartTime = std::chrono::system_clock::now();
387 // Set result to 0 microseconds. Display 0ms result when end macro is missing.
388 testCase->m_performanceTestDurationTime = std::chrono::microseconds::zero();
391 void TestRunner::endPerformanceTestTime()
393 TestCaseStruct* testCase = getCurrentTestCase();
397 testCase->m_performanceTestDurationTime = std::chrono::system_clock::now() -
398 testCase->m_performanceTestStartTime;
401 void TestRunner::getCurrentTestCasePerformanceResult(bool& isPerformanceTest,
402 std::chrono::system_clock::duration& result,
403 std::chrono::system_clock::duration& resultMax)
405 TestCaseStruct* testCase = getCurrentTestCase();
406 if (!testCase || !(testCase->m_isPerformanceTest)){
407 isPerformanceTest = false;
411 isPerformanceTest = testCase->m_isPerformanceTest;
412 result = testCase->m_performanceTestDurationTime;
413 resultMax = testCase->m_performanceMaxTime;
416 void TestRunner::setCurrentTestCasePerformanceResult(bool isPerformanceTest,
417 std::chrono::system_clock::duration result,
418 std::chrono::system_clock::duration resultMax)
420 TestCaseStruct* testCase = getCurrentTestCase();
424 testCase->m_isPerformanceTest = isPerformanceTest;
425 testCase->m_performanceTestDurationTime = result;
426 testCase->m_performanceMaxTime = resultMax;
430 void TestRunner::CollectResult(
431 const std::string& id,
432 const std::string& description,
433 const TestResultsCollectorBase::FailStatus::Type status,
434 const std::string& reason,
435 const bool& isPerformanceTest,
436 const std::chrono::system_clock::duration& performanceTestDurationTime,
437 const std::chrono::system_clock::duration& performanceMaxTime)
439 std::for_each(m_collectors.begin(),
441 [&](const TestResultsCollectors::value_type & collector)
443 collector.second->CollectResult(id,
448 performanceTestDurationTime,
453 void TestRunner::Banner()
455 using namespace DPL::Colors::Text;
469 void TestRunner::InvalidArgs(const std::string& message)
471 using namespace DPL::Colors::Text;
479 void TestRunner::Usage()
481 fprintf(stderr, "Usage: runner [options]\n\n");
482 fprintf(stderr, "Output type:\n");
483 fprintf(stderr, " --output=<output type> --output=<output type> ...\n");
484 fprintf(stderr, "\n possible output types:\n");
485 for (std::string &type : TestResultsCollectorBase::GetCollectorsNames()) {
486 fprintf(stderr, " --output=%s\n", type.c_str());
488 fprintf(stderr, "\n example:\n");
490 " test-binary --output=text --output=xml --file=output.xml\n\n");
491 fprintf(stderr, "Other parameters:\n");
493 " --regexp='regexp'\t Only selected tests"
494 " which names match regexp run\n\n");
495 fprintf(stderr, " --start=<test id>\tStart from concrete test id");
496 fprintf(stderr, " --group=<group name>\t Run tests only from one group\n");
497 fprintf(stderr, " --runignored\t Run also ignored tests\n");
498 fprintf(stderr, " --list\t Show a list of Test IDs\n");
499 fprintf(stderr, " --listgroups\t Show a list of Test Group names \n");
500 fprintf(stderr, " --only-from-xml=<xml file>\t Run only testcases specified in XML file \n"
501 " XML name is taken from attribute id=\"part1_part2\" as whole.\n"
502 " If part1 is not found (no _) then it is implicitily "
503 "set according to suite part1 from binary tests\n");
506 " --listingroup=<group name>\t Show a list of Test IDS in one group\n");
507 fprintf(stderr, " --allowchildlogs\t Allow to print logs from child process on screen.\n");
508 fprintf(stderr, " When active child process will be able to print logs on stdout and stderr.\n");
509 fprintf(stderr, " Both descriptors will be closed after test.\n");
510 fprintf(stderr, " --help\t This help\n\n");
511 std::for_each(m_collectors.begin(),
513 [] (const TestResultsCollectors::value_type & collector)
516 "Output %s has specific args:\n",
517 collector.first.c_str());
521 CollectorSpecificHelp().c_str());
523 fprintf(stderr, "For bug reporting, please write to:\n");
524 fprintf(stderr, "<p.dobrowolsk@samsung.com>\n");
527 int TestRunner::ExecTestRunner(int argc, char *argv[])
529 std::vector<std::string> args;
530 for (int i = 0; i < argc; ++i) {
531 args.push_back(argv[i]);
533 return ExecTestRunner(args);
536 void TestRunner::MarkAssertion()
541 int TestRunner::ExecTestRunner(ArgsList args)
543 m_runIgnored = false;
544 // Parse command line
546 args.erase(args.begin());
548 bool showHelp = false;
549 bool justList = false;
550 std::vector<std::string> xmlFiles;
552 TestResultsCollectorBasePtr currentCollector;
554 // Parse each argument
555 for(std::string &arg : args)
557 const std::string regexp = "--regexp=";
558 const std::string output = "--output=";
559 const std::string groupId = "--group=";
560 const std::string runIgnored = "--runignored";
561 const std::string listCmd = "--list";
562 const std::string startCmd = "--start=";
563 const std::string listGroupsCmd = "--listgroups";
564 const std::string listInGroup = "--listingroup=";
565 const std::string allowChildLogs = "--allowchildlogs";
566 const std::string onlyFromXML = "--only-from-xml=";
568 if (currentCollector) {
569 if (currentCollector->ParseCollectorSpecificArg(arg)) {
574 if (arg.find(startCmd) == 0) {
575 arg.erase(0, startCmd.length());
576 for (auto &group : m_testGroups) {
577 for (auto &tc : group.second) {
578 if (tc.name == arg) {
583 if (!m_startTestId.empty()) {
587 if (!m_startTestId.empty()) {
591 fprintf(stderr, "Start test id has not been found\n");
594 } else if (arg.find(groupId) == 0) {
595 arg.erase(0, groupId.length());
596 TestCaseGroupMap::iterator found = m_testGroups.find(arg);
597 if (found != m_testGroups.end()) {
598 std::string name = found->first;
599 TestCaseStructList newList = found->second;
600 m_testGroups.clear();
601 m_testGroups[name] = newList;
603 fprintf(stderr, "Group %s not found\n", arg.c_str());
608 } else if (arg == runIgnored) {
610 } else if (arg == listCmd) {
612 } else if (arg == listGroupsCmd) {
613 for (auto &group : m_testGroups) {
614 printf("GR:%s\n", group.first.c_str());
617 } else if (arg.find(listInGroup) == 0) {
618 arg.erase(0, listInGroup.length());
619 for (auto &test : m_testGroups[arg]) {
620 printf("ID:%s\n", test.name.c_str());
623 } else if (arg.find(allowChildLogs) == 0) {
624 arg.erase(0, allowChildLogs.length());
625 m_allowChildLogs = true;
626 } else if (arg == "--help") {
628 } else if (arg.find(output) == 0) {
629 arg.erase(0, output.length());
630 if (m_collectors.find(arg) != m_collectors.end()) {
632 "Multiple outputs of the same type are not supported!");
636 currentCollector.reset(TestResultsCollectorBase::Create(arg));
637 if (!currentCollector) {
638 InvalidArgs("Unsupported output type!");
642 m_collectors[arg] = currentCollector;
643 } else if (arg.find(regexp) == 0) {
644 arg.erase(0, regexp.length());
645 if (arg.length() == 0) {
651 if (arg[0] == '\'' && arg[arg.length() - 1] == '\'') {
653 arg.erase(arg.length() - 1);
656 if (arg.length() == 0) {
662 pcrecpp::RE re(arg.c_str());
663 for (auto &group : m_testGroups) {
664 TestCaseStructList newList;
665 for (auto &tc : group.second)
667 if (re.PartialMatch(tc.name)) {
668 newList.push_back(tc);
671 group.second = newList;
673 } else if(arg.find(onlyFromXML) == 0) {
674 arg.erase(0, onlyFromXML.length());
675 if (arg.length() == 0) {
681 if (arg[0] == '\'' && arg[arg.length() - 1] == '\'') {
683 arg.erase(arg.length() - 1);
686 if (arg.length() == 0) {
692 xmlFiles.push_back(arg);
700 if(!xmlFiles.empty())
702 if(!filterGroupsByXmls(xmlFiles))
704 fprintf(stderr, "XML file is not correct\n");
711 for (auto &group : m_testGroups) {
712 for (auto &tc : group.second) {
713 printf("ID:%s:%s\n", group.first.c_str(), tc.name.c_str());
719 currentCollector.reset();
727 if (m_collectors.empty()) {
728 TestResultsCollectorBasePtr collector(
729 TestResultsCollectorBase::Create("text"));
730 m_collectors["text"] = collector;
733 for (auto &collector : m_collectors) {
734 if (!collector.second->Configure()) {
735 fprintf(stderr, "Could not configure selected output");
746 bool TestRunner::getRunIgnored() const
751 void TestRunner::Terminate()
756 bool TestRunner::GetAllowChildLogs()
758 return m_allowChildLogs;