2 * Copyright (c) 2014-2015 Samsung Electronics Co., Ltd All Rights Reserved
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
17 * @file test_runner.cpp
18 * @author Przemyslaw Dobrowolski (p.dobrowolsk@samsung.com)
19 * @author Lukasz Wrzosek (l.wrzosek@samsung.com)
21 * @brief This file is the implementation file of test runner
24 #include <dpl/test/test_failed.h>
25 #include <dpl/test/test_ignored.h>
26 #include <dpl/test/test_runner.h>
27 #include <dpl/test/test_results_collector.h>
28 #include <dpl/exception.h>
29 #include <dpl/scoped_free.h>
30 #include <dpl/log/log.h>
31 #include <dpl/colors.h>
40 #include <libxml/xpath.h>
41 #include <libxml/xpathInternals.h>
42 #include <libxml/parser.h>
43 #include <libxml/tree.h>
45 #include <dpl/singleton_impl.h>
46 IMPLEMENT_SINGLETON(DPL::Test::TestRunner)
50 std::string getXMLNode(xmlNodePtr node)
53 xmlChar * value = xmlNodeGetContent(node);
54 ret = std::string(reinterpret_cast<char*>(value));
64 TestResult::FailStatus TryCatch(const std::function<void(void)> &func, std::string &reason) {
67 } catch (const DPL::Test::TestFailed &e) {
68 reason = e.GetMessage();
69 return TestResult::FailStatus::FAILED;
70 } catch (const DPL::Test::TestIgnored &e) {
71 reason = e.GetMessage();
72 return TestResult::FailStatus::IGNORED;
73 } catch (const std::exception &e) {
75 return TestResult::FailStatus::FAILED;
77 reason = "Unknown exception";
78 return TestResult::FailStatus::FAILED;
80 reason = std::string();
81 return TestResult::FailStatus::NONE;
84 void TestRunner::RegisterTest(TestCasePtr testCase)
86 m_testGroups[m_currentGroup].push_back(testCase);
87 m_testCaseSet.insert(testCase);
90 void TestRunner::InitGroup(const char* name)
92 m_currentGroup = name;
95 void TestRunner::normalizeXMLTag(std::string& str, const std::string& testcase)
97 //Add testcase if missing
98 std::string::size_type pos = str.find(testcase);
101 str = testcase + "_" + str;
104 //dpl test runner cannot have '-' character in name so it have to be replaced
105 // for TCT case to make comparision works
106 std::replace(str.begin(), str.end(), '-', '_');
109 bool TestRunner::filterGroupsByXmls(const std::vector<std::string> & files)
111 DECLARE_EXCEPTION_TYPE(DPL::Exception, XMLError)
113 const std::string idPath = "/test_definition/suite/set/testcase/@id";
116 std::map<std::string, bool> casesMap;
118 std::string testsuite;
119 if(!m_testGroups.empty())
121 for(TestCaseGroupMap::const_iterator cit = m_testGroups.begin(); cit != m_testGroups.end(); ++cit)
123 if(!cit->second.empty())
125 for(TestCaseList::const_iterator cj = cit->second.begin(); cj != cit->second.end(); ++cj)
127 std::string name = (*cj)->GetName();
128 std::string::size_type st = name.find('_');
129 if(st != std::string::npos)
131 name = name.substr(0, st);
136 if(!testsuite.empty()) break;
147 for (const std::string &file : files)
150 xmlXPathContextPtr xpathCtx;
152 doc = xmlReadFile(file.c_str(), nullptr, 0);
153 if (doc == nullptr) {
154 ThrowMsg(XMLError, "File Problem");
157 xpathCtx = xmlXPathNewContext(doc);
158 if (xpathCtx == nullptr) {
160 "Error: unable to create new XPath context\n");
162 xpathCtx->node = xmlDocGetRootElement(doc);
166 xmlXPathObjectPtr xpathObject;
167 //get requested node's values
168 xpathObject = xmlXPathEvalExpression(BAD_CAST idPath.c_str(), xpathCtx);
169 if (xpathObject == nullptr)
171 ThrowMsg(XMLError, "XPath evaluation failure: " << idPath);
173 xmlNodeSetPtr nodes = xpathObject->nodesetval;
174 unsigned size = (nodes) ? nodes->nodeNr : 0;
175 LogDebug("Found " << size << " nodes matching xpath");
176 for(unsigned i = 0; i < size; ++i)
178 LogPedantic("Type: " << nodes->nodeTab[i]->type);
179 if (nodes->nodeTab[i]->type == XML_ATTRIBUTE_NODE) {
180 xmlNodePtr curNode = nodes->nodeTab[i];
181 result = getXMLNode(curNode);
182 LogPedantic("Result: " << result);
183 normalizeXMLTag(result, testsuite);
184 casesMap.insert(make_pair(result, false));
187 //Cleanup of XPath data
188 xmlXPathFreeObject(xpathObject);
189 xmlXPathFreeContext(xpathCtx);
195 LogError("Libxml error: " << _rethrown_exception.DumpToString());
200 if(!filterByXML(casesMap))
208 bool TestRunner::filterByXML(std::map<std::string, bool> & casesMap)
210 for (auto &group : m_testGroups) {
211 TestCaseList newList;
212 for (auto &tc : group.second)
214 if (casesMap.find(tc->GetName()) != casesMap.end()) {
215 casesMap[tc->GetName()] = true;
216 newList.push_back(tc);
219 group.second = newList;
221 for (auto &cs : casesMap)
223 if(cs.second == false)
225 LogError("Cannot find testcase from XML file: " << cs.first);
232 void TestRunner::RunTestCase(TestCasePtr testCase)
234 setCurrentTestCase(testCase);
236 std::string initReason;
237 TestResult::FailStatus initStatus = TryCatch(std::bind(&TestCase::Init, testCase),
239 if (initStatus != TestResult::FailStatus::NONE) {
240 CollectResult(testCase->GetName(),
241 TestResult(initStatus, getConcatedFailReason(initReason), testCase->GetPerformance()));
242 setCurrentTestCase(nullptr);
246 std::string testReason;
247 TestResult::FailStatus testStatus = TryCatch(std::bind(&TestCase::Test, testCase),
249 testReason = getConcatedFailReason(testReason);
250 std::string finishReason;
251 TestResult::FailStatus finishStatus = TryCatch(std::bind(&TestCase::Finish, testCase),
253 finishReason = getConcatedFailReason(finishReason);
255 switch (finishStatus) {
256 case TestResult::FailStatus::FAILED:
257 testStatus = TestResult::FailStatus::FAILED;
258 if (!testReason.empty())
260 testReason += finishReason;
262 case TestResult::FailStatus::IGNORED:
263 if (testStatus == TestResult::FailStatus::NONE)
264 testStatus = TestResult::FailStatus::IGNORED;
265 if (!testReason.empty())
267 testReason += finishReason;
268 case TestResult::FailStatus::NONE:
271 Assert(false && "Unhandled fail status");
274 CollectResult(testCase->GetName(),
275 TestResult(testStatus, testReason, testCase->GetPerformance()));
276 setCurrentTestCase(nullptr);
279 void TestRunner::RunTests()
281 using namespace DPL::Colors::Text;
284 for (auto &collector : m_collectors) {
285 collector.second->Start();
289 for (auto &group : m_testGroups) {
290 count += group.second.size();
292 fprintf(stderr, "%sFound %d testcases...%s\n", GREEN_BEGIN, count, GREEN_END);
293 fprintf(stderr, "%s%s%s\n", GREEN_BEGIN, "Running tests...", GREEN_END);
294 for (auto &group : m_testGroups) {
295 TestCaseList list = group.second;
297 for (auto &collector : m_collectors) {
298 collector.second->CollectCurrentTestGroupName(group.first);
302 for (TestCaseList::const_iterator iterator = list.begin();
303 iterator != list.end();
306 TestCasePtr test = *iterator;
307 if (m_startTestId == test->GetName()) {
311 if (m_startTestId.empty()) {
314 if (m_terminate == true) {
315 // Terminate quietly without any logs
322 std::for_each(m_collectors.begin(),
324 [] (const TestResultsCollectors::value_type & collector)
326 collector.second->Finish();
330 fprintf(stderr, "%s%s%s\n\n", GREEN_BEGIN, "Finished", GREEN_END);
333 TestCasePtr TestRunner::getCurrentTestCase()
335 return m_currentTestCase;
338 void TestRunner::setCurrentTestCase(TestCasePtr testCase)
340 m_currentTestCase = testCase;
343 void TestRunner::beginPerformance(std::chrono::system_clock::duration maxDurationInMicroseconds)
345 TestCasePtr testCase = getCurrentTestCase();
349 if (!testCase->GetPerformance())
350 testCase->SetPerformance(
351 std::unique_ptr<PerformanceResult> \
352 (new PerformanceResult(maxDurationInMicroseconds)));
355 void TestRunner::endPerformance()
357 TestCasePtr testCase = getCurrentTestCase();
361 testCase->GetPerformance()->Finish();
364 ConstPerformanceResultPtr TestRunner::getCurrentTestCasePerformanceResult()
366 TestCasePtr testCase = getCurrentTestCase();
370 return testCase->GetPerformance();
373 void TestRunner::setCurrentTestCasePerformanceResult(const PerformanceResultPtr &performance)
375 TestCasePtr testCase = getCurrentTestCase();
379 testCase->SetPerformance(performance);
382 void TestRunner::addFailReason(const std::string &reason)
384 m_failReason.push(reason);
387 std::string TestRunner::getConcatedFailReason(const std::string &reason)
390 while (!m_failReason.empty())
392 ret += m_failReason.front();
398 TestRunner::~TestRunner()
400 for(auto &t : m_testCaseSet)
404 void TestRunner::CollectResult(const std::string& id, const TestResult& result)
406 if (result.GetFailStatus() == TestResult::FailStatus::IGNORED && m_runIgnored)
409 std::for_each(m_collectors.begin(),
411 [&](const TestResultsCollectors::value_type & collector)
413 collector.second->CollectResult(id, result);
417 void TestRunner::Banner()
419 using namespace DPL::Colors::Text;
433 void TestRunner::InvalidArgs(const std::string& message)
435 using namespace DPL::Colors::Text;
443 void TestRunner::Usage()
445 fprintf(stderr, "Usage: runner [options]\n\n");
446 fprintf(stderr, "Output type:\n");
447 fprintf(stderr, " --output=<output type> --output=<output type> ...\n");
448 fprintf(stderr, "\n possible output types:\n");
449 for (std::string &type : TestResultsCollectorBase::GetCollectorsNames()) {
450 fprintf(stderr, " --output=%s\n", type.c_str());
452 fprintf(stderr, "\n example:\n");
454 " test-binary --output=text --output=xml --file=output.xml\n\n");
455 fprintf(stderr, "Other parameters:\n");
457 " --regexp='regexp'\t Only selected tests"
458 " which names match regexp run\n\n");
459 fprintf(stderr, " --start=<test id>\tStart from concrete test id");
460 fprintf(stderr, " --group=<group name>\t Run tests only from one group\n");
461 fprintf(stderr, " --runignored\t Run also ignored tests\n");
462 fprintf(stderr, " --list\t Show a list of Test IDs\n");
463 fprintf(stderr, " --listgroups\t Show a list of Test Group names \n");
464 fprintf(stderr, " --only-from-xml=<xml file>\t Run only testcases specified in XML file \n"
465 " XML name is taken from attribute id=\"part1_part2\" as whole.\n"
466 " If part1 is not found (no _) then it is implicitily "
467 "set according to suite part1 from binary tests\n");
470 " --listingroup=<group name>\t Show a list of Test IDS in one group\n");
471 fprintf(stderr, " --allowchildlogs\t Allow to print logs from child process on screen.\n");
472 fprintf(stderr, " When active child process will be able to print logs on stdout and stderr.\n");
473 fprintf(stderr, " Both descriptors will be closed after test.\n");
474 fprintf(stderr, " --help\t This help\n\n");
475 std::for_each(m_collectors.begin(),
477 [] (const TestResultsCollectors::value_type & collector)
480 "Output %s has specific args:\n",
481 collector.first.c_str());
485 CollectorSpecificHelp().c_str());
487 fprintf(stderr, "For bug reporting, please write to:\n");
488 fprintf(stderr, "<p.dobrowolsk@samsung.com>\n");
491 int TestRunner::ExecTestRunner(int argc, char *argv[])
493 std::vector<std::string> args;
494 for (int i = 0; i < argc; ++i) {
495 args.push_back(argv[i]);
497 return ExecTestRunner(args);
500 int TestRunner::ExecTestRunner(ArgsList args)
502 m_runIgnored = false;
503 // Parse command line
505 args.erase(args.begin());
507 bool showHelp = false;
508 bool justList = false;
509 std::vector<std::string> xmlFiles;
511 TestResultsCollectorBasePtr currentCollector;
513 // Parse each argument
514 for(std::string &arg : args)
516 const std::string regexp = "--regexp=";
517 const std::string output = "--output=";
518 const std::string groupId = "--group=";
519 const std::string runIgnored = "--runignored";
520 const std::string listCmd = "--list";
521 const std::string startCmd = "--start=";
522 const std::string listGroupsCmd = "--listgroups";
523 const std::string listInGroup = "--listingroup=";
524 const std::string allowChildLogs = "--allowchildlogs";
525 const std::string onlyFromXML = "--only-from-xml=";
527 if (currentCollector) {
528 if (currentCollector->ParseCollectorSpecificArg(arg)) {
533 if (arg.find(startCmd) == 0) {
534 arg.erase(0, startCmd.length());
535 for (auto &group : m_testGroups) {
536 for (auto &tc : group.second) {
537 if (tc->GetName() == arg) {
542 if (!m_startTestId.empty()) {
546 if (!m_startTestId.empty()) {
550 fprintf(stderr, "Start test id has not been found\n");
553 } else if (arg.find(groupId) == 0) {
554 arg.erase(0, groupId.length());
555 TestCaseGroupMap::iterator found = m_testGroups.find(arg);
556 if (found != m_testGroups.end()) {
557 std::string name = found->first;
558 TestCaseList newList = found->second;
559 m_testGroups.clear();
560 m_testGroups[name] = newList;
562 fprintf(stderr, "Group %s not found\n", arg.c_str());
567 } else if (arg == runIgnored) {
569 } else if (arg == listCmd) {
571 } else if (arg == listGroupsCmd) {
572 for (auto &group : m_testGroups) {
573 printf("GR:%s\n", group.first.c_str());
576 } else if (arg.find(listInGroup) == 0) {
577 arg.erase(0, listInGroup.length());
578 for (auto &test : m_testGroups[arg]) {
579 printf("ID:%s\n", test->GetName().c_str());
582 } else if (arg.find(allowChildLogs) == 0) {
583 arg.erase(0, allowChildLogs.length());
584 m_allowChildLogs = true;
585 } else if (arg == "--help") {
587 } else if (arg.find(output) == 0) {
588 arg.erase(0, output.length());
589 if (m_collectors.find(arg) != m_collectors.end()) {
591 "Multiple outputs of the same type are not supported!");
595 currentCollector.reset(TestResultsCollectorBase::Create(arg));
596 if (!currentCollector) {
597 InvalidArgs("Unsupported output type!");
601 m_collectors[arg] = currentCollector;
602 } else if (arg.find(regexp) == 0) {
603 arg.erase(0, regexp.length());
604 if (arg.length() == 0) {
610 if (arg[0] == '\'' && arg[arg.length() - 1] == '\'') {
612 arg.erase(arg.length() - 1);
615 if (arg.length() == 0) {
621 pcrecpp::RE re(arg.c_str());
622 for (auto &group : m_testGroups) {
623 TestCaseList newList;
624 for (auto &tc : group.second)
626 if (re.PartialMatch(tc->GetName())) {
627 newList.push_back(tc);
630 group.second = newList;
632 } else if(arg.find(onlyFromXML) == 0) {
633 arg.erase(0, onlyFromXML.length());
634 if (arg.length() == 0) {
640 if (arg[0] == '\'' && arg[arg.length() - 1] == '\'') {
642 arg.erase(arg.length() - 1);
645 if (arg.length() == 0) {
651 xmlFiles.push_back(arg);
659 if(!xmlFiles.empty())
661 if(!filterGroupsByXmls(xmlFiles))
663 fprintf(stderr, "XML file is not correct\n");
670 for (auto &group : m_testGroups) {
671 for (auto &tc : group.second) {
672 printf("ID:%s:%s\n", group.first.c_str(), tc->GetName().c_str());
678 currentCollector.reset();
686 if (m_collectors.empty()) {
687 TestResultsCollectorBasePtr collector(
688 TestResultsCollectorBase::Create("text"));
689 m_collectors["text"] = collector;
692 for (auto &collector : m_collectors) {
693 if (!collector.second->Configure()) {
694 fprintf(stderr, "Could not configure selected output");
705 void TestRunner::Terminate()
710 bool TestRunner::GetAllowChildLogs()
712 return m_allowChildLogs;
715 void TestRunner::deferFailedException(const DPL::Test::TestFailed &ex)
717 if (m_deferDeepness <= 0)
720 if (m_deferredExceptionsMessages.empty()) {
721 m_firstDeferredFail = ex;
722 m_firstDeferredExceptionType = DeferredExceptionType::DEFERRED_FAILED;
724 m_deferredExceptionsMessages.push_back(ex.GetMessage());
727 void TestRunner::deferIgnoredException(const DPL::Test::TestIgnored &ex)
729 if (m_deferDeepness <= 0)
732 if (m_deferredExceptionsMessages.empty()) {
733 m_firstDeferredIgnore = ex;
734 m_firstDeferredExceptionType = DeferredExceptionType::DEFERRED_IGNORED;
736 m_deferredExceptionsMessages.push_back(ex.GetMessage());
739 void TestRunner::deferBegin()
744 void TestRunner::deferEnd()
746 if (m_deferDeepness > 0)
749 if (m_deferDeepness > 0)
752 bool oops = std::uncaught_exception();
753 size_t additionalExceptions = oops ? 0 : 1;
754 for (size_t i = additionalExceptions; i < m_deferredExceptionsMessages.size(); ++i)
755 addFailReason(m_deferredExceptionsMessages[i]);
757 if (!oops && !m_deferredExceptionsMessages.empty())
759 m_deferredExceptionsMessages.clear();
760 switch (m_firstDeferredExceptionType) {
761 case DeferredExceptionType::DEFERRED_FAILED:
762 throw m_firstDeferredFail;
763 case DeferredExceptionType::DEFERRED_IGNORED:
764 throw m_firstDeferredIgnore;
767 m_deferredExceptionsMessages.clear();