2 * Copyright (c) 2011 Samsung Electronics Co., Ltd All Rights Reserved
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
17 * @file test_runner.cpp
18 * @author Przemyslaw Dobrowolski (p.dobrowolsk@samsung.com)
19 * @author Lukasz Wrzosek (l.wrzosek@samsung.com)
21 * @brief This file is the implementation file of test runner
24 #include <dpl/test/test_runner.h>
25 #include <dpl/test/test_results_collector.h>
26 #include <dpl/exception.h>
27 #include <dpl/free_deleter.h>
29 #include <dpl/foreach.h>
30 #include <dpl/log/wrt_log.h>
31 #include <dpl/colors.h>
39 #include <dpl/utils/wrt_global_settings.h>
41 #include <libxml/xpath.h>
42 #include <libxml/xpathInternals.h>
43 #include <libxml/parser.h>
44 #include <libxml/tree.h>
46 #include <dpl/singleton_impl.h>
47 IMPLEMENT_SINGLETON(DPL::Test::TestRunner)
51 std::string getXMLNode(xmlNodePtr node)
54 xmlChar * value = xmlNodeGetContent(node);
55 ret = std::string(reinterpret_cast<char*>(value));
65 namespace // anonymous
67 std::string BaseName(std::string aPath)
69 std::unique_ptr<char[],free_deleter> path(strdup(aPath.c_str()));
70 if (NULL == path.get()) {
71 throw std::bad_alloc();
73 char* baseName = basename(path.get());
74 std::string retValue = baseName;
77 } // namespace anonymous
79 //! \brief Failed test message creator
81 //! \param[in] aTest string for tested expression
82 //! \param[in] aFile source file name
83 //! \param[in] aLine source file line
84 //! \param[in] aMessage error message
85 TestRunner::TestFailed::TestFailed(const char* aTest,
88 const std::string &aMessage)
90 std::ostringstream assertMsg;
91 assertMsg << "[" << BaseName(aFile) << ":" << aLine
92 << "] Assertion failed ("
93 << aTest << ") " << aMessage;
94 m_message = assertMsg.str();
97 TestRunner::TestFailed::TestFailed(const std::string &message)
102 void TestRunner::RegisterTest(const char *testName, TestCase proc)
104 m_testGroups[m_currentGroup].push_back(TestCaseStruct(testName, proc));
107 void TestRunner::InitGroup(const char* name)
109 m_currentGroup = name;
112 void TestRunner::normalizeXMLTag(std::string& str, const std::string& testcase)
114 //Add testcase if missing
115 std::string::size_type pos = str.find(testcase);
118 str = testcase + "_" + str;
121 //dpl test runner cannot have '-' character in name so it have to be replaced
122 // for TCT case to make comparision works
123 std::replace(str.begin(), str.end(), '-', '_');
126 bool TestRunner::filterGroupsByXmls(const std::vector<std::string> & files)
128 DECLARE_EXCEPTION_TYPE(DPL::Exception, XMLError)
130 const std::string idPath = "/test_definition/suite/set/testcase/@id";
133 std::map<std::string, bool> casesMap;
135 std::string testsuite;
136 if(!m_testGroups.empty())
138 for(TestCaseGroupMap::const_iterator cit = m_testGroups.begin(); cit != m_testGroups.end(); ++cit)
140 if(!cit->second.empty())
142 for(TestCaseStructList::const_iterator cj = cit->second.begin(); cj != cit->second.end(); ++cj)
144 std::string name = cj->name;
145 std::string::size_type st = name.find('_');
146 if(st != std::string::npos)
148 name = name.substr(0, st);
153 if(!testsuite.empty()) break;
167 xmlXPathContextPtr xpathCtx;
169 doc = xmlReadFile(file->c_str(), NULL, 0);
171 ThrowMsg(XMLError, "File Problem");
174 xpathCtx = xmlXPathNewContext(doc);
175 if (xpathCtx == NULL) {
177 "Error: unable to create new XPath context\n");
179 xpathCtx->node = xmlDocGetRootElement(doc);
183 xmlXPathObjectPtr xpathObject;
184 //get requested node's values
185 xpathObject = xmlXPathEvalExpression(BAD_CAST idPath.c_str(), xpathCtx);
186 if (xpathObject == NULL)
188 ThrowMsg(XMLError, "XPath evaluation failure: " << idPath);
190 xmlNodeSetPtr nodes = xpathObject->nodesetval;
191 unsigned size = (nodes) ? nodes->nodeNr : 0;
192 WrtLogD("Found %i nodes matching xpath", size);
193 for(unsigned i = 0; i < size; ++i)
195 WrtLogD("Type: %i", nodes->nodeTab[i]->type);
196 if (nodes->nodeTab[i]->type == XML_ATTRIBUTE_NODE) {
197 xmlNodePtr curNode = nodes->nodeTab[i];
198 result = getXMLNode(curNode);
199 WrtLogD("Result: %s", result.c_str());
200 normalizeXMLTag(result, testsuite);
201 casesMap.insert(make_pair(result, false));
204 //Cleanup of XPath data
205 xmlXPathFreeObject(xpathObject);
206 xmlXPathFreeContext(xpathCtx);
212 WrtLogE("Libxml error: %s", _rethrown_exception.DumpToString().c_str());
217 if(!filterByXML(casesMap))
225 bool TestRunner::filterByXML(std::map<std::string, bool> & casesMap)
227 FOREACH(group, m_testGroups) {
228 TestCaseStructList newList;
229 FOREACH(iterator, group->second)
231 if (casesMap.find(iterator->name) != casesMap.end()) {
232 casesMap[iterator->name] = true;
233 newList.push_back(*iterator);
236 group->second = newList;
238 FOREACH(cs, casesMap)
240 if(cs->second == false)
242 WrtLogE("Cannot find testcase from XML file: %s", cs->first.c_str());
249 TestRunner::Status TestRunner::RunTestCase(const TestCaseStruct& testCase)
253 } catch (const TestFailed &e) {
254 // Simple test failure
255 CollectResult(testCase.name,
257 TestResultsCollectorBase::FailStatus::FAILED,
260 } catch (const Ignored &e) {
262 // Simple test have to be implemented
263 CollectResult(testCase.name,
265 TestResultsCollectorBase::FailStatus::IGNORED,
270 } catch (const DPL::Exception &e) {
271 // DPL exception failure
272 CollectResult(testCase.name,
274 TestResultsCollectorBase::FailStatus::INTERNAL,
275 "DPL exception:" + e.GetMessage() + "\n" + e.DumpToString());
278 } catch (const std::exception &) {
279 // std exception failure
280 CollectResult(testCase.name,
282 TestResultsCollectorBase::FailStatus::INTERNAL,
287 // Unknown exception failure
288 CollectResult(testCase.name,
290 TestResultsCollectorBase::FailStatus::INTERNAL,
291 "unknown exception");
296 CollectResult(testCase.name,
298 TestResultsCollectorBase::FailStatus::NONE);
304 void TestRunner::RunTests()
306 using namespace DPL::Colors::Text;
311 FOREACH(group, m_testGroups) {
312 count += group->second.size();
315 std::for_each(m_collectors.begin(),
317 [count] (const TestResultsCollectors::value_type & collector)
319 collector.second->Start(count);
322 fprintf(stderr, "%sFound %d testcases...%s\n", GREEN_BEGIN, count, GREEN_END);
323 fprintf(stderr, "%s%s%s\n", GREEN_BEGIN, "Running tests...", GREEN_END);
324 FOREACH(group, m_testGroups) {
325 TestCaseStructList list = group->second;
328 m_collectors.begin(),
330 [&group](const TestResultsCollectors::value_type & collector)
333 CollectCurrentTestGroupName(group->first);
337 for (TestCaseStructList::const_iterator iterator = list.begin();
338 iterator != list.end();
341 TestCaseStruct test = *iterator;
342 if (m_startTestId == test.name) {
346 if (m_startTestId.empty()) {
349 if (m_terminate == true) {
350 // Terminate quietly without any logs
357 std::for_each(m_collectors.begin(),
359 [] (const TestResultsCollectors::value_type & collector)
361 collector.second->Finish();
365 fprintf(stderr, "%s%s%s\n\n", GREEN_BEGIN, "Finished", GREEN_END);
368 void TestRunner::CollectResult(
369 const std::string& id,
370 const std::string& description,
371 const TestResultsCollectorBase::FailStatus::Type status,
372 const std::string& reason)
374 std::for_each(m_collectors.begin(),
376 [&](const TestResultsCollectors::value_type & collector)
378 collector.second->CollectResult(id,
385 void TestRunner::Banner()
387 using namespace DPL::Colors::Text;
401 void TestRunner::InvalidArgs(const std::string& message)
403 using namespace DPL::Colors::Text;
411 void TestRunner::Usage()
413 fprintf(stderr, "Usage: runner [options]\n\n");
414 fprintf(stderr, "Output type:\n");
415 fprintf(stderr, " --output=<output type> --output=<output type> ...\n");
416 fprintf(stderr, "\n possible output types:\n");
417 FOREACH(type, TestResultsCollectorBase::GetCollectorsNames()) {
418 fprintf(stderr, " --output=%s\n", type->c_str());
420 fprintf(stderr, "\n example:\n");
422 " test-binary --output=text --output=xml --file=output.xml\n\n");
423 fprintf(stderr, "Other parameters:\n");
425 " --regexp='regexp'\t Only selected tests"
426 " which names match regexp run\n\n");
427 fprintf(stderr, " --start=<test id>\tStart from concrete test id");
428 fprintf(stderr, " --group=<group name>\t Run tests only from one group\n");
429 fprintf(stderr, " --runignored\t Run also ignored tests\n");
430 fprintf(stderr, " --list\t Show a list of Test IDs\n");
431 fprintf(stderr, " --listgroups\t Show a list of Test Group names \n");
432 fprintf(stderr, " --only-from-xml=<xml file>\t Run only testcases specified in XML file \n"
433 " XML name is taken from attribute id=\"part1_part2\" as whole.\n"
434 " If part1 is not found (no _) then it is implicitily "
435 "set according to suite part1 from binary tests\n");
438 " --listingroup=<group name>\t Show a list of Test IDS in one group\n");
439 fprintf(stderr, " --allowchildlogs\t Allow to print logs from child process on screen.\n");
440 fprintf(stderr, " When active child process will be able to print logs on stdout and stderr.\n");
441 fprintf(stderr, " Both descriptors will be closed after test.\n");
442 fprintf(stderr, " --help\t This help\n\n");
443 std::for_each(m_collectors.begin(),
445 [] (const TestResultsCollectors::value_type & collector)
448 "Output %s has specific args:\n",
449 collector.first.c_str());
453 CollectorSpecificHelp().c_str());
455 fprintf(stderr, "For bug reporting, please write to:\n");
456 fprintf(stderr, "<p.dobrowolsk@samsung.com>\n");
459 int TestRunner::ExecTestRunner(int argc, char *argv[])
461 std::vector<std::string> args;
462 for (int i = 0; i < argc; ++i) {
463 args.push_back(argv[i]);
465 return ExecTestRunner(args);
468 void TestRunner::MarkAssertion()
473 int TestRunner::ExecTestRunner(const ArgsList& value)
475 m_runIgnored = false;
476 ArgsList args = value;
477 // Parse command line
478 if (args.size() == 1) {
484 args.erase(args.begin());
486 bool showHelp = false;
487 bool justList = false;
488 std::vector<std::string> xmlFiles;
490 TestResultsCollectorBasePtr currentCollector;
492 // Parse each argument
495 std::string arg = *it;
496 const std::string regexp = "--regexp=";
497 const std::string output = "--output=";
498 const std::string groupId = "--group=";
499 const std::string runIgnored = "--runignored";
500 const std::string listCmd = "--list";
501 const std::string startCmd = "--start=";
502 const std::string listGroupsCmd = "--listgroups";
503 const std::string listInGroup = "--listingroup=";
504 const std::string allowChildLogs = "--allowchildlogs";
505 const std::string onlyFromXML = "--only-from-xml=";
507 if (currentCollector) {
508 if (currentCollector->ParseCollectorSpecificArg(arg)) {
513 if (arg.find(startCmd) == 0) {
514 arg.erase(0, startCmd.length());
515 FOREACH(group, m_testGroups) {
516 FOREACH(tc, group->second) {
517 if (tc->name == arg) {
522 if (!m_startTestId.empty()) {
526 if (!m_startTestId.empty()) {
530 fprintf(stderr, "Start test id has not been found\n");
533 } else if (arg.find(groupId) == 0) {
534 arg.erase(0, groupId.length());
535 TestCaseGroupMap::iterator found = m_testGroups.find(arg);
536 if (found != m_testGroups.end()) {
537 std::string name = found->first;
538 TestCaseStructList newList = found->second;
539 m_testGroups.clear();
540 m_testGroups[name] = newList;
542 fprintf(stderr, "Group %s not found\n", arg.c_str());
547 } else if (arg == runIgnored) {
549 } else if (arg == listCmd) {
551 } else if (arg == listGroupsCmd) {
552 FOREACH(group, m_testGroups) {
553 printf("GR:%s\n", group->first.c_str());
556 } else if (arg.find(listInGroup) == 0) {
557 arg.erase(0, listInGroup.length());
558 FOREACH(test, m_testGroups[arg]) {
559 printf("ID:%s\n", test->name.c_str());
562 } else if (arg.find(allowChildLogs) == 0) {
563 arg.erase(0, allowChildLogs.length());
564 m_allowChildLogs = true;
565 } else if (arg == "--help") {
567 } else if (arg.find(output) == 0) {
568 arg.erase(0, output.length());
569 if (m_collectors.find(arg) != m_collectors.end()) {
571 "Multiple outputs of the same type are not supported!");
575 currentCollector.reset(TestResultsCollectorBase::Create(arg));
576 if (!currentCollector) {
577 InvalidArgs("Unsupported output type!");
581 m_collectors[arg] = currentCollector;
582 } else if (arg.find(regexp) == 0) {
583 arg.erase(0, regexp.length());
584 if (arg.length() == 0) {
590 if (arg[0] == '\'' && arg[arg.length() - 1] == '\'') {
592 arg.erase(arg.length() - 1);
595 if (arg.length() == 0) {
601 pcrecpp::RE re(arg.c_str());
602 FOREACH(group, m_testGroups) {
603 TestCaseStructList newList;
604 FOREACH(iterator, group->second)
606 if (re.PartialMatch(iterator->name)) {
607 newList.push_back(*iterator);
610 group->second = newList;
612 } else if(arg.find(onlyFromXML) == 0) {
613 arg.erase(0, onlyFromXML.length());
614 if (arg.length() == 0) {
620 if (arg[0] == '\'' && arg[arg.length() - 1] == '\'') {
622 arg.erase(arg.length() - 1);
625 if (arg.length() == 0) {
631 xmlFiles.push_back(arg);
639 if(!xmlFiles.empty())
641 if(!filterGroupsByXmls(xmlFiles))
643 fprintf(stderr, "XML file is not correct\n");
650 FOREACH(group, m_testGroups) {
651 FOREACH(test, group->second) {
652 printf("ID:%s:%s\n", group->first.c_str(), test->name.c_str());
658 currentCollector.reset();
666 if (m_collectors.empty()) {
667 TestResultsCollectorBasePtr collector(
668 TestResultsCollectorBase::Create("text"));
669 m_collectors["text"] = collector;
672 for (auto it = m_collectors.begin(); it != m_collectors.end(); ++it) {
673 if (!it->second->Configure()) {
674 fprintf(stderr, "Could not configure selected output");
685 bool TestRunner::getRunIgnored() const
690 void TestRunner::Terminate()
695 bool TestRunner::GetAllowChildLogs()
697 return m_allowChildLogs;