2 * Copyright (c) 2011 Samsung Electronics Co., Ltd All Rights Reserved
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
17 * @file test_runner.cpp
18 * @author Przemyslaw Dobrowolski (p.dobrowolsk@samsung.com)
19 * @author Lukasz Wrzosek (l.wrzosek@samsung.com)
21 * @brief This file is the implementation file of test runner
24 #include <dpl/test/test_runner.h>
25 #include <dpl/test/test_results_collector.h>
26 #include <dpl/exception.h>
27 #include <dpl/scoped_free.h>
28 #include <dpl/foreach.h>
29 #include <dpl/log/log.h>
30 #include <dpl/colors.h>
38 #include <dpl/utils/wrt_global_settings.h>
40 #include <libxml/xpath.h>
41 #include <libxml/xpathInternals.h>
42 #include <libxml/parser.h>
43 #include <libxml/tree.h>
45 #include <dpl/singleton_impl.h>
46 IMPLEMENT_SINGLETON(DPL::Test::TestRunner)
50 std::string getXMLNode(xmlNodePtr node)
53 xmlChar * value = xmlNodeGetContent(node);
54 ret = std::string(reinterpret_cast<char*>(value));
64 namespace // anonymous
66 std::string BaseName(std::string aPath)
68 ScopedFree<char> path(strdup(aPath.c_str()));
69 if (NULL == path.Get()) {
70 throw std::bad_alloc();
72 char* baseName = basename(path.Get());
73 std::string retValue = baseName;
76 } // namespace anonymous
78 //! \brief Failed test message creator
80 //! \param[in] aTest string for tested expression
81 //! \param[in] aFile source file name
82 //! \param[in] aLine source file line
83 //! \param[in] aMessage error message
84 TestRunner::TestFailed::TestFailed(const char* aTest,
87 const std::string &aMessage)
89 std::ostringstream assertMsg;
90 assertMsg << "[" << BaseName(aFile) << ":" << aLine
91 << "] Assertion failed ("
92 << aTest << ") " << aMessage;
93 m_message = assertMsg.str();
96 TestRunner::TestFailed::TestFailed(const std::string &message)
101 void TestRunner::RegisterTest(const char *testName, TestCase proc)
103 m_testGroups[m_currentGroup].push_back(TestCaseStruct(testName, proc));
106 void TestRunner::InitGroup(const char* name)
108 m_currentGroup = name;
111 void TestRunner::normalizeXMLTag(std::string& str, const std::string& testcase)
113 //Add testcase if missing
114 std::string::size_type pos = str.find(testcase);
117 str = testcase + "_" + str;
120 //dpl test runner cannot have '-' character in name so it have to be replaced
121 // for TCT case to make comparision works
122 std::replace(str.begin(), str.end(), '-', '_');
125 bool TestRunner::filterGroupsByXmls(const std::vector<std::string> & files)
127 DECLARE_EXCEPTION_TYPE(DPL::Exception, XMLError)
129 const std::string idPath = "/test_definition/suite/set/testcase/@id";
132 std::map<std::string, bool> casesMap;
134 std::string testsuite;
135 if(!m_testGroups.empty())
137 for(TestCaseGroupMap::const_iterator cit = m_testGroups.begin(); cit != m_testGroups.end(); ++cit)
139 if(!cit->second.empty())
141 for(TestCaseStructList::const_iterator cj = cit->second.begin(); cj != cit->second.end(); ++cj)
143 std::string name = cj->name;
144 std::string::size_type st = name.find('_');
145 if(st != std::string::npos)
147 name = name.substr(0, st);
152 if(!testsuite.empty()) break;
166 xmlXPathContextPtr xpathCtx;
168 doc = xmlReadFile(file->c_str(), NULL, 0);
170 ThrowMsg(XMLError, "File Problem");
173 xpathCtx = xmlXPathNewContext(doc);
174 if (xpathCtx == NULL) {
176 "Error: unable to create new XPath context\n");
178 xpathCtx->node = xmlDocGetRootElement(doc);
182 xmlXPathObjectPtr xpathObject;
183 //get requested node's values
184 xpathObject = xmlXPathEvalExpression(BAD_CAST idPath.c_str(), xpathCtx);
185 if (xpathObject == NULL)
187 ThrowMsg(XMLError, "XPath evaluation failure: " << idPath);
189 xmlNodeSetPtr nodes = xpathObject->nodesetval;
190 unsigned size = (nodes) ? nodes->nodeNr : 0;
191 LogDebug("Found " << size << " nodes matching xpath");
192 for(unsigned i = 0; i < size; ++i)
194 LogPedantic("Type: " << nodes->nodeTab[i]->type);
195 if (nodes->nodeTab[i]->type == XML_ATTRIBUTE_NODE) {
196 xmlNodePtr curNode = nodes->nodeTab[i];
197 result = getXMLNode(curNode);
198 LogPedantic("Result: " << result);
199 normalizeXMLTag(result, testsuite);
200 casesMap.insert(make_pair(result, false));
203 //Cleanup of XPath data
204 xmlXPathFreeObject(xpathObject);
205 xmlXPathFreeContext(xpathCtx);
211 LogError("Libxml error: " << _rethrown_exception.DumpToString());
216 if(!filterByXML(casesMap))
224 bool TestRunner::filterByXML(std::map<std::string, bool> & casesMap)
226 FOREACH(group, m_testGroups) {
227 TestCaseStructList newList;
228 FOREACH(iterator, group->second)
230 if (casesMap.find(iterator->name) != casesMap.end()) {
231 casesMap[iterator->name] = true;
232 newList.push_back(*iterator);
235 group->second = newList;
237 FOREACH(cs, casesMap)
239 if(cs->second == false)
241 LogError("Cannot find testcase from XML file: " << cs->first);
248 TestRunner::Status TestRunner::RunTestCase(const TestCaseStruct& testCase)
252 } catch (const TestFailed &e) {
253 // Simple test failure
254 CollectResult(testCase.name,
256 TestResultsCollectorBase::FailStatus::FAILED,
259 } catch (const Ignored &e) {
261 // Simple test have to be implemented
262 CollectResult(testCase.name,
264 TestResultsCollectorBase::FailStatus::IGNORED,
269 } catch (const DPL::Exception &e) {
270 // DPL exception failure
271 CollectResult(testCase.name,
273 TestResultsCollectorBase::FailStatus::INTERNAL,
274 "DPL exception:" + e.GetMessage());
277 } catch (const std::exception &) {
278 // std exception failure
279 CollectResult(testCase.name,
281 TestResultsCollectorBase::FailStatus::INTERNAL,
286 // Unknown exception failure
287 CollectResult(testCase.name,
289 TestResultsCollectorBase::FailStatus::INTERNAL,
290 "unknown exception");
295 CollectResult(testCase.name,
297 TestResultsCollectorBase::FailStatus::NONE);
303 void TestRunner::RunTests()
305 using namespace DPL::Colors::Text;
308 std::for_each(m_collectors.begin(),
310 [] (const TestResultsCollectors::value_type & collector)
312 collector.second->Start();
316 FOREACH(group, m_testGroups) {
317 count += group->second.size();
319 fprintf(stderr, "%sFound %d testcases...%s\n", GREEN_BEGIN, count, GREEN_END);
320 fprintf(stderr, "%s%s%s\n", GREEN_BEGIN, "Running tests...", GREEN_END);
321 FOREACH(group, m_testGroups) {
322 TestCaseStructList list = group->second;
325 m_collectors.begin(),
327 [&group](const TestResultsCollectors::value_type & collector)
330 CollectCurrentTestGroupName(group->first);
334 for (TestCaseStructList::const_iterator iterator = list.begin();
335 iterator != list.end();
338 TestCaseStruct test = *iterator;
339 if (m_startTestId == test.name) {
343 if (m_startTestId.empty()) {
346 if (m_terminate == true) {
347 // Terminate quietly without any logs
354 std::for_each(m_collectors.begin(),
356 [] (const TestResultsCollectors::value_type & collector)
358 collector.second->Finish();
362 fprintf(stderr, "%s%s%s\n\n", GREEN_BEGIN, "Finished", GREEN_END);
365 void TestRunner::CollectResult(
366 const std::string& id,
367 const std::string& description,
368 const TestResultsCollectorBase::FailStatus::Type status,
369 const std::string& reason)
371 std::for_each(m_collectors.begin(),
373 [&](const TestResultsCollectors::value_type & collector)
375 collector.second->CollectResult(id,
382 void TestRunner::Banner()
384 using namespace DPL::Colors::Text;
398 void TestRunner::InvalidArgs(const std::string& message)
400 using namespace DPL::Colors::Text;
408 void TestRunner::Usage()
410 fprintf(stderr, "Usage: runner [options]\n\n");
411 fprintf(stderr, "Output type:\n");
412 fprintf(stderr, " --output=<output type> --output=<output type> ...\n");
413 fprintf(stderr, "\n possible output types:\n");
414 FOREACH(type, TestResultsCollectorBase::GetCollectorsNames()) {
415 fprintf(stderr, " --output=%s\n", type->c_str());
417 fprintf(stderr, "\n example:\n");
419 " test-binary --output=text --output=xml --file=output.xml\n\n");
420 fprintf(stderr, "Other parameters:\n");
422 " --regexp='regexp'\t Only selected tests"
423 " which names match regexp run\n\n");
424 fprintf(stderr, " --start=<test id>\tStart from concrete test id");
425 fprintf(stderr, " --group=<group name>\t Run tests only from one group\n");
426 fprintf(stderr, " --runignored\t Run also ignored tests\n");
427 fprintf(stderr, " --list\t Show a list of Test IDs\n");
428 fprintf(stderr, " --listgroups\t Show a list of Test Group names \n");
429 fprintf(stderr, " --only-from-xml=<xml file>\t Run only testcases specified in XML file \n"
430 " XML name is taken from attribute id=\"part1_part2\" as whole.\n"
431 " If part1 is not found (no _) then it is implicitily "
432 "set according to suite part1 from binary tests\n");
435 " --listingroup=<group name>\t Show a list of Test IDS in one group\n");
436 fprintf(stderr, " --allowchildlogs\t Allow to print logs from child process on screen.\n");
437 fprintf(stderr, " When active child process will be able to print logs on stdout and stderr.\n");
438 fprintf(stderr, " Both descriptors will be closed after test.\n");
439 fprintf(stderr, " --help\t This help\n\n");
440 std::for_each(m_collectors.begin(),
442 [] (const TestResultsCollectors::value_type & collector)
445 "Output %s has specific args:\n",
446 collector.first.c_str());
450 CollectorSpecificHelp().c_str());
452 fprintf(stderr, "For bug reporting, please write to:\n");
453 fprintf(stderr, "<p.dobrowolsk@samsung.com>\n");
456 int TestRunner::ExecTestRunner(int argc, char *argv[])
458 std::vector<std::string> args;
459 for (int i = 0; i < argc; ++i) {
460 args.push_back(argv[i]);
462 return ExecTestRunner(args);
465 void TestRunner::MarkAssertion()
470 int TestRunner::ExecTestRunner(const ArgsList& value)
472 m_runIgnored = false;
473 ArgsList args = value;
474 // Parse command line
475 if (args.size() == 1) {
481 args.erase(args.begin());
483 bool showHelp = false;
484 bool justList = false;
485 std::vector<std::string> xmlFiles;
487 TestResultsCollectorBasePtr currentCollector;
489 // Parse each argument
492 std::string arg = *it;
493 const std::string regexp = "--regexp=";
494 const std::string output = "--output=";
495 const std::string groupId = "--group=";
496 const std::string runIgnored = "--runignored";
497 const std::string listCmd = "--list";
498 const std::string startCmd = "--start=";
499 const std::string listGroupsCmd = "--listgroups";
500 const std::string listInGroup = "--listingroup=";
501 const std::string allowChildLogs = "--allowchildlogs";
502 const std::string onlyFromXML = "--only-from-xml=";
504 if (currentCollector) {
505 if (currentCollector->ParseCollectorSpecificArg(arg)) {
510 if (arg.find(startCmd) == 0) {
511 arg.erase(0, startCmd.length());
512 FOREACH(group, m_testGroups) {
513 FOREACH(tc, group->second) {
514 if (tc->name == arg) {
519 if (!m_startTestId.empty()) {
523 if (!m_startTestId.empty()) {
527 fprintf(stderr, "Start test id has not been found\n");
530 } else if (arg.find(groupId) == 0) {
531 arg.erase(0, groupId.length());
532 TestCaseGroupMap::iterator found = m_testGroups.find(arg);
533 if (found != m_testGroups.end()) {
534 std::string name = found->first;
535 TestCaseStructList newList = found->second;
536 m_testGroups.clear();
537 m_testGroups[name] = newList;
539 fprintf(stderr, "Group %s not found\n", arg.c_str());
544 } else if (arg == runIgnored) {
546 } else if (arg == listCmd) {
548 } else if (arg == listGroupsCmd) {
549 FOREACH(group, m_testGroups) {
550 printf("GR:%s\n", group->first.c_str());
553 } else if (arg.find(listInGroup) == 0) {
554 arg.erase(0, listInGroup.length());
555 FOREACH(test, m_testGroups[arg]) {
556 printf("ID:%s\n", test->name.c_str());
559 } else if (arg.find(allowChildLogs) == 0) {
560 arg.erase(0, allowChildLogs.length());
561 m_allowChildLogs = true;
562 } else if (arg == "--help") {
564 } else if (arg.find(output) == 0) {
565 arg.erase(0, output.length());
566 if (m_collectors.find(arg) != m_collectors.end()) {
568 "Multiple outputs of the same type are not supported!");
572 currentCollector.reset(TestResultsCollectorBase::Create(arg));
573 if (!currentCollector) {
574 InvalidArgs("Unsupported output type!");
578 m_collectors[arg] = currentCollector;
579 } else if (arg.find(regexp) == 0) {
580 arg.erase(0, regexp.length());
581 if (arg.length() == 0) {
587 if (arg[0] == '\'' && arg[arg.length() - 1] == '\'') {
589 arg.erase(arg.length() - 1);
592 if (arg.length() == 0) {
598 pcrecpp::RE re(arg.c_str());
599 FOREACH(group, m_testGroups) {
600 TestCaseStructList newList;
601 FOREACH(iterator, group->second)
603 if (re.PartialMatch(iterator->name)) {
604 newList.push_back(*iterator);
607 group->second = newList;
609 } else if(arg.find(onlyFromXML) == 0) {
610 arg.erase(0, onlyFromXML.length());
611 if (arg.length() == 0) {
617 if (arg[0] == '\'' && arg[arg.length() - 1] == '\'') {
619 arg.erase(arg.length() - 1);
622 if (arg.length() == 0) {
628 xmlFiles.push_back(arg);
636 if(!xmlFiles.empty())
638 if(!filterGroupsByXmls(xmlFiles))
640 fprintf(stderr, "XML file is not correct\n");
647 FOREACH(group, m_testGroups) {
648 FOREACH(test, group->second) {
649 printf("ID:%s:%s\n", group->first.c_str(), test->name.c_str());
655 currentCollector.reset();
663 if (m_collectors.empty()) {
664 TestResultsCollectorBasePtr collector(
665 TestResultsCollectorBase::Create("text"));
666 m_collectors["text"] = collector;
669 for (auto it = m_collectors.begin(); it != m_collectors.end(); ++it) {
670 if (!it->second->Configure()) {
671 fprintf(stderr, "Could not configure selected output");
682 bool TestRunner::getRunIgnored() const
687 void TestRunner::Terminate()
692 bool TestRunner::GetAllowChildLogs()
694 return m_allowChildLogs;