From 490411149e10904f2df6047df9ddbeb3a505b96f Mon Sep 17 00:00:00 2001 From: Tomasz Iwanek Date: Mon, 5 Aug 2013 10:50:45 +0200 Subject: [PATCH] Option --only-from-xml for dpl test framework [Issue#] LINUXWRT-743 [Feature] Reading testcase to be launched from TCT xml file [Cause] Need for automatic launching only TCT tests. [Solution] Additional option for test binary. Additonal changes: - printing number of testcases at startup - listing option will take into account --regexp and --only-from-xml [Verification] Build and run tests from change. Try this change with wrt-extra tests with commit "W3C testcases names adjust for TCT". Install TCT on target: - tct-widget01-w3c - tct-widget02-w3c - tct-wgtapi01-w3c - tct-wgtapi02-w3c - tct-widgetpolicy-w3c - tct-widgetdigsig-w3c All commands below should print requested list of tescases from xml file and do not complain about wrong XML file (note that list option prints now only test to run (after filtering them)): wrt-extra-tests-plugins-w3c --list --only-from-xml=/usr/share/tct-wgtapi02-w3c-tests/tests.xml wrt-extra-tests-plugins-w3c --list --only-from-xml=/usr/share/tct-wgtapi01-w3c-tests/tests.xml wrt-extra-tests-w3c --list --only-from-xml=/usr/share/tct-widget01-w3c-tests/tests.xml wrt-extra-tests-w3c --list --only-from-xml=/usr/share/tct-widget02-w3c-tests/tests.xml wrt-extra-tests-w3c-widget-policy --list --only-from-xml=/usr/share/tct-widgetpolicy-w3c-tests/tests.xml wrt-extra-tests-w3c-digsig --list --only-from-xml=/usr/share/tct-widgetdigsig-w3c-tests/tests.xml #no testcases anyway Change-Id: I1b62a5d17dc351c1f763fb1d9a877e1300bc6a9c --- build/test/CMakeLists.txt | 1 + build/test/dpl-test-efl.pc.in | 2 +- modules/test/include/dpl/test/test_runner.h | 4 + modules/test/src/test_runner.cpp | 214 +++++++++++++++++++++++++++- 4 files changed, 214 insertions(+), 7 deletions(-) diff --git a/build/test/CMakeLists.txt b/build/test/CMakeLists.txt index 710fc4e..d992952 100644 --- a/build/test/CMakeLists.txt +++ b/build/test/CMakeLists.txt @@ -21,6 +21,7 @@ INCLUDE(FindPkgConfig) PKG_CHECK_MODULES(SYS_EFL_TEST_ENGINE appcore-efl + libxml-2.0 REQUIRED) diff --git a/build/test/dpl-test-efl.pc.in b/build/test/dpl-test-efl.pc.in index 9d6f871..056ae09 100644 --- a/build/test/dpl-test-efl.pc.in +++ b/build/test/dpl-test-efl.pc.in @@ -6,6 +6,6 @@ includedir=${prefix}/include Name: dpl-test-efl Description: DPL Test Engine - EFL based Version: @VERSION@ -Requires: dpl-efl +Requires: dpl-efl libxml-2.0 Libs: -L${libdir} -ldpl-test-efl Cflags: -I${includedir}/dpl-efl diff --git a/modules/test/include/dpl/test/test_runner.h b/modules/test/include/dpl/test/test_runner.h index 99a2cb5..ae5c712 100644 --- a/modules/test/include/dpl/test/test_runner.h +++ b/modules/test/include/dpl/test/test_runner.h @@ -97,6 +97,10 @@ class TestRunner void InvalidArgs(const std::string& message = "Invalid arguments!"); void Usage(); + bool filterGroupsByXmls(const std::vector & files); + bool filterByXML(std::map & casesMap); + void normalizeXMLTag(std::string& str, const std::string& testcase); + enum Status { FAILED, IGNORED, PASS }; Status RunTestCase(const TestCaseStruct& testCase); diff --git a/modules/test/src/test_runner.cpp b/modules/test/src/test_runner.cpp index 126f1b0..594ba0c 100644 --- a/modules/test/src/test_runner.cpp +++ b/modules/test/src/test_runner.cpp @@ -37,9 +37,28 @@ #include #include +#include +#include +#include +#include + #include IMPLEMENT_SINGLETON(DPL::Test::TestRunner) +namespace { + +std::string getXMLNode(xmlNodePtr node) +{ + std::string ret; + xmlChar * value = xmlNodeGetContent(node); + ret = std::string(reinterpret_cast(value)); + xmlFree(value); + return ret; +} + +} + + namespace DPL { namespace Test { namespace // anonymous @@ -89,6 +108,143 @@ void TestRunner::InitGroup(const char* name) m_currentGroup = name; } +void TestRunner::normalizeXMLTag(std::string& str, const std::string& testcase) +{ + //Add testcase if missing + std::string::size_type pos = str.find(testcase); + if(pos != 0) + { + str = testcase + "_" + str; + } + + //dpl test runner cannot have '-' character in name so it have to be replaced + // for TCT case to make comparision works + std::replace(str.begin(), str.end(), '-', '_'); +} + +bool TestRunner::filterGroupsByXmls(const std::vector & files) +{ + DECLARE_EXCEPTION_TYPE(DPL::Exception, XMLError) + + const std::string idPath = "/test_definition/suite/set/testcase/@id"; + + bool success = true; + std::map casesMap; + + std::string testsuite; + if(!m_testGroups.empty()) + { + for(TestCaseGroupMap::const_iterator cit = m_testGroups.begin(); cit != m_testGroups.end(); ++cit) + { + if(!cit->second.empty()) + { + for(TestCaseStructList::const_iterator cj = cit->second.begin(); cj != cit->second.end(); ++cj) + { + std::string name = cj->name; + std::string::size_type st = name.find('_'); + if(st != std::string::npos) + { + name = name.substr(0, st); + testsuite = name; + break; + } + } + if(!testsuite.empty()) break; + } + } + } + + xmlInitParser(); + LIBXML_TEST_VERSION + xmlXPathInit(); + + Try + { + FOREACH(file, files) + { + xmlDocPtr doc; + xmlXPathContextPtr xpathCtx; + + doc = xmlReadFile(file->c_str(), NULL, 0); + if (doc == NULL) { + ThrowMsg(XMLError, "File Problem"); + } else { + //context + xpathCtx = xmlXPathNewContext(doc); + if (xpathCtx == NULL) { + ThrowMsg(XMLError, + "Error: unable to create new XPath context\n"); + } + xpathCtx->node = xmlDocGetRootElement(doc); + } + + std::string result; + xmlXPathObjectPtr xpathObject; + //get requested node's values + xpathObject = xmlXPathEvalExpression(BAD_CAST idPath.c_str(), xpathCtx); + if (xpathObject == NULL) + { + ThrowMsg(XMLError, "XPath evaluation failure: " << idPath); + } + xmlNodeSetPtr nodes = xpathObject->nodesetval; + unsigned size = (nodes) ? nodes->nodeNr : 0; + LogDebug("Found " << size << " nodes matching xpath"); + for(unsigned i = 0; i < size; ++i) + { + LogPedantic("Type: " << nodes->nodeTab[i]->type); + if (nodes->nodeTab[i]->type == XML_ATTRIBUTE_NODE) { + xmlNodePtr curNode = nodes->nodeTab[i]; + result = getXMLNode(curNode); + LogPedantic("Result: " << result); + normalizeXMLTag(result, testsuite); + casesMap.insert(make_pair(result, false)); + } + } + //Cleanup of XPath data + xmlXPathFreeObject(xpathObject); + xmlXPathFreeContext(xpathCtx); + xmlFreeDoc(doc); + } + } + Catch(XMLError) + { + LogError("Libxml error: " << _rethrown_exception.DumpToString()); + success = false; + } + xmlCleanupParser(); + + if(!filterByXML(casesMap)) + { + success = false; + } + + return success; +} + +bool TestRunner::filterByXML(std::map & casesMap) +{ + FOREACH(group, m_testGroups) { + TestCaseStructList newList; + FOREACH(iterator, group->second) + { + if (casesMap.find(iterator->name) != casesMap.end()) { + casesMap[iterator->name] = true; + newList.push_back(*iterator); + } + } + group->second = newList; + } + FOREACH(cs, casesMap) + { + if(cs->second == false) + { + LogError("Cannot find testcase from XML file: " << cs->first); + return false; + } + } + return true; +} + TestRunner::Status TestRunner::RunTestCase(const TestCaseStruct& testCase) { try { @@ -156,6 +312,11 @@ void TestRunner::RunTests() collector.second->Start(); }); + unsigned count = 0; + FOREACH(group, m_testGroups) { + count += group->second.size(); + } + fprintf(stderr, "%sFound %d testcases...%s\n", GREEN_BEGIN, count, GREEN_END); fprintf(stderr, "%s%s%s\n", GREEN_BEGIN, "Running tests...", GREEN_END); FOREACH(group, m_testGroups) { TestCaseStructList list = group->second; @@ -265,6 +426,10 @@ void TestRunner::Usage() fprintf(stderr, " --runignored\t Run also ignored tests\n"); fprintf(stderr, " --list\t Show a list of Test IDs\n"); fprintf(stderr, " --listgroups\t Show a list of Test Group names \n"); + fprintf(stderr, " --only-from-xml=\t Run only testcases specified in XML file \n" + " XML name is taken from attribute id=\"part1_part2\" as whole.\n" + " If part1 is not found (no _) then it is implicitily " + "set according to suite part1 from binary tests\n"); fprintf( stderr, " --listingroup=\t Show a list of Test IDS in one group\n"); @@ -316,6 +481,8 @@ int TestRunner::ExecTestRunner(const ArgsList& value) args.erase(args.begin()); bool showHelp = false; + bool justList = false; + std::vector xmlFiles; TestResultsCollectorBasePtr currentCollector; @@ -332,6 +499,7 @@ int TestRunner::ExecTestRunner(const ArgsList& value) const std::string listGroupsCmd = "--listgroups"; const std::string listInGroup = "--listingroup="; const std::string allowChildLogs = "--allowchildlogs"; + const std::string onlyFromXML = "--only-from-xml="; if (currentCollector) { if (currentCollector->ParseCollectorSpecificArg(arg)) { @@ -376,12 +544,7 @@ int TestRunner::ExecTestRunner(const ArgsList& value) } else if (arg == runIgnored) { m_runIgnored = true; } else if (arg == listCmd) { - FOREACH(group, m_testGroups) { - FOREACH(test, group->second) { - printf("ID:%s:%s\n", group->first.c_str(), test->name.c_str()); - } - } - return 0; + justList = true; } else if (arg == listGroupsCmd) { FOREACH(group, m_testGroups) { printf("GR:%s\n", group->first.c_str()); @@ -443,6 +606,26 @@ int TestRunner::ExecTestRunner(const ArgsList& value) } group->second = newList; } + } else if(arg.find(onlyFromXML) == 0) { + arg.erase(0, onlyFromXML.length()); + if (arg.length() == 0) { + InvalidArgs(); + Usage(); + return -1; + } + + if (arg[0] == '\'' && arg[arg.length() - 1] == '\'') { + arg.erase(0); + arg.erase(arg.length() - 1); + } + + if (arg.length() == 0) { + InvalidArgs(); + Usage(); + return -1; + } + + xmlFiles.push_back(arg); } else { InvalidArgs(); Usage(); @@ -450,6 +633,25 @@ int TestRunner::ExecTestRunner(const ArgsList& value) } } + if(!xmlFiles.empty()) + { + if(!filterGroupsByXmls(xmlFiles)) + { + fprintf(stderr, "XML file is not correct\n"); + return 0; + } + } + + if(justList) + { + FOREACH(group, m_testGroups) { + FOREACH(test, group->second) { + printf("ID:%s:%s\n", group->first.c_str(), test->name.c_str()); + } + } + return 0; + } + currentCollector.reset(); // Show help -- 2.7.4