Option --only-from-xml for dpl test framework

[Issue#]       LINUXWRT-743
[Feature]      Reading testcase to be launched from TCT xml file
[Cause]        Need for automatic launching only TCT tests.
[Solution]     Additional option for test binary.
Additonal changes:
 - printing number of testcases at startup
 - listing option will take into account --regexp and --only-from-xml
[Verification] Build and run tests from change. Try this change with wrt-extra tests with commit "W3C testcases names adjust for TCT".
Install TCT on target:
 - tct-widget01-w3c
 - tct-widget02-w3c
 - tct-wgtapi01-w3c
 - tct-wgtapi02-w3c
 - tct-widgetpolicy-w3c
 - tct-widgetdigsig-w3c

All commands below should print requested list of tescases from xml file
and do not complain about wrong XML file (note that list option prints now only test
to run (after filtering them)):
 wrt-extra-tests-plugins-w3c --list --only-from-xml=/usr/share/tct-wgtapi02-w3c-tests/tests.xml
 wrt-extra-tests-plugins-w3c --list --only-from-xml=/usr/share/tct-wgtapi01-w3c-tests/tests.xml
 wrt-extra-tests-w3c --list --only-from-xml=/usr/share/tct-widget01-w3c-tests/tests.xml
 wrt-extra-tests-w3c --list --only-from-xml=/usr/share/tct-widget02-w3c-tests/tests.xml
 wrt-extra-tests-w3c-widget-policy --list --only-from-xml=/usr/share/tct-widgetpolicy-w3c-tests/tests.xml
 wrt-extra-tests-w3c-digsig --list --only-from-xml=/usr/share/tct-widgetdigsig-w3c-tests/tests.xml #no testcases anyway

Change-Id: I1b62a5d17dc351c1f763fb1d9a877e1300bc6a9c

diff --git a/build/test/CMakeLists.txt b/build/test/CMakeLists.txt
index 710fc4e..d992952 100644 (file)
--- a/build/test/CMakeLists.txt
+++ b/build/test/CMakeLists.txt
@@ -21,6 +21,7 @@
 INCLUDE(FindPkgConfig)
 PKG_CHECK_MODULES(SYS_EFL_TEST_ENGINE
   appcore-efl
+  libxml-2.0
   REQUIRED)
 
 

diff --git a/build/test/dpl-test-efl.pc.in b/build/test/dpl-test-efl.pc.in
index 9d6f871..056ae09 100644 (file)
--- a/build/test/dpl-test-efl.pc.in
+++ b/build/test/dpl-test-efl.pc.in
@@ -6,6 +6,6 @@ includedir=${prefix}/include
 Name: dpl-test-efl
 Description: DPL Test Engine - EFL based
 Version: @VERSION@
-Requires: dpl-efl
+Requires: dpl-efl libxml-2.0
 Libs: -L${libdir} -ldpl-test-efl
 Cflags: -I${includedir}/dpl-efl

diff --git a/modules/test/include/dpl/test/test_runner.h b/modules/test/include/dpl/test/test_runner.h
index 99a2cb5..ae5c712 100644 (file)
--- a/modules/test/include/dpl/test/test_runner.h
+++ b/modules/test/include/dpl/test/test_runner.h
@@ -97,6 +97,10 @@ class TestRunner
     void InvalidArgs(const std::string& message = "Invalid arguments!");
     void Usage();
 
+    bool filterGroupsByXmls(const std::vector<std::string> & files);
+    bool filterByXML(std::map<std::string, bool> & casesMap);
+    void normalizeXMLTag(std::string& str, const std::string& testcase);
+
     enum Status { FAILED, IGNORED, PASS };
 
     Status RunTestCase(const TestCaseStruct& testCase);

diff --git a/modules/test/src/test_runner.cpp b/modules/test/src/test_runner.cpp
index 126f1b0..594ba0c 100644 (file)
--- a/modules/test/src/test_runner.cpp
+++ b/modules/test/src/test_runner.cpp
@@ -37,9 +37,28 @@
 #include <cstdlib>
 #include <dpl/utils/wrt_global_settings.h>
 
+#include <libxml/xpath.h>
+#include <libxml/xpathInternals.h>
+#include <libxml/parser.h>
+#include <libxml/tree.h>
+
 #include <dpl/singleton_impl.h>
 IMPLEMENT_SINGLETON(DPL::Test::TestRunner)
 
+namespace {
+
+std::string getXMLNode(xmlNodePtr node)
+{
+    std::string ret;
+    xmlChar * value = xmlNodeGetContent(node);
+    ret = std::string(reinterpret_cast<char*>(value));
+    xmlFree(value);
+    return ret;
+}
+
+}
+
+
 namespace DPL {
 namespace Test {
 namespace // anonymous
@@ -89,6 +108,143 @@ void TestRunner::InitGroup(const char* name)
     m_currentGroup = name;
 }
 
+void TestRunner::normalizeXMLTag(std::string& str, const std::string& testcase)
+{
+    //Add testcase if missing
+    std::string::size_type pos = str.find(testcase);
+    if(pos != 0)
+    {
+        str = testcase + "_" + str;
+    }
+
+    //dpl test runner cannot have '-' character in name so it have to be replaced
+    // for TCT case to make comparision works
+    std::replace(str.begin(), str.end(), '-', '_');
+}
+
+bool TestRunner::filterGroupsByXmls(const std::vector<std::string> & files)
+{
+    DECLARE_EXCEPTION_TYPE(DPL::Exception, XMLError)
+
+    const std::string idPath = "/test_definition/suite/set/testcase/@id";
+
+    bool success = true;
+    std::map<std::string, bool> casesMap;
+
+    std::string testsuite;
+    if(!m_testGroups.empty())
+    {
+        for(TestCaseGroupMap::const_iterator cit = m_testGroups.begin(); cit != m_testGroups.end(); ++cit)
+        {
+            if(!cit->second.empty())
+            {
+                for(TestCaseStructList::const_iterator cj = cit->second.begin(); cj != cit->second.end(); ++cj)
+                {
+                    std::string name = cj->name;
+                    std::string::size_type st = name.find('_');
+                    if(st != std::string::npos)
+                    {
+                        name = name.substr(0, st);
+                        testsuite = name;
+                        break;
+                    }
+                }
+                if(!testsuite.empty()) break;
+            }
+        }
+    }
+
+    xmlInitParser();
+    LIBXML_TEST_VERSION
+    xmlXPathInit();
+
+    Try
+    {
+        FOREACH(file, files)
+        {
+            xmlDocPtr doc;
+            xmlXPathContextPtr xpathCtx;
+
+            doc = xmlReadFile(file->c_str(), NULL, 0);
+            if (doc == NULL) {
+                ThrowMsg(XMLError, "File Problem");
+            } else {
+                //context
+                xpathCtx = xmlXPathNewContext(doc);
+                if (xpathCtx == NULL) {
+                    ThrowMsg(XMLError,
+                             "Error: unable to create new XPath context\n");
+                }
+                xpathCtx->node = xmlDocGetRootElement(doc);
+            }
+
+            std::string result;
+            xmlXPathObjectPtr xpathObject;
+            //get requested node's values
+            xpathObject = xmlXPathEvalExpression(BAD_CAST idPath.c_str(), xpathCtx);
+            if (xpathObject == NULL)
+            {
+                ThrowMsg(XMLError, "XPath evaluation failure: " << idPath);
+            }
+            xmlNodeSetPtr nodes = xpathObject->nodesetval;
+            unsigned size = (nodes) ? nodes->nodeNr : 0;
+            LogDebug("Found " << size << " nodes matching xpath");
+            for(unsigned i = 0; i < size; ++i)
+            {
+                LogPedantic("Type: " << nodes->nodeTab[i]->type);
+                if (nodes->nodeTab[i]->type == XML_ATTRIBUTE_NODE) {
+                    xmlNodePtr curNode = nodes->nodeTab[i];
+                    result = getXMLNode(curNode);
+                    LogPedantic("Result: " << result);
+                    normalizeXMLTag(result, testsuite);
+                    casesMap.insert(make_pair(result, false));
+                }
+            }
+            //Cleanup of XPath data
+            xmlXPathFreeObject(xpathObject);
+            xmlXPathFreeContext(xpathCtx);
+            xmlFreeDoc(doc);
+        }
+    }
+    Catch(XMLError)
+    {
+        LogError("Libxml error: " << _rethrown_exception.DumpToString());
+        success = false;
+    }
+    xmlCleanupParser();
+
+    if(!filterByXML(casesMap))
+    {
+        success = false;
+    }
+
+    return success;
+}
+
+bool TestRunner::filterByXML(std::map<std::string, bool> & casesMap)
+{
+    FOREACH(group, m_testGroups) {
+        TestCaseStructList newList;
+        FOREACH(iterator, group->second)
+        {
+            if (casesMap.find(iterator->name) != casesMap.end()) {
+                casesMap[iterator->name] = true;
+                newList.push_back(*iterator);
+            }
+        }
+        group->second = newList;
+    }
+    FOREACH(cs, casesMap)
+    {
+        if(cs->second == false)
+        {
+            LogError("Cannot find testcase from XML file: " << cs->first);
+            return false;
+        }
+    }
+    return true;
+}
+
 TestRunner::Status TestRunner::RunTestCase(const TestCaseStruct& testCase)
 {
     try {
@@ -156,6 +312,11 @@ void TestRunner::RunTests()
                       collector.second->Start();
                   });
 
+    unsigned count = 0;
+    FOREACH(group, m_testGroups) {
+        count += group->second.size();
+    }
+    fprintf(stderr, "%sFound %d testcases...%s\n", GREEN_BEGIN, count, GREEN_END);
     fprintf(stderr, "%s%s%s\n", GREEN_BEGIN, "Running tests...", GREEN_END);
     FOREACH(group, m_testGroups) {
         TestCaseStructList list = group->second;
@@ -265,6 +426,10 @@ void TestRunner::Usage()
     fprintf(stderr, "  --runignored\t Run also ignored tests\n");
     fprintf(stderr, "  --list\t Show a list of Test IDs\n");
     fprintf(stderr, "  --listgroups\t Show a list of Test Group names \n");
+    fprintf(stderr, "  --only-from-xml=<xml file>\t Run only testcases specified in XML file \n"
+                    "       XML name is taken from attribute id=\"part1_part2\" as whole.\n"
+                    "       If part1 is not found (no _) then it is implicitily "
+                           "set according to suite part1 from binary tests\n");
     fprintf(
         stderr,
         "  --listingroup=<group name>\t Show a list of Test IDS in one group\n");
@@ -316,6 +481,8 @@ int TestRunner::ExecTestRunner(const ArgsList& value)
     args.erase(args.begin());
 
     bool showHelp = false;
+    bool justList = false;
+    std::vector<std::string> xmlFiles;
 
     TestResultsCollectorBasePtr currentCollector;
 
@@ -332,6 +499,7 @@ int TestRunner::ExecTestRunner(const ArgsList& value)
         const std::string listGroupsCmd = "--listgroups";
         const std::string listInGroup = "--listingroup=";
         const std::string allowChildLogs = "--allowchildlogs";
+        const std::string onlyFromXML = "--only-from-xml=";
 
         if (currentCollector) {
             if (currentCollector->ParseCollectorSpecificArg(arg)) {
@@ -376,12 +544,7 @@ int TestRunner::ExecTestRunner(const ArgsList& value)
         } else if (arg == runIgnored) {
             m_runIgnored = true;
         } else if (arg == listCmd) {
-            FOREACH(group, m_testGroups) {
-                FOREACH(test, group->second) {
-                    printf("ID:%s:%s\n", group->first.c_str(), test->name.c_str());
-                }
-            }
-            return 0;
+            justList = true;
         } else if (arg == listGroupsCmd) {
             FOREACH(group, m_testGroups) {
                 printf("GR:%s\n", group->first.c_str());
@@ -443,6 +606,26 @@ int TestRunner::ExecTestRunner(const ArgsList& value)
                 }
                 group->second = newList;
             }
+        } else if(arg.find(onlyFromXML) == 0) {
+            arg.erase(0, onlyFromXML.length());
+            if (arg.length() == 0) {
+                InvalidArgs();
+                Usage();
+                return -1;
+            }
+
+            if (arg[0] == '\'' && arg[arg.length() - 1] == '\'') {
+                arg.erase(0);
+                arg.erase(arg.length() - 1);
+            }
+
+            if (arg.length() == 0) {
+                InvalidArgs();
+                Usage();
+                return -1;
+            }
+
+            xmlFiles.push_back(arg);
         } else {
             InvalidArgs();
             Usage();
@@ -450,6 +633,25 @@ int TestRunner::ExecTestRunner(const ArgsList& value)
         }
     }
 
+    if(!xmlFiles.empty())
+    {
+        if(!filterGroupsByXmls(xmlFiles))
+        {
+            fprintf(stderr, "XML file is not correct\n");
+            return 0;
+        }
+    }
+
+    if(justList)
+    {
+        FOREACH(group, m_testGroups) {
+            FOREACH(test, group->second) {
+                printf("ID:%s:%s\n", group->first.c_str(), test->name.c_str());
+            }
+        }
+        return 0;
+    }
+
     currentCollector.reset();
 
     // Show help