Add script creating colorful test output
authorLukasz Kostyra <l.kostyra@partner.samsung.com>
Wed, 5 Mar 2014 09:23:01 +0000 (10:23 +0100)
committerJan Olszak <j.olszak@samsung.com>
Mon, 19 May 2014 11:47:14 +0000 (13:47 +0200)
[Issue#]        PSDAC-170
[Feature]       Adds a script which parses XML test result into a pretty colorful page.
[Cause]         N/A
[Solution]      N/A
[Verification]  Build, install, run sc_tests_all. First, Boost output should show up and it should
                be colored according to following scheme:
                    * BOLD RED = line begins with tag [ERROR]
                    * BOLD YELLOW = line begins with tag [WARN ]
                    * BOLD BLUE = line begins with tag [INFO ]
                    * GREEN = line begins with tag [DEBUG]
                    * BLACK = line begins with tag [TRACE]
                Then, a summary containing all test results should show up.

                Additionally user can launch Valgrind and GDB using this script. Before entering
                binary with test add --valgrind, or --gdb option. Parsing arguments to test binary
                works as well.

                Example use with Valgrind:
               sc_test_launch --valgrind --leak-check=full -v security-containers-server-unit-tests

Change-Id: Ie183ae0ab799e896b7077f979c12c99416d54a84

packaging/security-containers.spec
src/scripts/sc_test_launch.py [new file with mode: 0755]
src/scripts/sc_test_parser.py [new file with mode: 0644]
src/scripts/sc_tests_all.py [new file with mode: 0755]
src/server/unit_tests/ut-scs-log.cpp [new file with mode: 0644]

index 1874ef5..aa26851 100644 (file)
@@ -1,3 +1,6 @@
+%define script_dir %{_libdir}/python/site-packages/sc_test_scripts/
+%define script_src_dir src/scripts
+
 Name:          security-containers
 Version:       0.1.0
 Release:       0
@@ -42,6 +45,15 @@ make -k %{?jobs:-j%jobs}
 %make_install
 mkdir -p %{buildroot}/etc/security-containers/config/libvirt-config/
 
+install -d %{buildroot}/%{_bindir}
+install -d %{buildroot}/%{script_dir}
+install -m 755 %{script_src_dir}/sc_tests_all.py %{buildroot}/%{script_dir}
+install -m 755 %{script_src_dir}/sc_test_launch.py %{buildroot}/%{script_dir}
+install -m 755 %{script_src_dir}/sc_test_parser.py %{buildroot}/%{script_dir}
+
+ln -sf %{script_dir}/sc_tests_all.py %{buildroot}/%{_bindir}/sc_tests_all
+ln -sf %{script_dir}/sc_test_launch.py %{buildroot}/%{_bindir}/sc_test_launch
+
 %clean
 rm -rf %{buildroot}
 
@@ -83,6 +95,7 @@ Summary:          Security Containers Unit Tests
 Group:            Development/Libraries
 Requires:         security-containers = %{version}-%{release}
 Requires:         security-containers-client = %{version}-%{release}
+Requires:         python
 Requires:         boost-test
 BuildRequires:    boost-devel
 
@@ -92,3 +105,8 @@ Unit tests for both: server and client.
 %files unit-tests
 %defattr(644,root,root,644)
 %attr(755,root,root) %{_bindir}/security-containers-server-unit-tests
+%attr(755,root,root) %{script_dir}/sc_tests_all.py
+%attr(755,root,root) %{script_dir}/sc_test_launch.py
+%{script_dir}/sc_test_parser.py
+%{_bindir}/sc_tests_all
+%{_bindir}/sc_test_launch
diff --git a/src/scripts/sc_test_launch.py b/src/scripts/sc_test_launch.py
new file mode 100755 (executable)
index 0000000..f80fe59
--- /dev/null
@@ -0,0 +1,90 @@
+#!/usr/bin/env python
+
+from xml.dom import minidom
+from sc_test_parser import Logger, Parser
+import subprocess
+import argparse
+import os
+
+_defLaunchArgs = ["--report_format=XML",
+                  "--catch_system_errors=no",
+                  "--log_level=test_suite",
+                  "--report_level=detailed",
+                 ]
+
+log = Logger()
+
+def _checkIfBinExists(binary):
+    paths = [s + "/" for s in os.environ["PATH"].split(os.pathsep)]
+    exists = any([os.path.isfile(path + binary) and os.access(path + binary, os.X_OK)
+                  for path in paths])
+
+    if not exists:
+        log.error(binary + " NOT FOUND.")
+
+    return exists
+
+
+
+def launchTest(cmd=[], externalToolCmd=[], parsing=True):
+    """Default function used to launch test binary.
+
+    Creates a new subprocess and parses it's output
+    """
+    if not _checkIfBinExists(cmd[0]):
+        return
+    if externalToolCmd and not _checkIfBinExists(externalToolCmd[0]):
+        return
+
+    log.info("Starting " + cmd[0] + "...")
+
+    if parsing:
+        parser = Parser()
+        p = subprocess.Popen(" ".join(externalToolCmd + cmd + _defLaunchArgs),
+                         shell=True,
+                         stdout=subprocess.PIPE,
+                         stderr=subprocess.STDOUT)
+        testResult = parser.parseOutputFromProcess(p)
+        if testResult != "":
+            domResult = minidom.parseString(testResult)
+            log.XMLSummary(domResult)
+    else:
+        # Launching process without coloring does not require report in XML form
+        # Avoid providing --report_format=XML, redirect std* by default to system's std*
+        p = subprocess.Popen(" ".join(externalToolCmd + cmd + _defLaunchArgs[1:]),
+                             shell=True)
+        p.wait()
+
+    log.info(cmd[0] + " finished.")
+
+
+
+_valgrindCmd = ["valgrind"]
+_gdbCmd = ["gdb", "--args"]
+
+def main():
+    argparser = argparse.ArgumentParser(description="Test binary launcher for security-containers.")
+    group = argparser.add_mutually_exclusive_group()
+    group.add_argument('--valgrind', action='store_true',
+                        help='Launch test binary inside Valgrind (assuming it is installed).')
+    group.add_argument('--gdb', action='store_true',
+                        help='Launch test binary with GDB (assuming it is installed).')
+    argparser.add_argument('binary', nargs=argparse.REMAINDER,
+                        help='Binary to be launched using script.')
+
+    args = argparser.parse_known_args()
+
+    if args[0].binary:
+        if args[0].gdb:
+            launchTest(args[0].binary, externalToolCmd=_gdbCmd + args[1], parsing=False)
+        elif args[0].valgrind:
+            launchTest(args[0].binary, externalToolCmd=_valgrindCmd + args[1])
+        else:
+            launchTest(args[0].binary, parsing=True)
+    else:
+        log.info("Test binary not provided! Exiting.")
+
+
+
+if __name__ == "__main__":
+    main()
diff --git a/src/scripts/sc_test_parser.py b/src/scripts/sc_test_parser.py
new file mode 100644 (file)
index 0000000..7fd13d4
--- /dev/null
@@ -0,0 +1,133 @@
+from xml.dom import minidom
+import sys
+
+
+
+BLACK = "\033[90m"
+RED = "\033[91m"
+GREEN = "\033[92m"
+YELLOW = "\033[93m"
+BLUE = "\033[94m"
+MAGENTA = "\033[95m"
+CYAN = "\033[96m"
+WHITE = "\033[97m"
+BOLD = "\033[1m"
+ENDC = "\033[0m"
+
+
+
+class Logger(object):
+    # Create summary of test providing DOM object with parsed XML
+    def info(self, msg):
+        print BOLD + msg + ENDC
+
+    def infoTitle(self, msg):
+        print CYAN + BOLD + msg + ENDC
+
+    def error(self, msg):
+        print RED + BOLD + msg + ENDC
+
+    def success(self, msg):
+        print GREEN + BOLD + msg + ENDC
+
+
+    __indentChar = "    "
+    def testCaseSummary(self, testName, testResult, recLevel):
+        msg = self.__indentChar * recLevel + BOLD + "{:<50}".format(testName + ":")
+
+        if testResult == "passed":
+            msg += GREEN
+        else:
+            msg += RED
+
+        print msg + testResult + ENDC
+
+    def testSuiteSummary(self, suite, recLevel=0, summarize=False):
+        indPrefix = self.__indentChar * recLevel
+
+        self.infoTitle(indPrefix + suite.attributes["name"].value + " results:")
+
+        for child in suite.childNodes:
+            if child.nodeName == "TestSuite":
+                self.testSuiteSummary(child, recLevel=recLevel + 1)
+            elif child.nodeName == "TestCase":
+                self.testCaseSummary(child.attributes["name"].value,
+                                     child.attributes["result"].value,
+                                     recLevel=recLevel + 1)
+
+        if summarize:
+            self.infoTitle(indPrefix + suite.attributes["name"].value + " summary:")
+            self.info(indPrefix + "Passed tests:  " + suite.attributes["test_cases_passed"].value)
+            self.info(indPrefix + "Failed tests:  " + suite.attributes["test_cases_failed"].value)
+            self.info(indPrefix + "Skipped tests: " + suite.attributes["test_cases_skipped"].value +
+                      "\n")
+
+    def XMLSummary(self, dom):
+        self.info("\n=========== SUMMARY ===========\n")
+
+        for result in dom.getElementsByTagName("TestResult"):
+            for child in result.childNodes:
+                if child.nodeName == "TestSuite":
+                    self.testSuiteSummary(child, summarize=True)
+
+
+
+class Colorizer(object):
+    # Add new types of errors/tags for parser here
+    lineTypeDict = {'[ERROR]': RED + BOLD,
+                    '[WARN ]': YELLOW + BOLD,
+                    '[INFO ]': BLUE + BOLD,
+                    '[DEBUG]': GREEN,
+                    '[TRACE]': BLACK
+                    }
+
+    # Looks for lineTypeDict keywords in provided line and paints such line appropriately
+    def paintLine(self, line):
+        for key in self.lineTypeDict.iterkeys():
+            if key in line:
+                return self.lineTypeDict[key] + line + ENDC
+
+        return line
+
+
+
+class Parser(object):
+    _testResultBegin = "<TestResult>"
+    _testResultEnd = "</TestResult>"
+
+    colorizer = Colorizer()
+
+    def __parseAndWriteLine(self, line):
+        result = ""
+        # Entire XML is kept in one line, if line begins with <TestResult>, extract it
+        if self._testResultBegin in line:
+            result += line[line.find(self._testResultBegin):
+                           line.find(self._testResultEnd) + len(self._testResultEnd)]
+            line = line[0:line.find(self._testResultBegin)] + line[line.find(self._testResultEnd) +
+                                                                   len(self._testResultEnd):]
+        sys.stdout.write(str(self.colorizer.paintLine(line)))
+        sys.stdout.flush()
+
+        return result
+
+
+    def parseOutputFromProcess(self, p):
+        """Parses stdout from given subprocess p - colors printed lines and looks for test results.
+        """
+        testResult = ""
+        # Dump test results
+        while True:
+            outline = p.stdout.readline()
+            testResult += self.__parseAndWriteLine(outline)
+            # If process returns a value, leave loop
+            if p.poll() != None:
+                break
+
+        # Sometimes the process might exit before we finish reading entire stdout
+        # Split ending of stdout in lines and finish reading it before ending the function
+        stdoutEnding = [s + '\n' for s in p.stdout.read().split('\n')]
+        for outline in stdoutEnding:
+            testResult += self.__parseAndWriteLine(outline)
+
+        return testResult
+
diff --git a/src/scripts/sc_tests_all.py b/src/scripts/sc_tests_all.py
new file mode 100755 (executable)
index 0000000..85c0d55
--- /dev/null
@@ -0,0 +1,9 @@
+#!/usr/bin/env python
+
+import sc_test_launch
+
+# insert other test binaries to this array
+_testCmdTable = ["security-containers-server-unit-tests"]
+
+for test in _testCmdTable:
+    sc_test_launch.launchTest([test])
diff --git a/src/server/unit_tests/ut-scs-log.cpp b/src/server/unit_tests/ut-scs-log.cpp
new file mode 100644 (file)
index 0000000..7f89ac1
--- /dev/null
@@ -0,0 +1,40 @@
+/*
+ *  Copyright (c) 2000 - 2014 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ *  Contact: Bumjin Im <bj.im@samsung.com>
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License
+ */
+
+
+/**
+ * @file    ut-scs-log.cpp
+ * @author  Lukasz Kostyra (l.kostyra@samsung.com)
+ * @brief   Unit tests for security-containers logging system
+ */
+
+#include "ut.hpp"
+#include "scs-log.hpp"
+
+BOOST_AUTO_TEST_SUITE(LogSuite)
+
+BOOST_AUTO_TEST_CASE(DumpAllLogTypes)
+{
+    LOGE("Logging an error message.");
+    LOGW("Logging a warning.");
+    LOGI("Logging some information.");
+    LOGD("Logging debug information.");
+    LOGT("Logging trace information.");
+}
+
+BOOST_AUTO_TEST_SUITE_END()