From: Nicolas Zingilé Date: Mon, 11 Aug 2014 13:26:35 +0000 (+0200) Subject: updated the common-suite-launcher tool X-Git-Tag: accepted/tizen/common/20140811.164821 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=refs%2Ftags%2Faccepted%2Ftizen%2Fcommon%2F20140811.164821;p=test%2Ftools%2Fcommon-suite-launcher.git updated the common-suite-launcher tool Change-Id: Ibb9628d75cf239358cf228af9b138f2157cde24d Signed-off-by: Nicolas Zingilé --- diff --git a/packaging/common-suite-launcher.spec b/packaging/common-suite-launcher.spec index 7167635..5312267 100644 --- a/packaging/common-suite-launcher.spec +++ b/packaging/common-suite-launcher.spec @@ -1,5 +1,5 @@ Name: common-suite-launcher -Version: 1.0.0 +Version: 2.0.0 Release: 0 License: GPL-2.0 Summary: Launcher of Tizen Common test suites @@ -12,7 +12,8 @@ BuildArch: noarch %description -Common Suite Launcher is the Launcher of the test suites of the Tizen Common profile +Common Suite Launcher is the launcher of the test suites that +are packaged in Tizen. %prep @@ -26,9 +27,11 @@ cp %{SOURCE1001} . %install install -d %{buildroot}/%{_bindir} install -m 0755 src/%{name} %{buildroot}/%{_bindir} +install -m 0755 src/result-format %{buildroot}/%{_bindir} %files %manifest %{name}.manifest %defattr(-,root,root) %{_bindir}/common-suite-launcher +%{_bindir}/result-format diff --git a/src/common-suite-launcher b/src/common-suite-launcher index 946386a..cf49492 100755 --- a/src/common-suite-launcher +++ b/src/common-suite-launcher @@ -1,5 +1,5 @@ -#!/bin/bash - +#!/usr/bin/python +# # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License # as published by the Free Software Foundation; either version 2 @@ -14,121 +14,214 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. # -# Authors: Nicolas Zingilé - -set -e - -script=$(basename $0) - -function error() { - echo ERROR: "$@" >&2 - cat << EOF >&2 -Usage: - $script list - $script launch [] - - list : list available suites - launch : launch a suite - |_ suite - suite to launch - |_ outdir - result directory (default : /tmp). Result files are testkit.result.xml and testkit.result.txt -EOF - exit 1 -} - -function list_suites() { - echo "### Available common suites :" - for suite in /usr/share/tests/* - do - if [[ $(basename $suite) == common-*-suite ]]; then - echo $(basename $suite) - fi - done -} - -function format_results() { - local outdir="" - echo "#### Generating test results in txt format" - if [[ -z $1 ]]; then - outdir=/tmp - else - outdir=$1 - cp /tmp/testkit.result.xml $outdir - fi - - if [ -e $outdir/testkit.result.xml ]; then - - starttime=$(xml sel -t -v //summary/start_at -n $outdir/testkit.result.xml) - stoptime=$(xml sel -t -v //summary/end_at -n $outdir/testkit.result.xml) - suitename=$(xml sel -t -v //suite/@name -n $outdir/testkit.result.xml) - totalcase=$(xml sel -t -v "count(//testcase)" -n $outdir/testkit.result.xml) - totalpass=$(xml sel -t -v "count(//testcase[@result='PASS'])" -n $outdir/testkit.result.xml) - totalfail=$(xml sel -t -v "count(//testcase[@result='FAIL'])" -n $outdir/testkit.result.xml) - totalna=$(xml sel -t -v "count(//testcase[@result='N/A'])" -n $outdir/testkit.result.xml) - sets=$(xml sel -t -v //set/@name -n $outdir/testkit.result.xml) - - cat << EOF > $outdir/testkit.result.txt -======================== Test Report ======================= - -Start time : $starttime -End time : $stoptime -Suite name : $suitename -Total cases : $totalcase -Pass cases : $totalpass -Fail cases : $totalfail -N/A cases : $totalna -EOF - for set_test in $sets - do - cat << EOF - -=== Set $set_test - -EOF - - xml sel -t -m "//testcase[@component='$set_test']" -v "concat(@name,'#',@purpose,'#',@result,'#',result_info/stdout)" -n $outdir/testkit.result.xml | - awk "-F#" 'NF>=4{ - o=""; for (i=4 ; i<=NF ; i=i+1) o = (o ? o" " : "") $i - printf "test id : %s\n", $1 - printf "%s\n", $2 - printf "result : %s\n", $3 - printf "stdout : %s\n", o - printf "\n" - }' - done | sed 's:\\n:\n:g' >> $outdir/testkit.result.txt - - echo "Test results are available in $outdir" - - else - echo "Cannot generate test results in other format. No testkit.result.xml file" - fi - -} - -function launch_suite() { - if [ -d /usr/share/tests/$1 ]; then - if [ -x /usr/share/tests/$1/runtest.sh ]; then - echo "#### Launching suite $1" - /usr/share/tests/$1/runtest.sh - format_results $2 - else - echo "Suite is corrupted ! cannot find runtest.sh script" - fi - else - echo "Cannot launch the suite $1. Suite directory doesn't exist" - error - fi -} - -case $1 in - list) - list_suites - ;; - launch) - [ -z $2 ] && error "No suite defined !" - [ ! -d $3 ] && error "Result directory doesn't exist. Please create it first !" - - launch_suite $2 $3 - ;; - *) - error "Command line doesn't have any option" -esac +# Authors: Nicolas Zingile + +import io +import os +import sys +import argparse +import shutil +import subprocess +import argparse +from lxml import etree + +#---- Global variables ----# +XSDFILE = '/usr/share/testkit-lite/xsd/test_definition.xsd' +GLOBALSUITEPATH = '/usr/share/tests/' +KNOWNPROFILES = ['ivi', 'common'] + +#---- Helper functions of the launch_suites function ----# +def check_runtest(suitedir): + """ Check if a test suite contains a valid runtest script + + Check if suitedir contains a runtest script and + if this script is executable. + + Args: + suitedir: Directory of the suite to check + + Returns: + A boolean + True if a runtest script is present and executable + False if no runtest script or runtest script is not executable + """ + result = False + runtestfile = os.path.join(suitedir, 'runtest') + if os.path.isfile(runtestfile) and os.access(runtestfile, os.X_OK): + result = True + + return result + +def check_xmlfile(suitedir, testxml): + """Check if a testkit xml file is well formed and valid + + Check if the syntax of testxml is in accordance with + the xml format. Then, check if testxml is in accordance + with the testkit lite schema definition. + Exits the program if testxml is not well formed. + + Args: + suitedir: Directory of the suite + testxml: Testkit xml file to check + + Returns: + A boolean if the document is well formed. + True if testxml is valid + False if testxml is not valid + """ + xmlschema_doc = etree.parse(XSDFILE) + xmlschema = etree.XMLSchema(xmlschema_doc) + + try: + xmlfile = etree.parse(os.path.join(suitedir,testxml)) + except etree.XMLSyntaxError as e: + print '-- xml file ' + testxml + ' of ' + os.path.basename(suitedir) + ' is not well-formed !' + print 'Error : ' + str(e) + sys.exit(1) + + valid = xmlschema.validate(xmlfile) + + return(valid, str(xmlschema.error_log)) + +def get_xmlfiles(folder): + """ Retrieves all .xml files of a directory. + + Get all the .xml files of folder. Sub folders + are not checked. + + Args: + folder: directory to borowse + + Returns: + A list of the xml files in folder + """ + xmlfiles = [] + for afile in os.listdir(folder): + if afile.endswith('.xml'): + xmlfiles.append(afile) + + return xmlfiles + +def print_output(process): + """Print the output of a process in stdout + + Args: + process: process for which to print the stdout + + Returns: + """ + while True: + nextline = process.stdout.readline() + if nextline == '' and process.poll() != None: + break + sys.stdout.write(nextline) + sys.stdout.flush() + + output = process.communicate()[0] + exitCode = process.returncode + + if (exitCode == 0): + return output + else: + exit(1) + +#---- subcommand function ----# + +def list_suites(args): + profiles = [] + + print '#---------- Available test suites ----------#' + if args.profile == 'all': + try: + for profile in os.listdir(GLOBALSUITEPATH): + if not profile.startswith('.') and isinstance(KNOWNPROFILES.index(profile), int): + profiles.append(profile) + except OSError: + print '-' + except ValueError: + print 'Error : \'' + profile + '\' is not a supported Tizen profile !' + else: + profiles.append(args.profile) + + for profile in profiles: + print '\n##-- List of ' + profile + ' test suites' + suitepath = os.path.join(GLOBALSUITEPATH, profile) + suitelist = os.listdir(suitepath) + try: + if not suitelist: + print '-' + else: + for suite in os.listdir(suitepath): + print suite + except OSError: + print '-' + +def launch_suites(args): + + print '##-- Checking integrity of the test suites' + if len(args.suites) != len(set(args.suites)): + print '-- List of given test suites is invalid !' + print 'Error : the list of the test suites to launch should not contains duplicates !' + exit (1) + for suite in args.suites: + profile = suite.split('-', 2)[0] + suitepath = os.path.join(GLOBALSUITEPATH, profile, suite) + print suite + if not os.path.isdir(suitepath): + print '-- The test suite is invalid !' + print 'Error : the test suite \''+ os.path.basename(suitepath) + '\' doesn\'t exist.' + exit(1) + if check_runtest(suitepath): + print '-- runtest script of ' + suite + ' is present and executable. Ok' + testkitxmlfiles = get_xmlfiles(suitepath) + if not testkitxmlfiles: + print '-- No xml file found !' + print 'Error : there is no testkit xml file in the suite directory.' + for axmlfile in testkitxmlfiles: + result = check_xmlfile(suitepath, axmlfile) + if result[0]: + print '-- xml file ' + axmlfile + ' of ' + suite + ' is valid. Ok' + else: + print '-- xml file ' + axmlfile + ' of ' + suite + ' is invalid !' + print 'Error : ' + result[1] + exit(1) + else: + print '-- runtest script of ' + suite + ' is corrupted !' + print 'Error : \'runtest\' script is not present and/or is not executable' + sys.exit(1) + + print '\n##-- Executing the suite : ' + suite + "\n" + finaloutdir = os.path.join(args.outdir, suite) + if not os.path.isdir(finaloutdir): + os.makedirs(finaloutdir, 0755) + else: + shutil.rmtree(finaloutdir) + + process = subprocess.Popen([os.path.join(suitepath, 'runtest'), finaloutdir], cwd=suitepath, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + print_output(process) + for afile in os.listdir(finaloutdir): + if afile.endswith(".xml"): + cmd = "result-format -f " + os.path.join(finaloutdir, afile) + " -o " + finaloutdir + os.system(cmd) + +def main (): + + parser = argparse.ArgumentParser(description='Tool to manage Tizen test suites') + subparser = parser.add_subparsers(dest='subcommand') + sp_list = subparser.add_parser('list', help='list available test suites') + sp_list.add_argument('--profile', help='list the available test suites of Tizen PROFILE', choices=['common', 'ivi', 'all'], default='all') + sp_launch = subparser.add_parser('launch', help='launch a test suites') + sp_launch.add_argument('--suites', help='test suite(s) to launch', required=True, nargs='+') + sp_launch.add_argument('--outdir', help='output directory for the results', default='/var/log/qa') + + args = parser.parse_args() + + if args.subcommand == 'list': + list_suites(args) + elif args.subcommand == 'launch': + launch_suites(args) + else: + parser.print_usage() + +if __name__ == "__main__": + main() diff --git a/src/result-format b/src/result-format new file mode 100755 index 0000000..9e6a580 --- /dev/null +++ b/src/result-format @@ -0,0 +1,135 @@ +#!/usr/bin/python +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# Authors: Nicolas Zingile + +import io +import os +import argparse +import sys +from lxml import etree + +def printsatats(xmltree, suitename="", setname=""): + """Get some data information on a testkit result xml tree + + Get the total number, the pass, the failed and the N/A test + cases. + If suitename is not defined, information are retrieved in the + whole document. + If suitename is defined, information are retrieved in the + given suite. + If both suitename and setname are defined, information are + retrieved in the set of the suite which have been entered + + Args: + xmltree: testkit result xml tree to analyse + suitename: name of the suite to analyse + setname: name of the set to analyse + + Returns: + A string with the information + """ + stringbuf = "" + xpathtotal = "" + xpathpass = "" + xpathfail = "" + xpathna = "" + + if not suitename: + xpathtotal = "count(//suite/set/testcase" + else: + if not setname: + xpathtotal = "count(//suite[@name='" + suitename + "']/set/testcase" + else: + xpathtotal = "count(//suite[@name='" + suitename + "']/set[@name='" + setname + "']/testcase" + + xpathpass = xpathtotal + "[@result='PASS'])" + xpathfail = xpathtotal + "[@result='FAIL'])" + xpathna = xpathtotal + "[@result='N/A'])" + xpathtotal += ")" + stringbuf += "\nTotal cases : " + str(int((xmltree.xpath(xpathtotal)))) + stringbuf += "\nPass cases : " + str(int((xmltree.xpath(xpathpass)))) + stringbuf += "\nFail cases : " + str(int((xmltree.xpath(xpathfail)))) + stringbuf += "\nN/A cases : " + str(int((xmltree.xpath(xpathna)))) + + return stringbuf + +def format_result(resultxml, outputdir): + """ Format a testkit xml result file in a testkit txt result file + + Creates a testkit txt result file in outputdir with resultxml + + Args: + resultxml: path of the testkit xml result file + outputdir: folder for the future testkit resuklt txt file + + Returns: + """ + if not os.path.exists(resultxml) or not os.path.isfile(resultxml): + print "Error : Testkit result file doesn't exist !" + exit(1) + else: + resultname = os.path.basename(os.path.splitext(resultxml)[0] + '.txt') + + if not os.path.isdir(outputdir): + print "Error : Output directory doesn't exist. You should create it first" + exit(1) + else: + resultpath = os.path.join(outputdir, resultname) + if os.path.exists(resultpath): + os.remove(resultpath) + + parser = etree.XMLParser(strip_cdata=False) + + try: + xmldoc = etree.parse(resultxml, parser) + except etree.XMLSyntaxError: + print "Error : testkit xml file is not well formed !" + exit(1) + + filedesc = open(resultpath, 'w+') + filedesc.write('======================== Test Report =======================\n') + filedesc.write(printsatats(xmldoc)) + + suiteiter = xmldoc.iter('suite') + + for asuite in suiteiter: + suitename = asuite.get('name') + filedesc.write('\n\n\n====== Suite : ' + suitename + ' ======\n') + filedesc.write(printsatats(xmldoc, suitename)) + setiter = asuite.iter('set') + for aset in setiter: + setname = aset.get('name') + filedesc.write('\n\n=== Set : ' + setname + ' ===\n') + filedesc.write(printsatats(xmldoc, suitename, setname)) + tcaseiter = aset.iter('testcase') + for atcase in tcaseiter: + filedesc.write('\n\ntest id : ' + atcase.get('id')) + filedesc.write('\nobjective : ' + atcase.get('purpose')) + filedesc.write('\nresult : ' + atcase.get('result')) + filedesc.write('\nstdout : ' + atcase.find('result_info/stdout').text.decode('string_escape')) + filedesc.write('stderr : ' + atcase.find('result_info/stderr').text.decode('string_escape')) + +def main (): + parser = argparse.ArgumentParser(description='Tool to convert a testkit xml file in a txt format') + parser.add_argument('-f', '--file', required=True, help='testkit xml result file') + parser.add_argument('-o', '--outdir', metavar='DIR', default='/tmp', help='output directory') + args = parser.parse_args() + + format_result(args.file, args.outdir) + +if __name__=="__main__": + main()