2 # -*- coding: utf-8 -*-
4 #Copyright (C) 2010 Nokia Corporation and/or its subsidiary(-ies)
6 #This library is free software; you can redistribute it and/or
7 #modify it under the terms of the GNU Library General Public
8 #License as published by the Free Software Foundation; either
9 #version 2 of the License, or (at your option) any later version.
11 #This library is distributed in the hope that it will be useful,
12 #but WITHOUT ANY WARRANTY; without even the implied warranty of
13 #MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 #Library General Public License for more details.
16 #You should have received a copy of the GNU Library General Public License
17 #along with this library; see the file COPYING.LIB. If not, write to
18 #the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
19 #Boston, MA 02110-1301, USA.
21 from __future__ import with_statement
27 from subprocess import Popen, PIPE, STDOUT
28 from optparse import OptionParser
32 def __init__(self, name):
33 self._log = logging.getLogger(name)
34 self.debug = self._log.debug
35 self.warn = self._log.warn
36 self.error = self._log.error
37 self.exception = self._log.exception
38 self.info = self._log.info
42 """ Option manager. It parses and checks script's parameters, sets an internal variable. """
44 def __init__(self, args):
45 Log.__init__(self, "Options")
47 opt = OptionParser("%prog [options] PathToSearch.\nTry -h or --help.")
48 opt.add_option("-j", "--parallel-level", action="store", type="int",
49 dest="parallel_level", default=None,
50 help="Number of parallel processes executing the Qt's tests. Default: cpu count.")
51 opt.add_option("-v", "--verbose", action="store", type="int",
52 dest="verbose", default=2,
53 help="Verbose level (0 - quiet, 1 - errors only, 2 - infos and warnings, 3 - debug information). Default: %default.")
54 opt.add_option("", "--tests-options", action="store", type="string",
55 dest="tests_options", default="",
56 help="Parameters passed to Qt's tests (for example '-eventdelay 123').")
57 opt.add_option("-o", "--output-file", action="store", type="string",
58 dest="output_file", default="/tmp/qtwebkittests.html",
59 help="File where results will be stored. The file will be overwritten. Default: %default.")
60 opt.add_option("-b", "--browser", action="store", dest="browser",
62 help="Browser in which results will be opened. Default %default.")
63 opt.add_option("", "--do-not-open-results", action="store_false",
64 dest="open_results", default=True,
65 help="The results shouldn't pop-up in a browser automatically")
66 opt.add_option("-d", "--developer-mode", action="store_true",
67 dest="developer", default=False,
68 help="Special mode for debugging. In general it simulates human behavior, running all autotests. In the mode everything is executed synchronously, no html output will be generated, no changes or transformation will be applied to stderr or stdout. In this mode options; parallel-level, output-file, browser and do-not-open-results will be ignored.")
69 opt.add_option("-t", "--timeout", action="store", type="int",
70 dest="timeout", default=0,
71 help="Timeout in seconds for each testsuite. Zero value means that there is not timeout. Default: %default.")
73 self._o, self._a = opt.parse_args(args)
74 verbose = self._o.verbose
76 logging.basicConfig(level=logging.CRITICAL,)
78 logging.basicConfig(level=logging.ERROR,)
80 logging.basicConfig(level=logging.INFO,)
82 logging.basicConfig(level=logging.DEBUG,)
84 logging.basicConfig(level=logging.INFO,)
85 log.warn("Bad verbose level, switching to default.")
87 if not os.path.exists(self._a[0]):
88 raise Exception("Given path doesn't exist.")
90 raise IndexError("Only one directory could be provided.")
91 self._o.path = self._a[0]
93 log.error("Bad usage. Please try -h or --help.")
96 log.error("Path '%s' doesn't exist", self._a[0])
99 if not self._o.parallel_level is None:
100 log.warn("Developer mode sets parallel-level option to one.")
101 self._o.parallel_level = 1
102 self._o.open_results = False
104 def __getattr__(self, attr):
105 """ Maps all options properties into this object (remove one level of indirection). """
106 return getattr(self._o, attr)
110 """ Runs one given test.
111 args should contain a tuple with 3 elements;
112 TestSuiteResult containing full file name of an autotest executable.
113 str with options that should be passed to the autotest executable
114 bool if true then the stdout will be buffered and separated from the stderr, if it is false
115 then the stdout and the stderr will be merged together and left unbuffered (the TestSuiteResult output will be None).
116 int time after which the autotest executable would be killed
118 log = logging.getLogger("Exec")
119 test_suite, options, buffered, timeout = args
122 log.info("Running... %s", test_suite.test_file_name())
124 tst = Popen([test_suite.test_file_name()] + options.split(), stdout=PIPE, stderr=None)
126 tst = Popen([test_suite.test_file_name()] + options.split(), stdout=None, stderr=STDOUT)
128 from threading import Timer
129 log.debug("Setting timeout timer %i sec on %s (process %s)", timeout, test_suite.test_file_name(), tst.pid)
130 def process_killer():
134 except AttributeError:
135 # Workaround for python version < 2.6 it can be removed as soon as we drop support for python2.5
138 PROCESS_TERMINATE = 1
139 handle = ctypes.windll.kernel32.OpenProcess(PROCESS_TERMINATE, False, tst.pid)
140 ctypes.windll.kernel32.TerminateProcess(handle, -1)
141 ctypes.windll.kernel32.CloseHandle(handle)
142 except AttributeError:
143 # windll is not accessible so we are on *nix like system
145 os.kill(tst.pid, signal.SIGTERM)
146 log.error("Timeout, process '%s' (%i) was terminated", test_suite.test_file_name(), tst.pid)
148 # the process was finished before got killed
150 timer = Timer(timeout, process_killer)
153 log.exception("Can't open an autotest file: '%s'. Skipping the test...", e.filename)
155 test_suite.set_output(tst.communicate()[0]) # takes stdout only, in developer mode it would be None.
156 log.info("Finished %s", test_suite.test_file_name())
160 class TestSuiteResult(object):
161 """ Keeps information about a test. """
165 self._test_file_name = None
167 def set_output(self, xml):
169 self._output = xml.strip()
174 def set_test_file_name(self, file_name):
175 self._test_file_name = file_name
177 def test_file_name(self):
178 return self._test_file_name
182 """ The main script. All real work is done in run() method. """
184 def __init__(self, options):
185 Log.__init__(self, "Main")
186 self._options = options
187 if options.parallel_level > 1 or options.parallel_level is None:
189 from multiprocessing import Pool
191 self.warn("Import Error: the multiprocessing module couldn't be loaded (may be lack of python-multiprocessing package?). The Qt autotests will be executed one by one.")
192 options.parallel_level = 1
193 if options.parallel_level == 1:
196 """ A hack, created to avoid problems with multiprocessing module, this class is single thread replacement for the multiprocessing.Pool class. """
197 def __init__(self, processes):
200 def imap_unordered(self, func, files):
201 return map(func, files)
203 def map(self, func, files):
204 return map(func, files)
209 """ Find && execute && publish results of all test. "All in one" function. """
210 # This is needed for Qt finding our QML modules. The current code makes our
211 # two existing API tests (WK1 API and WK2 UI process API) work correctly.
212 qml_import_path = self._options.path + "../../../../imports"
213 qml_import_path += ":" + self._options.path + "../../../../../../imports"
214 os.putenv("QML_IMPORT_PATH", qml_import_path)
215 path = os.getenv("PATH")
216 path += ":" + self._options.path + "../../../../../../bin"
217 os.putenv("PATH", path)
218 self.debug("Searching executables...")
219 tests_executables = self.find_tests_paths(self._options.path)
220 self.debug("Found: %s", len(tests_executables))
221 self.debug("Executing tests...")
222 results = self.run_tests(tests_executables)
223 if not self._options.developer:
224 self.debug("Transforming...")
225 transformed_results = self.transform(results)
226 self.debug("Publishing...")
227 self.announce_results(transformed_results)
229 def find_tests_paths(self, path):
230 """ Finds all tests executables inside the given path. """
232 for root, dirs, files in os.walk(path):
233 # Check only for a file that name starts from 'tst_' and that we can execute.
234 filtered_path = filter(lambda w: w.startswith('tst_') and os.access(os.path.join(root, w), os.X_OK), files)
235 filtered_path = map(lambda w: os.path.join(root, w), filtered_path)
236 for file_name in filtered_path:
237 r = TestSuiteResult()
238 r.set_test_file_name(file_name)
239 executables.append(r)
242 def run_tests(self, files):
243 """ Executes given files by using a pool of workers. """
244 workers = self._Pool(processes=self._options.parallel_level)
245 # to each file add options.
246 self.debug("Using %s the workers pool, number of workers %i", repr(workers), self._options.parallel_level)
247 package = map(lambda w: [w, self._options.tests_options, not self._options.developer, self._options.timeout], files)
248 self.debug("Generated packages for workers: %s", repr(package))
249 results = workers.map(run_test, package) # Collects results.
252 def transform(self, results):
253 """ Transforms list of the results to specialized versions. """
254 stdout = self.convert_to_stdout(results)
255 html = self.convert_to_html(results)
256 return {"stdout": stdout, "html": html}
258 def announce_results(self, results):
259 """ Shows the results. """
260 self.announce_results_stdout(results['stdout'])
261 self.announce_results_html(results['html'])
263 def announce_results_stdout(self, results):
264 """ Show the results by printing to the stdout."""
267 def announce_results_html(self, results):
268 """ Shows the result by creating a html file and calling a web browser to render it. """
269 with file(self._options.output_file, 'w') as f:
271 if self._options.open_results:
272 Popen(self._options.browser + " " + self._options.output_file, stdout=None, stderr=None, shell=True)
274 def convert_to_stdout(self, results):
275 """ Converts results, that they could be nicely presented in the stdout. """
276 # Join all results into one piece.
277 txt = "\n\n".join(map(lambda w: w.output(), results))
278 # Find total count of failed, skipped and passed tests.
279 totals = re.findall(r"([0-9]+) passed, ([0-9]+) failed, ([0-9]+) skipped", txt)
280 totals = reduce(lambda x, y: (int(x[0]) + int(y[0]), int(x[1]) + int(y[1]), int(x[2]) + int(y[2])), totals)
281 totals = map(str, totals)
282 totals = totals[0] + " passed, " + totals[1] + " failed, " + totals[2] + " skipped"
284 txt += '\n\n\n' + '*' * 70
285 txt += "\n**" + ("TOTALS: " + totals).center(66) + '**'
286 txt += '\n' + '*' * 70 + '\n'
289 def convert_to_html(self, results):
290 """ Converts results, that they could showed as a html page. """
291 # Join results into one piece.
292 txt = "\n\n".join(map(lambda w: w.output(), results))
293 txt = txt.replace('&', '&').replace('<', "<").replace('>', ">")
294 # Add a color and a style.
295 txt = re.sub(r"([* ]+(Finished)[ a-z_A-Z0-9]+[*]+)",
298 txt = re.sub(r"([*]+[ a-z_A-Z0-9]+[*]+)",
299 lambda w: "<case class='good'><br><br><b>" + w.group(0) + r"</b></case>",
301 txt = re.sub(r"(Config: Using QTest library)((.)+)",
302 lambda w: "\n<case class='good'><br><i>" + w.group(0) + r"</i> ",
304 txt = re.sub(r"\n(PASS)((.)+)",
305 lambda w: "</case>\n<case class='good'><br><status class='pass'>" + w.group(1) + r"</status>" + w.group(2),
307 txt = re.sub(r"\n(FAIL!)((.)+)",
308 lambda w: "</case>\n<case class='bad'><br><status class='fail'>" + w.group(1) + r"</status>" + w.group(2),
310 txt = re.sub(r"\n(XPASS)((.)+)",
311 lambda w: "</case>\n<case class='bad'><br><status class='xpass'>" + w.group(1) + r"</status>" + w.group(2),
313 txt = re.sub(r"\n(XFAIL)((.)+)",
314 lambda w: "</case>\n<case class='good'><br><status class='xfail'>" + w.group(1) + r"</status>" + w.group(2),
316 txt = re.sub(r"\n(SKIP)((.)+)",
317 lambda w: "</case>\n<case class='good'><br><status class='xfail'>" + w.group(1) + r"</status>" + w.group(2),
319 txt = re.sub(r"\n(QWARN)((.)+)",
320 lambda w: "</case>\n<case class='bad'><br><status class='warn'>" + w.group(1) + r"</status>" + w.group(2),
322 txt = re.sub(r"\n(RESULT)((.)+)",
323 lambda w: "</case>\n<case class='good'><br><status class='benchmark'>" + w.group(1) + r"</status>" + w.group(2),
325 txt = re.sub(r"\n(QFATAL)((.)+)",
326 lambda w: "</case>\n<case class='bad'><br><status class='crash'>" + w.group(1) + r"</status>" + w.group(2),
328 txt = re.sub(r"\n(Totals:)([0-9', a-z]*)",
329 lambda w: "</case>\n<case class='good'><br><b>" + w.group(1) + r"</b>" + w.group(2) + "</case>",
331 # Find total count of failed, skipped and passed tests.
332 totals = re.findall(r"([0-9]+) passed, ([0-9]+) failed, ([0-9]+) skipped", txt)
333 totals = reduce(lambda x, y: (int(x[0]) + int(y[0]), int(x[1]) + int(y[1]), int(x[2]) + int(y[2])), totals)
334 totals = map(str, totals)
335 totals = totals[0] + " passed, " + totals[1] + " failed, " + totals[2] + " skipped."
336 # Create a header of the html source.
342 // Try to find the right styleSheet (this document could be embedded in an other html doc)
343 for (i = document.styleSheets.length - 1; i >= 0; --i) {
344 if (document.styleSheets[i].cssRules[0].selectorText == "case.good") {
345 resultStyleSheet = i;
349 // The styleSheet hasn't been found, but it should be the last one.
350 resultStyleSheet = document.styleSheets.length - 1;
354 document.styleSheets[resultStyleSheet].cssRules[0].style.display='none';
358 document.styleSheets[resultStyleSheet].cssRules[0].style.display='';
362 <style type="text/css">
363 case.good {color:black}
364 case.bad {color:black}
365 status.pass {color:green}
366 status.crash {color:red}
367 status.fail {color:red}
368 status.xpass {color:663300}
369 status.xfail {color:004500}
370 status.benchmark {color:000088}
371 status.warn {color:orange}
372 status.crash {color:red; text-decoration:blink; background-color:black}
375 <body onload="init()">
377 <h1>Qt's autotests results</h1>%(totals)s<br>
380 <input type="button" value="Show failures only" onclick="hide()"/>
382 <input type="button" value="Show all" onclick="show()"/>
388 </html>""" % {"totals": totals, "results": txt}
392 if __name__ == '__main__':
393 options = Options(sys.argv[1:])