Tools/gtk/run-api-tests

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (C) 2011, 2012 Igalia S.L.
   4 #
   5 # This library is free software; you can redistribute it and/or
   6 # modify it under the terms of the GNU Library General Public
   7 # License as published by the Free Software Foundation; either
   8 # version 2 of the License, or (at your option) any later version.
   9 #
  10 # This library is distributed in the hope that it will be useful,
  11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13 # Library General Public License for more details.
  14 #
  15 # You should have received a copy of the GNU Library General Public License
  16 # along with this library; see the file COPYING.LIB.  If not, write to
  17 # the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
  18 # Boston, MA 02110-1301, USA.
  19
  20 import common
  21 import subprocess
  22 import os
  23 import sys
  24 import optparse
  25 import re
  26 from signal import alarm, signal, SIGALRM, SIGKILL
  27 from gi.repository import Gio, GLib
  28
  29 class SkippedTest:
  30     ENTIRE_SUITE = None
  31
  32     def __init__(self, test, test_case, reason, bug=None):
  33         self.test = test
  34         self.test_case = test_case
  35         self.reason = reason
  36         self.bug = bug
  37
  38     def __str__(self):
  39         skipped_test_str = "%s" % self.test
  40
  41         if not(self.skip_entire_suite()):
  42             skipped_test_str += " [%s]" % self.test_case
  43
  44         skipped_test_str += ": %s " % self.reason
  45         if self.bug is not None:
  46             skipped_test_str += "(https://bugs.webkit.org/show_bug.cgi?id=%d)" % self.bug
  47         return skipped_test_str
  48
  49     def skip_entire_suite(self):
  50         return self.test_case == SkippedTest.ENTIRE_SUITE
  51
  52 class TestTimeout(Exception):
  53     pass
  54
  55 class TestRunner:
  56     TEST_DIRS = [ "unittests", "WebKit2APITests", "TestWebKitAPI" ]
  57
  58     SKIPPED = [
  59         SkippedTest("unittests/testdownload", "/webkit/download/not-found", "Test fails in GTK Linux 64-bit Release bot", 82329),
  60         SkippedTest("unittests/testwebview", "/webkit/webview/icon-uri", "Test times out in GTK Linux 64-bit Release bot", 82328),
  61         SkippedTest("unittests/testwebresource", "/webkit/webresource/sub_resource_loading", "Test fails in GTK Linux 64-bit Release bot", 82330),
  62         SkippedTest("unittests/testwebinspector", "/webkit/webinspector/close-and-inspect", "Test is flaky in GTK Linux 32-bit Release bot", 82869),
  63         SkippedTest("WebKit2APITests/TestWebKitWebView", "/webkit2/WebKitWebView/mouse-target", "Test is flaky in GTK Linux 32-bit Release bot", 82866),
  64         SkippedTest("WebKit2APITests/TestResources", "/webkit2/WebKitWebView/resources", "Test is flaky in GTK Linux 32-bit Release bot", 82868),
  65         SkippedTest("WebKit2APITests/TestWebKitFindController", "/webkit2/WebKitFindController/next", "Test fails ", 91083),
  66         SkippedTest("WebKit2APITests/TestWebKitFindController", "/webkit2/WebKitFindController/previous", "Test fails", 91083),
  67         SkippedTest("WebKit2APITests/TestWebKitFindController", "/webkit2/WebKitFindController/hide", "Test  always fails in Xvfb", 89810),
  68         SkippedTest("TestWebKitAPI/TestWebKit2", "WebKit2.WKConnection", "Tests fail and time out out", 84959),
  69         SkippedTest("TestWebKitAPI/TestWebKit2", "WebKit2.RestoreSessionStateContainingFormData", "Session State is not implemented in GTK+ port", 84960),
  70         SkippedTest("TestWebKitAPI/TestWebKit2", "WebKit2.SpacebarScrolling", "Test fails", 84961),
  71         SkippedTest("TestWebKitAPI/TestWebKit2", "WebKit2.NewFirstVisuallyNonEmptyLayoutForImages", "Test is flaky", 85066),
  72         SkippedTest("TestWebKitAPI/TestWebKit2", "WebKit2.NewFirstVisuallyNonEmptyLayoutFrames", "Test fails", 85037),
  73         SkippedTest("TestWebKitAPI/TestWebKit2", "WebKit2.MouseMoveAfterCrash", "Test is flaky", 85066),
  74         SkippedTest("TestWebKitAPI/TestWebKit2", "WebKit2.CanHandleRequest", "Test fails", 88453),
  75         SkippedTest("TestWebKitAPI/TestWebKit2", "WebKit2.WKPageGetScaleFactorNotZero", "Test fails and times out", 88455),
  76     ]
  77
  78     def __init__(self, options, tests=[]):
  79         self._options = options
  80         self._programs_path = common.build_path("Programs")
  81         self._tests = self._get_tests(tests)
  82         self._skipped_tests = TestRunner.SKIPPED
  83
  84         # These SPI daemons need to be active for the accessibility tests to work.
  85         self._spi_registryd = None
  86         self._spi_bus_launcher = None
  87
  88     def _get_tests(self, tests):
  89         if tests:
  90             return tests
  91
  92         tests = []
  93         for test_dir in self.TEST_DIRS:
  94             absolute_test_dir = os.path.join(self._programs_path, test_dir)
  95             if not os.path.isdir(absolute_test_dir):
  96                 continue
  97             for test_file in os.listdir(absolute_test_dir):
  98                 if not test_file.lower().startswith("test"):
  99                     continue
 100                 test_path = os.path.join(self._programs_path, test_dir, test_file)
 101                 if os.path.isfile(test_path) and os.access(test_path, os.X_OK):
 102                     tests.append(test_path)
 103         return tests
 104
 105     def _lookup_atspi2_binary(self, filename):
 106         exec_prefix = common.pkg_config_file_variable('atspi-2', 'exec_prefix')
 107         if not exec_prefix:
 108             return None
 109         for path in ['libexec', 'lib/at-spi2-core', 'lib32/at-spi2-core', 'lib64/at-spi2-core']:
 110             filepath = os.path.join(exec_prefix, path, filename)
 111             if os.path.isfile(filepath):
 112                 return filepath
 113
 114         return None
 115
 116     def _start_accessibility_daemons(self):
 117         return False
 118         spi_bus_launcher_path = self._lookup_atspi2_binary('at-spi-bus-launcher')
 119         spi_registryd_path = self._lookup_atspi2_binary('at-spi2-registryd')
 120         if not spi_bus_launcher_path or not spi_registryd_path:
 121             return False
 122
 123         try:
 124             self._ally_bus_launcher = subprocess.Popen([spi_bus_launcher_path], env=self._test_env)
 125         except:
 126             sys.stderr.write("Failed to launch the accessibility bus\n")
 127             sys.stderr.flush()
 128             return False
 129
 130         # We need to wait until the SPI bus is launched before trying to start the SPI
 131         # registry, so we spin a main loop until the bus name appears on DBus.
 132         loop = GLib.MainLoop()
 133         Gio.bus_watch_name(Gio.BusType.SESSION, 'org.a11y.Bus', Gio.BusNameWatcherFlags.NONE,
 134                            lambda *args: loop.quit(), None)
 135         loop.run()
 136
 137         try:
 138             self._spi_registryd = subprocess.Popen([spi_registryd_path], env=self._test_env)
 139         except:
 140             sys.stderr.write("Failed to launch the accessibility registry\n")
 141             sys.stderr.flush()
 142             return False
 143
 144         return True
 145
 146     def _setup_testing_environment(self):
 147         self._test_env = os.environ
 148         self._test_env["DISPLAY"] = self._options.display
 149         self._test_env["WEBKIT_INSPECTOR_PATH"] = os.path.abspath(os.path.join(self._programs_path, 'resources', 'inspector'))
 150         self._test_env['GSETTINGS_BACKEND'] = 'memory'
 151         self._test_env["TEST_WEBKIT_API_WEBKIT2_RESOURCES_PATH"] = common.top_level_path("Tools", "TestWebKitAPI", "Tests", "WebKit2")
 152         self._test_env["TEST_WEBKIT_API_WEBKIT2_INJECTED_BUNDLE_PATH"] = common.build_path("Libraries")
 153         self._test_env["WEBKIT_EXEC_PATH"] = self._programs_path
 154
 155         try:
 156             self._xvfb = subprocess.Popen(["Xvfb", self._options.display, "-screen", "0", "800x600x24", "-nolisten", "tcp"],
 157                                           stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 158         except Exception as e:
 159             sys.stderr.write("Failed to run Xvfb: %s\n" % e)
 160             sys.stderr.flush()
 161             return False
 162
 163         # If we cannot start the accessibility daemons, we can just skip the accessibility tests.
 164         if not self._start_accessibility_daemons():
 165             print "Could not start accessibility bus, so skipping TestWebKitAccessibility"
 166             self._skipped_tests.append(SkippedTest("WebKit2APITests/TestWebKitAccessibility", SkippedTest.ENTIRE_SUITE, "Could not start accessibility bus"))
 167         return True
 168
 169     def _tear_down_testing_environment(self):
 170         if self._spi_registryd:
 171             self._spi_registryd.terminate()
 172         if self._spi_bus_launcher:
 173             self._spi_bus_launcher.terminate()
 174         self._xvfb.terminate()
 175
 176     def _test_cases_to_skip(self, test_program):
 177         if self._options.skipped_action != 'skip':
 178             return []
 179
 180         test_cases = []
 181         for skipped in self._skipped_tests:
 182             if test_program.endswith(skipped.test) and not skipped.skip_entire_suite():
 183                 test_cases.append(skipped.test_case)
 184         return test_cases
 185
 186     def _should_run_test_program(self, test_program):
 187         # This is not affected by the command-line arguments, since programs are skipped for
 188         # problems in the harness, such as failing to start the accessibility bus.
 189         for skipped in self._skipped_tests:
 190             if test_program.endswith(skipped.test) and skipped.skip_entire_suite():
 191                 return False
 192         return True
 193
 194     def _get_child_pid_from_test_output(self, output):
 195         if not output:
 196             return -1
 197         match = re.search(r'\(pid=(?P<child_pid>[0-9]+)\)', output)
 198         if not match:
 199             return -1
 200         return int(match.group('child_pid'))
 201
 202     def _kill_process(self, pid):
 203         try:
 204             os.kill(pid, SIGKILL)
 205         except OSError:
 206             # Process already died.
 207             pass
 208
 209     def _run_test_command(self, command, timeout=-1):
 210         def alarm_handler(signum, frame):
 211             raise TestTimeout
 212
 213         p = subprocess.Popen(command, stdout=subprocess.PIPE, env=self._test_env)
 214         if timeout > 0:
 215             signal(SIGALRM, alarm_handler)
 216             alarm(timeout)
 217
 218         stdout = ""
 219         try:
 220             stdout = p.communicate()[0]
 221             if timeout > 0:
 222                 alarm(0)
 223             sys.stdout.write(stdout)
 224             sys.stdout.flush()
 225         except TestTimeout:
 226             self._kill_process(p.pid)
 227             child_pid = self._get_child_pid_from_test_output(stdout)
 228             if child_pid > 0:
 229                 self._kill_process(child_pid)
 230             raise
 231
 232         return not p.returncode
 233
 234     def _run_test_glib(self, test_program):
 235         tester_command = ['gtester']
 236         if self._options.verbose:
 237             tester_command.append('--verbose')
 238         for test_case in self._test_cases_to_skip(test_program):
 239             tester_command.extend(['-s', test_case])
 240         tester_command.append(test_program)
 241
 242         return self._run_test_command(tester_command, self._options.timeout)
 243
 244     def _run_test_google(self, test_program):
 245         tester_command = [test_program]
 246         skipped_tests_cases = self._test_cases_to_skip(test_program)
 247         if skipped_tests_cases:
 248             tester_command.append("--gtest_filter=-%s" % ":".join(skipped_tests_cases))
 249
 250         return self._run_test_command(tester_command, self._options.timeout)
 251
 252     def _run_test(self, test_program):
 253         if "unittests" in test_program or "WebKit2APITests" in test_program:
 254             return self._run_test_glib(test_program)
 255
 256         if "TestWebKitAPI" in test_program:
 257             return self._run_test_google(test_program)
 258
 259         return False
 260
 261     def run_tests(self):
 262         if not self._tests:
 263             sys.stderr.write("ERROR: tests not found in %s.\n" % (self._programs_path))
 264             sys.stderr.flush()
 265             return 1
 266
 267         if not self._setup_testing_environment():
 268             return 1
 269
 270         # Remove skipped tests now instead of when we find them, because
 271         # some tests might be skipped while setting up the test environment.
 272         self._tests = [test for test in self._tests if self._should_run_test_program(test)]
 273
 274         failed_tests = []
 275         timed_out_tests = []
 276         try:
 277             for test in self._tests:
 278                 success = True
 279                 try:
 280                     success = self._run_test(test)
 281                 except TestTimeout:
 282                     sys.stdout.write("TEST: %s: TIMEOUT\n" % test)
 283                     sys.stdout.flush()
 284                     timed_out_tests.append(test)
 285
 286                 if not success:
 287                     failed_tests.append(test)
 288         finally:
 289             self._tear_down_testing_environment()
 290
 291         if failed_tests:
 292             names = [test.replace(self._programs_path, '', 1) for test in failed_tests]
 293             sys.stdout.write("Tests failed: %s\n" % ", ".join(names))
 294             sys.stdout.flush()
 295
 296         if timed_out_tests:
 297             names = [test.replace(self._programs_path, '', 1) for test in timed_out_tests]
 298             sys.stdout.write("Tests that timed out: %s\n" % ", ".join(names))
 299             sys.stdout.flush()
 300
 301         if self._skipped_tests and self._options.skipped_action == 'skip':
 302             sys.stdout.write("Tests skipped:\n%s\n" % "\n".join([str(skipped) for skipped in self._skipped_tests]))
 303             sys.stdout.flush()
 304
 305         return len(failed_tests)
 306
 307 if __name__ == "__main__":
 308     option_parser = optparse.OptionParser(usage='usage: %prog [options] [test...]')
 309     option_parser.add_option('-r', '--release',
 310                              action='store_true', dest='release',
 311                              help='Run in Release')
 312     option_parser.add_option('-d', '--debug',
 313                              action='store_true', dest='debug',
 314                              help='Run in Debug')
 315     option_parser.add_option('-v', '--verbose',
 316                              action='store_true', dest='verbose',
 317                              help='Run gtester in verbose mode')
 318     option_parser.add_option('--display', action='store', dest='display', default=':55',
 319                              help='Display to run Xvfb')
 320     option_parser.add_option('--skipped', action='store', dest='skipped_action',
 321                              choices=['skip', 'ignore', 'only'], default='skip',
 322                              metavar='skip|ignore|only',
 323                              help='Specifies how to treat the skipped tests')
 324     option_parser.add_option('-t', '--timeout',
 325                              action='store', type='int', dest='timeout', default=10,
 326                              help='Time in seconds until a test times out')
 327     options, args = option_parser.parse_args()
 328
 329     sys.exit(TestRunner(options, args).run_tests())