Tools/Scripts/webkitpy/layout_tests/run_webkit_tests.py

   1 #!/usr/bin/env python
   2 # Copyright (C) 2010 Google Inc. All rights reserved.
   3 # Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Szeged
   4 # Copyright (C) 2011 Apple Inc. All rights reserved.
   5 #
   6 # Redistribution and use in source and binary forms, with or without
   7 # modification, are permitted provided that the following conditions are
   8 # met:
   9 #
  10 #     * Redistributions of source code must retain the above copyright
  11 # notice, this list of conditions and the following disclaimer.
  12 #     * Redistributions in binary form must reproduce the above
  13 # copyright notice, this list of conditions and the following disclaimer
  14 # in the documentation and/or other materials provided with the
  15 # distribution.
  16 #     * Neither the name of Google Inc. nor the names of its
  17 # contributors may be used to endorse or promote products derived from
  18 # this software without specific prior written permission.
  19 #
  20 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  21 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  22 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  23 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  24 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  25 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  26 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  27 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  28 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  29 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  30 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  31
  32 import errno
  33 import logging
  34 import optparse
  35 import os
  36 import signal
  37 import sys
  38
  39 from webkitpy.common.host import Host
  40 from webkitpy.layout_tests.controllers.manager import Manager, WorkerException
  41 from webkitpy.layout_tests.views import printing
  42
  43
  44 _log = logging.getLogger(__name__)
  45
  46
  47 def run(port, options, args, regular_output=sys.stderr, buildbot_output=sys.stdout):
  48     warnings = _set_up_derived_options(port, options)
  49
  50     printer = printing.Printer(port, options, regular_output, buildbot_output, configure_logging=True)
  51
  52     for warning in warnings:
  53         _log.warning(warning)
  54
  55     if options.help_printing:
  56         printer.help_printing()
  57         printer.cleanup()
  58         return 0
  59
  60     # We wrap any parts of the run that are slow or likely to raise exceptions
  61     # in a try/finally to ensure that we clean up the logging configuration.
  62     unexpected_result_count = -1
  63     try:
  64         manager = Manager(port, options, printer)
  65         manager.print_config()
  66
  67         printer.print_update("Collecting tests ...")
  68         try:
  69             manager.collect_tests(args)
  70         except IOError, e:
  71             if e.errno == errno.ENOENT:
  72                 return -1
  73             raise
  74
  75         if options.lint_test_files:
  76             return manager.lint()
  77
  78         printer.print_update("Checking build ...")
  79         if not port.check_build(manager.needs_servers()):
  80             _log.error("Build check failed")
  81             return -1
  82
  83         printer.print_update("Parsing expectations ...")
  84         manager.parse_expectations()
  85
  86         result_summary = manager.set_up_run()
  87         if result_summary:
  88             unexpected_result_count = manager.run(result_summary)
  89             manager.clean_up_run()
  90             _log.debug("Testing completed, Exit status: %d" % unexpected_result_count)
  91     finally:
  92         printer.cleanup()
  93
  94     return unexpected_result_count
  95
  96
  97 def _set_up_derived_options(port, options):
  98     """Sets the options values that depend on other options values."""
  99     # We return a list of warnings to print after the printer is initialized.
 100     warnings = []
 101
 102     if options.worker_model is None:
 103         options.worker_model = port.default_worker_model()
 104
 105     if options.worker_model == 'inline':
 106         if options.child_processes and int(options.child_processes) > 1:
 107             warnings.append("--worker-model=inline overrides --child-processes")
 108         options.child_processes = "1"
 109     if not options.child_processes:
 110         options.child_processes = os.environ.get("WEBKIT_TEST_CHILD_PROCESSES",
 111                                                  str(port.default_child_processes()))
 112
 113     if not options.configuration:
 114         options.configuration = port.default_configuration()
 115
 116     if options.pixel_tests is None:
 117         options.pixel_tests = True
 118
 119     if not options.time_out_ms:
 120         if options.configuration == "Debug":
 121             options.time_out_ms = str(2 * Manager.DEFAULT_TEST_TIMEOUT_MS)
 122         else:
 123             options.time_out_ms = str(Manager.DEFAULT_TEST_TIMEOUT_MS)
 124
 125     options.slow_time_out_ms = str(5 * int(options.time_out_ms))
 126
 127     if options.additional_platform_directory:
 128         normalized_platform_directories = []
 129         for path in options.additional_platform_directory:
 130             if not port.filesystem.isabs(path):
 131                 warnings.append("--additional-platform-directory=%s is ignored since it is not absolute" % path)
 132                 continue
 133             normalized_platform_directories.append(port.filesystem.normpath(path))
 134         options.additional_platform_directory = normalized_platform_directories
 135
 136     if not options.http and options.force:
 137         warnings.append("--no-http is ignored since --force is also provided")
 138         options.http = True
 139
 140     return warnings
 141
 142
 143 def _compat_shim_callback(option, opt_str, value, parser):
 144     print "Ignoring unsupported option: %s" % opt_str
 145
 146
 147 def _compat_shim_option(option_name, **kwargs):
 148     return optparse.make_option(option_name, action="callback",
 149         callback=_compat_shim_callback,
 150         help="Ignored, for old-run-webkit-tests compat only.", **kwargs)
 151
 152
 153 def parse_args(args=None):
 154     """Provides a default set of command line args.
 155
 156     Returns a tuple of options, args from optparse"""
 157
 158     # FIXME: All of these options should be stored closer to the code which
 159     # FIXME: actually uses them. configuration_options should move
 160     # FIXME: to WebKitPort and be shared across all scripts.
 161     configuration_options = [
 162         optparse.make_option("-t", "--target", dest="configuration",
 163                              help="(DEPRECATED)"),
 164         # FIXME: --help should display which configuration is default.
 165         optparse.make_option('--debug', action='store_const', const='Debug',
 166                              dest="configuration",
 167                              help='Set the configuration to Debug'),
 168         optparse.make_option('--release', action='store_const',
 169                              const='Release', dest="configuration",
 170                              help='Set the configuration to Release'),
 171         # old-run-webkit-tests also accepts -c, --configuration CONFIGURATION.
 172         optparse.make_option("--platform", help="Override port/platform being tested (i.e. chromium-mac)"),
 173         optparse.make_option('--qt', action='store_const', const='qt', dest="platform", help='Alias for --platform=qt'),
 174         optparse.make_option('--gtk', action='store_const', const='gtk', dest="platform", help='Alias for --platform=gtk'),
 175     ]
 176
 177     print_options = printing.print_options()
 178
 179     # FIXME: These options should move onto the ChromiumPort.
 180     chromium_options = [
 181         optparse.make_option("--chromium", action="store_true", default=False,
 182             help="use the Chromium port"),
 183         optparse.make_option("--startup-dialog", action="store_true",
 184             default=False, help="create a dialog on DumpRenderTree startup"),
 185         optparse.make_option("--gp-fault-error-box", action="store_true",
 186             default=False, help="enable Windows GP fault error box"),
 187         optparse.make_option("--js-flags",
 188             type="string", help="JavaScript flags to pass to tests"),
 189         optparse.make_option("--stress-opt", action="store_true",
 190             default=False,
 191             help="Enable additional stress test to JavaScript optimization"),
 192         optparse.make_option("--stress-deopt", action="store_true",
 193             default=False,
 194             help="Enable additional stress test to JavaScript optimization"),
 195         optparse.make_option("--nocheck-sys-deps", action="store_true",
 196             default=False,
 197             help="Don't check the system dependencies (themes)"),
 198         optparse.make_option("--accelerated-video",
 199             action="store_true",
 200             help="Use hardware-accelerated compositing for video"),
 201         optparse.make_option("--no-accelerated-video",
 202             action="store_false",
 203             dest="accelerated_video",
 204             help="Don't use hardware-accelerated compositing for video"),
 205         optparse.make_option("--threaded-compositing",
 206             action="store_true",
 207             help="Use threaded compositing for rendering"),
 208         optparse.make_option("--accelerated-2d-canvas",
 209             action="store_true",
 210             help="Use hardware-accelerated 2D Canvas calls"),
 211         optparse.make_option("--no-accelerated-2d-canvas",
 212             action="store_false",
 213             dest="accelerated_2d_canvas",
 214             help="Don't use hardware-accelerated 2D Canvas calls"),
 215         optparse.make_option("--accelerated-drawing",
 216             action="store_true",
 217             default=False,
 218             help="Use hardware accelerated drawing of composited pages"),
 219         optparse.make_option("--enable-hardware-gpu",
 220             action="store_true",
 221             default=False,
 222             help="Run graphics tests on real GPU hardware vs software"),
 223     ]
 224
 225     webkit_options = [
 226         optparse.make_option("--gc-between-tests", action="store_true", default=False,
 227             help="Force garbage collection between each test"),
 228         optparse.make_option("--complex-text", action="store_true", default=False,
 229             help="Use the complex text code path for all text (Mac OS X and Windows only)"),
 230         optparse.make_option("-l", "--leaks", action="store_true", default=False,
 231             help="Enable leaks checking (Mac OS X only)"),
 232         optparse.make_option("-g", "--guard-malloc", action="store_true", default=False,
 233             help="Enable malloc guard (Mac OS X only)"),
 234         optparse.make_option("--threaded", action="store_true", default=False,
 235             help="Run a concurrent JavaScript thread with each test"),
 236         optparse.make_option("--webkit-test-runner", "-2", action="store_true",
 237             help="Use WebKitTestRunner rather than DumpRenderTree."),
 238         optparse.make_option("--root", action="store",
 239             help="Path to a pre-built root of WebKit (for running tests using a nightly build of WebKit)"),
 240     ]
 241
 242     old_run_webkit_tests_compat = [
 243         # FIXME: Remove this option once the bots don't refer to it.
 244         # results.html is smart enough to figure this out itself.
 245         _compat_shim_option("--use-remote-links-to-tests"),
 246     ]
 247
 248     results_options = [
 249         optparse.make_option("-p", "--pixel-tests", action="store_true",
 250             dest="pixel_tests", help="Enable pixel-to-pixel PNG comparisons"),
 251         optparse.make_option("--no-pixel-tests", action="store_false",
 252             dest="pixel_tests", help="Disable pixel-to-pixel PNG comparisons"),
 253         optparse.make_option("--no-sample-on-timeout", action="store_false",
 254             dest="sample_on_timeout", help="Don't run sample on timeout (Mac OS X only)"),
 255         optparse.make_option("--no-ref-tests", action="store_true",
 256             dest="no_ref_tests", help="Skip all ref tests"),
 257         optparse.make_option("--tolerance",
 258             help="Ignore image differences less than this percentage (some "
 259                 "ports may ignore this option)", type="float"),
 260         optparse.make_option("--results-directory", help="Location of test results"),
 261         optparse.make_option("--build-directory",
 262             help="Path to the directory under which build files are kept (should not include configuration)"),
 263         optparse.make_option("--new-baseline", action="store_true",
 264             default=False, help="Save all generated results as new baselines "
 265                  "into the platform directory, overwriting whatever's "
 266                  "already there."),
 267         optparse.make_option("--reset-results", action="store_true",
 268             default=False, help="Reset any existing baselines to the "
 269                  "generated results"),
 270         optparse.make_option("--no-new-test-results", action="store_false",
 271             dest="new_test_results", default=True,
 272             help="Don't create new baselines when no expected results exist"),
 273         optparse.make_option("--skip-failing-tests", action="store_true",
 274             default=False, help="Skip tests that are expected to fail. "
 275                  "Note: When using this option, you might miss new crashes "
 276                  "in these tests."),
 277         optparse.make_option("--additional-drt-flag", action="append",
 278             default=[], help="Additional command line flag to pass to DumpRenderTree "
 279                  "Specify multiple times to add multiple flags."),
 280         optparse.make_option("--additional-platform-directory", action="append",
 281             default=[], help="Additional directory where to look for test "
 282                  "baselines (will take precendence over platform baselines). "
 283                  "Specify multiple times to add multiple search path entries."),
 284         optparse.make_option("--no-show-results", action="store_false",
 285             default=True, dest="show_results",
 286             help="Don't launch a browser with results after the tests "
 287                  "are done"),
 288         # FIXME: We should have a helper function to do this sort of
 289         # deprectated mapping and automatically log, etc.
 290         optparse.make_option("--noshow-results", action="store_false", dest="show_results", help="Deprecated, same as --no-show-results."),
 291         optparse.make_option("--no-launch-safari", action="store_false", dest="show_results", help="Deprecated, same as --no-show-results."),
 292         optparse.make_option("--full-results-html", action="store_true",
 293             default=False,
 294             help="Show all failures in results.html, rather than only regressions"),
 295         optparse.make_option("--clobber-old-results", action="store_true",
 296             default=False, help="Clobbers test results from previous runs."),
 297         optparse.make_option("--no-record-results", action="store_false",
 298             default=True, dest="record_results",
 299             help="Don't record the results."),
 300         optparse.make_option("--http", action="store_true", dest="http",
 301             default=True, help="Run HTTP and WebSocket tests (default)"),
 302         optparse.make_option("--no-http", action="store_false", dest="http",
 303             help="Don't run HTTP and WebSocket tests"),
 304     ]
 305
 306     test_options = [
 307         optparse.make_option("--build", dest="build",
 308             action="store_true", default=True,
 309             help="Check to ensure the DumpRenderTree build is up-to-date "
 310                  "(default)."),
 311         optparse.make_option("--no-build", dest="build",
 312             action="store_false", help="Don't check to see if the "
 313                                        "DumpRenderTree build is up-to-date."),
 314         optparse.make_option("-n", "--dry-run", action="store_true",
 315             default=False,
 316             help="Do everything but actually run the tests or upload results."),
 317         # old-run-webkit-tests has --valgrind instead of wrapper.
 318         optparse.make_option("--wrapper",
 319             help="wrapper command to insert before invocations of "
 320                  "DumpRenderTree; option is split on whitespace before "
 321                  "running. (Example: --wrapper='valgrind --smc-check=all')"),
 322         # old-run-webkit-tests:
 323         # -i|--ignore-tests               Comma-separated list of directories
 324         #                                 or tests to ignore
 325         optparse.make_option("--test-list", action="append",
 326             help="read list of tests to run from file", metavar="FILE"),
 327         # old-run-webkit-tests uses --skipped==[default|ignore|only]
 328         # instead of --force:
 329         optparse.make_option("--force", action="store_true", default=False,
 330             help="Run all tests, even those marked SKIP in the test list"),
 331         optparse.make_option("--time-out-ms",
 332             help="Set the timeout for each test"),
 333         # old-run-webkit-tests calls --randomize-order --random:
 334         optparse.make_option("--randomize-order", action="store_true",
 335             default=False, help=("Run tests in random order (useful "
 336                                 "for tracking down corruption)")),
 337         optparse.make_option("--run-chunk",
 338             help=("Run a specified chunk (n:l), the nth of len l, "
 339                  "of the layout tests")),
 340         optparse.make_option("--run-part", help=("Run a specified part (n:m), "
 341                   "the nth of m parts, of the layout tests")),
 342         # old-run-webkit-tests calls --batch-size: --nthly n
 343         #   Restart DumpRenderTree every n tests (default: 1000)
 344         optparse.make_option("--batch-size",
 345             help=("Run a the tests in batches (n), after every n tests, "
 346                   "DumpRenderTree is relaunched."), type="int", default=None),
 347         # old-run-webkit-tests calls --run-singly: -1|--singly
 348         # Isolate each test case run (implies --nthly 1 --verbose)
 349         optparse.make_option("--run-singly", action="store_true",
 350             default=False, help="run a separate DumpRenderTree for each test"),
 351         optparse.make_option("--child-processes",
 352             help="Number of DumpRenderTrees to run in parallel."),
 353         # FIXME: Display default number of child processes that will run.
 354         optparse.make_option("--worker-model", action="store",
 355             default=None, help=("controls worker model. Valid values are "
 356                                 "'inline' and 'processes'.")),
 357         optparse.make_option("-f", "--experimental-fully-parallel",
 358             action="store_true",
 359             help="run all tests in parallel"),
 360         optparse.make_option("--no-experimental-fully-parallel",
 361             action="store_false",
 362             dest="experimental_fully_parallel",
 363             help="do not run all tests in parallel"),
 364         optparse.make_option("--exit-after-n-failures", type="int", default=500,
 365             help="Exit after the first N failures instead of running all "
 366             "tests"),
 367         optparse.make_option("--exit-after-n-crashes-or-timeouts", type="int",
 368             default=20, help="Exit after the first N crashes instead of "
 369             "running all tests"),
 370         optparse.make_option("--iterations", type="int", help="Number of times to run the set of tests (e.g. ABCABCABC)"),
 371         optparse.make_option("--repeat-each", type="int", help="Number of times to run each test (e.g. AAABBBCCC)"),
 372         optparse.make_option("--retry-failures", action="store_true",
 373             default=True,
 374             help="Re-try any tests that produce unexpected results (default)"),
 375         optparse.make_option("--no-retry-failures", action="store_false",
 376             dest="retry_failures",
 377             help="Don't re-try any tests that produce unexpected results."),
 378     ]
 379
 380     misc_options = [
 381         optparse.make_option("--lint-test-files", action="store_true",
 382         default=False, help=("Makes sure the test files parse for all "
 383                             "configurations. Does not run any tests.")),
 384     ]
 385
 386     # FIXME: Move these into json_results_generator.py
 387     results_json_options = [
 388         optparse.make_option("--master-name", help="The name of the buildbot master."),
 389         optparse.make_option("--builder-name", default="DUMMY_BUILDER_NAME",
 390             help=("The name of the builder shown on the waterfall running "
 391                   "this script e.g. WebKit.")),
 392         optparse.make_option("--build-name", default="DUMMY_BUILD_NAME",
 393             help=("The name of the builder used in its path, e.g. "
 394                   "webkit-rel.")),
 395         optparse.make_option("--build-number", default="DUMMY_BUILD_NUMBER",
 396             help=("The build number of the builder running this script.")),
 397         optparse.make_option("--test-results-server", default="",
 398             help=("If specified, upload results json files to this appengine "
 399                   "server.")),
 400     ]
 401
 402     option_list = (configuration_options + print_options +
 403                    chromium_options + webkit_options + results_options + test_options +
 404                    misc_options + results_json_options + old_run_webkit_tests_compat)
 405     option_parser = optparse.OptionParser(option_list=option_list)
 406
 407     return option_parser.parse_args(args)
 408
 409
 410 def main():
 411     options, args = parse_args()
 412     host = Host()
 413     host._initialize_scm()
 414     port = host.port_factory.get(options.platform, options)
 415     return run(port, options, args)
 416
 417
 418 if '__main__' == __name__:
 419     try:
 420         sys.exit(main())
 421     except KeyboardInterrupt:
 422         # This mirrors what the shell normally does.
 423         INTERRUPTED_EXIT_STATUS = signal.SIGINT + 128
 424         sys.exit(INTERRUPTED_EXIT_STATUS)
 425     except WorkerException:
 426         # This is a randomly chosen exit code that can be tested against to
 427         # indicate that an unexpected exception occurred.
 428         EXCEPTIONAL_EXIT_STATUS = 254
 429         sys.exit(EXCEPTIONAL_EXIT_STATUS)