src/v8/tools/run-tests.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright 2012 the V8 project authors. All rights reserved.
   4 # Redistribution and use in source and binary forms, with or without
   5 # modification, are permitted provided that the following conditions are
   6 # met:
   7 #
   8 #     * Redistributions of source code must retain the above copyright
   9 #       notice, this list of conditions and the following disclaimer.
  10 #     * Redistributions in binary form must reproduce the above
  11 #       copyright notice, this list of conditions and the following
  12 #       disclaimer in the documentation and/or other materials provided
  13 #       with the distribution.
  14 #     * Neither the name of Google Inc. nor the names of its
  15 #       contributors may be used to endorse or promote products derived
  16 #       from this software without specific prior written permission.
  17 #
  18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  29
  30
  31 import itertools
  32 import multiprocessing
  33 import optparse
  34 import os
  35 from os.path import join
  36 import platform
  37 import shlex
  38 import subprocess
  39 import sys
  40 import time
  41
  42 from testrunner.local import execution
  43 from testrunner.local import progress
  44 from testrunner.local import testsuite
  45 from testrunner.local import utils
  46 from testrunner.local import verbose
  47 from testrunner.network import network_execution
  48 from testrunner.objects import context
  49
  50
  51 ARCH_GUESS = utils.DefaultArch()
  52 DEFAULT_TESTS = ["mjsunit", "cctest", "message", "preparser"]
  53 TIMEOUT_DEFAULT = 60
  54 TIMEOUT_SCALEFACTOR = {"debug"   : 4,
  55                        "release" : 1 }
  56
  57 # Use this to run several variants of the tests.
  58 VARIANT_FLAGS = {
  59     "default": [],
  60     "stress": ["--stress-opt", "--always-opt"],
  61     "nocrankshaft": ["--nocrankshaft"]}
  62
  63 VARIANTS = ["default", "stress", "nocrankshaft"]
  64
  65 MODE_FLAGS = {
  66     "debug"   : ["--nobreak-on-abort", "--nodead-code-elimination",
  67                  "--nofold-constants", "--enable-slow-asserts",
  68                  "--debug-code", "--verify-heap"],
  69     "release" : ["--nobreak-on-abort", "--nodead-code-elimination",
  70                  "--nofold-constants"]}
  71
  72 GC_STRESS_FLAGS = ["--gc-interval=500", "--stress-compaction",
  73                    "--concurrent-recompilation-queue-length=64",
  74                    "--concurrent-recompilation-delay=500",
  75                    "--concurrent-recompilation"]
  76
  77 SUPPORTED_ARCHS = ["android_arm",
  78                    "android_ia32",
  79                    "arm",
  80                    "ia32",
  81                    "mipsel",
  82                    "nacl_ia32",
  83                    "nacl_x64",
  84                    "x64"]
  85 # Double the timeout for these:
  86 SLOW_ARCHS = ["android_arm",
  87               "android_ia32",
  88               "arm",
  89               "mipsel",
  90               "nacl_ia32",
  91               "nacl_x64"]
  92
  93
  94 def BuildOptions():
  95   result = optparse.OptionParser()
  96   result.add_option("--arch",
  97                     help=("The architecture to run tests for, "
  98                           "'auto' or 'native' for auto-detect"),
  99                     default="ia32,x64,arm")
 100   result.add_option("--arch-and-mode",
 101                     help="Architecture and mode in the format 'arch.mode'",
 102                     default=None)
 103   result.add_option("--asan",
 104                     help="Regard test expectations for ASAN",
 105                     default=False, action="store_true")
 106   result.add_option("--buildbot",
 107                     help="Adapt to path structure used on buildbots",
 108                     default=False, action="store_true")
 109   result.add_option("--cat", help="Print the source of the tests",
 110                     default=False, action="store_true")
 111   result.add_option("--flaky-tests",
 112                     help="Regard tests marked as flaky (run|skip|dontcare)",
 113                     default="dontcare")
 114   result.add_option("--slow-tests",
 115                     help="Regard slow tests (run|skip|dontcare)",
 116                     default="dontcare")
 117   result.add_option("--pass-fail-tests",
 118                     help="Regard pass|fail tests (run|skip|dontcare)",
 119                     default="dontcare")
 120   result.add_option("--gc-stress",
 121                     help="Switch on GC stress mode",
 122                     default=False, action="store_true")
 123   result.add_option("--command-prefix",
 124                     help="Prepended to each shell command used to run a test",
 125                     default="")
 126   result.add_option("--download-data", help="Download missing test suite data",
 127                     default=False, action="store_true")
 128   result.add_option("--extra-flags",
 129                     help="Additional flags to pass to each test command",
 130                     default="")
 131   result.add_option("--isolates", help="Whether to test isolates",
 132                     default=False, action="store_true")
 133   result.add_option("-j", help="The number of parallel tasks to run",
 134                     default=0, type="int")
 135   result.add_option("-m", "--mode",
 136                     help="The test modes in which to run (comma-separated)",
 137                     default="release,debug")
 138   result.add_option("--no-i18n", "--noi18n",
 139                     help="Skip internationalization tests",
 140                     default=False, action="store_true")
 141   result.add_option("--no-network", "--nonetwork",
 142                     help="Don't distribute tests on the network",
 143                     default=(utils.GuessOS() != "linux"),
 144                     dest="no_network", action="store_true")
 145   result.add_option("--no-presubmit", "--nopresubmit",
 146                     help='Skip presubmit checks',
 147                     default=False, dest="no_presubmit", action="store_true")
 148   result.add_option("--no-stress", "--nostress",
 149                     help="Don't run crankshaft --always-opt --stress-op test",
 150                     default=False, dest="no_stress", action="store_true")
 151   result.add_option("--no-variants", "--novariants",
 152                     help="Don't run any testing variants",
 153                     default=False, dest="no_variants", action="store_true")
 154   result.add_option("--variants",
 155                     help="Comma-separated list of testing variants")
 156   result.add_option("--outdir", help="Base directory with compile output",
 157                     default="out")
 158   result.add_option("-p", "--progress",
 159                     help=("The style of progress indicator"
 160                           " (verbose, dots, color, mono)"),
 161                     choices=progress.PROGRESS_INDICATORS.keys(), default="mono")
 162   result.add_option("--quickcheck", default=False, action="store_true",
 163                     help=("Quick check mode (skip slow/flaky tests)"))
 164   result.add_option("--report", help="Print a summary of the tests to be run",
 165                     default=False, action="store_true")
 166   result.add_option("--shard-count",
 167                     help="Split testsuites into this number of shards",
 168                     default=1, type="int")
 169   result.add_option("--shard-run",
 170                     help="Run this shard from the split up tests.",
 171                     default=1, type="int")
 172   result.add_option("--shell", help="DEPRECATED! use --shell-dir", default="")
 173   result.add_option("--shell-dir", help="Directory containing executables",
 174                     default="")
 175   result.add_option("--stress-only",
 176                     help="Only run tests with --always-opt --stress-opt",
 177                     default=False, action="store_true")
 178   result.add_option("--time", help="Print timing information after running",
 179                     default=False, action="store_true")
 180   result.add_option("-t", "--timeout", help="Timeout in seconds",
 181                     default= -1, type="int")
 182   result.add_option("-v", "--verbose", help="Verbose output",
 183                     default=False, action="store_true")
 184   result.add_option("--valgrind", help="Run tests through valgrind",
 185                     default=False, action="store_true")
 186   result.add_option("--warn-unused", help="Report unused rules",
 187                     default=False, action="store_true")
 188   result.add_option("--junitout", help="File name of the JUnit output")
 189   result.add_option("--junittestsuite",
 190                     help="The testsuite name in the JUnit output file",
 191                     default="v8tests")
 192   return result
 193
 194
 195 def ProcessOptions(options):
 196   global VARIANT_FLAGS
 197   global VARIANTS
 198
 199   # Architecture and mode related stuff.
 200   if options.arch_and_mode:
 201     options.arch_and_mode = [arch_and_mode.split(".")
 202         for arch_and_mode in options.arch_and_mode.split(",")]
 203     options.arch = ",".join([tokens[0] for tokens in options.arch_and_mode])
 204     options.mode = ",".join([tokens[1] for tokens in options.arch_and_mode])
 205   options.mode = options.mode.split(",")
 206   for mode in options.mode:
 207     if not mode.lower() in ["debug", "release", "optdebug"]:
 208       print "Unknown mode %s" % mode
 209       return False
 210   if options.arch in ["auto", "native"]:
 211     options.arch = ARCH_GUESS
 212   options.arch = options.arch.split(",")
 213   for arch in options.arch:
 214     if not arch in SUPPORTED_ARCHS:
 215       print "Unknown architecture %s" % arch
 216       return False
 217
 218   # Store the final configuration in arch_and_mode list. Don't overwrite
 219   # predefined arch_and_mode since it is more expressive than arch and mode.
 220   if not options.arch_and_mode:
 221     options.arch_and_mode = itertools.product(options.arch, options.mode)
 222
 223   # Special processing of other options, sorted alphabetically.
 224
 225   if options.buildbot:
 226     # Buildbots run presubmit tests as a separate step.
 227     options.no_presubmit = True
 228     options.no_network = True
 229   if options.command_prefix:
 230     print("Specifying --command-prefix disables network distribution, "
 231           "running tests locally.")
 232     options.no_network = True
 233   options.command_prefix = shlex.split(options.command_prefix)
 234   options.extra_flags = shlex.split(options.extra_flags)
 235
 236   if options.gc_stress:
 237     options.extra_flags += GC_STRESS_FLAGS
 238
 239   if options.j == 0:
 240     options.j = multiprocessing.cpu_count()
 241
 242   def excl(*args):
 243     """Returns true if zero or one of multiple arguments are true."""
 244     return reduce(lambda x, y: x + y, args) <= 1
 245
 246   if not excl(options.no_stress, options.stress_only, options.no_variants,
 247               bool(options.variants), options.quickcheck):
 248     print("Use only one of --no-stress, --stress-only, --no-variants, "
 249           "--variants, or --quickcheck.")
 250     return False
 251   if options.no_stress:
 252     VARIANTS = ["default", "nocrankshaft"]
 253   if options.no_variants:
 254     VARIANTS = ["default"]
 255   if options.stress_only:
 256     VARIANTS = ["stress"]
 257   if options.variants:
 258     VARIANTS = options.variants.split(",")
 259     if not set(VARIANTS).issubset(VARIANT_FLAGS.keys()):
 260       print "All variants must be in %s" % str(VARIANT_FLAGS.keys())
 261       return False
 262   if options.quickcheck:
 263     VARIANTS = ["default", "stress"]
 264     options.flaky_tests = "skip"
 265     options.slow_tests = "skip"
 266     options.pass_fail_tests = "skip"
 267
 268   if not options.shell_dir:
 269     if options.shell:
 270       print "Warning: --shell is deprecated, use --shell-dir instead."
 271       options.shell_dir = os.path.dirname(options.shell)
 272   if options.valgrind:
 273     run_valgrind = os.path.join("tools", "run-valgrind.py")
 274     # This is OK for distributed running, so we don't need to set no_network.
 275     options.command_prefix = (["python", "-u", run_valgrind] +
 276                               options.command_prefix)
 277   def CheckTestMode(name, option):
 278     if not option in ["run", "skip", "dontcare"]:
 279       print "Unknown %s mode %s" % (name, option)
 280       return False
 281     return True
 282   if not CheckTestMode("flaky test", options.flaky_tests):
 283     return False
 284   if not CheckTestMode("slow test", options.slow_tests):
 285     return False
 286   if not CheckTestMode("pass|fail test", options.pass_fail_tests):
 287     return False
 288   if not options.no_i18n:
 289     DEFAULT_TESTS.append("intl")
 290   return True
 291
 292
 293 def ShardTests(tests, shard_count, shard_run):
 294   if shard_count < 2:
 295     return tests
 296   if shard_run < 1 or shard_run > shard_count:
 297     print "shard-run not a valid number, should be in [1:shard-count]"
 298     print "defaulting back to running all tests"
 299     return tests
 300   count = 0
 301   shard = []
 302   for test in tests:
 303     if count % shard_count == shard_run - 1:
 304       shard.append(test)
 305     count += 1
 306   return shard
 307
 308
 309 def Main():
 310   parser = BuildOptions()
 311   (options, args) = parser.parse_args()
 312   if not ProcessOptions(options):
 313     parser.print_help()
 314     return 1
 315
 316   exit_code = 0
 317   workspace = os.path.abspath(join(os.path.dirname(sys.argv[0]), ".."))
 318   if not options.no_presubmit:
 319     print ">>> running presubmit tests"
 320     code = subprocess.call(
 321         [sys.executable, join(workspace, "tools", "presubmit.py")])
 322     exit_code = code
 323
 324   suite_paths = utils.GetSuitePaths(join(workspace, "test"))
 325
 326   if len(args) == 0:
 327     suite_paths = [ s for s in DEFAULT_TESTS if s in suite_paths ]
 328   else:
 329     args_suites = set()
 330     for arg in args:
 331       suite = arg.split(os.path.sep)[0]
 332       if not suite in args_suites:
 333         args_suites.add(suite)
 334     suite_paths = [ s for s in args_suites if s in suite_paths ]
 335
 336   suites = []
 337   for root in suite_paths:
 338     suite = testsuite.TestSuite.LoadTestSuite(
 339         os.path.join(workspace, "test", root))
 340     if suite:
 341       suites.append(suite)
 342
 343   if options.download_data:
 344     for s in suites:
 345       s.DownloadData()
 346
 347   for (arch, mode) in options.arch_and_mode:
 348     try:
 349       code = Execute(arch, mode, args, options, suites, workspace)
 350     except KeyboardInterrupt:
 351       return 2
 352     exit_code = exit_code or code
 353   return exit_code
 354
 355
 356 def Execute(arch, mode, args, options, suites, workspace):
 357   print(">>> Running tests for %s.%s" % (arch, mode))
 358
 359   shell_dir = options.shell_dir
 360   if not shell_dir:
 361     if options.buildbot:
 362       shell_dir = os.path.join(workspace, options.outdir, mode)
 363       mode = mode.lower()
 364     else:
 365       shell_dir = os.path.join(workspace, options.outdir,
 366                                "%s.%s" % (arch, mode))
 367   shell_dir = os.path.relpath(shell_dir)
 368
 369   if mode == "optdebug":
 370     mode = "debug"  # "optdebug" is just an alias.
 371
 372   # Populate context object.
 373   mode_flags = MODE_FLAGS[mode]
 374   timeout = options.timeout
 375   if timeout == -1:
 376     # Simulators are slow, therefore allow a longer default timeout.
 377     if arch in SLOW_ARCHS:
 378       timeout = 2 * TIMEOUT_DEFAULT;
 379     else:
 380       timeout = TIMEOUT_DEFAULT;
 381
 382   timeout *= TIMEOUT_SCALEFACTOR[mode]
 383   ctx = context.Context(arch, mode, shell_dir,
 384                         mode_flags, options.verbose,
 385                         timeout, options.isolates,
 386                         options.command_prefix,
 387                         options.extra_flags,
 388                         options.no_i18n)
 389
 390   # Find available test suites and read test cases from them.
 391   variables = {
 392     "arch": arch,
 393     "asan": options.asan,
 394     "deopt_fuzzer": False,
 395     "gc_stress": options.gc_stress,
 396     "isolates": options.isolates,
 397     "mode": mode,
 398     "no_i18n": options.no_i18n,
 399     "simulator": utils.UseSimulator(arch),
 400     "system": utils.GuessOS(),
 401   }
 402   all_tests = []
 403   num_tests = 0
 404   test_id = 0
 405   for s in suites:
 406     s.ReadStatusFile(variables)
 407     s.ReadTestCases(ctx)
 408     if len(args) > 0:
 409       s.FilterTestCasesByArgs(args)
 410     all_tests += s.tests
 411     s.FilterTestCasesByStatus(options.warn_unused, options.flaky_tests,
 412                               options.slow_tests, options.pass_fail_tests)
 413     if options.cat:
 414       verbose.PrintTestSource(s.tests)
 415       continue
 416     variant_flags = [VARIANT_FLAGS[var] for var in VARIANTS]
 417     s.tests = [ t.CopyAddingFlags(v)
 418                 for t in s.tests
 419                 for v in s.VariantFlags(t, variant_flags) ]
 420     s.tests = ShardTests(s.tests, options.shard_count, options.shard_run)
 421     num_tests += len(s.tests)
 422     for t in s.tests:
 423       t.id = test_id
 424       test_id += 1
 425
 426   if options.cat:
 427     return 0  # We're done here.
 428
 429   if options.report:
 430     verbose.PrintReport(all_tests)
 431
 432   if num_tests == 0:
 433     print "No tests to run."
 434     return 0
 435
 436   # Run the tests, either locally or distributed on the network.
 437   try:
 438     start_time = time.time()
 439     progress_indicator = progress.PROGRESS_INDICATORS[options.progress]()
 440     if options.junitout:
 441       progress_indicator = progress.JUnitTestProgressIndicator(
 442           progress_indicator, options.junitout, options.junittestsuite)
 443
 444     run_networked = not options.no_network
 445     if not run_networked:
 446       print("Network distribution disabled, running tests locally.")
 447     elif utils.GuessOS() != "linux":
 448       print("Network distribution is only supported on Linux, sorry!")
 449       run_networked = False
 450     peers = []
 451     if run_networked:
 452       peers = network_execution.GetPeers()
 453       if not peers:
 454         print("No connection to distribution server; running tests locally.")
 455         run_networked = False
 456       elif len(peers) == 1:
 457         print("No other peers on the network; running tests locally.")
 458         run_networked = False
 459       elif num_tests <= 100:
 460         print("Less than 100 tests, running them locally.")
 461         run_networked = False
 462
 463     if run_networked:
 464       runner = network_execution.NetworkedRunner(suites, progress_indicator,
 465                                                  ctx, peers, workspace)
 466     else:
 467       runner = execution.Runner(suites, progress_indicator, ctx)
 468
 469     exit_code = runner.Run(options.j)
 470     if runner.terminate:
 471       return exit_code
 472     overall_duration = time.time() - start_time
 473   except KeyboardInterrupt:
 474     raise
 475
 476   if options.time:
 477     verbose.PrintTestDurations(suites, overall_duration)
 478   return exit_code
 479
 480
 481 if __name__ == "__main__":
 482   sys.exit(Main())