src/v8/tools/run-tests.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright 2012 the V8 project authors. All rights reserved.
   4 # Redistribution and use in source and binary forms, with or without
   5 # modification, are permitted provided that the following conditions are
   6 # met:
   7 #
   8 #     * Redistributions of source code must retain the above copyright
   9 #       notice, this list of conditions and the following disclaimer.
  10 #     * Redistributions in binary form must reproduce the above
  11 #       copyright notice, this list of conditions and the following
  12 #       disclaimer in the documentation and/or other materials provided
  13 #       with the distribution.
  14 #     * Neither the name of Google Inc. nor the names of its
  15 #       contributors may be used to endorse or promote products derived
  16 #       from this software without specific prior written permission.
  17 #
  18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  29
  30
  31 import itertools
  32 import multiprocessing
  33 import optparse
  34 import os
  35 from os.path import join
  36 import platform
  37 import random
  38 import shlex
  39 import subprocess
  40 import sys
  41 import time
  42
  43 from testrunner.local import execution
  44 from testrunner.local import progress
  45 from testrunner.local import testsuite
  46 from testrunner.local import utils
  47 from testrunner.local import verbose
  48 from testrunner.network import network_execution
  49 from testrunner.objects import context
  50
  51
  52 ARCH_GUESS = utils.DefaultArch()
  53 DEFAULT_TESTS = ["mjsunit", "cctest", "message", "preparser"]
  54 TIMEOUT_DEFAULT = 60
  55 TIMEOUT_SCALEFACTOR = {"debug"   : 4,
  56                        "release" : 1 }
  57
  58 # Use this to run several variants of the tests.
  59 VARIANT_FLAGS = {
  60     "default": [],
  61     "stress": ["--stress-opt", "--always-opt"],
  62     "nocrankshaft": ["--nocrankshaft"]}
  63
  64 VARIANTS = ["default", "stress", "nocrankshaft"]
  65
  66 MODE_FLAGS = {
  67     "debug"   : ["--nohard-abort", "--nodead-code-elimination",
  68                  "--nofold-constants", "--enable-slow-asserts",
  69                  "--debug-code", "--verify-heap"],
  70     "release" : ["--nohard-abort", "--nodead-code-elimination",
  71                  "--nofold-constants"]}
  72
  73 GC_STRESS_FLAGS = ["--gc-interval=500", "--stress-compaction",
  74                    "--concurrent-recompilation-queue-length=64",
  75                    "--concurrent-recompilation-delay=500",
  76                    "--concurrent-recompilation"]
  77
  78 SUPPORTED_ARCHS = ["android_arm",
  79                    "android_arm64",
  80                    "android_ia32",
  81                    "arm",
  82                    "ia32",
  83                    "mips",
  84                    "mipsel",
  85                    "nacl_ia32",
  86                    "nacl_x64",
  87                    "x64",
  88                    "arm64"]
  89 # Double the timeout for these:
  90 SLOW_ARCHS = ["android_arm",
  91               "android_arm64",
  92               "android_ia32",
  93               "arm",
  94               "mips",
  95               "mipsel",
  96               "nacl_ia32",
  97               "nacl_x64",
  98               "arm64"]
  99
 100
 101 def BuildOptions():
 102   result = optparse.OptionParser()
 103   result.add_option("--arch",
 104                     help=("The architecture to run tests for, "
 105                           "'auto' or 'native' for auto-detect"),
 106                     default="ia32,x64,arm")
 107   result.add_option("--arch-and-mode",
 108                     help="Architecture and mode in the format 'arch.mode'",
 109                     default=None)
 110   result.add_option("--asan",
 111                     help="Regard test expectations for ASAN",
 112                     default=False, action="store_true")
 113   result.add_option("--buildbot",
 114                     help="Adapt to path structure used on buildbots",
 115                     default=False, action="store_true")
 116   result.add_option("--cat", help="Print the source of the tests",
 117                     default=False, action="store_true")
 118   result.add_option("--flaky-tests",
 119                     help="Regard tests marked as flaky (run|skip|dontcare)",
 120                     default="dontcare")
 121   result.add_option("--slow-tests",
 122                     help="Regard slow tests (run|skip|dontcare)",
 123                     default="dontcare")
 124   result.add_option("--pass-fail-tests",
 125                     help="Regard pass|fail tests (run|skip|dontcare)",
 126                     default="dontcare")
 127   result.add_option("--gc-stress",
 128                     help="Switch on GC stress mode",
 129                     default=False, action="store_true")
 130   result.add_option("--command-prefix",
 131                     help="Prepended to each shell command used to run a test",
 132                     default="")
 133   result.add_option("--download-data", help="Download missing test suite data",
 134                     default=False, action="store_true")
 135   result.add_option("--extra-flags",
 136                     help="Additional flags to pass to each test command",
 137                     default="")
 138   result.add_option("--isolates", help="Whether to test isolates",
 139                     default=False, action="store_true")
 140   result.add_option("-j", help="The number of parallel tasks to run",
 141                     default=0, type="int")
 142   result.add_option("-m", "--mode",
 143                     help="The test modes in which to run (comma-separated)",
 144                     default="release,debug")
 145   result.add_option("--no-i18n", "--noi18n",
 146                     help="Skip internationalization tests",
 147                     default=False, action="store_true")
 148   result.add_option("--no-network", "--nonetwork",
 149                     help="Don't distribute tests on the network",
 150                     default=(utils.GuessOS() != "linux"),
 151                     dest="no_network", action="store_true")
 152   result.add_option("--no-presubmit", "--nopresubmit",
 153                     help='Skip presubmit checks',
 154                     default=False, dest="no_presubmit", action="store_true")
 155   result.add_option("--no-snap", "--nosnap",
 156                     help='Test a build compiled without snapshot.',
 157                     default=False, dest="no_snap", action="store_true")
 158   result.add_option("--no-stress", "--nostress",
 159                     help="Don't run crankshaft --always-opt --stress-op test",
 160                     default=False, dest="no_stress", action="store_true")
 161   result.add_option("--no-variants", "--novariants",
 162                     help="Don't run any testing variants",
 163                     default=False, dest="no_variants", action="store_true")
 164   result.add_option("--variants",
 165                     help="Comma-separated list of testing variants")
 166   result.add_option("--outdir", help="Base directory with compile output",
 167                     default="out")
 168   result.add_option("-p", "--progress",
 169                     help=("The style of progress indicator"
 170                           " (verbose, dots, color, mono)"),
 171                     choices=progress.PROGRESS_INDICATORS.keys(), default="mono")
 172   result.add_option("--quickcheck", default=False, action="store_true",
 173                     help=("Quick check mode (skip slow/flaky tests)"))
 174   result.add_option("--report", help="Print a summary of the tests to be run",
 175                     default=False, action="store_true")
 176   result.add_option("--shard-count",
 177                     help="Split testsuites into this number of shards",
 178                     default=1, type="int")
 179   result.add_option("--shard-run",
 180                     help="Run this shard from the split up tests.",
 181                     default=1, type="int")
 182   result.add_option("--shell", help="DEPRECATED! use --shell-dir", default="")
 183   result.add_option("--shell-dir", help="Directory containing executables",
 184                     default="")
 185   result.add_option("--dont-skip-slow-simulator-tests",
 186                     help="Don't skip more slow tests when using a simulator.",
 187                     default=False, action="store_true",
 188                     dest="dont_skip_simulator_slow_tests")
 189   result.add_option("--stress-only",
 190                     help="Only run tests with --always-opt --stress-opt",
 191                     default=False, action="store_true")
 192   result.add_option("--time", help="Print timing information after running",
 193                     default=False, action="store_true")
 194   result.add_option("-t", "--timeout", help="Timeout in seconds",
 195                     default= -1, type="int")
 196   result.add_option("-v", "--verbose", help="Verbose output",
 197                     default=False, action="store_true")
 198   result.add_option("--valgrind", help="Run tests through valgrind",
 199                     default=False, action="store_true")
 200   result.add_option("--warn-unused", help="Report unused rules",
 201                     default=False, action="store_true")
 202   result.add_option("--junitout", help="File name of the JUnit output")
 203   result.add_option("--junittestsuite",
 204                     help="The testsuite name in the JUnit output file",
 205                     default="v8tests")
 206   result.add_option("--random-seed", default=0, dest="random_seed",
 207                     help="Default seed for initializing random generator")
 208   return result
 209
 210
 211 def ProcessOptions(options):
 212   global VARIANT_FLAGS
 213   global VARIANTS
 214
 215   # Architecture and mode related stuff.
 216   if options.arch_and_mode:
 217     options.arch_and_mode = [arch_and_mode.split(".")
 218         for arch_and_mode in options.arch_and_mode.split(",")]
 219     options.arch = ",".join([tokens[0] for tokens in options.arch_and_mode])
 220     options.mode = ",".join([tokens[1] for tokens in options.arch_and_mode])
 221   options.mode = options.mode.split(",")
 222   for mode in options.mode:
 223     if not mode.lower() in ["debug", "release", "optdebug"]:
 224       print "Unknown mode %s" % mode
 225       return False
 226   if options.arch in ["auto", "native"]:
 227     options.arch = ARCH_GUESS
 228   options.arch = options.arch.split(",")
 229   for arch in options.arch:
 230     if not arch in SUPPORTED_ARCHS:
 231       print "Unknown architecture %s" % arch
 232       return False
 233
 234   # Store the final configuration in arch_and_mode list. Don't overwrite
 235   # predefined arch_and_mode since it is more expressive than arch and mode.
 236   if not options.arch_and_mode:
 237     options.arch_and_mode = itertools.product(options.arch, options.mode)
 238
 239   # Special processing of other options, sorted alphabetically.
 240
 241   if options.buildbot:
 242     # Buildbots run presubmit tests as a separate step.
 243     options.no_presubmit = True
 244     options.no_network = True
 245   if options.command_prefix:
 246     print("Specifying --command-prefix disables network distribution, "
 247           "running tests locally.")
 248     options.no_network = True
 249   options.command_prefix = shlex.split(options.command_prefix)
 250   options.extra_flags = shlex.split(options.extra_flags)
 251
 252   if options.gc_stress:
 253     options.extra_flags += GC_STRESS_FLAGS
 254
 255   if options.j == 0:
 256     options.j = multiprocessing.cpu_count()
 257
 258   while options.random_seed == 0:
 259     options.random_seed = random.SystemRandom().randint(-2147483648, 2147483647)
 260
 261   def excl(*args):
 262     """Returns true if zero or one of multiple arguments are true."""
 263     return reduce(lambda x, y: x + y, args) <= 1
 264
 265   if not excl(options.no_stress, options.stress_only, options.no_variants,
 266               bool(options.variants), options.quickcheck):
 267     print("Use only one of --no-stress, --stress-only, --no-variants, "
 268           "--variants, or --quickcheck.")
 269     return False
 270   if options.no_stress:
 271     VARIANTS = ["default", "nocrankshaft"]
 272   if options.no_variants:
 273     VARIANTS = ["default"]
 274   if options.stress_only:
 275     VARIANTS = ["stress"]
 276   if options.variants:
 277     VARIANTS = options.variants.split(",")
 278     if not set(VARIANTS).issubset(VARIANT_FLAGS.keys()):
 279       print "All variants must be in %s" % str(VARIANT_FLAGS.keys())
 280       return False
 281   if options.quickcheck:
 282     VARIANTS = ["default", "stress"]
 283     options.flaky_tests = "skip"
 284     options.slow_tests = "skip"
 285     options.pass_fail_tests = "skip"
 286
 287   if not options.shell_dir:
 288     if options.shell:
 289       print "Warning: --shell is deprecated, use --shell-dir instead."
 290       options.shell_dir = os.path.dirname(options.shell)
 291   if options.valgrind:
 292     run_valgrind = os.path.join("tools", "run-valgrind.py")
 293     # This is OK for distributed running, so we don't need to set no_network.
 294     options.command_prefix = (["python", "-u", run_valgrind] +
 295                               options.command_prefix)
 296   def CheckTestMode(name, option):
 297     if not option in ["run", "skip", "dontcare"]:
 298       print "Unknown %s mode %s" % (name, option)
 299       return False
 300     return True
 301   if not CheckTestMode("flaky test", options.flaky_tests):
 302     return False
 303   if not CheckTestMode("slow test", options.slow_tests):
 304     return False
 305   if not CheckTestMode("pass|fail test", options.pass_fail_tests):
 306     return False
 307   if not options.no_i18n:
 308     DEFAULT_TESTS.append("intl")
 309   return True
 310
 311
 312 def ShardTests(tests, shard_count, shard_run):
 313   if shard_count < 2:
 314     return tests
 315   if shard_run < 1 or shard_run > shard_count:
 316     print "shard-run not a valid number, should be in [1:shard-count]"
 317     print "defaulting back to running all tests"
 318     return tests
 319   count = 0
 320   shard = []
 321   for test in tests:
 322     if count % shard_count == shard_run - 1:
 323       shard.append(test)
 324     count += 1
 325   return shard
 326
 327
 328 def Main():
 329   parser = BuildOptions()
 330   (options, args) = parser.parse_args()
 331   if not ProcessOptions(options):
 332     parser.print_help()
 333     return 1
 334
 335   exit_code = 0
 336   workspace = os.path.abspath(join(os.path.dirname(sys.argv[0]), ".."))
 337   if not options.no_presubmit:
 338     print ">>> running presubmit tests"
 339     code = subprocess.call(
 340         [sys.executable, join(workspace, "tools", "presubmit.py")])
 341     exit_code = code
 342
 343   suite_paths = utils.GetSuitePaths(join(workspace, "test"))
 344
 345   if len(args) == 0:
 346     suite_paths = [ s for s in DEFAULT_TESTS if s in suite_paths ]
 347   else:
 348     args_suites = set()
 349     for arg in args:
 350       suite = arg.split(os.path.sep)[0]
 351       if not suite in args_suites:
 352         args_suites.add(suite)
 353     suite_paths = [ s for s in args_suites if s in suite_paths ]
 354
 355   suites = []
 356   for root in suite_paths:
 357     suite = testsuite.TestSuite.LoadTestSuite(
 358         os.path.join(workspace, "test", root))
 359     if suite:
 360       suites.append(suite)
 361
 362   if options.download_data:
 363     for s in suites:
 364       s.DownloadData()
 365
 366   for (arch, mode) in options.arch_and_mode:
 367     try:
 368       code = Execute(arch, mode, args, options, suites, workspace)
 369     except KeyboardInterrupt:
 370       return 2
 371     exit_code = exit_code or code
 372   return exit_code
 373
 374
 375 def Execute(arch, mode, args, options, suites, workspace):
 376   print(">>> Running tests for %s.%s" % (arch, mode))
 377
 378   shell_dir = options.shell_dir
 379   if not shell_dir:
 380     if options.buildbot:
 381       shell_dir = os.path.join(workspace, options.outdir, mode)
 382       mode = mode.lower()
 383     else:
 384       shell_dir = os.path.join(workspace, options.outdir,
 385                                "%s.%s" % (arch, mode))
 386   shell_dir = os.path.relpath(shell_dir)
 387
 388   if mode == "optdebug":
 389     mode = "debug"  # "optdebug" is just an alias.
 390
 391   # Populate context object.
 392   mode_flags = MODE_FLAGS[mode]
 393   timeout = options.timeout
 394   if timeout == -1:
 395     # Simulators are slow, therefore allow a longer default timeout.
 396     if arch in SLOW_ARCHS:
 397       timeout = 2 * TIMEOUT_DEFAULT;
 398     else:
 399       timeout = TIMEOUT_DEFAULT;
 400
 401   timeout *= TIMEOUT_SCALEFACTOR[mode]
 402   ctx = context.Context(arch, mode, shell_dir,
 403                         mode_flags, options.verbose,
 404                         timeout, options.isolates,
 405                         options.command_prefix,
 406                         options.extra_flags,
 407                         options.no_i18n,
 408                         options.random_seed)
 409
 410   # TODO(all): Combine "simulator" and "simulator_run".
 411   simulator_run = not options.dont_skip_simulator_slow_tests and \
 412       arch in ['arm64', 'arm', 'mips'] and ARCH_GUESS and arch != ARCH_GUESS
 413   # Find available test suites and read test cases from them.
 414   variables = {
 415     "arch": arch,
 416     "asan": options.asan,
 417     "deopt_fuzzer": False,
 418     "gc_stress": options.gc_stress,
 419     "isolates": options.isolates,
 420     "mode": mode,
 421     "no_i18n": options.no_i18n,
 422     "no_snap": options.no_snap,
 423     "simulator_run": simulator_run,
 424     "simulator": utils.UseSimulator(arch),
 425     "system": utils.GuessOS(),
 426   }
 427   all_tests = []
 428   num_tests = 0
 429   test_id = 0
 430   for s in suites:
 431     s.ReadStatusFile(variables)
 432     s.ReadTestCases(ctx)
 433     if len(args) > 0:
 434       s.FilterTestCasesByArgs(args)
 435     all_tests += s.tests
 436     s.FilterTestCasesByStatus(options.warn_unused, options.flaky_tests,
 437                               options.slow_tests, options.pass_fail_tests)
 438     if options.cat:
 439       verbose.PrintTestSource(s.tests)
 440       continue
 441     variant_flags = [VARIANT_FLAGS[var] for var in VARIANTS]
 442     s.tests = [ t.CopyAddingFlags(v)
 443                 for t in s.tests
 444                 for v in s.VariantFlags(t, variant_flags) ]
 445     s.tests = ShardTests(s.tests, options.shard_count, options.shard_run)
 446     num_tests += len(s.tests)
 447     for t in s.tests:
 448       t.id = test_id
 449       test_id += 1
 450
 451   if options.cat:
 452     return 0  # We're done here.
 453
 454   if options.report:
 455     verbose.PrintReport(all_tests)
 456
 457   if num_tests == 0:
 458     print "No tests to run."
 459     return 0
 460
 461   # Run the tests, either locally or distributed on the network.
 462   try:
 463     start_time = time.time()
 464     progress_indicator = progress.PROGRESS_INDICATORS[options.progress]()
 465     if options.junitout:
 466       progress_indicator = progress.JUnitTestProgressIndicator(
 467           progress_indicator, options.junitout, options.junittestsuite)
 468
 469     run_networked = not options.no_network
 470     if not run_networked:
 471       print("Network distribution disabled, running tests locally.")
 472     elif utils.GuessOS() != "linux":
 473       print("Network distribution is only supported on Linux, sorry!")
 474       run_networked = False
 475     peers = []
 476     if run_networked:
 477       peers = network_execution.GetPeers()
 478       if not peers:
 479         print("No connection to distribution server; running tests locally.")
 480         run_networked = False
 481       elif len(peers) == 1:
 482         print("No other peers on the network; running tests locally.")
 483         run_networked = False
 484       elif num_tests <= 100:
 485         print("Less than 100 tests, running them locally.")
 486         run_networked = False
 487
 488     if run_networked:
 489       runner = network_execution.NetworkedRunner(suites, progress_indicator,
 490                                                  ctx, peers, workspace)
 491     else:
 492       runner = execution.Runner(suites, progress_indicator, ctx)
 493
 494     exit_code = runner.Run(options.j)
 495     if runner.terminate:
 496       return exit_code
 497     overall_duration = time.time() - start_time
 498   except KeyboardInterrupt:
 499     raise
 500
 501   if options.time:
 502     verbose.PrintTestDurations(suites, overall_duration)
 503   return exit_code
 504
 505
 506 if __name__ == "__main__":
 507   sys.exit(Main())