deps/v8/tools/run-tests.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright 2012 the V8 project authors. All rights reserved.
   4 # Redistribution and use in source and binary forms, with or without
   5 # modification, are permitted provided that the following conditions are
   6 # met:
   7 #
   8 #     * Redistributions of source code must retain the above copyright
   9 #       notice, this list of conditions and the following disclaimer.
  10 #     * Redistributions in binary form must reproduce the above
  11 #       copyright notice, this list of conditions and the following
  12 #       disclaimer in the documentation and/or other materials provided
  13 #       with the distribution.
  14 #     * Neither the name of Google Inc. nor the names of its
  15 #       contributors may be used to endorse or promote products derived
  16 #       from this software without specific prior written permission.
  17 #
  18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  29
  30
  31 from collections import OrderedDict
  32 import itertools
  33 import multiprocessing
  34 import optparse
  35 import os
  36 from os.path import join
  37 import platform
  38 import random
  39 import shlex
  40 import subprocess
  41 import sys
  42 import time
  43
  44 from testrunner.local import execution
  45 from testrunner.local import progress
  46 from testrunner.local import testsuite
  47 from testrunner.local.testsuite import VARIANT_FLAGS
  48 from testrunner.local import utils
  49 from testrunner.local import verbose
  50 from testrunner.network import network_execution
  51 from testrunner.objects import context
  52
  53
  54 ARCH_GUESS = utils.DefaultArch()
  55 DEFAULT_TESTS = [
  56   "mjsunit",
  57   "unittests",
  58   "cctest",
  59   "message",
  60   "preparser",
  61 ]
  62
  63 # Map of test name synonyms to lists of test suites. Should be ordered by
  64 # expected runtimes (suites with slow test cases first). These groups are
  65 # invoked in seperate steps on the bots.
  66 TEST_MAP = {
  67   "default": [
  68     "mjsunit",
  69     "cctest",
  70     "message",
  71     "preparser",
  72   ],
  73   "optimize_for_size": [
  74     "mjsunit",
  75     "cctest",
  76     "webkit",
  77   ],
  78   "unittests": [
  79     "unittests",
  80   ],
  81 }
  82
  83 TIMEOUT_DEFAULT = 60
  84
  85 VARIANTS = ["default", "stress", "turbofan", "nocrankshaft"]
  86
  87 DEBUG_FLAGS = ["--nohard-abort", "--nodead-code-elimination",
  88                "--nofold-constants", "--enable-slow-asserts",
  89                "--debug-code", "--verify-heap"]
  90 RELEASE_FLAGS = ["--nohard-abort", "--nodead-code-elimination",
  91                  "--nofold-constants"]
  92
  93 MODES = {
  94   "debug": {
  95     "flags": DEBUG_FLAGS,
  96     "timeout_scalefactor": 4,
  97     "status_mode": "debug",
  98     "execution_mode": "debug",
  99     "output_folder": "debug",
 100   },
 101   "optdebug": {
 102     "flags": DEBUG_FLAGS,
 103     "timeout_scalefactor": 4,
 104     "status_mode": "debug",
 105     "execution_mode": "debug",
 106     "output_folder": "optdebug",
 107   },
 108   "release": {
 109     "flags": RELEASE_FLAGS,
 110     "timeout_scalefactor": 1,
 111     "status_mode": "release",
 112     "execution_mode": "release",
 113     "output_folder": "release",
 114   },
 115   # This mode requires v8 to be compiled with dchecks and slow dchecks.
 116   "tryrelease": {
 117     "flags": RELEASE_FLAGS + ["--enable-slow-asserts"],
 118     "timeout_scalefactor": 2,
 119     "status_mode": "debug",
 120     "execution_mode": "release",
 121     "output_folder": "release",
 122   },
 123 }
 124
 125 GC_STRESS_FLAGS = ["--gc-interval=500", "--stress-compaction",
 126                    "--concurrent-recompilation-queue-length=64",
 127                    "--concurrent-recompilation-delay=500",
 128                    "--concurrent-recompilation"]
 129
 130 SUPPORTED_ARCHS = ["android_arm",
 131                    "android_arm64",
 132                    "android_ia32",
 133                    "android_x64",
 134                    "arm",
 135                    "ia32",
 136                    "x87",
 137                    "mips",
 138                    "mipsel",
 139                    "mips64el",
 140                    "nacl_ia32",
 141                    "nacl_x64",
 142                    "ppc",
 143                    "ppc64",
 144                    "x64",
 145                    "x32",
 146                    "arm64"]
 147 # Double the timeout for these:
 148 SLOW_ARCHS = ["android_arm",
 149               "android_arm64",
 150               "android_ia32",
 151               "android_x64",
 152               "arm",
 153               "mips",
 154               "mipsel",
 155               "mips64el",
 156               "nacl_ia32",
 157               "nacl_x64",
 158               "x87",
 159               "arm64"]
 160
 161
 162 def BuildOptions():
 163   result = optparse.OptionParser()
 164   result.add_option("--arch",
 165                     help=("The architecture to run tests for, "
 166                           "'auto' or 'native' for auto-detect"),
 167                     default="ia32,x64,arm")
 168   result.add_option("--arch-and-mode",
 169                     help="Architecture and mode in the format 'arch.mode'",
 170                     default=None)
 171   result.add_option("--asan",
 172                     help="Regard test expectations for ASAN",
 173                     default=False, action="store_true")
 174   result.add_option("--buildbot",
 175                     help="Adapt to path structure used on buildbots",
 176                     default=False, action="store_true")
 177   result.add_option("--dcheck-always-on",
 178                     help="Indicates that V8 was compiled with DCHECKs enabled",
 179                     default=False, action="store_true")
 180   result.add_option("--cat", help="Print the source of the tests",
 181                     default=False, action="store_true")
 182   result.add_option("--flaky-tests",
 183                     help="Regard tests marked as flaky (run|skip|dontcare)",
 184                     default="dontcare")
 185   result.add_option("--slow-tests",
 186                     help="Regard slow tests (run|skip|dontcare)",
 187                     default="dontcare")
 188   result.add_option("--pass-fail-tests",
 189                     help="Regard pass|fail tests (run|skip|dontcare)",
 190                     default="dontcare")
 191   result.add_option("--gc-stress",
 192                     help="Switch on GC stress mode",
 193                     default=False, action="store_true")
 194   result.add_option("--command-prefix",
 195                     help="Prepended to each shell command used to run a test",
 196                     default="")
 197   result.add_option("--download-data", help="Download missing test suite data",
 198                     default=False, action="store_true")
 199   result.add_option("--extra-flags",
 200                     help="Additional flags to pass to each test command",
 201                     default="")
 202   result.add_option("--isolates", help="Whether to test isolates",
 203                     default=False, action="store_true")
 204   result.add_option("-j", help="The number of parallel tasks to run",
 205                     default=0, type="int")
 206   result.add_option("-m", "--mode",
 207                     help="The test modes in which to run (comma-separated)",
 208                     default="release,debug")
 209   result.add_option("--no-harness", "--noharness",
 210                     help="Run without test harness of a given suite",
 211                     default=False, action="store_true")
 212   result.add_option("--no-i18n", "--noi18n",
 213                     help="Skip internationalization tests",
 214                     default=False, action="store_true")
 215   result.add_option("--no-network", "--nonetwork",
 216                     help="Don't distribute tests on the network",
 217                     default=(utils.GuessOS() != "linux"),
 218                     dest="no_network", action="store_true")
 219   result.add_option("--no-presubmit", "--nopresubmit",
 220                     help='Skip presubmit checks',
 221                     default=False, dest="no_presubmit", action="store_true")
 222   result.add_option("--no-snap", "--nosnap",
 223                     help='Test a build compiled without snapshot.',
 224                     default=False, dest="no_snap", action="store_true")
 225   result.add_option("--no-sorting", "--nosorting",
 226                     help="Don't sort tests according to duration of last run.",
 227                     default=False, dest="no_sorting", action="store_true")
 228   result.add_option("--no-stress", "--nostress",
 229                     help="Don't run crankshaft --always-opt --stress-op test",
 230                     default=False, dest="no_stress", action="store_true")
 231   result.add_option("--no-variants", "--novariants",
 232                     help="Don't run any testing variants",
 233                     default=False, dest="no_variants", action="store_true")
 234   result.add_option("--variants",
 235                     help="Comma-separated list of testing variants")
 236   result.add_option("--outdir", help="Base directory with compile output",
 237                     default="out")
 238   result.add_option("--predictable",
 239                     help="Compare output of several reruns of each test",
 240                     default=False, action="store_true")
 241   result.add_option("-p", "--progress",
 242                     help=("The style of progress indicator"
 243                           " (verbose, dots, color, mono)"),
 244                     choices=progress.PROGRESS_INDICATORS.keys(), default="mono")
 245   result.add_option("--quickcheck", default=False, action="store_true",
 246                     help=("Quick check mode (skip slow/flaky tests)"))
 247   result.add_option("--report", help="Print a summary of the tests to be run",
 248                     default=False, action="store_true")
 249   result.add_option("--json-test-results",
 250                     help="Path to a file for storing json results.")
 251   result.add_option("--rerun-failures-count",
 252                     help=("Number of times to rerun each failing test case. "
 253                           "Very slow tests will be rerun only once."),
 254                     default=0, type="int")
 255   result.add_option("--rerun-failures-max",
 256                     help="Maximum number of failing test cases to rerun.",
 257                     default=100, type="int")
 258   result.add_option("--shard-count",
 259                     help="Split testsuites into this number of shards",
 260                     default=1, type="int")
 261   result.add_option("--shard-run",
 262                     help="Run this shard from the split up tests.",
 263                     default=1, type="int")
 264   result.add_option("--shell", help="DEPRECATED! use --shell-dir", default="")
 265   result.add_option("--shell-dir", help="Directory containing executables",
 266                     default="")
 267   result.add_option("--dont-skip-slow-simulator-tests",
 268                     help="Don't skip more slow tests when using a simulator.",
 269                     default=False, action="store_true",
 270                     dest="dont_skip_simulator_slow_tests")
 271   result.add_option("--stress-only",
 272                     help="Only run tests with --always-opt --stress-opt",
 273                     default=False, action="store_true")
 274   result.add_option("--time", help="Print timing information after running",
 275                     default=False, action="store_true")
 276   result.add_option("-t", "--timeout", help="Timeout in seconds",
 277                     default= -1, type="int")
 278   result.add_option("--tsan",
 279                     help="Regard test expectations for TSAN",
 280                     default=False, action="store_true")
 281   result.add_option("-v", "--verbose", help="Verbose output",
 282                     default=False, action="store_true")
 283   result.add_option("--valgrind", help="Run tests through valgrind",
 284                     default=False, action="store_true")
 285   result.add_option("--warn-unused", help="Report unused rules",
 286                     default=False, action="store_true")
 287   result.add_option("--junitout", help="File name of the JUnit output")
 288   result.add_option("--junittestsuite",
 289                     help="The testsuite name in the JUnit output file",
 290                     default="v8tests")
 291   result.add_option("--random-seed", default=0, dest="random_seed",
 292                     help="Default seed for initializing random generator")
 293   result.add_option("--msan",
 294                     help="Regard test expectations for MSAN",
 295                     default=False, action="store_true")
 296   return result
 297
 298
 299 def ProcessOptions(options):
 300   global VARIANT_FLAGS
 301   global VARIANTS
 302
 303   # Architecture and mode related stuff.
 304   if options.arch_and_mode:
 305     options.arch_and_mode = [arch_and_mode.split(".")
 306         for arch_and_mode in options.arch_and_mode.split(",")]
 307     options.arch = ",".join([tokens[0] for tokens in options.arch_and_mode])
 308     options.mode = ",".join([tokens[1] for tokens in options.arch_and_mode])
 309   options.mode = options.mode.split(",")
 310   for mode in options.mode:
 311     if not mode.lower() in MODES:
 312       print "Unknown mode %s" % mode
 313       return False
 314   if options.arch in ["auto", "native"]:
 315     options.arch = ARCH_GUESS
 316   options.arch = options.arch.split(",")
 317   for arch in options.arch:
 318     if not arch in SUPPORTED_ARCHS:
 319       print "Unknown architecture %s" % arch
 320       return False
 321
 322   # Store the final configuration in arch_and_mode list. Don't overwrite
 323   # predefined arch_and_mode since it is more expressive than arch and mode.
 324   if not options.arch_and_mode:
 325     options.arch_and_mode = itertools.product(options.arch, options.mode)
 326
 327   # Special processing of other options, sorted alphabetically.
 328
 329   if options.buildbot:
 330     # Buildbots run presubmit tests as a separate step.
 331     options.no_presubmit = True
 332     options.no_network = True
 333   if options.command_prefix:
 334     print("Specifying --command-prefix disables network distribution, "
 335           "running tests locally.")
 336     options.no_network = True
 337   options.command_prefix = shlex.split(options.command_prefix)
 338   options.extra_flags = shlex.split(options.extra_flags)
 339
 340   if options.gc_stress:
 341     options.extra_flags += GC_STRESS_FLAGS
 342
 343   if options.asan:
 344     options.extra_flags.append("--invoke-weak-callbacks")
 345
 346   if options.tsan:
 347     VARIANTS = ["default"]
 348     suppressions_file = os.path.join(os.path.dirname(os.path.abspath(__file__)),
 349                                      'sanitizers', 'tsan_suppressions.txt')
 350     tsan_options = '%s suppressions=%s' % (
 351         os.environ.get('TSAN_OPTIONS', ''), suppressions_file)
 352     os.environ['TSAN_OPTIONS'] = tsan_options
 353
 354   if options.j == 0:
 355     options.j = multiprocessing.cpu_count()
 356
 357   while options.random_seed == 0:
 358     options.random_seed = random.SystemRandom().randint(-2147483648, 2147483647)
 359
 360   def excl(*args):
 361     """Returns true if zero or one of multiple arguments are true."""
 362     return reduce(lambda x, y: x + y, args) <= 1
 363
 364   if not excl(options.no_stress, options.stress_only, options.no_variants,
 365               bool(options.variants)):
 366     print("Use only one of --no-stress, --stress-only, --no-variants, "
 367           "or --variants.")
 368     return False
 369   if options.quickcheck:
 370     VARIANTS = ["default", "stress"]
 371     options.flaky_tests = "skip"
 372     options.slow_tests = "skip"
 373     options.pass_fail_tests = "skip"
 374   if options.no_stress:
 375     VARIANTS = ["default", "nocrankshaft"]
 376   if options.no_variants:
 377     VARIANTS = ["default"]
 378   if options.stress_only:
 379     VARIANTS = ["stress"]
 380   if options.variants:
 381     VARIANTS = options.variants.split(",")
 382     if not set(VARIANTS).issubset(VARIANT_FLAGS.keys()):
 383       print "All variants must be in %s" % str(VARIANT_FLAGS.keys())
 384       return False
 385   if options.predictable:
 386     VARIANTS = ["default"]
 387     options.extra_flags.append("--predictable")
 388     options.extra_flags.append("--verify_predictable")
 389     options.extra_flags.append("--no-inline-new")
 390
 391   if not options.shell_dir:
 392     if options.shell:
 393       print "Warning: --shell is deprecated, use --shell-dir instead."
 394       options.shell_dir = os.path.dirname(options.shell)
 395   if options.valgrind:
 396     run_valgrind = os.path.join("tools", "run-valgrind.py")
 397     # This is OK for distributed running, so we don't need to set no_network.
 398     options.command_prefix = (["python", "-u", run_valgrind] +
 399                               options.command_prefix)
 400   def CheckTestMode(name, option):
 401     if not option in ["run", "skip", "dontcare"]:
 402       print "Unknown %s mode %s" % (name, option)
 403       return False
 404     return True
 405   if not CheckTestMode("flaky test", options.flaky_tests):
 406     return False
 407   if not CheckTestMode("slow test", options.slow_tests):
 408     return False
 409   if not CheckTestMode("pass|fail test", options.pass_fail_tests):
 410     return False
 411   if not options.no_i18n:
 412     DEFAULT_TESTS.append("intl")
 413   return True
 414
 415
 416 def ShardTests(tests, shard_count, shard_run):
 417   if shard_count < 2:
 418     return tests
 419   if shard_run < 1 or shard_run > shard_count:
 420     print "shard-run not a valid number, should be in [1:shard-count]"
 421     print "defaulting back to running all tests"
 422     return tests
 423   count = 0
 424   shard = []
 425   for test in tests:
 426     if count % shard_count == shard_run - 1:
 427       shard.append(test)
 428     count += 1
 429   return shard
 430
 431
 432 def Main():
 433   parser = BuildOptions()
 434   (options, args) = parser.parse_args()
 435   if not ProcessOptions(options):
 436     parser.print_help()
 437     return 1
 438
 439   exit_code = 0
 440   workspace = os.path.abspath(join(os.path.dirname(sys.argv[0]), ".."))
 441   if not options.no_presubmit:
 442     print ">>> running presubmit tests"
 443     exit_code = subprocess.call(
 444         [sys.executable, join(workspace, "tools", "presubmit.py")])
 445
 446   suite_paths = utils.GetSuitePaths(join(workspace, "test"))
 447
 448   # Expand arguments with grouped tests. The args should reflect the list of
 449   # suites as otherwise filters would break.
 450   def ExpandTestGroups(name):
 451     if name in TEST_MAP:
 452       return [suite for suite in TEST_MAP[arg]]
 453     else:
 454       return [name]
 455   args = reduce(lambda x, y: x + y,
 456          [ExpandTestGroups(arg) for arg in args],
 457          [])
 458
 459   if len(args) == 0:
 460     suite_paths = [ s for s in DEFAULT_TESTS if s in suite_paths ]
 461   else:
 462     args_suites = OrderedDict() # Used as set
 463     for arg in args:
 464       args_suites[arg.split(os.path.sep)[0]] = True
 465     suite_paths = [ s for s in args_suites if s in suite_paths ]
 466
 467   suites = []
 468   for root in suite_paths:
 469     suite = testsuite.TestSuite.LoadTestSuite(
 470         os.path.join(workspace, "test", root))
 471     if suite:
 472       suites.append(suite)
 473
 474   if options.download_data:
 475     for s in suites:
 476       s.DownloadData()
 477
 478   for (arch, mode) in options.arch_and_mode:
 479     try:
 480       code = Execute(arch, mode, args, options, suites, workspace)
 481     except KeyboardInterrupt:
 482       return 2
 483     exit_code = exit_code or code
 484   return exit_code
 485
 486
 487 def Execute(arch, mode, args, options, suites, workspace):
 488   print(">>> Running tests for %s.%s" % (arch, mode))
 489
 490   shell_dir = options.shell_dir
 491   if not shell_dir:
 492     if options.buildbot:
 493       # TODO(machenbach): Get rid of different output folder location on
 494       # buildbot. Currently this is capitalized Release and Debug.
 495       shell_dir = os.path.join(workspace, options.outdir, mode)
 496       mode = mode.lower()
 497     else:
 498       shell_dir = os.path.join(
 499           workspace,
 500           options.outdir,
 501           "%s.%s" % (arch, MODES[mode]["output_folder"]),
 502       )
 503   shell_dir = os.path.relpath(shell_dir)
 504
 505   # Populate context object.
 506   mode_flags = MODES[mode]["flags"]
 507   timeout = options.timeout
 508   if timeout == -1:
 509     # Simulators are slow, therefore allow a longer default timeout.
 510     if arch in SLOW_ARCHS:
 511       timeout = 2 * TIMEOUT_DEFAULT;
 512     else:
 513       timeout = TIMEOUT_DEFAULT;
 514
 515   timeout *= MODES[mode]["timeout_scalefactor"]
 516
 517   if options.predictable:
 518     # Predictable mode is slower.
 519     timeout *= 2
 520
 521   ctx = context.Context(arch, MODES[mode]["execution_mode"], shell_dir,
 522                         mode_flags, options.verbose,
 523                         timeout, options.isolates,
 524                         options.command_prefix,
 525                         options.extra_flags,
 526                         options.no_i18n,
 527                         options.random_seed,
 528                         options.no_sorting,
 529                         options.rerun_failures_count,
 530                         options.rerun_failures_max,
 531                         options.predictable,
 532                         options.no_harness)
 533
 534   # TODO(all): Combine "simulator" and "simulator_run".
 535   simulator_run = not options.dont_skip_simulator_slow_tests and \
 536       arch in ['arm64', 'arm', 'mipsel', 'mips', 'mips64el', \
 537                'ppc', 'ppc64'] and \
 538       ARCH_GUESS and arch != ARCH_GUESS
 539   # Find available test suites and read test cases from them.
 540   variables = {
 541     "arch": arch,
 542     "asan": options.asan,
 543     "deopt_fuzzer": False,
 544     "gc_stress": options.gc_stress,
 545     "isolates": options.isolates,
 546     "mode": MODES[mode]["status_mode"],
 547     "no_i18n": options.no_i18n,
 548     "no_snap": options.no_snap,
 549     "simulator_run": simulator_run,
 550     "simulator": utils.UseSimulator(arch),
 551     "system": utils.GuessOS(),
 552     "tsan": options.tsan,
 553     "msan": options.msan,
 554     "dcheck_always_on": options.dcheck_always_on,
 555     "byteorder": sys.byteorder,
 556   }
 557   all_tests = []
 558   num_tests = 0
 559   test_id = 0
 560   for s in suites:
 561     s.ReadStatusFile(variables)
 562     s.ReadTestCases(ctx)
 563     if len(args) > 0:
 564       s.FilterTestCasesByArgs(args)
 565     all_tests += s.tests
 566     s.FilterTestCasesByStatus(options.warn_unused, options.flaky_tests,
 567                               options.slow_tests, options.pass_fail_tests)
 568     if options.cat:
 569       verbose.PrintTestSource(s.tests)
 570       continue
 571     variant_flags = [VARIANT_FLAGS[var] for var in VARIANTS]
 572     s.tests = [ t.CopyAddingFlags(v)
 573                 for t in s.tests
 574                 for v in s.VariantFlags(t, variant_flags) ]
 575     s.tests = ShardTests(s.tests, options.shard_count, options.shard_run)
 576     num_tests += len(s.tests)
 577     for t in s.tests:
 578       t.id = test_id
 579       test_id += 1
 580
 581   if options.cat:
 582     return 0  # We're done here.
 583
 584   if options.report:
 585     verbose.PrintReport(all_tests)
 586
 587   if num_tests == 0:
 588     print "No tests to run."
 589     return 0
 590
 591   # Run the tests, either locally or distributed on the network.
 592   start_time = time.time()
 593   progress_indicator = progress.PROGRESS_INDICATORS[options.progress]()
 594   if options.junitout:
 595     progress_indicator = progress.JUnitTestProgressIndicator(
 596         progress_indicator, options.junitout, options.junittestsuite)
 597   if options.json_test_results:
 598     progress_indicator = progress.JsonTestProgressIndicator(
 599         progress_indicator, options.json_test_results, arch,
 600         MODES[mode]["execution_mode"])
 601
 602   run_networked = not options.no_network
 603   if not run_networked:
 604     print("Network distribution disabled, running tests locally.")
 605   elif utils.GuessOS() != "linux":
 606     print("Network distribution is only supported on Linux, sorry!")
 607     run_networked = False
 608   peers = []
 609   if run_networked:
 610     peers = network_execution.GetPeers()
 611     if not peers:
 612       print("No connection to distribution server; running tests locally.")
 613       run_networked = False
 614     elif len(peers) == 1:
 615       print("No other peers on the network; running tests locally.")
 616       run_networked = False
 617     elif num_tests <= 100:
 618       print("Less than 100 tests, running them locally.")
 619       run_networked = False
 620
 621   if run_networked:
 622     runner = network_execution.NetworkedRunner(suites, progress_indicator,
 623                                                ctx, peers, workspace)
 624   else:
 625     runner = execution.Runner(suites, progress_indicator, ctx)
 626
 627   exit_code = runner.Run(options.j)
 628   overall_duration = time.time() - start_time
 629
 630   if options.time:
 631     verbose.PrintTestDurations(suites, overall_duration)
 632   return exit_code
 633
 634
 635 if __name__ == "__main__":
 636   sys.exit(Main())