3 # Copyright 2012 the V8 project authors. All rights reserved.
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
8 # * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 # * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following
12 # disclaimer in the documentation and/or other materials provided
13 # with the distribution.
14 # * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived
16 # from this software without specific prior written permission.
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
33 import multiprocessing
36 from os.path import join
43 from testrunner.local import execution
44 from testrunner.local import progress
45 from testrunner.local import testsuite
46 from testrunner.local import utils
47 from testrunner.local import verbose
48 from testrunner.objects import context
51 ARCH_GUESS = utils.DefaultArch()
52 DEFAULT_TESTS = ["mjsunit", "webkit"]
54 TIMEOUT_SCALEFACTOR = {"debug" : 4,
58 "debug" : ["--nohard-abort", "--nodead-code-elimination",
59 "--nofold-constants", "--enable-slow-asserts",
60 "--debug-code", "--verify-heap",
61 "--noconcurrent-recompilation"],
62 "release" : ["--nohard-abort", "--nodead-code-elimination",
63 "--nofold-constants", "--noconcurrent-recompilation"]}
65 SUPPORTED_ARCHS = ["android_arm",
73 # Double the timeout for these:
74 SLOW_ARCHS = ["android_arm",
80 MAX_DEOPT = 1000000000
81 DISTRIBUTION_MODES = ["smooth", "random"]
84 class RandomDistribution:
85 def __init__(self, seed=None):
86 seed = seed or random.randint(1, sys.maxint)
87 print "Using random distribution with seed %d" % seed
88 self._random = random.Random(seed)
90 def Distribute(self, n, m):
93 return self._random.sample(xrange(1, m + 1), n)
96 class SmoothDistribution:
97 """Distribute n numbers into the interval [1:m].
98 F1: Factor of the first derivation of the distribution function.
99 F2: Factor of the second derivation of the distribution function.
100 With F1 and F2 set to 0, the distribution will be equal.
102 def __init__(self, factor1=2.0, factor2=0.2):
103 self._factor1 = factor1
104 self._factor2 = factor2
106 def Distribute(self, n, m):
117 for i in range(0, n):
123 # Project the distribution into the interval [0:M].
124 result = [ x * m / result[-1] for x in result ]
126 # Equalize by n. The closer n is to m, the more equal will be the
128 for (i, x) in enumerate(result):
129 # The value of x if it was equally distributed.
130 equal_x = i / float(n - 1) * float(m - 1) + 1
132 # Difference factor between actual and equal distribution.
133 diff = 1 - (x / equal_x)
135 # Equalize x dependent on the number of values to distribute.
136 result[i] = int(x + (i + 1) * diff)
140 def Distribution(options):
141 if options.distribution_mode == "random":
142 return RandomDistribution(options.seed)
143 if options.distribution_mode == "smooth":
144 return SmoothDistribution(options.distribution_factor1,
145 options.distribution_factor2)
149 result = optparse.OptionParser()
150 result.add_option("--arch",
151 help=("The architecture to run tests for, "
152 "'auto' or 'native' for auto-detect"),
153 default="ia32,x64,arm")
154 result.add_option("--arch-and-mode",
155 help="Architecture and mode in the format 'arch.mode'",
157 result.add_option("--asan",
158 help="Regard test expectations for ASAN",
159 default=False, action="store_true")
160 result.add_option("--buildbot",
161 help="Adapt to path structure used on buildbots",
162 default=False, action="store_true")
163 result.add_option("--command-prefix",
164 help="Prepended to each shell command used to run a test",
166 result.add_option("--coverage", help=("Exponential test coverage "
167 "(range 0.0, 1.0) -- 0.0: one test, 1.0 all tests (slow)"),
168 default=0.4, type="float")
169 result.add_option("--coverage-lift", help=("Lifts test coverage for tests "
170 "with a small number of deopt points (range 0, inf)"),
171 default=20, type="int")
172 result.add_option("--download-data", help="Download missing test suite data",
173 default=False, action="store_true")
174 result.add_option("--distribution-factor1", help=("Factor of the first "
175 "derivation of the distribution function"), default=2.0,
177 result.add_option("--distribution-factor2", help=("Factor of the second "
178 "derivation of the distribution function"), default=0.7,
180 result.add_option("--distribution-mode", help=("How to select deopt points "
181 "for a given test (smooth|random)"),
183 result.add_option("--dump-results-file", help=("Dump maximum number of "
184 "deopt points per test to a file"))
185 result.add_option("--extra-flags",
186 help="Additional flags to pass to each test command",
188 result.add_option("--isolates", help="Whether to test isolates",
189 default=False, action="store_true")
190 result.add_option("-j", help="The number of parallel tasks to run",
191 default=0, type="int")
192 result.add_option("-m", "--mode",
193 help="The test modes in which to run (comma-separated)",
194 default="release,debug")
195 result.add_option("--outdir", help="Base directory with compile output",
197 result.add_option("-p", "--progress",
198 help=("The style of progress indicator"
199 " (verbose, dots, color, mono)"),
200 choices=progress.PROGRESS_INDICATORS.keys(),
202 result.add_option("--shard-count",
203 help="Split testsuites into this number of shards",
204 default=1, type="int")
205 result.add_option("--shard-run",
206 help="Run this shard from the split up tests.",
207 default=1, type="int")
208 result.add_option("--shell-dir", help="Directory containing executables",
210 result.add_option("--seed", help="The seed for the random distribution",
212 result.add_option("-t", "--timeout", help="Timeout in seconds",
213 default= -1, type="int")
214 result.add_option("-v", "--verbose", help="Verbose output",
215 default=False, action="store_true")
219 def ProcessOptions(options):
222 # Architecture and mode related stuff.
223 if options.arch_and_mode:
224 tokens = options.arch_and_mode.split(".")
225 options.arch = tokens[0]
226 options.mode = tokens[1]
227 options.mode = options.mode.split(",")
228 for mode in options.mode:
229 if not mode.lower() in ["debug", "release"]:
230 print "Unknown mode %s" % mode
232 if options.arch in ["auto", "native"]:
233 options.arch = ARCH_GUESS
234 options.arch = options.arch.split(",")
235 for arch in options.arch:
236 if not arch in SUPPORTED_ARCHS:
237 print "Unknown architecture %s" % arch
240 # Special processing of other options, sorted alphabetically.
241 options.command_prefix = shlex.split(options.command_prefix)
242 options.extra_flags = shlex.split(options.extra_flags)
244 options.j = multiprocessing.cpu_count()
245 if not options.distribution_mode in DISTRIBUTION_MODES:
246 print "Unknown distribution mode %s" % options.distribution_mode
248 if options.distribution_factor1 < 0.0:
249 print ("Distribution factor1 %s is out of range. Defaulting to 0.0"
250 % options.distribution_factor1)
251 options.distribution_factor1 = 0.0
252 if options.distribution_factor2 < 0.0:
253 print ("Distribution factor2 %s is out of range. Defaulting to 0.0"
254 % options.distribution_factor2)
255 options.distribution_factor2 = 0.0
256 if options.coverage < 0.0 or options.coverage > 1.0:
257 print ("Coverage %s is out of range. Defaulting to 0.4"
259 options.coverage = 0.4
260 if options.coverage_lift < 0:
261 print ("Coverage lift %s is out of range. Defaulting to 0"
262 % options.coverage_lift)
263 options.coverage_lift = 0
267 def ShardTests(tests, shard_count, shard_run):
270 if shard_run < 1 or shard_run > shard_count:
271 print "shard-run not a valid number, should be in [1:shard-count]"
272 print "defaulting back to running all tests"
277 if count % shard_count == shard_run - 1:
284 parser = BuildOptions()
285 (options, args) = parser.parse_args()
286 if not ProcessOptions(options):
291 workspace = os.path.abspath(join(os.path.dirname(sys.argv[0]), ".."))
293 suite_paths = utils.GetSuitePaths(join(workspace, "test"))
296 suite_paths = [ s for s in suite_paths if s in DEFAULT_TESTS ]
300 suite = arg.split(os.path.sep)[0]
301 if not suite in args_suites:
302 args_suites.add(suite)
303 suite_paths = [ s for s in suite_paths if s in args_suites ]
306 for root in suite_paths:
307 suite = testsuite.TestSuite.LoadTestSuite(
308 os.path.join(workspace, "test", root))
312 if options.download_data:
316 for mode in options.mode:
317 for arch in options.arch:
318 code = Execute(arch, mode, args, options, suites, workspace)
319 exit_code = exit_code or code
323 def CalculateNTests(m, options):
324 """Calculates the number of tests from m deopt points with exponential
326 The coverage is expected to be between 0.0 and 1.0.
327 The 'coverage lift' lifts the coverage for tests with smaller m values.
329 c = float(options.coverage)
330 l = float(options.coverage_lift)
331 return int(math.pow(m, (m * c + l) / (m + l)))
334 def Execute(arch, mode, args, options, suites, workspace):
335 print(">>> Running tests for %s.%s" % (arch, mode))
337 dist = Distribution(options)
339 shell_dir = options.shell_dir
342 shell_dir = os.path.join(workspace, options.outdir, mode)
345 shell_dir = os.path.join(workspace, options.outdir,
346 "%s.%s" % (arch, mode))
347 shell_dir = os.path.relpath(shell_dir)
349 # Populate context object.
350 mode_flags = MODE_FLAGS[mode]
351 timeout = options.timeout
353 # Simulators are slow, therefore allow a longer default timeout.
354 if arch in SLOW_ARCHS:
355 timeout = 2 * TIMEOUT_DEFAULT;
357 timeout = TIMEOUT_DEFAULT;
359 timeout *= TIMEOUT_SCALEFACTOR[mode]
360 ctx = context.Context(arch, mode, shell_dir,
361 mode_flags, options.verbose,
362 timeout, options.isolates,
363 options.command_prefix,
367 # Find available test suites and read test cases from them.
370 "asan": options.asan,
371 "deopt_fuzzer": True,
373 "isolates": options.isolates,
376 "simulator": utils.UseSimulator(arch),
377 "system": utils.GuessOS(),
383 # Remember test case prototypes for the fuzzing phase.
384 test_backup = dict((s, []) for s in suites)
387 s.ReadStatusFile(variables)
390 s.FilterTestCasesByArgs(args)
392 s.FilterTestCasesByStatus(False)
393 test_backup[s] = s.tests
394 analysis_flags = ["--deopt-every-n-times", "%d" % MAX_DEOPT,
395 "--print-deopt-stress"]
396 s.tests = [ t.CopyAddingFlags(analysis_flags) for t in s.tests ]
397 num_tests += len(s.tests)
403 print "No tests to run."
407 print(">>> Collection phase")
408 progress_indicator = progress.PROGRESS_INDICATORS[options.progress]()
409 runner = execution.Runner(suites, progress_indicator, ctx)
411 exit_code = runner.Run(options.j)
415 except KeyboardInterrupt:
418 print(">>> Analysis phase")
424 for line in t.output.stdout.splitlines():
425 if line.startswith("=== Stress deopt counter: "):
426 test_results[t.path] = MAX_DEOPT - int(line.split(" ")[-1])
428 if t.path not in test_results:
429 print "Missing results for %s" % t.path
430 if options.dump_results_file:
431 results_dict = dict((t.path, n) for (t, n) in test_results.iteritems())
432 with file("%s.%d.txt" % (dump_results_file, time.time()), "w") as f:
433 f.write(json.dumps(results_dict))
435 # Reset tests and redistribute the prototypes from the collection phase.
438 print "Test distributions:"
439 for t in test_backup[s]:
440 max_deopt = test_results.get(t.path, 0)
443 n_deopt = CalculateNTests(max_deopt, options)
444 distribution = dist.Distribute(n_deopt, max_deopt)
446 print "%s %s" % (t.path, distribution)
447 for i in distribution:
448 fuzzing_flags = ["--deopt-every-n-times", "%d" % i]
449 s.tests.append(t.CopyAddingFlags(fuzzing_flags))
450 num_tests += len(s.tests)
456 print "No tests to run."
460 print(">>> Deopt fuzzing phase (%d test cases)" % num_tests)
461 progress_indicator = progress.PROGRESS_INDICATORS[options.progress]()
462 runner = execution.Runner(suites, progress_indicator, ctx)
464 exit_code = runner.Run(options.j)
468 except KeyboardInterrupt:
474 if __name__ == "__main__":