options.random_seed,
True, # No sorting of test cases.
0, # Don't rerun failing tests.
- 0) # No use of a rerun-failing-tests maximum.
+ 0, # No use of a rerun-failing-tests maximum.
+ False) # No predictable mode.
# Find available test suites and read test cases from them.
variables = {
help="Comma-separated list of testing variants")
result.add_option("--outdir", help="Base directory with compile output",
default="out")
+ result.add_option("--predictable",
+ help="Compare output of several reruns of each test",
+ default=False, action="store_true")
result.add_option("-p", "--progress",
help=("The style of progress indicator"
" (verbose, dots, color, mono)"),
options.flaky_tests = "skip"
options.slow_tests = "skip"
options.pass_fail_tests = "skip"
+ if options.predictable:
+ VARIANTS = ["default"]
+ options.extra_flags.append("--predictable")
+ options.extra_flags.append("--verify_predictable")
+ options.extra_flags.append("--no-inline-new")
if not options.shell_dir:
if options.shell:
timeout = TIMEOUT_DEFAULT;
timeout *= TIMEOUT_SCALEFACTOR[mode]
+
+ if options.predictable:
+ # Predictable mode is slower.
+ timeout *= 2
+
ctx = context.Context(arch, mode, shell_dir,
mode_flags, options.verbose,
timeout, options.isolates,
options.random_seed,
options.no_sorting,
options.rerun_failures_count,
- options.rerun_failures_max)
+ options.rerun_failures_max,
+ options.predictable)
# TODO(all): Combine "simulator" and "simulator_run".
simulator_run = not options.dont_skip_simulator_slow_tests and \
pool.add([self._GetJob(test)])
self.remaining += 1
+ def _ProcessTestNormal(self, test, result, pool):
+ self.indicator.AboutToRun(test)
+ test.output = result[1]
+ test.duration = result[2]
+ has_unexpected_output = test.suite.HasUnexpectedOutput(test)
+ if has_unexpected_output:
+ self.failed.append(test)
+ if test.output.HasCrashed():
+ self.crashed += 1
+ else:
+ self.succeeded += 1
+ self.remaining -= 1
+ self.indicator.HasRun(test, has_unexpected_output)
+ if has_unexpected_output:
+ # Rerun test failures after the indicator has processed the results.
+ self._MaybeRerun(pool, test)
+ # Update the perf database if the test succeeded.
+ return not has_unexpected_output
+
+ def _ProcessTestPredictable(self, test, result, pool):
+ # Always pass the test duration for the database update.
+ test.duration = result[2]
+ if test.run == 1 and result[1].HasTimedOut():
+ # If we get a timeout in the first run, we are already in an
+ # unpredictable state. Just report it as a failure and don't rerun.
+ self.indicator.AboutToRun(test)
+ test.output = result[1]
+ self.remaining -= 1
+ self.failed.append(test)
+ self.indicator.HasRun(test, True)
+ if test.run > 1 and test.output != result[1]:
+ # From the second run on, check for differences. If a difference is
+ # found, call the indicator twice to report both tests. All runs of each
+ # test are counted as one for the statistic.
+ self.indicator.AboutToRun(test)
+ self.remaining -= 1
+ self.failed.append(test)
+ self.indicator.HasRun(test, True)
+ self.indicator.AboutToRun(test)
+ test.output = result[1]
+ self.indicator.HasRun(test, True)
+ elif test.run >= 3:
+ # No difference on the third run -> report a success.
+ self.indicator.AboutToRun(test)
+ self.remaining -= 1
+ self.succeeded += 1
+ test.output = result[1]
+ self.indicator.HasRun(test, False)
+ else:
+ # No difference yet and less than three runs -> add another run and
+ # remember the output for comparison.
+ test.run += 1
+ test.output = result[1]
+ pool.add([self._GetJob(test)])
+ # Always update the perf database.
+ return True
+
def Run(self, jobs):
self.indicator.Starting()
self._RunInternal(jobs)
it = pool.imap_unordered(RunTest, queue)
for result in it:
test = test_map[result[0]]
- self.indicator.AboutToRun(test)
- test.output = result[1]
- test.duration = result[2]
- has_unexpected_output = test.suite.HasUnexpectedOutput(test)
- if has_unexpected_output:
- self.failed.append(test)
- if test.output.HasCrashed():
- self.crashed += 1
+ if self.context.predictable:
+ update_perf = self._ProcessTestPredictable(test, result, pool)
else:
+ update_perf = self._ProcessTestNormal(test, result, pool)
+ if update_perf:
self._RunPerfSafe(lambda: self.perfdata.UpdatePerfData(test))
- self.succeeded += 1
- self.remaining -= 1
- self.indicator.HasRun(test, has_unexpected_output)
- if has_unexpected_output:
- # Rerun test failures after the indicator has processed the results.
- self._MaybeRerun(pool, test)
finally:
pool.terminate()
self._RunPerfSafe(lambda: self.perf_data_manager.close())
class Context():
def __init__(self, arch, mode, shell_dir, mode_flags, verbose, timeout,
isolates, command_prefix, extra_flags, noi18n, random_seed,
- no_sorting, rerun_failures_count, rerun_failures_max):
+ no_sorting, rerun_failures_count, rerun_failures_max,
+ predictable):
self.arch = arch
self.mode = mode
self.shell_dir = shell_dir
self.no_sorting = no_sorting
self.rerun_failures_count = rerun_failures_count
self.rerun_failures_max = rerun_failures_max
+ self.predictable = predictable
def Pack(self):
return [self.arch, self.mode, self.mode_flags, self.timeout, self.isolates,
self.command_prefix, self.extra_flags, self.noi18n,
self.random_seed, self.no_sorting, self.rerun_failures_count,
- self.rerun_failures_max]
+ self.rerun_failures_max, self.predictable]
@staticmethod
def Unpack(packed):
# For the order of the fields, refer to Pack() above.
return Context(packed[0], packed[1], None, packed[2], False,
packed[3], packed[4], packed[5], packed[6], packed[7],
- packed[8], packed[9], packed[10], packed[11])
+ packed[8], packed[9], packed[10], packed[11], packed[12])
self.stdout = stdout
self.stderr = stderr
+ def __ne__(self, other):
+ return (self.exit_code != other.exit_code or
+ self.timed_out != other.timed_out or
+ self.stdout != other.stdout or
+ self.stderr != other.stderr)
+
def HasCrashed(self):
if utils.IsWindows():
return 0x80000000 & self.exit_code and not (0x3FFFFF00 & self.exit_code)
projects / platform / upstream / v8.git / commitdiff