import subprocess
import tarfile
+from testrunner.local import statusfile
from testrunner.local import testsuite
from testrunner.objects import testcase
+class BenchmarksVariantGenerator(testsuite.VariantGenerator):
+ # Both --nocrankshaft and --stressopt are very slow. Add TF but without
+ # always opt to match the way the benchmarks are run for performance
+ # testing.
+ def FilterVariantsByTest(self, testcase):
+ if testcase.outcomes and statusfile.OnlyStandardVariant(
+ testcase.outcomes):
+ return self.standard_variant
+ return self.fast_variants
+
+ def GetFlagSets(self, testcase, variant):
+ return testsuite.FAST_VARIANT_FLAGS[variant]
+
+
class BenchmarksTestSuite(testsuite.TestSuite):
def __init__(self, name, root):
os.chdir(old_cwd)
- def VariantFlags(self, testcase, default_flags):
- # Both --nocrankshaft and --stressopt are very slow. Add TF but without
- # always opt to match the way the benchmarks are run for performance
- # testing.
- return [[], ["--turbo"]]
+ def _VariantGeneratorFactory(self):
+ return BenchmarksVariantGenerator
def GetSuite(name, root):
with open(testcase.flags[0]) as f:
return f.read()
- def VariantFlags(self, testcase, default_flags):
- return [[]];
+ def _VariantGeneratorFactory(self):
+ return testsuite.StandardVariantGenerator
def GetSuite(name, root):
TEST_262_HARNESS_PATH = ["data", "harness"]
TEST_262_TOOLS_PATH = ["data", "tools", "packaging"]
+ALL_VARIANT_FLAGS_STRICT = dict(
+ (v, [flags + ["--use-strict"] for flags in flag_sets])
+ for v, flag_sets in testsuite.ALL_VARIANT_FLAGS.iteritems()
+)
+
+FAST_VARIANT_FLAGS_STRICT = dict(
+ (v, [flags + ["--use-strict"] for flags in flag_sets])
+ for v, flag_sets in testsuite.FAST_VARIANT_FLAGS.iteritems()
+)
+
+ALL_VARIANT_FLAGS_BOTH = dict(
+ (v, [flags for flags in testsuite.ALL_VARIANT_FLAGS[v] +
+ ALL_VARIANT_FLAGS_STRICT[v]])
+ for v in testsuite.ALL_VARIANT_FLAGS
+)
+
+FAST_VARIANT_FLAGS_BOTH = dict(
+ (v, [flags for flags in testsuite.FAST_VARIANT_FLAGS[v] +
+ FAST_VARIANT_FLAGS_STRICT[v]])
+ for v in testsuite.FAST_VARIANT_FLAGS
+)
+
+ALL_VARIANTS = {
+ 'nostrict': testsuite.ALL_VARIANT_FLAGS,
+ 'strict': ALL_VARIANT_FLAGS_STRICT,
+ 'both': ALL_VARIANT_FLAGS_BOTH,
+}
+
+FAST_VARIANTS = {
+ 'nostrict': testsuite.FAST_VARIANT_FLAGS,
+ 'strict': FAST_VARIANT_FLAGS_STRICT,
+ 'both': FAST_VARIANT_FLAGS_BOTH,
+}
+
+class Test262VariantGenerator(testsuite.VariantGenerator):
+ def GetFlagSets(self, testcase, variant):
+ if testcase.outcomes and statusfile.OnlyFastVariants(testcase.outcomes):
+ variant_flags = FAST_VARIANTS
+ else:
+ variant_flags = ALL_VARIANTS
+
+ test_record = self.suite.GetTestRecord(testcase)
+ if "noStrict" in test_record:
+ return variant_flags["nostrict"][variant]
+ if "onlyStrict" in test_record:
+ return variant_flags["strict"][variant]
+ return variant_flags["both"][variant]
+
+
class Test262TestSuite(testsuite.TestSuite):
def __init__(self, name, root):
self.GetIncludesForTest(testcase) + ["--harmony"] +
[os.path.join(self.testroot, testcase.path + ".js")])
- def VariantFlags(self, testcase, default_flags):
- flags = super(Test262TestSuite, self).VariantFlags(testcase, default_flags)
- test_record = self.GetTestRecord(testcase)
- if "noStrict" in test_record:
- return flags
- strict_flags = [f + ["--use-strict"] for f in flags]
- if "onlyStrict" in test_record:
- return strict_flags
- return flags + strict_flags
+ def _VariantGeneratorFactory(self):
+ return Test262VariantGenerator
def LoadParseTestRecord(self):
if not self.ParseTestRecord:
from testrunner.local import execution
from testrunner.local import progress
from testrunner.local import testsuite
-from testrunner.local.testsuite import VARIANT_FLAGS
+from testrunner.local.testsuite import ALL_VARIANTS
from testrunner.local import utils
from testrunner.local import verbose
from testrunner.network import network_execution
return mode.lower()
def ProcessOptions(options):
- global VARIANT_FLAGS
+ global ALL_VARIANTS
global VARIANTS
# Architecture and mode related stuff.
VARIANTS = ["stress"]
if options.variants:
VARIANTS = options.variants.split(",")
- if not set(VARIANTS).issubset(VARIANT_FLAGS.keys()):
- print "All variants must be in %s" % str(VARIANT_FLAGS.keys())
+ if not set(VARIANTS).issubset(ALL_VARIANTS.keys()):
+ print "All variants must be in %s" % str(ALL_VARIANTS.keys())
return False
if options.predictable:
VARIANTS = ["default"]
if options.cat:
verbose.PrintTestSource(s.tests)
continue
- variant_flags = [VARIANT_FLAGS[var] for var in VARIANTS]
- variant_tests = [ t.CopyAddingFlags(v)
+ variant_gen = s.CreateVariantGenerator(VARIANTS)
+ variant_tests = [ t.CopyAddingFlags(flags)
for t in s.tests
- for v in s.VariantFlags(t, variant_flags) ]
+ for v in variant_gen.FilterVariantsByTest(t)
+ for flags in variant_gen.GetFlagSets(t, v) ]
if options.random_seed_stress_count > 1:
# Duplicate test for random seed stress mode.
else:
yield ["--random-seed=%d" % RandomSeed()]
s.tests = [
- t.CopyAddingFlags(v)
+ t.CopyAddingFlags(flags)
for t in variant_tests
- for v in iter_seed_flags()
+ for flags in iter_seed_flags()
]
else:
s.tests = variant_tests
from ..objects import testcase
# Use this to run several variants of the tests.
-VARIANT_FLAGS = {
- "default": [],
- "stress": ["--stress-opt", "--always-opt"],
- "turbofan": ["--turbo", "--always-opt"],
- "nocrankshaft": ["--nocrankshaft"]}
+ALL_VARIANT_FLAGS = {
+ "default": [[]],
+ "stress": [["--stress-opt", "--always-opt"]],
+ "turbofan": [["--turbo", "--always-opt"]],
+ "nocrankshaft": [["--nocrankshaft"]],
+}
+
+# FAST_VARIANTS implies no --always-opt.
+FAST_VARIANT_FLAGS = {
+ "default": [[]],
+ "stress": [["--stress-opt"]],
+ "turbofan": [["--turbo"]],
+ "nocrankshaft": [["--nocrankshaft"]],
+}
+
+ALL_VARIANTS = set(["default", "stress", "turbofan", "nocrankshaft"])
+FAST_VARIANTS = set(["default", "turbofan"])
+STANDARD_VARIANT = set(["default"])
+
+
+class VariantGenerator(object):
+ def __init__(self, suite, variants):
+ self.suite = suite
+ self.all_variants = ALL_VARIANTS & variants
+ self.fast_variants = FAST_VARIANTS & variants
+ self.standard_variant = STANDARD_VARIANT & variants
+
+ def FilterVariantsByTest(self, testcase):
+ if testcase.outcomes and statusfile.OnlyStandardVariant(
+ testcase.outcomes):
+ return self.standard_variant
+ if testcase.outcomes and statusfile.OnlyFastVariants(testcase.outcomes):
+ return self.fast_variants
+ return self.all_variants
+
+ def GetFlagSets(self, testcase, variant):
+ if testcase.outcomes and statusfile.OnlyFastVariants(testcase.outcomes):
+ return FAST_VARIANT_FLAGS[variant]
+ else:
+ return ALL_VARIANT_FLAGS[variant]
-FAST_VARIANT_FLAGS = [
- f for v, f in VARIANT_FLAGS.iteritems() if v in ["default", "turbofan"]
-]
class TestSuite(object):
def ListTests(self, context):
raise NotImplementedError
- def VariantFlags(self, testcase, default_flags):
- if testcase.outcomes and statusfile.OnlyStandardVariant(testcase.outcomes):
- return [[]]
- if testcase.outcomes and statusfile.OnlyFastVariants(testcase.outcomes):
- # FAST_VARIANTS implies no --always-opt.
- return [ filter(lambda flag: flag != "--always-opt", f)
- for f in filter(lambda flags: flags in FAST_VARIANT_FLAGS,
- default_flags) ]
- return default_flags
+ def _VariantGeneratorFactory(self):
+ """The variant generator class to be used."""
+ return VariantGenerator
+
+ def CreateVariantGenerator(self, variants):
+ """Return a generator for the testing variants of this suite.
+
+ Args:
+ variants: List of variant names to be run as specified by the test
+ runner.
+ Returns: An object of type VariantGenerator.
+ """
+ return self._VariantGeneratorFactory()(self, set(variants))
def DownloadData(self):
pass
return self.total_duration
+class StandardVariantGenerator(VariantGenerator):
+ def FilterVariantsByTest(self, testcase):
+ return self.standard_variant
+
+
class GoogleTestSuite(TestSuite):
def __init__(self, name, root):
super(GoogleTestSuite, self).__init__(name, root)
["--gtest_print_time=0"] +
context.mode_flags)
- def VariantFlags(self, testcase, default_flags):
- return [[]]
+ def _VariantGeneratorFactory(self):
+ return StandardVariantGenerator
def shell(self):
return self.name