From 3543dd53a44f02829dd64250f13f472e21944cee Mon Sep 17 00:00:00 2001 From: "ricow@chromium.org" Date: Tue, 24 Aug 2010 13:34:59 +0000 Subject: [PATCH] Refactor the tools/test.py script and related testcfg.py files. This makes it possible to run several variants of the tests (with different flags that is) by adding extra lists to the VARIANT_FLAGS list. In addition, there is a number of smaller refactorings. Review URL: http://codereview.chromium.org/3164023 git-svn-id: http://v8.googlecode.com/svn/branches/bleeding_edge@5329 ce2b1a6d-e550-0410-aec6-3dcde31c8c00 --- test/cctest/testcfg.py | 8 ++--- test/es5conform/testcfg.py | 5 ++-- test/message/testcfg.py | 5 ++-- test/mjsunit/testcfg.py | 12 +++----- test/mozilla/testcfg.py | 7 ++--- test/sputnik/testcfg.py | 10 +++---- tools/test.py | 73 +++++++++++++++++++++++++++++++++++++--------- 7 files changed, 77 insertions(+), 43 deletions(-) diff --git a/test/cctest/testcfg.py b/test/cctest/testcfg.py index c2427c8..485f2cf 100644 --- a/test/cctest/testcfg.py +++ b/test/cctest/testcfg.py @@ -31,15 +31,12 @@ from os.path import join, dirname, exists import platform import utils -CCTEST_DEBUG_FLAGS = ['--enable-slow-asserts', '--debug-code', '--verify-heap'] - class CcTestCase(test.TestCase): def __init__(self, path, executable, mode, raw_name, dependency, context): - super(CcTestCase, self).__init__(context, path) + super(CcTestCase, self).__init__(context, path, mode) self.executable = executable - self.mode = mode self.raw_name = raw_name self.dependency = dependency @@ -54,8 +51,7 @@ class CcTestCase(test.TestCase): serialization_file += '_' + self.GetName() serialization_option = '--testing_serialization_file=' + serialization_file result = [ self.executable, name, serialization_option ] - if self.mode == 'debug': - result += CCTEST_DEBUG_FLAGS + result += self.context.GetVmFlags(self, self.mode) return result def GetCommand(self): diff --git a/test/es5conform/testcfg.py b/test/es5conform/testcfg.py index d1f23aa..43d6104 100644 --- a/test/es5conform/testcfg.py +++ b/test/es5conform/testcfg.py @@ -37,9 +37,8 @@ HARNESS_FILES = ['sth.js'] class ES5ConformTestCase(test.TestCase): def __init__(self, filename, path, context, root, mode, framework): - super(ES5ConformTestCase, self).__init__(context, path) + super(ES5ConformTestCase, self).__init__(context, path, mode) self.filename = filename - self.mode = mode self.framework = framework self.root = root @@ -55,7 +54,7 @@ class ES5ConformTestCase(test.TestCase): return 'FAILED!' in output.stdout def GetCommand(self): - result = [self.context.GetVm(self.mode)] + result = self.context.GetVmCommand(self, self.mode) result += ['-e', 'var window = this'] result += self.framework result.append(self.filename) diff --git a/test/message/testcfg.py b/test/message/testcfg.py index 6004282..7dae047 100644 --- a/test/message/testcfg.py +++ b/test/message/testcfg.py @@ -35,11 +35,10 @@ FLAGS_PATTERN = re.compile(r"//\s+Flags:(.*)") class MessageTestCase(test.TestCase): def __init__(self, path, file, expected, mode, context, config): - super(MessageTestCase, self).__init__(context, path) + super(MessageTestCase, self).__init__(context, path, mode) self.file = file self.expected = expected self.config = config - self.mode = mode def IgnoreLine(self, str): """Ignore empty lines and valgrind output.""" @@ -79,7 +78,7 @@ class MessageTestCase(test.TestCase): return self.path[-1] def GetCommand(self): - result = [self.config.context.GetVm(self.mode)] + result = self.config.context.GetVmCommand(self, self.mode) source = open(self.file).read() flags_match = FLAGS_PATTERN.search(source) if flags_match: diff --git a/test/mjsunit/testcfg.py b/test/mjsunit/testcfg.py index 49064b1..d8fe24d 100644 --- a/test/mjsunit/testcfg.py +++ b/test/mjsunit/testcfg.py @@ -31,7 +31,6 @@ from os.path import join, dirname, exists import re import tempfile -MJSUNIT_DEBUG_FLAGS = ['--enable-slow-asserts', '--debug-code', '--verify-heap'] FLAGS_PATTERN = re.compile(r"//\s+Flags:(.*)") FILES_PATTERN = re.compile(r"//\s+Files:(.*)") SELF_SCRIPT_PATTERN = re.compile(r"//\s+Env: TEST_FILE_NAME") @@ -40,10 +39,9 @@ SELF_SCRIPT_PATTERN = re.compile(r"//\s+Env: TEST_FILE_NAME") class MjsunitTestCase(test.TestCase): def __init__(self, path, file, mode, context, config): - super(MjsunitTestCase, self).__init__(context, path) + super(MjsunitTestCase, self).__init__(context, path, mode) self.file = file self.config = config - self.mode = mode self.self_script = False def GetLabel(self): @@ -53,13 +51,11 @@ class MjsunitTestCase(test.TestCase): return self.path[-1] def GetCommand(self): - result = [self.config.context.GetVm(self.mode)] + result = self.config.context.GetVmCommand(self, self.mode) source = open(self.file).read() flags_match = FLAGS_PATTERN.search(source) if flags_match: result += flags_match.group(1).strip().split() - if self.mode == 'debug': - result += MJSUNIT_DEBUG_FLAGS additional_files = [] files_match = FILES_PATTERN.search(source); # Accept several lines of 'Files:' @@ -94,8 +90,8 @@ class MjsunitTestCase(test.TestCase): self.self_script = self_script return self_script - def Cleanup(self): - if self.self_script: + def AfterRun(self, result): + if self.self_script and (not result.HasPreciousOutput()): test.CheckedUnlink(self.self_script) class MjsunitTestConfiguration(test.TestConfiguration): diff --git a/test/mozilla/testcfg.py b/test/mozilla/testcfg.py index d1c1767..7a6438f 100644 --- a/test/mozilla/testcfg.py +++ b/test/mozilla/testcfg.py @@ -57,9 +57,8 @@ TEST_DIRS = """ class MozillaTestCase(test.TestCase): def __init__(self, filename, path, context, root, mode, framework): - super(MozillaTestCase, self).__init__(context, path) + super(MozillaTestCase, self).__init__(context, path, mode) self.filename = filename - self.mode = mode self.framework = framework self.root = root @@ -75,8 +74,8 @@ class MozillaTestCase(test.TestCase): return 'FAILED!' in output.stdout def GetCommand(self): - result = [self.context.GetVm(self.mode), '--expose-gc', - join(self.root, 'mozilla-shell-emulation.js')] + result = self.context.GetVmCommand(self, self.mode) + \ + [ '--expose-gc', join(self.root, 'mozilla-shell-emulation.js') ] result += self.framework result.append(self.filename) return result diff --git a/test/sputnik/testcfg.py b/test/sputnik/testcfg.py index 6592382..f7a5edc 100644 --- a/test/sputnik/testcfg.py +++ b/test/sputnik/testcfg.py @@ -36,9 +36,8 @@ import time class SputnikTestCase(test.TestCase): def __init__(self, case, path, context, mode): - super(SputnikTestCase, self).__init__(context, path) + super(SputnikTestCase, self).__init__(context, path, mode) self.case = case - self.mode = mode self.tmpfile = None self.source = None @@ -56,12 +55,13 @@ class SputnikTestCase(test.TestCase): self.tmpfile.Write(self.GetSource()) self.tmpfile.Close() - def AfterRun(self): - self.tmpfile.Dispose() + def AfterRun(self, result): + # Dispose the temporary file if everything looks okay. + if not result.HasPreciousOutput(): self.tmpfile.Dispose() self.tmpfile = None def GetCommand(self): - result = [self.context.GetVm(self.mode)] + result = self.context.GetVmCommand(self, self.mode) result.append(self.tmpfile.name) return result diff --git a/tools/test.py b/tools/test.py index f17e9b1..4b916f8 100755 --- a/tools/test.py +++ b/tools/test.py @@ -331,10 +331,11 @@ class CommandOutput(object): class TestCase(object): - def __init__(self, context, path): + def __init__(self, context, path, mode): self.path = path self.context = context self.duration = None + self.mode = mode def IsNegative(self): return False @@ -355,14 +356,19 @@ class TestCase(object): def RunCommand(self, command): full_command = self.context.processor(command) - output = Execute(full_command, self.context, self.context.timeout) + output = Execute(full_command, + self.context, + self.context.GetTimeout(self.mode)) self.Cleanup() - return TestOutput(self, full_command, output) + return TestOutput(self, + full_command, + output, + self.context.store_unexpected_output) def BeforeRun(self): pass - def AfterRun(self): + def AfterRun(self, result): pass def Run(self): @@ -370,7 +376,7 @@ class TestCase(object): try: result = self.RunCommand(self.GetCommand()) finally: - self.AfterRun() + self.AfterRun(result) return result def Cleanup(self): @@ -379,10 +385,11 @@ class TestCase(object): class TestOutput(object): - def __init__(self, test, command, output): + def __init__(self, test, command, output, store_unexpected_output): self.test = test self.command = command self.output = output + self.store_unexpected_output = store_unexpected_output def UnexpectedOutput(self): if self.HasCrashed(): @@ -395,6 +402,9 @@ class TestOutput(object): outcome = PASS return not outcome in self.test.outcomes + def HasPreciousOutput(self): + return self.UnexpectedOutput() and self.store_unexpected_output + def HasCrashed(self): if utils.IsWindows(): return 0x80000000 & self.output.exit_code and not (0x3FFFFF00 & self.output.exit_code) @@ -557,6 +567,11 @@ class TestSuite(object): return self.name +# Use this to run several variants of the tests, e.g.: +# VARIANT_FLAGS = [[], ['--always_compact', '--noflush_code']] +VARIANT_FLAGS = [[]] + + class TestRepository(TestSuite): def __init__(self, path): @@ -583,8 +598,12 @@ class TestRepository(TestSuite): def GetBuildRequirements(self, path, context): return self.GetConfiguration(context).GetBuildRequirements() - def ListTests(self, current_path, path, context, mode): - return self.GetConfiguration(context).ListTests(current_path, path, mode) + def AddTestsToList(self, result, current_path, path, context, mode): + for v in VARIANT_FLAGS: + tests = self.GetConfiguration(context).ListTests(current_path, path, mode) + for t in tests: t.variant_flags = v + result += tests + def GetTestStatus(self, context, sections, defs): self.GetConfiguration(context).GetTestStatus(sections, defs) @@ -611,7 +630,7 @@ class LiteralTestSuite(TestSuite): test_name = test.GetName() if not name or name.match(test_name): full_path = current_path + [test_name] - result += test.ListTests(full_path, path, context, mode) + test.AddTestsToList(result, full_path, path, context, mode) return result def GetTestStatus(self, context, sections, defs): @@ -619,12 +638,20 @@ class LiteralTestSuite(TestSuite): test.GetTestStatus(context, sections, defs) -SUFFIX = {'debug': '_g', 'release': ''} +SUFFIX = { + 'debug' : '_g', + 'release' : '' } +FLAGS = { + 'debug' : ['--enable-slow-asserts', '--debug-code', '--verify-heap'], + 'release' : []} +TIMEOUT_SCALEFACTOR = { + 'debug' : 4, + 'release' : 1 } class Context(object): - def __init__(self, workspace, buildspace, verbose, vm, timeout, processor, suppress_dialogs): + def __init__(self, workspace, buildspace, verbose, vm, timeout, processor, suppress_dialogs, store_unexpected_output): self.workspace = workspace self.buildspace = buildspace self.verbose = verbose @@ -632,6 +659,7 @@ class Context(object): self.timeout = timeout self.processor = processor self.suppress_dialogs = suppress_dialogs + self.store_unexpected_output = store_unexpected_output def GetVm(self, mode): name = self.vm_root + SUFFIX[mode] @@ -639,6 +667,15 @@ class Context(object): name = name + '.exe' return name + def GetVmCommand(self, testcase, mode): + return [self.GetVm(mode)] + self.GetVmFlags(testcase, mode) + + def GetVmFlags(self, testcase, mode): + return testcase.variant_flags + FLAGS[mode] + + def GetTimeout(self, mode): + return self.timeout * TIMEOUT_SCALEFACTOR[mode] + def RunTestCases(cases_to_run, progress, tasks): progress = PROGRESS_INDICATORS[progress](cases_to_run) return progress.Run(tasks) @@ -1121,7 +1158,13 @@ def BuildOptions(): dest="suppress_dialogs", default=True, action="store_true") result.add_option("--no-suppress-dialogs", help="Display Windows dialogs for crashing tests", dest="suppress_dialogs", action="store_false") - result.add_option("--shell", help="Path to V8 shell", default="shell"); + result.add_option("--shell", help="Path to V8 shell", default="shell") + result.add_option("--store-unexpected-output", + help="Store the temporary JS files from tests that fails", + dest="store_unexpected_output", default=True, action="store_true") + result.add_option("--no-store-unexpected-output", + help="Deletes the temporary JS files from tests that fails", + dest="store_unexpected_output", action="store_false") return result @@ -1258,11 +1301,13 @@ def Main(): shell = abspath(options.shell) buildspace = dirname(shell) + context = Context(workspace, buildspace, VERBOSE, shell, options.timeout, GetSpecialCommandProcessor(options.special_command), - options.suppress_dialogs) + options.suppress_dialogs, + options.store_unexpected_output) # First build the required targets if not options.no_build: reqs = [ ] @@ -1278,7 +1323,7 @@ def Main(): # Just return if we are only building the targets for running the tests. if options.build_only: return 0 - + # Get status for tests sections = [ ] defs = { } -- 2.7.4