From 213bac1fd28bf8d7a8ba1b16177a254df00d573c Mon Sep 17 00:00:00 2001 From: "mstarzinger@chromium.org" Date: Fri, 7 Oct 2011 07:30:45 +0000 Subject: [PATCH] Update test harness and status file for Test262. R=rossberg@chromium.org TEST=test262 Review URL: http://codereview.chromium.org/8171005 git-svn-id: http://v8.googlecode.com/svn/branches/bleeding_edge@9543 ce2b1a6d-e550-0410-aec6-3dcde31c8c00 --- test/test262/README | 4 +- test/test262/test262.status | 355 +++++++++++++++++++++++++++++++------------- test/test262/testcfg.py | 34 ++--- 3 files changed, 259 insertions(+), 134 deletions(-) diff --git a/test/test262/README b/test/test262/README index ea6b4a7..094356f 100644 --- a/test/test262/README +++ b/test/test262/README @@ -4,11 +4,11 @@ tests from http://hg.ecmascript.org/tests/test262 -at revision 128 as 'data' in this directory. Using later version +at revision 271 as 'data' in this directory. Using later version may be possible but the tests are only known to pass (and indeed run) with that revision. -hg clone -r 128 http://hg.ecmascript.org/tests/test262 data +hg clone -r 271 http://hg.ecmascript.org/tests/test262 data If you do update to a newer revision you may have to change the test harness adapter code since it uses internal functionality from the diff --git a/test/test262/test262.status b/test/test262/test262.status index 8cee210..4c58e00 100644 --- a/test/test262/test262.status +++ b/test/test262/test262.status @@ -25,98 +25,244 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +prefix test262 +def FAIL_OK = FAIL, OKAY -# -# ietestcenter tests. -# +############################### BUGS ################################### -prefix ietestcenter +# A bound function should fail on access to 'caller' and 'arguments'. +S15.3.4.5_A1: FAIL +S15.3.4.5_A2: FAIL +# '__proto__' should be treated as a normal property in JSON. +S15.12.2_A1: FAIL -# -# Deliberate differences for compatibility with other browsers -# -# 15.9.5.43-0-9 and 15.9.5.43-0-10. V8 doesn't throw RangeError -# from Date.prototype.toISOString when string is not a finite number. -# This is compatible with Firefox and Safari. -15.9.5.43-0-9: PASS || FAIL -15.9.5.43-0-10: PASS || FAIL +# V8 Bug: http://code.google.com/p/v8/issues/detail?id=1196 +S8.7_A5_T2: FAIL -# -# Unanalyzed failures which may be bugs or deliberate differences -# +# V8 Bug: http://code.google.com/p/v8/issues/detail?id=1624 +S10.4.2.1_A1: FAIL -# Bug? Strict Mode - TypeError is thrown when changing the value of a Value -# Property of the Global Object under strict mode (NaN) -10.2.1.1.3-4-16-s: FAIL -# Bug? Strict Mode - TypeError is thrown when changing the value of a Value -# Property of the Global Object under strict mode (undefined) -10.2.1.1.3-4-18-s: FAIL -# Invalid test: https://bugs.ecmascript.org/show_bug.cgi?id=76 -10.4.2-2-c-1: FAIL -# BUG: 11.8.2 Greater-than Operator - Partial left to right order enforced -# when using Greater-than operator: valueOf > valueOf +# V8 Bug: http://code.google.com/p/v8/issues/detail?id=1752 +S11.8.2_A2.3_T1: FAIL +S11.8.3_A2.3_T1: FAIL 11.8.2-1: FAIL -# BUG: 11.8.2 Greater-than Operator - Partial left to right order enforced -# when using Greater-than operator: valueOf > toString 11.8.2-2: FAIL -# BUG: 11.8.2 Greater-than Operator - Partial left to right order enforced -# when using Greater-than operator: toString > valueOf 11.8.2-3: FAIL -# BUG: 11.8.2 Greater-than Operator - Partial left to right order enforced -# when using Greater-than operator: toString > toString 11.8.2-4: FAIL -# BUG: 11.8.3 Less-than-or-equal Operator - Partial left to right order -# enforced when using Less-than-or-equal operator: valueOf <= valueOf 11.8.3-1: FAIL -# BUG: 11.8.3 Less-than-or-equal Operator - Partial left to right order -# enforced when using Less-than-or-equal operator: valueOf <= toString 11.8.3-2: FAIL -# BUG: 11.8.3 Less-than-or-equal Operator - Partial left to right order -# enforced when using Less-than-or-equal operator: toString <= valueOf 11.8.3-3: FAIL -# BUG: 11.8.3 Less-than-or-equal Operator - Partial left to right order -# enforced when using Less-than-or-equal operator: toString <= toString 11.8.3-4: FAIL -# BUG: 11.8.3 Less-than-or-equal Operator - Partial left to right order -# enforced when using Less-than-or-equal operator: valueOf <= valueOf 11.8.3-5: FAIL -# Bug? simple assignment throws TypeError if LeftHandSide is a readonly property -# in strict mode (Global.undefined) -11.13.1-4-27-s: FAIL -# Bug? simple assignment throws TypeError if LeftHandSide is a readonly property -# in strict mode (Global.Infinity) -11.13.1-4-3-s: FAIL -# BUG: Global.NaN is a data property with default attribute values -15.1.1.1-0: FAIL -# BUG: Global.Infinity is a data property with default attribute values -15.1.1.2-0: FAIL -# BUG: Global.undefined is a data property with default attribute values -15.1.1.3-0: FAIL -# BUG: Object.getOwnPropertyDescriptor returns data desc (all false) -# for properties on built-ins (Global.NaN) -15.2.3.3-4-178: FAIL -# BUG: Object.getOwnPropertyDescriptor returns data desc (all false) -# for properties on built-ins (Global.Infinity) -15.2.3.3-4-179: FAIL -# BUG: Object.getOwnPropertyDescriptor returns data desc (all false) -# for properties on built-ins (Global.undefined) -15.2.3.3-4-180: FAIL -# BUG: Object.getOwnPropertyDescriptor returns data desc (all false) -# for properties on built-ins (RegExp.prototype.source) -# There is no RegExp.prototype.source -15.2.3.3-4-212: FAIL -# BUG: Object.getOwnPropertyDescriptor returns data desc (all false) -# for properties on built-ins (RegExp.prototype.global) -# There is no RegExp.prototype.global -15.2.3.3-4-213: FAIL -# BUG: Object.getOwnPropertyDescriptor returns data desc (all false) -# for properties on built-ins (RegExp.prototype.ignoreCase) -# There is no RegExp.prototype.ignoreCase -15.2.3.3-4-214: FAIL -# BUG: Object.getOwnPropertyDescriptor returns data desc (all false) -# for properties on built-ins (RegExp.prototype.multiline) -15.2.3.3-4-215: FAIL + +# V8 Bug. +S13.2.3_A1: FAIL + +# V8 Bug: http://code.google.com/p/v8/issues/detail?id=1530 +S15.3.3.1_A4: FAIL + +# Invalid test cases (recent change adding var changes semantics) +S8.3_A1_T1: FAIL +S15.3_A3_T1: FAIL +S15.3_A3_T3: FAIL + +##################### DELIBERATE INCOMPATIBILITIES ##################### + +# 15.9.5.43-0-9 and 15.9.5.43-0-10. V8 doesn't throw RangeError +# from Date.prototype.toISOString when string is not a finite number. +# This is compatible with Firefox and Safari. +15.9.5.43-0-9: PASS || FAIL +15.9.5.43-0-10: PASS || FAIL + +# We deliberately treat arguments to parseInt() with a leading zero as +# octal numbers in order to not break the web. +S15.1.2.2_A5.1_T1: FAIL_OK + +# This tests precision of trignometric functions. We're slightly off +# from the implementation in libc (~ 1e-17) but it's not clear if we +# or they are closer to the right answer, or if it even matters. +S15.8.2.16_A7: PASS || FAIL_OK +S15.8.2.18_A7: PASS || FAIL_OK +S15.8.2.13_A23: PASS || FAIL_OK + +# We allow calls to regexp exec() with no arguments to fail for +# compatibility reasons. +S15.10.6.2_A1_T16: FAIL_OK +S15.10.6.2_A12: FAIL_OK +S15.10.6.3_A1_T16: FAIL_OK + +# Sputnik tests (r97) assume RegExp.prototype is an Object, not a RegExp. +S15.10.6_A2: FAIL_OK + +# We are silent in some regexp cases where the spec wants us to give +# errors, for compatibility. +S15.10.2.11_A1_T2: FAIL +S15.10.2.11_A1_T3: FAIL + +# We are more lenient in which string character escapes we allow than +# the spec (7.8.4 p. 19) wants us to be. This is for compatibility. +S7.8.4_A4.3_T3: FAIL_OK +S7.8.4_A4.3_T4: FAIL_OK +S7.8.4_A4.3_T5: FAIL_OK +S7.8.4_A4.3_T6: FAIL_OK +S7.8.4_A6.1_T4: FAIL_OK +S7.8.4_A6.2_T1: FAIL_OK +S7.8.4_A6.2_T2: FAIL_OK +S7.8.4_A6.4_T1: FAIL_OK +S7.8.4_A6.4_T2: FAIL_OK +S7.8.4_A7.1_T4: FAIL_OK +S7.8.4_A7.2_T1: FAIL_OK +S7.8.4_A7.2_T2: FAIL_OK +S7.8.4_A7.2_T3: FAIL_OK +S7.8.4_A7.2_T4: FAIL_OK +S7.8.4_A7.2_T5: FAIL_OK +S7.8.4_A7.2_T6: FAIL_OK +S7.8.4_A7.4_T1: FAIL_OK +S7.8.4_A7.4_T2: FAIL_OK + +# Sputnik expects unicode escape sequences in RegExp flags to be interpreted. +# The specification requires them to be passed uninterpreted to the RegExp +# constructor. We now implement that. +S7.8.5_A3.1_T7: FAIL_OK +S7.8.5_A3.1_T8: FAIL_OK +S7.8.5_A3.1_T9: FAIL_OK + +# We allow some keywords to be used as identifiers. +S7.5.3_A1.15: FAIL_OK +S7.5.3_A1.18: FAIL_OK +S7.5.3_A1.21: FAIL_OK +S7.5.3_A1.22: FAIL_OK +S7.5.3_A1.23: FAIL_OK +S7.5.3_A1.24: FAIL_OK +S7.5.3_A1.26: FAIL_OK + +# This checks for non-262 behavior +S7.6_D1: PASS || FAIL_OK +S7.6_D2: PASS || FAIL_OK +S8.4_D1.1: PASS || FAIL_OK +S8.4_D2.1: PASS || FAIL_OK +S8.4_D2.2: PASS || FAIL_OK +S8.4_D2.3: PASS || FAIL_OK +S8.4_D2.4: PASS || FAIL_OK +S8.4_D2.5: PASS || FAIL_OK +S8.4_D2.6: PASS || FAIL_OK +S8.4_D2.7: PASS || FAIL_OK +S11.4.3_D1.2: PASS || FAIL_OK +S12.6.4_A14_T1: PASS || FAIL_OK +S12.6.4_D1: PASS || FAIL_OK +S12.6.4_R1: PASS || FAIL_OK +S12.6.4_R2: PASS || FAIL_OK +S13.2_D1.2: PASS || FAIL_OK +S13_D1_T1: PASS || FAIL_OK +S14_D4_T3: PASS || FAIL_OK +S14_D7: PASS || FAIL_OK +S15.1.2.2_D1.2: PASS || FAIL_OK +S15.5.2_D2: PASS || FAIL_OK +S15.5.4.11_D1.1_T1: PASS || FAIL_OK +S15.5.4.11_D1.1_T2: PASS || FAIL_OK +S15.5.4.11_D1.1_T3: PASS || FAIL_OK +S15.5.4.11_D1.1_T4: PASS || FAIL_OK + +# We allow function declarations within statements +S12.6.2_A13_T1: FAIL_OK +S12.6.2_A13_T2: FAIL_OK +S12.6.4_A13_T1: FAIL_OK +S12.6.4_A13_T2: FAIL_OK +S15.3.4.2_A1_T1: FAIL_OK + +# Linux and Mac defaults to extended 80 bit floating point format in the FPU. +# We follow the other major JS engines by keeping this default. +S8.5_A2.2: PASS, FAIL if $system == linux, FAIL if $system == macos +S8.5_A2.1: PASS, FAIL if $system == linux, FAIL if $system == macos + +# These tests fail because we had to add bugs to be compatible with JSC. See +# http://code.google.com/p/chromium/issues/detail?id=1717 +S15.5.4.1_A1_T2: FAIL_OK +S15.5.4_A1: FAIL_OK +S15.5.4_A3: FAIL_OK +S15.9.5.10_A1_T2: FAIL_OK +S15.9.5.11_A1_T2: FAIL_OK +S15.9.5.12_A1_T2: FAIL_OK +S15.9.5.13_A1_T2: FAIL_OK +S15.9.5.14_A1_T2: FAIL_OK +S15.9.5.15_A1_T2: FAIL_OK +S15.9.5.16_A1_T2: FAIL_OK +S15.9.5.17_A1_T2: FAIL_OK +S15.9.5.18_A1_T2: FAIL_OK +S15.9.5.19_A1_T2: FAIL_OK +S15.9.5.20_A1_T2: FAIL_OK +S15.9.5.21_A1_T2: FAIL_OK +S15.9.5.22_A1_T2: FAIL_OK +S15.9.5.23_A1_T2: FAIL_OK +S15.9.5.24_A1_T2: FAIL_OK +S15.9.5.25_A1_T2: FAIL_OK +S15.9.5.26_A1_T2: FAIL_OK +S15.9.5.27_A1_T2: FAIL_OK +S15.9.5.28_A1_T2: FAIL_OK +S15.9.5.29_A1_T2: FAIL_OK +S15.9.5.2_A1_T2: FAIL_OK +S15.9.5.30_A1_T2: FAIL_OK +S15.9.5.31_A1_T2: FAIL_OK +S15.9.5.32_A1_T2: FAIL_OK +S15.9.5.33_A1_T2: FAIL_OK +S15.9.5.34_A1_T2: FAIL_OK +S15.9.5.35_A1_T2: FAIL_OK +S15.9.5.36_A1_T2: FAIL_OK +S15.9.5.37_A1_T2: FAIL_OK +S15.9.5.38_A1_T2: FAIL_OK +S15.9.5.39_A1_T2: FAIL_OK +S15.9.5.3_A1_T2: FAIL_OK +S15.9.5.40_A1_T2: FAIL_OK +S15.9.5.41_A1_T2: FAIL_OK +S15.9.5.42_A1_T2: FAIL_OK +S15.9.5.4_A1_T2: FAIL_OK +S15.9.5.5_A1_T2: FAIL_OK +S15.9.5.6_A1_T2: FAIL_OK +S15.9.5.7_A1_T2: FAIL_OK +S15.9.5.8_A1_T2: FAIL_OK +S15.9.5.9_A1_T2: FAIL_OK + +############################# ES3 TESTS ################################ +# These tests check for ES3 semantics, and differ from ES5. +# When we follow ES5 semantics, it's ok to fail the test. + +# Allow keywords as names of properties in object initialisers and +# in dot-notation property access. +S11.1.5_A4.1: FAIL_OK +S11.1.5_A4.2: FAIL_OK + +# Calls builtins without an explicit receiver which means that +# undefined is passed to the builtin. The tests expect the global +# object to be passed which was true in ES3 but not in ES5. +S11.1.1_A2: FAIL_OK +S15.5.4.4_A1_T3: FAIL_OK +S15.5.4.5_A1_T3: FAIL_OK +S15.5.4.6_A1_T3: FAIL_OK +S15.5.4.7_A1_T3: FAIL_OK +S15.5.4.8_A1_T3: FAIL_OK +S15.5.4.9_A1_T3: FAIL_OK +S15.5.4.10_A1_T3: FAIL_OK +S15.5.4.11_A1_T3: FAIL_OK +S15.5.4.12_A1_T3: FAIL_OK +S15.5.4.13_A1_T3: FAIL_OK +S15.5.4.14_A1_T3: FAIL_OK +S15.5.4.15_A1_T3: FAIL_OK + +# NaN, Infinity and undefined are read-only according to ES5. +S15.1.1.1_A2_T1: FAIL_OK # NaN +S15.1.1.1_A2_T2: FAIL_OK # NaN +S15.1.1.2_A2_T1: FAIL_OK # Infinity +# S15.1.1.2_A2_T2 would fail if it weren't bogus in r97. sputnik bug #45. +S15.1.1.3_A2_T1: FAIL_OK # undefined +S15.1.1.3_A2_T2: FAIL_OK # undefined + +# Array.prototype.to[Locale]String is generic in ES5. +S15.4.4.2_A2_T1: FAIL_OK +S15.4.4.3_A2_T1: FAIL_OK + +######################### UNANALYZED FAILURES ########################## + # Bug? Object.defineProperty - Update [[Enumerable]] attribute of 'name' # property to true successfully when [[Enumerable]] attribute of 'name' # is false and [[Configurable]] attribute of 'name' is true, the 'desc' @@ -1319,36 +1465,31 @@ prefix ietestcenter # Bug? Date.prototype.toISOString - value of year is Infinity # Date.prototype.toISOString throw the RangeError 15.9.5.43-0-15: FAIL -# Bug? RegExp - the thrown error is SyntaxError instead of RegExpError when 'F' -# contains any character other than 'g', 'i', or 'm' -15.10.4.1-3: FAIL -# Bug? RegExp.prototype is itself a RegExp -15.10.6: FAIL -# Bug? RegExp.prototype.source is of type String -15.10.7.1-1: FAIL -# Bug? RegExp.prototype.source is a data property with default attribute values -# (false) -15.10.7.1-2: FAIL -# Bug? RegExp.prototype.global is of type Boolean -15.10.7.2-1: FAIL -# Bug? RegExp.prototype.global is a data property with default attribute values -# (false) -15.10.7.2-2: FAIL -# Bug? RegExp.prototype.ignoreCase is of type Boolean -15.10.7.3-1: FAIL -# Bug? RegExp.prototype.ignoreCase is a data property with default attribute -# values (false) -15.10.7.3-2: FAIL -# Bug? RegExp.prototype.multiline is of type Boolean -15.10.7.4-1: FAIL -# Bug? RegExp.prototype.multiline is a data property with default attribute -# values (false) -15.10.7.4-2: FAIL -# Bug? RegExp.prototype.lastIndex is of type Number -15.10.7.5-1: FAIL -# Bug? RegExp.prototype.lastIndex is a data property with specified attribute -# values -15.10.7.5-2: FAIL # Bug? Error.prototype.toString return the value of 'msg' when 'name' is empty # string and 'msg' isn't undefined 15.11.4.4-8-1: FAIL + +############################ SKIPPED TESTS ############################# + +# These tests take a looong time to run in debug mode. +S15.1.3.2_A2.5_T1: PASS, SKIP if $mode == debug +S15.1.3.1_A2.5_T1: PASS, SKIP if $mode == debug + +[ $arch == arm ] + +# BUG(3251225): Tests that timeout with --nocrankshaft. +S15.1.3.1_A2.5_T1: SKIP +S15.1.3.2_A2.5_T1: SKIP +S15.1.3.1_A2.4_T1: SKIP +S15.1.3.1_A2.5_T1: SKIP +S15.1.3.2_A2.4_T1: SKIP +S15.1.3.2_A2.5_T1: SKIP +S15.1.3.3_A2.3_T1: SKIP +S15.1.3.4_A2.3_T1: SKIP +S15.1.3.1_A2.5_T1: SKIP +S15.1.3.2_A2.5_T1: SKIP + +[ $arch == mips ] + +# Skip all tests on MIPS. +*: SKIP diff --git a/test/test262/testcfg.py b/test/test262/testcfg.py index 9482046..52127cd 100644 --- a/test/test262/testcfg.py +++ b/test/test262/testcfg.py @@ -43,10 +43,10 @@ class Test262TestCase(test.TestCase): self.root = root def IsNegative(self): - return self.filename.endswith('-n.js') + return '@negative' in self.GetSource() def GetLabel(self): - return "%s test262 %s %s" % (self.mode, self.GetGroup(), self.GetName()) + return "%s test262 %s" % (self.mode, self.GetName()) def IsFailureOutput(self, output): if output.exit_code != 0: @@ -63,9 +63,6 @@ class Test262TestCase(test.TestCase): def GetName(self): return self.path[-1] - def GetGroup(self): - return self.path[0] - def GetSource(self): return open(self.filename).read() @@ -75,13 +72,14 @@ class Test262TestConfiguration(test.TestConfiguration): def __init__(self, context, root): super(Test262TestConfiguration, self).__init__(context, root) - def AddIETestCenter(self, tests, current_path, path, mode): - current_root = join(self.root, 'data', 'test', 'suite', 'ietestcenter') + def ListTests(self, current_path, path, mode, variant_flags): + testroot = join(self.root, 'data', 'test', 'suite') harness = [join(self.root, 'data', 'test', 'harness', f) for f in TEST_262_HARNESS] harness += [join(self.root, 'harness-adapt.js')] - for root, dirs, files in os.walk(current_root): - for dotted in [x for x in dirs if x.startswith('.')]: + tests = [] + for root, dirs, files in os.walk(testroot): + for dotted in [x for x in dirs if x.startswith('.')]: dirs.remove(dotted) dirs.sort() root_path = root[len(self.root):].split(os.path.sep) @@ -89,25 +87,11 @@ class Test262TestConfiguration(test.TestConfiguration): files.sort() for file in files: if file.endswith('.js'): - if self.Contains(path, root_path): - test_path = ['ietestcenter', file[:-3]] + test_path = ['test262', file[:-3]] + if self.Contains(path, test_path): test = Test262TestCase(join(root, file), test_path, self.context, self.root, mode, harness) tests.append(test) - - def AddSputnikConvertedTests(self, tests, current_path, path, mode): - # To be enabled - pass - - def AddSputnikTests(self, tests, current_path, path, mode): - # To be enabled - pass - - def ListTests(self, current_path, path, mode, variant_flags): - tests = [] - self.AddIETestCenter(tests, current_path, path, mode) - self.AddSputnikConvertedTests(tests, current_path, path, mode) - self.AddSputnikTests(tests, current_path, path, mode) return tests def GetBuildRequirements(self): -- 2.7.4