Simplify test262.py work-flow

Make test262.py work by default again like the original tool with
regards to TestExpectations, i.e. not use them at all. Instead make
it an option that is now passed via "make check".

So after a change in code we run "make check" to check for any regressions or
fixes. If there are any regressions, then we call tests/test262.py <name of
test> to debug it and by not reading TestExpectations by default the test will
fail.

Change-Id: I00b43c5d09c17c296dfa958293f769663ef49de1
Reviewed-by: Lars Knoll <lars.knoll@digia.com>

diff --git a/tests/test262.py b/tests/test262.py
index e8df475..7f95808 100755 (executable)
--- a/tests/test262.py
+++ b/tests/test262.py
@@ -55,10 +55,12 @@ def ReportError(s):
 
 
 class TestExpectations:
-    def __init__(self):
-        f = open(rootDir + "/TestExpectations")
+    def __init__(self, enabled):
         self.testsToSkip = []
         self.failingTests = []
+        f = open(rootDir + "/TestExpectations")
+        if not enabled:
+            return
         for line in f.read().splitlines():
             line = line.strip()
             if len(line) == 0 or line[0] == "#":
@@ -126,6 +128,8 @@ def BuildOptions():
                     help="Test only non-strict mode")
   result.add_option("--parallel", default=False, action="store_true",
                     help="Run tests in parallel")
+  result.add_option("--with-test-expectations", default=False, action="store_true",
+                    help="Parse TestExpectations to deal with tests known to fail")
   result.add_option("--update-expectations", default=False, action="store_true",
                     help="Update test expectations fail when a test passes that was expected to fail")
   # TODO: Once enough tests are made strict compat, change the default
@@ -369,7 +373,7 @@ def MakePlural(n):
 
 class TestSuite(object):
 
-  def __init__(self, root, strict_only, non_strict_only, unmarked_default):
+  def __init__(self, root, strict_only, non_strict_only, unmarked_default, load_expectations):
     # TODO: derive from packagerConfig.py
     self.test_root = path.join(root, 'test', 'suite')
     self.lib_root = path.join(root, 'test', 'harness')
@@ -377,7 +381,7 @@ class TestSuite(object):
     self.non_strict_only = non_strict_only
     self.unmarked_default = unmarked_default
     self.include_cache = { }
-    self.expectations = TestExpectations()
+    self.expectations = TestExpectations(load_expectations)
 
   def Validate(self):
     if not path.exists(self.test_root):
@@ -525,7 +529,8 @@ def Main():
   test_suite = TestSuite(options.tests, 
                          options.strict_only, 
                          options.non_strict_only,
-                         options.unmarked_default)
+                         options.unmarked_default,
+                         options.with_test_expectations)
   test_suite.Validate()
   if options.cat:
     test_suite.Print(args)

diff --git a/v4.pro b/v4.pro
index e947da9..75da88a 100644 (file)
--- a/v4.pro
+++ b/v4.pro
@@ -87,7 +87,7 @@ DEFINES += QMLJS_NO_LLVM
 }
 
 checktarget.target = check
-checktarget.commands = python tests/test262.py --parallel --update-expectations
+checktarget.commands = python tests/test262.py --parallel --with-test-expectations --update-expectations
 checktarget.depends = all
 QMAKE_EXTRA_TARGETS += checktarget