tools: support testing known issues

This commit adds a known_issues directory to the test directory
for scripts that reproduce known bugs. Since these scripts are
expected to fail, it also adds a --expect-fail flag to test.py
which reports tests as successful when they fail.

Refs: https://github.com/nodejs/testing/issues/18
Backport-URL: https://github.com/nodejs/node/pull/5785
PR-URL: https://github.com/nodejs/node/pull/5528
Reviewed-By: James M Snell <jasnell@gmail.com>
Reviewed-By: Wyatt Preul <wpreul@gmail.com>
Reviewed-By: Rich Trott <rtrott@gmail.com>
Conflicts:
	tools/test.py

diff --git a/Makefile b/Makefile
index e526ad5242d5197cddbf0d16377645a22f1af716..5ad0000f7d6d39dff60a980da99f482df6929f63 100644 (file)
--- a/Makefile
+++ b/Makefile
@@ -168,6 +168,9 @@ test-internet: all
 test-debugger: all
        $(PYTHON) tools/test.py debugger
 
+test-known-issues: all
+       $(PYTHON) tools/test.py known_issues --expect-fail
+
 test-npm: $(NODE_EXE)
        NODE=$(NODE) tools/test-npm.sh
 

diff --git a/test/known_issues/testcfg.py b/test/known_issues/testcfg.py
new file mode 100644 (file)
index 0000000..af4f565
--- /dev/null
+++ b/test/known_issues/testcfg.py
@@ -0,0 +1,6 @@
+import sys, os
+sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
+import testpy
+
+def GetConfiguration(context, root):
+  return testpy.SimpleTestConfiguration(context, root, 'known_issues')

diff --git a/tools/test.py b/tools/test.py
index 217e72b870a6f835476c16759460cfd9e1720f87..d5caff92e1232c3350985491bc0ecbcce9d06763 100755 (executable)
--- a/tools/test.py
+++ b/tools/test.py
@@ -430,7 +430,7 @@ class TestCase(object):
     self.thread_id = 0
 
   def IsNegative(self):
-    return False
+    return self.context.expect_fail
 
   def CompareTime(self, other):
     return cmp(other.duration, self.duration)
@@ -774,11 +774,13 @@ TIMEOUT_SCALEFACTOR = {
 
 class Context(object):
 
-  def __init__(self, workspace, buildspace, verbose, vm, timeout, processor, suppress_dialogs, store_unexpected_output):
+  def __init__(self, workspace, buildspace, verbose, vm, expect_fail,
+               timeout, processor, suppress_dialogs, store_unexpected_output):
     self.workspace = workspace
     self.buildspace = buildspace
     self.verbose = verbose
     self.vm_root = vm
+    self.expect_fail = expect_fail
     self.timeout = timeout
     self.processor = processor
     self.suppress_dialogs = suppress_dialogs
@@ -1281,6 +1283,8 @@ def BuildOptions():
   result.add_option("--snapshot", help="Run the tests with snapshot turned on",
       default=False, action="store_true")
   result.add_option("--special-command", default=None)
+  result.add_option("--expect-fail", dest="expect_fail",
+      help="Expect test cases to fail", default=False, action="store_true")
   result.add_option("--valgrind", help="Run tests through valgrind",
       default=False, action="store_true")
   result.add_option("--cat", help="Print the source of the tests",
@@ -1471,6 +1475,7 @@ def Main():
                     buildspace,
                     VERBOSE,
                     shell,
+                    options.expect_fail,
                     options.timeout,
                     processor,
                     options.suppress_dialogs,

diff --git a/vcbuild.bat b/vcbuild.bat
index e955860b8a464e337a176b1f350a1ecc4861f777..cd3f758a3933a15cf7cf486c0bf684d3399e9ab5 100644 (file)
--- a/vcbuild.bat
+++ b/vcbuild.bat
@@ -61,6 +61,7 @@ if /i "%1"=="test-gc"       set test_args=%test_args% gc&set buildnodeweak=1&got
 if /i "%1"=="test-internet" set test_args=%test_args% internet&goto arg-ok
 if /i "%1"=="test-pummel"   set test_args=%test_args% pummel&goto arg-ok
 if /i "%1"=="test-all"      set test_args=%test_args% sequential parallel message gc internet pummel&set buildnodeweak=1&set jslint=1&goto arg-ok
+if /i "%1"=="test-known-issues" set test_args=%test_args% known_issues --expect-fail&goto arg-ok
 if /i "%1"=="jslint"        set jslint=1&goto arg-ok
 if /i "%1"=="msi"           set msi=1&set licensertf=1&set download_arg="--download=all"&set i18n_arg=small-icu&goto arg-ok
 if /i "%1"=="build-release" set build_release=1&goto arg-ok