Fix per-test timeout

1. Add "timeout" option to runtest.cmd to set the per-test timeout, in ms.
2. Change NETCI to use this new option.

This should fix various random test failures that frequently occur
in Debug test runs due to timeout (e.g., chaos and type generator tests).

diff --git a/netci.groovy b/netci.groovy
index 56cf9b0..9185d93 100755 (executable)
--- a/netci.groovy
+++ b/netci.groovy
@@ -473,12 +473,6 @@ def static getR2RDisplayName(def scenario) {
 def static envScriptCreate(def os, def stepScriptLocation) {
     def stepScript = ''
     if (os == 'Windows_NT') {
-        // Timeout in ms, default is 10 minutes. For stress modes up this to 30 minutes.
-        // BUG?: it seems this is ignored, as this script will be run in an environment where
-        //       environment variables will be discarded before this is used.
-        def timeout = 1800000
-        stepScript += "set __TestTimeout=${timeout}\r\n"
-
         stepScript += "echo Creating TestEnv script\r\n"
         stepScript += "if exist ${stepScriptLocation} del ${stepScriptLocation}\r\n"
 
@@ -1528,6 +1522,16 @@ def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR
                             testOpts += " link %WORKSPACE%\\linker\\linker\\bin\\netcore_Release\\netcoreapp2.0\\win10-${arch}\\publish\\illink.exe"
                         }
 
+                        // Default per-test timeout is 10 minutes. For stress modes and Debug scenarios, increase this
+                        // to 30 minutes (30 * 60 * 1000 = 180000). The "timeout" argument to runtest.cmd sets this, by
+                        // taking a timeout value in milliseconds. (Note that it sets the __TestTimeout environment variable,
+                        // which is read by the xunit harness.)
+                        if (isJitStressScenario(scenario) || isR2RStressScenario(scenario) || (lowerConfiguration == 'debug'))
+                        {
+                            def timeout = 1800000
+                            testOpts += " timeout ${timeout}"
+                        }
+
                         // If we are running a stress mode, we should write out the set of key
                         // value env pairs to a file at this point and then we'll pass that to runtest.cmd
 

diff --git a/tests/runtest.cmd b/tests/runtest.cmd
index 6774a65..d98cdbb 100644 (file)
--- a/tests/runtest.cmd
+++ b/tests/runtest.cmd
@@ -110,6 +110,7 @@ if /i "%1" == "runcrossgentests"      (set RunCrossGen=true&shift&goto Arg_Loop)
 if /i "%1" == "link"                  (set DoLink=true&set ILLINK=%2&shift&shift&goto Arg_Loop)
 if /i "%1" == "tieredcompilation"     (set COMPLUS_EXPERIMENTAL_TieredCompilation=1&shift&goto Arg_Loop)
 if /i "%1" == "gcname"                (set COMPlus_GCName=%2&shift&shift&goto Arg_Loop)
+if /i "%1" == "timeout"               (set __TestTimeout=%2&shift&shift&goto Arg_Loop)
 
 REM change it to COMPlus_GCStress when we stop using xunit harness
 if /i "%1" == "gcstresslevel"         (set __GCSTRESSLEVEL=%2&set __TestTimeout=1800000&shift&shift&goto Arg_Loop)
@@ -500,7 +501,9 @@ echo                               4: GC on every allowable JITed instruction
 echo                               8: GC on every allowable NGEN instruction
 echo                              16: GC only on a unique stack trace
 echo tieredcompilation         - Run the tests with COMPlus_EXPERIMENTAL_TieredCompilation=1
-echo gcname ^<n^>              - Runs the tests with COMPlus_GCName=n
+echo gcname ^<name^>             - Runs the tests with COMPlus_GCName=name
+echo timeout ^<n^>               - Sets the per-test timeout in milliseconds ^(default is 10 minutes = 10 * 60 * 1000 = 600000^).
+echo                             Note: some options override this ^(gcstresslevel, longgc, gcsimulator^).
 echo msbuildargs ^<args...^>     - Pass all subsequent args directly to msbuild invocations.
 echo ^<CORE_ROOT^>               - Path to the runtime to test (if specified).
 echo.