Enable fine-grained and universal exclusions for CoreFX tests

One problem we've had in coreclr Jenkins when running corefx
tests (in the coreclr outerloop runs -- the ones using the
run-corefx-tests.py harness) is the inability to exclude
failing tests for x86 or x64, and the inability to have
fine-grained exclusions. This has led to problems like
https://github.com/dotnet/coreclr/issues/22442 where
virtually all Windows x86 and x64 corefx runs fail due to
timeouts. And issues like https://github.com/dotnet/coreclr/issues/24159
where all corefx test legs fail, and we have no control
over exclusions to get them all passing again quickly.

Now that the corefx RunTests.cmd/sh wrapper scripts
parse named arguments, including a response file that
is passed on to xunit, and also that the corefx used
xunit has support for this response file as well as
fine-grained exclusions (per-method/per-class/per-namespace),
we can take advantage of it.

This change adds a single, global, corefx xunit exclusion
response file, that will be used for all platforms. Since this
run-corefx-tests.py mechanism is not expected to live much longer,
this seems sufficient.

diff --git a/netci.groovy b/netci.groovy
index 22ee7ff..fde15f9 100755 (executable)
--- a/netci.groovy
+++ b/netci.groovy
@@ -2169,8 +2169,9 @@ def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR
                                 def workspaceRelativeFxRoot = "_/fx"
                                 def absoluteFxRoot = "%WORKSPACE%\\_\\fx"
                                 def fxBranch = getFxBranch(branch)
+                                def exclusionRspPath = "%WORKSPACE%\\tests\\scripts\\run-corefx-tests-exclusions.txt"
 
-                                buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-corefx-tests.py -arch ${arch} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${fxBranch} -env_script ${envScriptPath}"
+                                buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-corefx-tests.py -arch ${arch} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${fxBranch} -env_script ${envScriptPath} -exclusion_rsp_file ${exclusionRspPath}"
 
                                 // Archive and process (only) the test results
                                 Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/artifacts/bin/**/testResults.xml", "", /* doNotFailIfNothingArchived */ true, /* archiveOnlyIfSuccessful */ false)
@@ -2403,8 +2404,9 @@ def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR
                             def workspaceRelativeFxRoot = "_/fx"
                             def absoluteFxRoot = "\$WORKSPACE/${workspaceRelativeFxRoot}"
                             def fxBranch = getFxBranch(branch)
+                            def exclusionRspPath = "\$WORKSPACE%/tests/scripts/run-corefx-tests-exclusions.txt"
 
-                            buildCommands += "python -u \$WORKSPACE/tests/scripts/run-corefx-tests.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${fxBranch} -env_script ${scriptFileName}"
+                            buildCommands += "python -u \$WORKSPACE/tests/scripts/run-corefx-tests.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${fxBranch} -env_script ${scriptFileName} -exclusion_rsp_file ${exclusionRspPath}"
 
                             // Archive and process (only) the test results
                             Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/artifacts/bin/**/testResults.xml", "", /* doNotFailIfNothingArchived */ true, /* archiveOnlyIfSuccessful */ false)
@@ -2989,7 +2991,8 @@ def static CreateWindowsArmTestJob(def dslFactory, def project, def architecture
                 def corefx_runtime_path   = "%WORKSPACE%\\_\\fx\\artifacts\\bin\\testhost\\netcoreapp-Windows_NT-Release-${architecture}"
                 def corefx_tests_dir      = "%WORKSPACE%\\_\\fx\\artifacts\\bin\\tests"
                 def corefx_exclusion_file = "%WORKSPACE%\\tests\\${architecture}\\corefx_test_exclusions.txt"
-                batchFile("call %WORKSPACE%\\tests\\scripts\\run-corefx-tests.bat ${corefx_runtime_path} ${corefx_tests_dir} ${corefx_exclusion_file} ${architecture}")
+                def exclusionRspPath      = "%WORKSPACE%\\tests\\scripts\\run-corefx-tests-exclusions.txt"
+                batchFile("call %WORKSPACE%\\tests\\scripts\\run-corefx-tests.bat ${corefx_runtime_path} ${corefx_tests_dir} ${corefx_exclusion_file} ${architecture} ${exclusionRspPath}")
 
             } else { // !isCoreFxScenario(scenario)
 
@@ -3374,8 +3377,9 @@ python -u \${WORKSPACE}/tests/scripts/run-pmi-diffs.py -arch ${architecture} -ci
                 shell("tar -czf dasm.${os}.${architecture}.${configuration}.tgz ./_/pmi/asm")
             }
             else if (doCoreFxTesting) {
+                def exclusionRspPath = "\${WORKSPACE}/tests/scripts/run-corefx-tests-exclusions.txt"
                 shell("""\
-\${WORKSPACE}/tests/scripts/run-corefx-tests.sh --test-exclude-file \${WORKSPACE}/tests/${architecture}/corefx_linux_test_exclusions.txt --runtime \${WORKSPACE}/${workspaceRelativeFxRootLinux}/artifacts/bin/testhost/netcoreapp-Linux-Release-${architecture} --arch ${architecture} --corefx-tests \${WORKSPACE}/${workspaceRelativeFxRootLinux}/artifacts/bin --configurationGroup Release""")
+\${WORKSPACE}/tests/scripts/run-corefx-tests.sh --test-exclude-file \${WORKSPACE}/tests/${architecture}/corefx_linux_test_exclusions.txt --runtime \${WORKSPACE}/${workspaceRelativeFxRootLinux}/artifacts/bin/testhost/netcoreapp-Linux-Release-${architecture} --arch ${architecture} --corefx-tests \${WORKSPACE}/${workspaceRelativeFxRootLinux}/artifacts/bin --configurationGroup Release --exclusion-rsp-file ${exclusionRspPath}""")
             }
             else {
                 def runScript = "${dockerCmd}./tests/runtest.sh"

diff --git a/tests/scripts/run-corefx-tests-exclusions.txt b/tests/scripts/run-corefx-tests-exclusions.txt
new file mode 100644 (file)
index 0000000..e9fe4f9
--- /dev/null
+++ b/tests/scripts/run-corefx-tests-exclusions.txt
@@ -0,0 +1,14 @@
+# This is a "response file" used to pass fine-grained test exclusions to the
+# corefx xunit test wrapper scripts, RunTests.cmd or RunTests.sh. Lines here
+# should be in a format that xunit understands. Lines beginning with '#' are
+# comment lines and are ignored.
+#
+# Please list a GitHub issue for every exclusion.
+
+# https://github.com/dotnet/coreclr/issues/24159
+-nomethod "System.Buffers.Tests.ArrayBufferWriterTests_Byte.Advance"
+-nomethod "System.Buffers.Tests.ArrayBufferWriterTests_Char.Advance"
+
+# https://github.com/dotnet/coreclr/issues/22442
+-nomethod "Microsoft.Win32.SystemEventsTests.CreateTimerTests.ConcurrentTimers"
+-nomethod "Microsoft.Win32.SystemEventsTests.SessionSwitchTests.SignalsSessionSwitch"

diff --git a/tests/scripts/run-corefx-tests.bat b/tests/scripts/run-corefx-tests.bat
index 34f86d2..18758fd 100644 (file)
--- a/tests/scripts/run-corefx-tests.bat
+++ b/tests/scripts/run-corefx-tests.bat
@@ -18,6 +18,8 @@ echo ^<tests dir^>           -- Path to corefx test tree, e.g., _\fx\bin\tests
 echo ^<test exclusion file^> -- Path to test exclusion file, e.g., C:\coreclr\tests\arm\corefx_test_exclusions.txt
 echo ^<architecture^>        -- Architecture to run, either ARM or ARM64. (We can't depend on PROCESSOR_ARCHITECTURE because
 echo                            the batch script might be invoked with an ARM64 CMD but we need to run ARM.)
+echo ^<exclusion rsp file^>  -- Path to test exclusion response file, passed to RunTests.cmd and then xunit, e.g.,
+echo                            C:\coreclr\tests\scripts\run-corefx-tests-exclusions.txt
 echo.
 echo The ^<test exclusion file^> is a file with a list of assemblies for which the
 echo tests should not be run. This allows excluding failing tests by excluding the
@@ -28,22 +30,26 @@ echo.
 echo     System.Console.Tests
 echo     System.Data.SqlClient.Tests
 echo     System.Diagnostics.Process.Tests
+echo.
+echo The ^<exclusion rsp file^> is in the form expected by xunit.console.dll as a response file.
 goto :eof
 
 :start
-if "%4"=="" goto usage
-if not "%5"=="" goto usage
+if "%5"=="" goto usage
+if not "%6"=="" goto usage
 
 set _runtime_path=%1
 set _tests_dir=%2
 set _exclusion_file=%3
 set _architecture=%4
+set _exclusion_rsp_file=%5
 
 echo Running CoreFX tests
 echo Using runtime: %_runtime_path%
 echo Using tests: %_tests_dir%
 echo Using test exclusion file: %_exclusion_file%
 echo Using architecture: %_architecture%
+echo Using exclusion response file: %_exclusion_rsp_file%
 
 set _pass=0
 set _fail=0
@@ -81,7 +87,7 @@ if %errorlevel% EQU 0 (
     echo COREFX TEST %_t3% EXCLUDED
     set /A _skipped=_skipped + 1
 ) else (
-    call :run %1\RunTests.cmd --runtime-path %_runtime_path%
+    call :run %1\RunTests.cmd --runtime-path %_runtime_path% --rsp-file %_exclusion_rsp_file%
 )
 goto :eof
 

diff --git a/tests/scripts/run-corefx-tests.py b/tests/scripts/run-corefx-tests.py
index 0222953..0c66b6f 100644 (file)
--- a/tests/scripts/run-corefx-tests.py
+++ b/tests/scripts/run-corefx-tests.py
@@ -69,6 +69,7 @@ parser.add_argument('-fx_root', dest='fx_root', default=None)
 parser.add_argument('-fx_branch', dest='fx_branch', default='master')
 parser.add_argument('-fx_commit', dest='fx_commit', default=None)
 parser.add_argument('-env_script', dest='env_script', default=None)
+parser.add_argument('-exclusion_rsp_file', dest='exclusion_rsp_file', default=None)
 parser.add_argument('-no_run_tests', dest='no_run_tests', action="store_true", default=False)
 
 
@@ -81,7 +82,7 @@ def validate_args(args):
     Args:
         args (argparser.ArgumentParser): Args parsed by the argument parser.
     Returns:
-        (arch, ci_arch, build_type, clr_root, fx_root, fx_branch, fx_commit, env_script, no_run_tests)
+        (arch, ci_arch, build_type, clr_root, fx_root, fx_branch, fx_commit, env_script, exclusion_rsp_file, no_run_tests)
             (str, str, str, str, str, str, str, str, str)
     Notes:
     If the arguments are valid then return them all in a tuple. If not, raise
@@ -96,6 +97,7 @@ def validate_args(args):
     fx_branch = args.fx_branch
     fx_commit = args.fx_commit
     env_script = args.env_script
+    exclusion_rsp_file = args.exclusion_rsp_file
     no_run_tests = args.no_run_tests
 
     def validate_arg(arg, check):
@@ -142,7 +144,11 @@ def validate_args(args):
         validate_arg(env_script, lambda item: os.path.isfile(env_script))
         env_script = os.path.abspath(env_script)
 
-    args = (arch, ci_arch, build_type, clr_root, fx_root, fx_branch, fx_commit, env_script, no_run_tests)
+    if exclusion_rsp_file is not None:
+        validate_arg(exclusion_rsp_file, lambda item: os.path.isfile(exclusion_rsp_file))
+        exclusion_rsp_file = os.path.abspath(exclusion_rsp_file)
+
+    args = (arch, ci_arch, build_type, clr_root, fx_root, fx_branch, fx_commit, env_script, exclusion_rsp_file, no_run_tests)
 
     log('Configuration:')
     log(' arch: %s' % arch)
@@ -153,6 +159,7 @@ def validate_args(args):
     log(' fx_branch: %s' % fx_branch)
     log(' fx_commit: %s' % fx_commit)
     log(' env_script: %s' % env_script)
+    log(' exclusion_rsp_file: %s' % exclusion_rsp_file)
     log(' no_run_tests: %s' % no_run_tests)
 
     return args
@@ -215,7 +222,7 @@ def main(args):
     global Unix_name_map
     global testing
 
-    arch, ci_arch, build_type, clr_root, fx_root, fx_branch, fx_commit, env_script, no_run_tests = validate_args(
+    arch, ci_arch, build_type, clr_root, fx_root, fx_branch, fx_commit, env_script, exclusion_rsp_file, no_run_tests = validate_args(
         args)
 
     clr_os = 'Windows_NT' if Is_windows else Unix_name_map[os.uname()[0]]
@@ -380,6 +387,9 @@ def main(args):
         # It is needed under docker where LC_ALL is not configured.
         command += ' --warnAsError false'
 
+    if exclusion_rsp_file is not None:
+        command += (' /p:TestRspFile=%s' % exclusion_rsp_file)
+
     # Run the corefx test build and run the tests themselves.
 
     log(command)

diff --git a/tests/scripts/run-corefx-tests.sh b/tests/scripts/run-corefx-tests.sh
index 4a04f0a..8604ecf 100755 (executable)
--- a/tests/scripts/run-corefx-tests.sh
+++ b/tests/scripts/run-corefx-tests.sh
@@ -327,9 +327,9 @@ run_test()
 
     echo
     echo "Running tests in $dirName"
-    echo "${TimeoutTool}./RunTests.sh --runtime-path $Runtime"
+    echo "${TimeoutTool}./RunTests.sh --runtime-path $Runtime --rsp-file $ExclusionRspFile"
     echo
-    ${TimeoutTool}./RunTests.sh --runtime-path "$Runtime"
+    ${TimeoutTool}./RunTests.sh --runtime-path "$Runtime" --rsp-file "$ExclusionRspFile"
     exitCode=$?
 
     if [ $exitCode -ne 0 ] ; then
@@ -459,6 +459,10 @@ do
             TestExcludeFile=$2
             ;;
 
+        --exclusion-rsp-file)
+            ExclusionRspFile=$2
+            ;;
+
         --timeout)
             TimeoutTime=$2
             ;;