def workspaceRelativeFxRoot = "_/fx"
def absoluteFxRoot = "%WORKSPACE%\\_\\fx"
def fxBranch = getFxBranch(branch)
+ def exclusionRspPath = "%WORKSPACE%\\tests\\scripts\\run-corefx-tests-exclusions.txt"
- buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-corefx-tests.py -arch ${arch} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${fxBranch} -env_script ${envScriptPath}"
+ buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-corefx-tests.py -arch ${arch} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${fxBranch} -env_script ${envScriptPath} -exclusion_rsp_file ${exclusionRspPath}"
// Archive and process (only) the test results
Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/artifacts/bin/**/testResults.xml", "", /* doNotFailIfNothingArchived */ true, /* archiveOnlyIfSuccessful */ false)
def workspaceRelativeFxRoot = "_/fx"
def absoluteFxRoot = "\$WORKSPACE/${workspaceRelativeFxRoot}"
def fxBranch = getFxBranch(branch)
+ def exclusionRspPath = "\$WORKSPACE%/tests/scripts/run-corefx-tests-exclusions.txt"
- buildCommands += "python -u \$WORKSPACE/tests/scripts/run-corefx-tests.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${fxBranch} -env_script ${scriptFileName}"
+ buildCommands += "python -u \$WORKSPACE/tests/scripts/run-corefx-tests.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${fxBranch} -env_script ${scriptFileName} -exclusion_rsp_file ${exclusionRspPath}"
// Archive and process (only) the test results
Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/artifacts/bin/**/testResults.xml", "", /* doNotFailIfNothingArchived */ true, /* archiveOnlyIfSuccessful */ false)
def corefx_runtime_path = "%WORKSPACE%\\_\\fx\\artifacts\\bin\\testhost\\netcoreapp-Windows_NT-Release-${architecture}"
def corefx_tests_dir = "%WORKSPACE%\\_\\fx\\artifacts\\bin\\tests"
def corefx_exclusion_file = "%WORKSPACE%\\tests\\${architecture}\\corefx_test_exclusions.txt"
- batchFile("call %WORKSPACE%\\tests\\scripts\\run-corefx-tests.bat ${corefx_runtime_path} ${corefx_tests_dir} ${corefx_exclusion_file} ${architecture}")
+ def exclusionRspPath = "%WORKSPACE%\\tests\\scripts\\run-corefx-tests-exclusions.txt"
+ batchFile("call %WORKSPACE%\\tests\\scripts\\run-corefx-tests.bat ${corefx_runtime_path} ${corefx_tests_dir} ${corefx_exclusion_file} ${architecture} ${exclusionRspPath}")
} else { // !isCoreFxScenario(scenario)
shell("tar -czf dasm.${os}.${architecture}.${configuration}.tgz ./_/pmi/asm")
}
else if (doCoreFxTesting) {
+ def exclusionRspPath = "\${WORKSPACE}/tests/scripts/run-corefx-tests-exclusions.txt"
shell("""\
-\${WORKSPACE}/tests/scripts/run-corefx-tests.sh --test-exclude-file \${WORKSPACE}/tests/${architecture}/corefx_linux_test_exclusions.txt --runtime \${WORKSPACE}/${workspaceRelativeFxRootLinux}/artifacts/bin/testhost/netcoreapp-Linux-Release-${architecture} --arch ${architecture} --corefx-tests \${WORKSPACE}/${workspaceRelativeFxRootLinux}/artifacts/bin --configurationGroup Release""")
+\${WORKSPACE}/tests/scripts/run-corefx-tests.sh --test-exclude-file \${WORKSPACE}/tests/${architecture}/corefx_linux_test_exclusions.txt --runtime \${WORKSPACE}/${workspaceRelativeFxRootLinux}/artifacts/bin/testhost/netcoreapp-Linux-Release-${architecture} --arch ${architecture} --corefx-tests \${WORKSPACE}/${workspaceRelativeFxRootLinux}/artifacts/bin --configurationGroup Release --exclusion-rsp-file ${exclusionRspPath}""")
}
else {
def runScript = "${dockerCmd}./tests/runtest.sh"
--- /dev/null
+# This is a "response file" used to pass fine-grained test exclusions to the
+# corefx xunit test wrapper scripts, RunTests.cmd or RunTests.sh. Lines here
+# should be in a format that xunit understands. Lines beginning with '#' are
+# comment lines and are ignored.
+#
+# Please list a GitHub issue for every exclusion.
+
+# https://github.com/dotnet/coreclr/issues/24159
+-nomethod "System.Buffers.Tests.ArrayBufferWriterTests_Byte.Advance"
+-nomethod "System.Buffers.Tests.ArrayBufferWriterTests_Char.Advance"
+
+# https://github.com/dotnet/coreclr/issues/22442
+-nomethod "Microsoft.Win32.SystemEventsTests.CreateTimerTests.ConcurrentTimers"
+-nomethod "Microsoft.Win32.SystemEventsTests.SessionSwitchTests.SignalsSessionSwitch"
echo ^<test exclusion file^> -- Path to test exclusion file, e.g., C:\coreclr\tests\arm\corefx_test_exclusions.txt
echo ^<architecture^> -- Architecture to run, either ARM or ARM64. (We can't depend on PROCESSOR_ARCHITECTURE because
echo the batch script might be invoked with an ARM64 CMD but we need to run ARM.)
+echo ^<exclusion rsp file^> -- Path to test exclusion response file, passed to RunTests.cmd and then xunit, e.g.,
+echo C:\coreclr\tests\scripts\run-corefx-tests-exclusions.txt
echo.
echo The ^<test exclusion file^> is a file with a list of assemblies for which the
echo tests should not be run. This allows excluding failing tests by excluding the
echo System.Console.Tests
echo System.Data.SqlClient.Tests
echo System.Diagnostics.Process.Tests
+echo.
+echo The ^<exclusion rsp file^> is in the form expected by xunit.console.dll as a response file.
goto :eof
:start
-if "%4"=="" goto usage
-if not "%5"=="" goto usage
+if "%5"=="" goto usage
+if not "%6"=="" goto usage
set _runtime_path=%1
set _tests_dir=%2
set _exclusion_file=%3
set _architecture=%4
+set _exclusion_rsp_file=%5
echo Running CoreFX tests
echo Using runtime: %_runtime_path%
echo Using tests: %_tests_dir%
echo Using test exclusion file: %_exclusion_file%
echo Using architecture: %_architecture%
+echo Using exclusion response file: %_exclusion_rsp_file%
set _pass=0
set _fail=0
echo COREFX TEST %_t3% EXCLUDED
set /A _skipped=_skipped + 1
) else (
- call :run %1\RunTests.cmd --runtime-path %_runtime_path%
+ call :run %1\RunTests.cmd --runtime-path %_runtime_path% --rsp-file %_exclusion_rsp_file%
)
goto :eof
parser.add_argument('-fx_branch', dest='fx_branch', default='master')
parser.add_argument('-fx_commit', dest='fx_commit', default=None)
parser.add_argument('-env_script', dest='env_script', default=None)
+parser.add_argument('-exclusion_rsp_file', dest='exclusion_rsp_file', default=None)
parser.add_argument('-no_run_tests', dest='no_run_tests', action="store_true", default=False)
Args:
args (argparser.ArgumentParser): Args parsed by the argument parser.
Returns:
- (arch, ci_arch, build_type, clr_root, fx_root, fx_branch, fx_commit, env_script, no_run_tests)
+ (arch, ci_arch, build_type, clr_root, fx_root, fx_branch, fx_commit, env_script, exclusion_rsp_file, no_run_tests)
(str, str, str, str, str, str, str, str, str)
Notes:
If the arguments are valid then return them all in a tuple. If not, raise
fx_branch = args.fx_branch
fx_commit = args.fx_commit
env_script = args.env_script
+ exclusion_rsp_file = args.exclusion_rsp_file
no_run_tests = args.no_run_tests
def validate_arg(arg, check):
validate_arg(env_script, lambda item: os.path.isfile(env_script))
env_script = os.path.abspath(env_script)
- args = (arch, ci_arch, build_type, clr_root, fx_root, fx_branch, fx_commit, env_script, no_run_tests)
+ if exclusion_rsp_file is not None:
+ validate_arg(exclusion_rsp_file, lambda item: os.path.isfile(exclusion_rsp_file))
+ exclusion_rsp_file = os.path.abspath(exclusion_rsp_file)
+
+ args = (arch, ci_arch, build_type, clr_root, fx_root, fx_branch, fx_commit, env_script, exclusion_rsp_file, no_run_tests)
log('Configuration:')
log(' arch: %s' % arch)
log(' fx_branch: %s' % fx_branch)
log(' fx_commit: %s' % fx_commit)
log(' env_script: %s' % env_script)
+ log(' exclusion_rsp_file: %s' % exclusion_rsp_file)
log(' no_run_tests: %s' % no_run_tests)
return args
global Unix_name_map
global testing
- arch, ci_arch, build_type, clr_root, fx_root, fx_branch, fx_commit, env_script, no_run_tests = validate_args(
+ arch, ci_arch, build_type, clr_root, fx_root, fx_branch, fx_commit, env_script, exclusion_rsp_file, no_run_tests = validate_args(
args)
clr_os = 'Windows_NT' if Is_windows else Unix_name_map[os.uname()[0]]
# It is needed under docker where LC_ALL is not configured.
command += ' --warnAsError false'
+ if exclusion_rsp_file is not None:
+ command += (' /p:TestRspFile=%s' % exclusion_rsp_file)
+
# Run the corefx test build and run the tests themselves.
log(command)
echo
echo "Running tests in $dirName"
- echo "${TimeoutTool}./RunTests.sh --runtime-path $Runtime"
+ echo "${TimeoutTool}./RunTests.sh --runtime-path $Runtime --rsp-file $ExclusionRspFile"
echo
- ${TimeoutTool}./RunTests.sh --runtime-path "$Runtime"
+ ${TimeoutTool}./RunTests.sh --runtime-path "$Runtime" --rsp-file "$ExclusionRspFile"
exitCode=$?
if [ $exitCode -ne 0 ] ; then
TestExcludeFile=$2
;;
+ --exclusion-rsp-file)
+ ExclusionRspFile=$2
+ ;;
+
--timeout)
TimeoutTime=$2
;;