5 echo 'CoreCLR test runner script.'
7 echo 'Typical command line:'
9 echo 'coreclr/tests/runtest.sh'
10 echo ' --testRootDir="temp/Windows_NT.x64.Debug"'
11 echo ' --testNativeBinDir="coreclr/bin/obj/Linux.x64.Debug/tests"'
12 echo ' --coreOverlayDir="coreclr/bin/tests/Linux.x64.Debug/Tests/Core_Root"'
13 echo ' --copyNativeTestBin'
15 echo 'Required arguments:'
16 echo ' --testRootDir=<path> : Root directory of the test build (e.g. coreclr/bin/tests/Windows_NT.x64.Debug).'
17 echo ' --testNativeBinDir=<path> : Directory of the native CoreCLR test build (e.g. coreclr/bin/obj/Linux.x64.Debug/tests).'
18 echo ' (Also required: Either --coreOverlayDir, or all of the switches --coreOverlayDir overrides)'
20 echo 'Optional arguments:'
21 echo ' --coreOverlayDir=<path> : Directory containing core binaries and test dependencies. If not specified, the'
22 echo ' default is testRootDir/Tests/coreoverlay. This switch overrides --coreClrBinDir,'
23 echo ' --mscorlibDir, and --coreFxBinDir.'
24 echo ' --coreClrBinDir=<path> : Directory of the CoreCLR build (e.g. coreclr/bin/Product/Linux.x64.Debug).'
25 echo ' --mscorlibDir=<path> : Directory containing the built mscorlib.dll. If not specified, it is expected to be'
26 echo ' in the directory specified by --coreClrBinDir.'
27 echo ' --coreFxBinDir="<path>" : Directory with CoreFX build outputs'
28 echo ' (e.g. "corefx/bin/runtime/netcoreapp-Linux-Debug-x64")'
29 echo ' If files with the same name are present in multiple directories, the first one wins.'
30 echo ' --testDir=<path> : Run tests only in the specified directory. The path is relative to the directory'
31 echo ' specified by --testRootDir. Multiple of this switch may be specified.'
32 echo ' --testDirFile=<path> : Run tests only in the directories specified by the file at <path>. Paths are listed'
33 echo ' one line, relative to the directory specified by --testRootDir.'
34 echo ' --build-overlay-only : Build coreoverlay only, and skip running tests.'
35 echo ' --runFailingTestsOnly : Run only the tests that are disabled on this platform due to unexpected failures.'
36 echo ' Failing tests are listed in coreclr/tests/failingTestsOutsideWindows.txt, one per'
37 echo ' line, as paths to .sh files relative to the directory specified by --testRootDir.'
38 echo ' --disableEventLogging : Disable the events logged by both VM and Managed Code'
39 echo ' --sequential : Run tests sequentially (default is to run in parallel).'
40 echo ' --playlist=<path> : Run only the tests that are specified in the file at <path>, in the same format as'
41 echo ' runFailingTestsOnly'
42 echo ' -v, --verbose : Show output from each test.'
43 echo ' -h|--help : Show usage information.'
44 echo ' --useServerGC : Enable server GC for this test run'
45 echo ' --test-env : Script to set environment variables for tests'
46 echo ' --copyNativeTestBin : Explicitly copy native test components into the test dir'
47 echo ' --crossgen : Precompiles the framework managed assemblies'
48 echo ' --runcrossgentests : Runs the ready to run tests'
49 echo ' --jitstress=<n> : Runs the tests with COMPlus_JitStress=n'
50 echo ' --jitstressregs=<n> : Runs the tests with COMPlus_JitStressRegs=n'
51 echo ' --jitminopts : Runs the tests with COMPlus_JITMinOpts=1'
52 echo ' --jitforcerelocs : Runs the tests with COMPlus_ForceRelocs=1'
53 echo ' --jitdisasm : Runs jit-dasm on the tests'
54 echo ' --gcstresslevel=<n> : Runs the tests with COMPlus_GCStress=n'
55 echo ' --gcname=<n> : Runs the tests with COMPlus_GCName=n'
56 echo ' --ilasmroundtrip : Runs ilasm round trip on the tests'
57 echo ' 0: None 1: GC on all allocs and '"'easy'"' places'
58 echo ' 2: GC on transitions to preemptive GC 4: GC on every allowable JITed instr'
59 echo ' 8: GC on every allowable NGEN instr 16: GC only on a unique stack trace'
60 echo ' --long-gc : Runs the long GC tests'
61 echo ' --gcsimulator : Runs the GCSimulator tests'
62 echo ' --tieredcompilation : Runs the tests with COMPlus_TieredCompilation=1'
63 echo ' --link <ILlink> : Runs the tests after linking via ILlink'
64 echo ' --show-time : Print execution sequence and running time for each test'
65 echo ' --no-lf-conversion : Do not execute LF conversion before running test script'
66 echo ' --limitedDumpGeneration : Enables the generation of a limited number of core dumps if test(s) crash, even if ulimit'
67 echo ' is zero when launching this script. This option is intended for use in CI.'
68 echo ' --xunitOutputPath=<path> : Create xUnit XML report at the specifed path (default: <test root>/coreclrtests.xml)'
70 echo 'CoreFX Test Options '
71 echo ' --corefxtests : Runs CoreFX tests'
72 echo ' --corefxtestsall : Runs all available CoreFX tests'
73 echo ' --corefxtestlist=<path> : Runs the CoreFX tests specified in the passed list'
74 echo ' --testHostDir=<path> : Directory containing a built test host including core binaries, test dependencies'
75 echo ' and a dotnet executable'
77 echo 'Runtime Code Coverage options:'
78 echo ' --coreclr-coverage : Optional argument to get coreclr code coverage reports'
79 echo ' --coreclr-objs=<path> : Location of root of the object directory'
80 echo ' containing the linux/mac coreclr build'
81 echo ' --coreclr-src=<path> : Location of root of the directory'
82 echo ' containing the coreclr source files'
83 echo ' --coverage-output-dir=<path> : Directory where coverage output will be written to'
87 function print_results {
89 echo "======================="
91 echo "======================="
92 echo "# CoreCLR Bin Dir : $coreClrBinDir"
93 echo "# Tests Discovered : $countTotalTests"
94 echo "# Passed : $countPassedTests"
95 echo "# Failed : $countFailedTests"
96 echo "# Skipped : $countSkippedTests"
97 echo "======================="
100 # Initialize counters for bookkeeping.
106 # Variables for xUnit-style XML output. XML format: https://xunit.github.io/docs/format-xml-v2.html
110 # Variables for text file output. These can be passed back to runtest.sh using the "--playlist" argument
111 # to rerun specific tests.
116 # libExtension determines extension for dynamic library files
117 # runtimeName determines where CoreFX Runtime files will be located
134 echo "Unsupported OS $OSName detected, configuring as if for Linux"
139 function xunit_output_begin {
140 if [ -z "$xunitOutputPath" ]; then
141 xunitOutputPath=$testRootDir/coreclrtests.xml
143 if ! [ -e $(basename "$xunitOutputPath") ]; then
144 xunitOutputPath=$testRootDir/coreclrtests.xml
146 xunitTestOutputPath=${xunitOutputPath}.test
147 if [ -e "$xunitOutputPath" ]; then
148 rm -f -r "$xunitOutputPath"
150 if [ -e "$xunitTestOutputPath" ]; then
151 rm -f -r "$xunitTestOutputPath"
155 function xunit_output_add_test {
159 # <test .../> <!-- Write this element here -->
161 local scriptFilePath=$1
162 local outputFilePath=$2
163 local testResult=$3 # Pass, Fail, or Skip
164 local testScriptExitCode=$4
165 local testRunningTime=$5
167 local testPath=${scriptFilePath%.sh} # Remove trailing ".sh"
168 local testDir=$(dirname "$testPath")
169 local testName=$(basename "$testPath")
171 # Replace '/' with '.'
172 testPath=$(echo "$testPath" | tr / .)
173 testDir=$(echo "$testDir" | tr / .)
179 line="${line} name=\"${testPath}\""
180 line="${line} type=\"${testDir}\""
181 line="${line} method=\"${testName}\""
182 line="${line} result=\"${testResult}\""
183 if [ -n "$testRunningTime" ] && [ "$testResult" != "Skip" ]; then
184 line="${line} time=\"${testRunningTime}\""
187 if [ "$testResult" == "Pass" ]; then
189 echo "$line" >>"$xunitTestOutputPath"
194 echo "$line" >>"$xunitTestOutputPath"
197 if [ "$testResult" == "Skip" ]; then
198 line="${line}<reason><![CDATA[$(cat "$outputFilePath")]]></reason>"
199 echo "$line" >>"$xunitTestOutputPath"
201 line="${line}<failure exception-type=\"Exit code: ${testScriptExitCode}\">"
202 echo "$line" >>"$xunitTestOutputPath"
205 line="${line}<message>"
206 echo "$line" >>"$xunitTestOutputPath"
208 line="${line}<![CDATA["
209 echo "$line" >>"$xunitTestOutputPath"
210 cat "$outputFilePath" >>"$xunitTestOutputPath"
213 echo "$line" >>"$xunitTestOutputPath"
215 line="${line}</message>"
216 echo "$line" >>"$xunitTestOutputPath"
219 line="${line}</failure>"
220 echo "$line" >>"$xunitTestOutputPath"
224 line="${line}</test>"
225 echo "$line" >>"$xunitTestOutputPath"
228 function xunit_output_end {
230 local errorMessage=$2
233 if [ -z "$errorSource" ]; then
239 echo '<?xml version="1.0" encoding="utf-8"?>' >>"$xunitOutputPath"
240 echo '<assemblies>' >>"$xunitOutputPath"
246 line="${line}<assembly"
247 line="${line} name=\"CoreClrTestAssembly\""
248 line="${line} total=\"${countTotalTests}\""
249 line="${line} passed=\"${countPassedTests}\""
250 line="${line} failed=\"${countFailedTests}\""
251 line="${line} skipped=\"${countSkippedTests}\""
252 line="${line} errors=\"${errorCount}\""
254 echo "$line" >>"$xunitOutputPath"
258 line="${line}<collection"
259 line="${line} name=\"CoreClrTestCollection\""
260 line="${line} total=\"${countTotalTests}\""
261 line="${line} passed=\"${countPassedTests}\""
262 line="${line} failed=\"${countFailedTests}\""
263 line="${line} skipped=\"${countSkippedTests}\""
265 echo "$line" >>"$xunitOutputPath"
267 # <test .../> <test .../> ...
268 if [ -f "$xunitTestOutputPath" ]; then
269 cat "$xunitTestOutputPath" >>"$xunitOutputPath"
270 rm -f "$xunitTestOutputPath"
275 line="${line}</collection>"
276 echo "$line" >>"$xunitOutputPath"
278 if [ -n "$errorSource" ]; then
281 line="${line}<errors>"
282 echo "$line" >>"$xunitOutputPath"
287 line="${line} type=\"TestHarnessError\""
288 line="${line} name=\"${errorSource}\""
290 echo "$line" >>"$xunitOutputPath"
294 line="${line}<failure>${errorMessage}</failure>"
295 echo "$line" >>"$xunitOutputPath"
299 line="${line}</error>"
300 echo "$line" >>"$xunitOutputPath"
304 line="${line}</errors>"
305 echo "$line" >>"$xunitOutputPath"
310 line="${line}</assembly>"
311 echo "$line" >>"$xunitOutputPath"
314 echo '</assemblies>' >>"$xunitOutputPath"
317 function text_file_output_begin {
318 if [ -z "$testsPassOutputPath" ]; then
319 testsPassOutputPath=$testRootDir/coreclrtests.pass.txt
321 if ! [ -e $(basename "$testsPassOutputPath") ]; then
322 testsPassOutputPath=$testRootDir/coreclrtests.pass.txt
324 if [ -e "$testsPassOutputPath" ]; then
325 rm -f "$testsPassOutputPath"
327 if [ -z "$testsFailOutputPath" ]; then
328 testsFailOutputPath=$testRootDir/coreclrtests.fail.txt
330 if ! [ -e $(basename "$testsFailOutputPath") ]; then
331 testsFailOutputPath=$testRootDir/coreclrtests.fail.txt
333 if [ -e "$testsFailOutputPath" ]; then
334 rm -f "$testsFailOutputPath"
336 if [ -z "$testsSkipOutputPath" ]; then
337 testsSkipOutputPath=$testRootDir/coreclrtests.skip.txt
339 if ! [ -e $(basename "$testsSkipOutputPath") ]; then
340 testsSkipOutputPath=$testRootDir/coreclrtests.skip.txt
342 if [ -e "$testsSkipOutputPath" ]; then
343 rm -f "$testsSkipOutputPath"
347 function text_file_output_add_test {
348 local scriptFilePath=$1
349 local testResult=$2 # Pass, Fail, or Skip
351 if [ "$testResult" == "Pass" ]; then
352 echo "$scriptFilePath" >>"$testsPassOutputPath"
353 elif [ "$testResult" == "Skip" ]; then
354 echo "$scriptFilePath" >>"$testsSkipOutputPath"
356 echo "$scriptFilePath" >>"$testsFailOutputPath"
360 function exit_with_error {
362 local errorMessage=$2
365 if [ -z "$printUsage" ]; then
370 xunit_output_end "$errorSource" "$errorMessage"
371 if ((printUsage != 0)); then
374 exit $EXIT_CODE_EXCEPTION
377 # Handle Ctrl-C. We will stop execution and print the results that
378 # we gathered so far.
379 function handle_ctrl_c {
380 local errorSource='handle_ctrl_c'
383 echo "*** Stopping... ***"
385 exit_with_error "$errorSource" "Test run aborted by Ctrl+C."
388 # Register the Ctrl-C handler
389 trap handle_ctrl_c INT
391 function create_core_overlay {
392 local errorSource='create_core_overlay'
395 if [ -n "$coreOverlayDir" ]; then
396 export CORE_ROOT="$coreOverlayDir"
398 if [ -n "$copyNativeTestBin" ]; then
399 copy_test_native_bin_to_test_root $coreOverlayDir
405 # Check inputs to make sure we have enough information to create the core layout. $testRootDir/Tests/Core_Root should
406 # already exist and contain test dependencies that are not built.
407 local testDependenciesDir=$testRootDir/Tests/Core_Root
408 if [ ! -d "$testDependenciesDir" ]; then
409 exit_with_error "$errorSource" "Did not find the test dependencies directory: $testDependenciesDir"
411 if [ -z "$coreClrBinDir" ]; then
412 exit_with_error "$errorSource" "One of --coreOverlayDir or --coreClrBinDir must be specified." "$printUsage"
414 if [ ! -d "$coreClrBinDir" ]; then
415 exit_with_error "$errorSource" "Directory specified by --coreClrBinDir does not exist: $coreClrBinDir"
419 coreOverlayDir=$testRootDir/Tests/coreoverlay
420 export CORE_ROOT="$coreOverlayDir"
421 if [ -e "$coreOverlayDir" ]; then
422 rm -f -r "$coreOverlayDir"
424 mkdir "$coreOverlayDir"
426 cp -f -v "$coreFxBinDir/"* "$coreOverlayDir/" 2>/dev/null
427 cp -f -p -v "$coreClrBinDir/"* "$coreOverlayDir/" 2>/dev/null
428 if [ -d "$mscorlibDir/bin" ]; then
429 cp -f -v "$mscorlibDir/bin/"* "$coreOverlayDir/" 2>/dev/null
431 cp -f -v "$testDependenciesDir/"xunit* "$coreOverlayDir/" 2>/dev/null
432 cp -n -v "$testDependenciesDir/"* "$coreOverlayDir/" 2>/dev/null
433 if [ -f "$coreOverlayDir/mscorlib.ni.dll" ]; then
434 # Test dependencies come from a Windows build, and mscorlib.ni.dll would be the one from Windows
435 rm -f "$coreOverlayDir/mscorlib.ni.dll"
437 if [ -f "$coreOverlayDir/System.Private.CoreLib.ni.dll" ]; then
438 # Test dependencies come from a Windows build, and System.Private.CoreLib.ni.dll would be the one from Windows
439 rm -f "$coreOverlayDir/System.Private.CoreLib.ni.dll"
441 copy_test_native_bin_to_test_root $coreOverlayDir
444 function create_testhost
446 if [ ! -d "$testHostDir" ]; then
447 exit_with_error "$errorSource" "Did not find the test host directory: $testHostDir"
450 # Initialize test variables
451 local buildToolsDir=$coreClrSrc/Tools
452 local dotnetExe=$coreClrSrc/dotnet.sh
453 local coreClrSrcTestDir=$coreClrSrc/tests
455 if [ -z $coreClrBinDir ]; then
456 local coreClrBinDir=${coreClrSrc}/bin
457 export __CoreFXTestDir=${coreClrSrc}/bin/tests/CoreFX
459 export __CoreFXTestDir=${coreClrBinDir}/tests/CoreFX
462 local coreFXTestSetupUtilityName=CoreFX.TestUtils.TestFileSetup
463 local coreFXTestSetupUtility="${coreClrSrcTestDir}/src/Common/CoreFX/TestFileSetup/${coreFXTestSetupUtilityName}.csproj"
464 local coreFXTestSetupUtilityOutputPath=${__CoreFXTestDir}/TestUtilities
465 local coreFXTestBinariesOutputPath=${__CoreFXTestDir}/tests_downloaded
467 if [ -z $CoreFXTestList]; then
468 local CoreFXTestList="${coreClrSrcTestDir}/CoreFX/CoreFX.issues.json"
472 # Check if we're running under OSX
474 local coreFXTestRemoteURL=$(<${coreClrSrcTestDir}/CoreFX/CoreFXTestListURL_OSX.txt)
475 local coreFXTestExclusionDef=nonosxtests
479 local coreFXTestRemoteURL=$(<${coreClrSrcTestDir}/CoreFX/CoreFXTestListURL_Linux.txt)
480 local coreFXTestExclusionDef=nonlinuxtests
484 local coreFXTestExecutable=xunit.console.netcore.exe
485 local coreFXLogDir=${coreClrBinDir}/Logs/CoreFX/
486 local coreFXTestExecutableArgs="--notrait category=nonnetcoreapptests --notrait category=${coreFXTestExclusionDef} --notrait category=failing --notrait category=IgnoreForCI --notrait category=OuterLoop --notrait Benchmark=true"
488 chmod +x ${dotnetExe}
489 resetCommandArgs=("msbuild /t:Restore ${coreFXTestSetupUtility}")
490 echo "${dotnetExe} $resetCommandArgs"
491 "${dotnetExe}" $resetCommandArgs
493 buildCommandArgs=("msbuild ${coreFXTestSetupUtility} /p:OutputPath=${coreFXTestSetupUtilityOutputPath} /p:Platform=${_arch} /p:Configuration=Release")
494 echo "${dotnetExe} $buildCommandArgs"
495 "${dotnetExe}" $buildCommandArgs
497 if [ "${RunCoreFXTestsAll}" == "1" ]; then
498 local coreFXRunCommand=--runAllTests
500 local coreFXRunCommand=--runSpecifiedTests
503 local buildTestSetupUtilArgs=("${coreFXTestSetupUtilityOutputPath}/${coreFXTestSetupUtilityName}.dll --clean --outputDirectory ${coreFXTestBinariesOutputPath} --testListJsonPath ${CoreFXTestList} ${coreFXRunCommand} --dotnetPath ${testHostDir}/dotnet --testUrl ${coreFXTestRemoteURL} --executable ${coreFXTestExecutable} --log ${coreFXLogDir} ${coreFXTestExecutableArgs}")
504 echo "${dotnetExe} $buildTestSetupUtilArgs"
505 "${dotnetExe}" $buildTestSetupUtilArgs
508 if [ $exitCode != 0 ]; then
509 echo Running CoreFX tests finished with failures
511 echo Running CoreFX tests finished successfully
514 echo Check ${coreFXLogDir} for test run logs
519 declare -a skipCrossGenFiles
521 function is_skip_crossgen_test {
522 for skip in "${skipCrossGenFiles[@]}"; do
523 if [ "$1" == "$skip" ]; then
530 function precompile_overlay_assemblies {
531 skipCrossGenFiles=($(read_array "$(dirname "$0")/skipCrossGenFiles.$ARCH.txt"))
533 if [ $doCrossgen == 1 ]; then
534 local overlayDir=$CORE_ROOT
536 filesToPrecompile=$(find -L $overlayDir -iname \*.dll -not -iname \*.ni.dll -not -iname \*-ms-win-\* -type f )
537 for fileToPrecompile in ${filesToPrecompile}
539 local filename=${fileToPrecompile}
540 if [ $jitdisasm == 1 ]; then
541 $overlayDir/corerun $overlayDir/jit-dasm.dll --crossgen $overlayDir/crossgen --platform $overlayDir --output $testRootDir/dasm $filename
543 if [ $exitCode != 0 ]; then
544 echo Unable to generate dasm for $filename
547 if is_skip_crossgen_test "$(basename $filename)"; then
550 echo Precompiling $filename
551 $overlayDir/crossgen /Platform_Assemblies_Paths $overlayDir $filename 1> $filename.stdout 2>$filename.stderr
553 if [[ $exitCode != 0 ]]; then
554 if grep -q -e '0x80131018' $filename.stderr; then
555 printf "\n\t$filename is not a managed assembly.\n\n"
557 echo Unable to precompile $filename.
563 rm $filename.{stdout,stderr}
568 echo Skipping crossgen of FX assemblies.
572 function copy_test_native_bin_to_test_root {
573 local errorSource='copy_test_native_bin_to_test_root'
576 if [ -z "$testNativeBinDir" ]; then
577 exit_with_error "$errorSource" "--testNativeBinDir is required."
579 testNativeBinDir=$testNativeBinDir/src
580 if [ ! -d "$testNativeBinDir" ]; then
581 exit_with_error "$errorSource" "Directory specified by --testNativeBinDir does not exist: $testNativeBinDir"
584 # Copy native test components from the native test build into the respective test directory in the test root directory
585 find "$testNativeBinDir" -type f -iname "*.$libExtension" |
586 while IFS='' read -r filePath || [ -n "$filePath" ]; do
587 local dirPath=$(dirname "$filePath")
588 cp -f "$filePath" "$coreRootDir"
592 # Variables for unsupported and failing tests
593 declare -a unsupportedTests
594 declare -a failingTests
595 declare -a playlistTests
596 ((runFailingTestsOnly = 0))
598 # Get an array of items by reading the specified file line by line.
599 function read_array {
602 if [ ! -f "$1" ]; then
606 # bash in Mac OS X doesn't support 'readarray', so using alternate way instead.
607 # readarray -t theArray < "$1"
608 # Any line that starts with '#' is ignored.
609 while IFS='' read -r line || [ -n "$line" ]; do
610 if [[ $line != "#"* ]]; then
611 theArray[${#theArray[@]}]=$line
617 function load_unsupported_tests {
618 # Load the list of tests that are not supported on this platform. These tests are disabled (skipped) permanently.
619 unsupportedTests=($(read_array "$(dirname "$0")/testsUnsupportedOutsideWindows.txt"))
620 unsupportedTests+=($(read_array "$(dirname "$0")/testsUnsupported.$ARCH.txt"))
623 function load_failing_tests {
624 # Load the list of tests that fail on this platform. These tests are disabled (skipped) temporarily, pending investigation.
625 failingTests=($(read_array "$(dirname "$0")/testsFailingOutsideWindows.txt"))
626 failingTests+=($(read_array "$(dirname "$0")/testsFailing.$ARCH.txt"))
629 function load_playlist_tests {
630 # Load the list of tests that are enabled as a part of this test playlist.
631 playlistTests=($(read_array "${playlistFile}"))
634 function is_unsupported_test {
635 for unsupportedTest in "${unsupportedTests[@]}"; do
636 if [ "$1" == "$unsupportedTest" ]; then
643 function is_failing_test {
644 for failingTest in "${failingTests[@]}"; do
645 if [ "$1" == "$failingTest" ]; then
652 function is_playlist_test {
653 for playlistTest in "${playlistTests[@]}"; do
654 if [ "$1" == "$playlistTest" ]; then
661 function skip_unsupported_test {
662 # This function runs in a background process. It should not echo anything, and should not use global variables. This
663 # function is analogous to run_test, and causes the test to be skipped with the message below.
665 local scriptFilePath=$1
666 local outputFilePath=$2
668 echo "Not supported on this platform." >"$outputFilePath"
669 return 2 # skip the test
672 function skip_failing_test {
673 # This function runs in a background process. It should not echo anything, and should not use global variables. This
674 # function is analogous to run_test, and causes the test to be skipped with the message below.
676 local scriptFilePath=$1
677 local outputFilePath=$2
679 echo "Temporarily disabled on this platform due to unexpected failures." >"$outputFilePath"
680 return 2 # skip the test
683 function skip_non_playlist_test {
684 # This function runs in a background process. It should not echo anything, and should not use global variables. This
685 # function is analogous to run_test, and causes the test to be skipped with the message below.
687 local scriptFilePath=$1
688 local outputFilePath=$2
690 echo "Test is not included in the running playlist." >"$outputFilePath"
691 return 2 # skip the test
694 function set_up_core_dump_generation {
695 # We will only enable dump generation here if we're on Mac or Linux
696 if [[ ! ( "$(uname -s)" == "Darwin" || "$(uname -s)" == "Linux" ) ]]; then
700 # We won't enable dump generation on OS X/macOS if the machine hasn't been
701 # configured with the kern.corefile pattern we expect.
702 if [[ ( "$(uname -s)" == "Darwin" && "$(sysctl -n kern.corefile)" != "core.%P" ) ]]; then
703 echo "WARNING: Core dump generation not being enabled due to unexpected kern.corefile value."
707 # Allow dump generation
710 if [ "$(uname -s)" == "Linux" ]; then
711 if [ -e /proc/self/coredump_filter ]; then
712 # Include memory in private and shared file-backed mappings in the dump.
713 # This ensures that we can see disassembly from our shared libraries when
714 # inspecting the contents of the dump. See 'man core' for details.
715 echo 0x3F > /proc/self/coredump_filter
720 function print_info_from_core_file {
723 if [ "$ARCH" == "arm64" ]; then
724 echo "Not inspecting core dumps on arm64 at the moment."
729 local core_file_name=$1
730 local executable_name=$2
732 if ! [ -e $executable_name ]; then
733 echo "Unable to find executable $executable_name"
735 elif ! [ -e $core_file_name ]; then
736 echo "Unable to find core file $core_file_name"
740 # Use LLDB to inspect the core dump on Mac, and GDB everywhere else.
741 if [[ "$OSName" == "Darwin" ]]; then
742 hash lldb 2>/dev/null || { echo >&2 "LLDB was not found. Unable to print core file."; return; }
744 echo "Printing info from core file $core_file_name"
745 lldb -c $core_file_name -b -o 'bt'
747 # Use GDB to print the backtrace from the core file.
748 hash gdb 2>/dev/null || { echo >&2 "GDB was not found. Unable to print core file."; return; }
750 echo "Printing info from core file $core_file_name"
751 gdb --batch -ex "thread apply all bt full" -ex "quit" $executable_name $core_file_name
755 function download_dumpling_script {
756 echo "Downloading latest version of dumpling script."
757 wget "https://dumpling.azurewebsites.net/api/client/dumpling.py"
759 local dumpling_script="dumpling.py"
760 chmod +x $dumpling_script
763 function upload_core_file_to_dumpling {
764 local core_file_name=$1
765 local dumpling_script="dumpling.py"
766 local dumpling_file="local_dumplings.txt"
768 # dumpling requires that the file exist before appending.
769 touch ./$dumpling_file
771 if [ ! -x $dumpling_script ]; then
772 download_dumpling_script
775 if [ ! -x $dumpling_script ]; then
776 echo "Failed to download dumpling script. Dump cannot be uploaded."
780 echo "Uploading $core_file_name to dumpling service."
782 local paths_to_add=""
783 if [ -d "$coreClrBinDir" ]; then
784 echo "Uploading CoreCLR binaries with dump."
785 paths_to_add=$coreClrBinDir
788 # Ensure the script has Unix line endings
789 perl -pi -e 's/\r\n|\n|\r/\n/g' "$dumpling_script"
791 # The output from this will include a unique ID for this dump.
792 ./$dumpling_script "upload" "--dumppath" "$core_file_name" "--incpaths" $paths_to_add "--properties" "Project=CoreCLR" "--squelch" | tee -a $dumpling_file
795 function preserve_core_file {
796 local core_file_name=$1
797 local storage_location="/tmp/coredumps_coreclr"
799 # Create the directory (this shouldn't fail even if it already exists).
800 mkdir -p $storage_location
802 # Only preserve the dump if the directory is empty. Otherwise, do nothing.
803 # This is a way to prevent us from storing/uploading too many dumps.
804 if [ ! "$(ls -A $storage_location)" ]; then
805 echo "Copying core file $core_file_name to $storage_location"
806 cp $core_file_name $storage_location
808 upload_core_file_to_dumpling $core_file_name
812 function inspect_and_delete_core_files {
813 # This function prints some basic information from core files in the current
814 # directory and deletes them immediately. Based on the state of the system, it may
815 # also upload a core file to the dumpling service.
816 # (see preserve_core_file).
818 # Depending on distro/configuration, the core files may either be named "core"
819 # or "core.<PID>" by default. We will read /proc/sys/kernel/core_uses_pid to
820 # determine which one it is.
821 # On OS X/macOS, we checked the kern.corefile value before enabling core dump
822 # generation, so we know it always includes the PID.
823 local core_name_uses_pid=0
824 if [[ (( -e /proc/sys/kernel/core_uses_pid ) && ( "1" == $(cat /proc/sys/kernel/core_uses_pid) ))
825 || ( "$(uname -s)" == "Darwin" ) ]]; then
829 if [ $core_name_uses_pid == "1" ]; then
830 # We don't know what the PID of the process was, so let's look at all core
831 # files whose name matches core.NUMBER
833 [[ $f =~ core.[0-9]+ ]] && print_info_from_core_file "$f" $CORE_ROOT/"corerun" && preserve_core_file "$f" && rm "$f"
835 elif [ -f core ]; then
836 print_info_from_core_file "core" $CORE_ROOT/"corerun"
837 preserve_core_file "core"
843 # This function runs in a background process. It should not echo anything, and should not use global variables.
845 local scriptFilePath=$1
846 local outputFilePath=$2
848 # Switch to directory where the script is
849 cd "$(dirname "$scriptFilePath")"
851 local scriptFileName=$(basename "$scriptFilePath")
852 local outputFileName=$(basename "$outputFilePath")
854 if [ "$limitedCoreDumps" == "ON" ]; then
855 set_up_core_dump_generation
858 "./$scriptFileName" >"$outputFileName" 2>&1
859 local testScriptExitCode=$?
861 # We will try to print some information from generated core dumps if a debugger
862 # is available, and possibly store a dump in a non-transient location.
863 if [ "$limitedCoreDumps" == "ON" ]; then
864 inspect_and_delete_core_files
867 return $testScriptExitCode
870 # Variables for running tests in the background
871 if [ `uname` = "NetBSD" ]; then
872 NumProc=$(getconf NPROCESSORS_ONLN)
873 elif [ `uname` = "Darwin" ]; then
874 NumProc=$(getconf _NPROCESSORS_ONLN)
876 if [ -x "$(command -v nproc)" ]; then
877 NumProc=$(nproc --all)
878 elif [ -x "$(command -v getconf)" ]; then
879 NumProc=$(getconf _NPROCESSORS_ONLN)
884 ((maxProcesses = $NumProc * 3 / 2)) # long tests delay process creation, use a few more processors
887 declare -a scriptFilePaths
888 declare -a outputFilePaths
889 declare -a processIds
890 declare -a testStartTimes
898 for (( i=0; i<$maxProcesses; i++ )); do
899 pid=${processIds[$i]}
900 if [ -z "$pid" ] || [ "$pid" == "$pidNone" ]; then
903 if ! kill -0 $pid 2>/dev/null; then
907 processIds[$i]=$pidNone
915 function get_available_process_index {
918 for (( i=0; i<$maxProcesses; i++ )); do
919 pid=${processIds[$i]}
920 if [ -z "$pid" ] || [ "$pid" == "$pidNone" ]; then
927 function finish_test {
929 local testScriptExitCode=$?
930 local finishedProcessIndex=$waitProcessIndex
933 local scriptFilePath=${scriptFilePaths[$finishedProcessIndex]}
934 local outputFilePath=${outputFilePaths[$finishedProcessIndex]}
935 local scriptFileName=$(basename "$scriptFilePath")
938 local testRunningTime=
941 if ((verbose == 1)); then
942 header=$(printf "[%4d]" $countTotalTests)
945 if [ "$showTime" == "ON" ]; then
946 testEndTime=$(date +%s)
947 testRunningTime=$(( $testEndTime - ${testStartTimes[$finishedProcessIndex]} ))
948 header=$header$(printf "[%4ds]" $testRunningTime)
952 case $testScriptExitCode in
954 let countPassedTests++
956 if ((verbose == 1 || runFailingTestsOnly == 1)); then
957 echo "PASSED - ${header}${scriptFilePath}"
959 echo " - ${header}${scriptFilePath}"
963 let countSkippedTests++
965 echo "SKIPPED - ${header}${scriptFilePath}"
968 let countFailedTests++
970 echo "FAILED - ${header}${scriptFilePath}"
973 let countTotalTests++
975 if ((verbose == 1 || testScriptExitCode != 0)); then
976 while IFS='' read -r line || [ -n "$line" ]; do
978 done <"$outputFilePath"
981 xunit_output_add_test "$scriptFilePath" "$outputFilePath" "$testResult" "$testScriptExitCode" "$testRunningTime"
982 text_file_output_add_test "$scriptFilePath" "$testResult"
985 function finish_remaining_tests {
986 # Finish the remaining tests in the order in which they were started
987 while ((processCount > 0)); do
993 local scriptFilePath=$1
994 local scriptFileDir=$(dirname "$scriptFilePath")
996 test "$verbose" == 1 && echo "Preparing $scriptFilePath"
998 if [ ! "$noLFConversion" == "ON" ]; then
999 # Convert DOS line endings to Unix if needed
1000 perl -pi -e 's/\r\n|\n|\r/\n/g' "$scriptFilePath"
1003 # Add executable file mode bit if needed
1004 chmod +x "$scriptFilePath"
1006 #remove any NI and Locks
1007 rm -f $scriptFileDir/*.ni.*
1008 rm -rf $scriptFileDir/lock
1011 function start_test {
1012 local nextProcessIndex=$(get_available_process_index)
1013 local scriptFilePath=$1
1014 if ((runFailingTestsOnly == 1)) && ! is_failing_test "$scriptFilePath"; then
1018 # Skip any test that's not in the current playlist, if a playlist was
1020 if [ -n "$playlistFile" ] && ! is_playlist_test "$scriptFilePath"; then
1024 if ((nextProcessIndex == maxProcesses)); then
1026 nextProcessIndex=$(get_available_process_index)
1029 scriptFilePaths[$nextProcessIndex]=$scriptFilePath
1030 local scriptFileName=$(basename "$scriptFilePath")
1031 local outputFilePath=$(dirname "$scriptFilePath")/${scriptFileName}.out
1032 outputFilePaths[$nextProcessIndex]=$outputFilePath
1034 if [ "$showTime" == "ON" ]; then
1035 testStartTimes[$nextProcessIndex]=$(date +%s)
1038 test "$verbose" == 1 && echo "Starting $scriptFilePath"
1039 if is_unsupported_test "$scriptFilePath"; then
1040 skip_unsupported_test "$scriptFilePath" "$outputFilePath" &
1041 elif ((runFailingTestsOnly == 0)) && is_failing_test "$scriptFilePath"; then
1042 skip_failing_test "$scriptFilePath" "$outputFilePath" &
1044 run_test "$scriptFilePath" "$outputFilePath" &
1046 processIds[$nextProcessIndex]=$!
1051 # Get a list of directories in which to scan for tests by reading the
1052 # specified file line by line.
1053 function set_test_directories {
1054 local errorSource='set_test_directories'
1056 local listFileName=$1
1058 if [ ! -f "$listFileName" ]
1060 exit_with_error "$errorSource" "Test directories file not found at $listFileName"
1062 testDirectories=($(read_array "$listFileName"))
1065 function run_tests_in_directory {
1068 # Recursively search through directories for .sh files to prepare them.
1069 # Note: This needs to occur before any test runs as some of the .sh files
1070 # depend on other .sh files
1071 for scriptFilePath in $(find "$testDir" -type f -iname '*.sh' | sort)
1073 prep_test "${scriptFilePath:2}"
1075 echo "The tests have been prepared"
1076 # Recursively search through directories for .sh files to run.
1077 for scriptFilePath in $(find "$testDir" -type f -iname '*.sh' | sort)
1079 start_test "${scriptFilePath:2}"
1083 function coreclr_code_coverage {
1084 local coverageDir="$coverageOutputDir/Coverage"
1085 local toolsDir="$coverageOutputDir/Coverage/tools"
1086 local reportsDir="$coverageOutputDir/Coverage/reports"
1087 local packageName="unix-code-coverage-tools.1.0.0.nupkg"
1090 mkdir -p $coverageDir
1092 mkdir -p $reportsDir
1093 pushd $toolsDir > /dev/null
1095 echo "Pulling down code coverage tools"
1096 wget -q https://www.myget.org/F/dotnet-buildtools/api/v2/package/unix-code-coverage-tools/1.0.0 -O $packageName
1097 echo "Unzipping to $toolsDir"
1098 unzip -q -o $packageName
1102 chmod a+rwx ./$OSName/llvm-cov
1105 echo "Generating coreclr code coverage reports at $reportsDir/coreclr.html"
1106 echo "./gcovr $coreClrObjs --gcov-executable=$toolsDir/$OS/llvm-cov -r $coreClrSrc --html --html-details -o $reportsDir/coreclr.html"
1108 ./gcovr $coreClrObjs --gcov-executable=$toolsDir/$OSName/llvm-cov -r $coreClrSrc --html --html-details -o $reportsDir/coreclr.html
1114 function check_cpu_architecture {
1115 local CPUName=$(uname -m)
1132 echo "Unknown CPU $CPUName detected, configuring as if for x64"
1140 ARCH=$(check_cpu_architecture)
1141 echo "Running on CPU- $ARCH"
1143 # Exit code constants
1144 readonly EXIT_CODE_SUCCESS=0 # Script ran normally.
1145 readonly EXIT_CODE_EXCEPTION=1 # Script exited because something exceptional happened (e.g. bad arguments, Ctrl-C interrupt).
1146 readonly EXIT_CODE_TEST_FAILURE=2 # Script completed successfully, but one or more tests failed.
1148 # Argument variables
1167 ((disableEventLogging = 0))
1181 exit $EXIT_CODE_SUCCESS
1190 export COMPlus_JitStress=${i#*=}
1193 export COMPlus_JitStressRegs=${i#*=}
1196 export COMPlus_JITMinOpts=1
1198 --copyNativeTestBin)
1199 export copyNativeTestBin=1
1202 export COMPlus_ForceRelocs=1
1205 export ILLINK=${i#*=}
1208 --tieredcompilation)
1209 export COMPlus_TieredCompilation=1
1215 ((ilasmroundtrip = 1))
1220 --testNativeBinDir=*)
1221 testNativeBinDir=${i#*=}
1224 coreOverlayDir=${i#*=}
1227 coreClrBinDir=${i#*=}
1233 coreFxBinDir=${i#*=}
1236 testDirectories[${#testDirectories[@]}]=${i#*=}
1239 set_test_directories "${i#*=}"
1241 --runFailingTestsOnly)
1242 ((runFailingTestsOnly = 1))
1244 --disableEventLogging)
1245 ((disableEventLogging = 1))
1248 export RunCrossGen=1
1251 export RunCoreFXTests=1
1254 export RunCoreFXTests=1
1255 export RunCoreFXTestsAll=1
1258 export RunCoreFXTests=1
1259 export CoreFXTestList=${i#*=}
1262 export testHostDir=${i#*=}
1265 ((maxProcesses = 1))
1277 playlistFile=${i#*=}
1288 --coverage-output-dir=*)
1289 coverageOutputDir=${i#*=}
1295 export COMPlus_GCStress=${i#*=}
1298 export COMPlus_GCName=${i#*=}
1306 --build-overlay-only)
1309 --limitedDumpGeneration)
1312 --xunitOutputPath=*)
1313 xunitOutputPath=${i#*=}
1316 echo "Unknown switch: $i"
1318 exit $EXIT_CODE_SUCCESS
1323 if [ -n "$coreOverlayDir" ] && [ "$buildOverlayOnly" == "ON" ]; then
1324 echo "Can not use \'--coreOverlayDir=<path>\' and \'--build-overlay-only\' at the same time."
1325 exit $EXIT_CODE_EXCEPTION
1328 if ((disableEventLogging == 0)); then
1329 export COMPlus_EnableEventLog=1
1332 export COMPlus_gcServer="$serverGC"
1334 if [ "$RunCoreFXTests" == 1 ];
1336 if [ -z "$coreClrSrc" ]
1338 echo "Coreclr src files are required to run CoreFX tests"
1339 echo "Coreclr src files root path can be passed using '--coreclr-src' argument"
1341 exit $EXIT_CODE_EXCEPTION
1344 if [ -z "$testHostDir" ]; then
1345 echo "--testHostDir is required to run CoreFX tests"
1347 exit $EXIT_CODE_EXCEPTION
1350 if [ ! -f "$testHostDir/dotnet" ]; then
1351 echo "Executable dotnet not found in $testHostDir"
1352 exit $EXIT_CODE_EXCEPTION
1355 if [ ! -d "$testHostDir" ]; then
1356 echo "Directory specified by --testHostDir does not exist: $testRootDir"
1357 exit $EXIT_CODE_EXCEPTION
1364 if [ -z "$testRootDir" ]; then
1365 echo "--testRootDir is required."
1367 exit $EXIT_CODE_EXCEPTION
1369 if [ ! -d "$testRootDir" ]; then
1370 echo "Directory specified by --testRootDir does not exist: $testRootDir"
1371 exit $EXIT_CODE_EXCEPTION
1374 # Copy native interop test libraries over to the mscorlib path in
1375 # order for interop tests to run on linux.
1376 if [ -z "$mscorlibDir" ]; then
1377 mscorlibDir=$coreClrBinDir
1380 if [ ! -z "$longgc" ]; then
1381 echo "Running Long GC tests"
1382 export RunningLongGCTests=1
1385 if [ ! -z "$gcsimulator" ]; then
1386 echo "Running GC simulator tests"
1387 export RunningGCSimulatorTests=1
1390 if [[ ! "$jitdisasm" -eq 0 ]]; then
1391 echo "Running jit disasm"
1392 export RunningJitDisasm=1
1395 if [ ! -z "$ilasmroundtrip" ]; then
1396 echo "Running Ilasm round trip"
1397 export RunningIlasmRoundTrip=1
1400 # If this is a coverage run, make sure the appropriate args have been passed
1401 if [ "$CoreClrCoverage" == "ON" ]
1403 echo "Code coverage is enabled for this run"
1405 if [ ! "$OSName" == "Darwin" ] && [ ! "$OSName" == "Linux" ]
1407 echo "Code Coverage not supported on $OS"
1411 if [ -z "$coreClrObjs" ]
1413 echo "Coreclr obj files are required to generate code coverage reports"
1414 echo "Coreclr obj files root path can be passed using '--coreclr-obj' argument"
1418 if [ -z "$coreClrSrc" ]
1420 echo "Coreclr src files are required to generate code coverage reports"
1421 echo "Coreclr src files root path can be passed using '--coreclr-src' argument"
1425 if [ -z "$coverageOutputDir" ]
1427 echo "Output directory for coverage results must be specified"
1428 echo "Output path can be specified '--coverage-output-dir' argument"
1434 text_file_output_begin
1436 precompile_overlay_assemblies
1438 if [ "$buildOverlayOnly" == "ON" ];
1440 echo "Build overlay directory '$coreOverlayDir' complete."
1444 if [ -n "$playlistFile" ]
1446 # Use a playlist file exclusively, if it was provided
1447 echo "Executing playlist $playlistFile"
1450 load_unsupported_tests
1454 scriptPath=$(dirname $0)
1455 ${scriptPath}/setup-stress-dependencies.sh --arch=$ARCH --outputDir=$coreOverlayDir
1457 export __TestEnv=$testEnv
1461 dumplingsListPath="$testRootDir/dumplings.txt"
1463 # clean up any existing dumpling remnants from previous runs.
1464 rm -f "$dumplingsListPath"
1465 find $testRootDir -type f -name "local_dumplings.txt" -exec rm {} \;
1467 time_start=$(date +"%s")
1468 if [ -z "$testDirectories" ]
1470 # No test directories were specified, so run everything in the current
1471 # directory and its subdirectories.
1472 run_tests_in_directory "."
1474 # Otherwise, run all the tests in each specified test directory.
1475 for testDir in "${testDirectories[@]}"
1477 if [ ! -d "$testDir" ]; then
1478 echo "Test directory does not exist: $testDir"
1480 run_tests_in_directory "./$testDir"
1484 finish_remaining_tests
1488 find $testRootDir -type f -name "local_dumplings.txt" -exec cat {} \; > $dumplingsListPath
1490 if [ -s $dumplingsListPath ]; then
1491 cat $dumplingsListPath
1493 rm $dumplingsListPath
1496 time_end=$(date +"%s")
1497 time_diff=$(($time_end-$time_start))
1498 echo "$(($time_diff / 60)) minutes and $(($time_diff % 60)) seconds taken to run CoreCLR tests."
1502 if [ "$CoreClrCoverage" == "ON" ]
1504 coreclr_code_coverage
1507 if ((countFailedTests > 0)); then
1508 exit $EXIT_CODE_TEST_FAILURE
1511 exit $EXIT_CODE_SUCCESS