From 3b40c7cfc4519a95e9d03ac85a2a2de68299aa7f Mon Sep 17 00:00:00 2001 From: =?utf8?q?Tom=C3=A1=C5=A1=20Rylek?= Date: Thu, 12 Aug 2021 15:54:17 +0200 Subject: [PATCH] Add support for building just a subset of runtime tests (#57142) This change adds four new options to the runtime test build command: 1) test - build just that one test. 2) dir - build all tests in the directory. 3) tree - build all tests in the given subtree. 4) all - use clean rebuild (i.e. don't apply incrementalism) when building the tests. Thanks Tomas --- .../testing/coreclr/unix-test-instructions.md | 32 ++++++++++++++++ .../testing/coreclr/windows-test-instructions.md | 33 +++++++++++++++++ src/tests/Common/dirs.proj | 22 +++++++++++ src/tests/build.cmd | 24 ++++++++++++ src/tests/build.sh | 43 ++++++++++++++++++---- 5 files changed, 146 insertions(+), 8 deletions(-) diff --git a/docs/workflow/testing/coreclr/unix-test-instructions.md b/docs/workflow/testing/coreclr/unix-test-instructions.md index 18842af..981ce5a 100644 --- a/docs/workflow/testing/coreclr/unix-test-instructions.md +++ b/docs/workflow/testing/coreclr/unix-test-instructions.md @@ -40,6 +40,38 @@ The `src/tests/build.sh` script generates the Core_Root folder, which contains t The output will be at `/artifacts/tests/coreclr/../Tests/Core_Root`. +## Building Test Subsets + +The `src/tests/build.sh` script supports three options that let you limit the set of tests to build; +when none of these is specified, the entire test tree (under `src/tests`) gets built but that can be +lengthy (especially in `-priority1` mode) and unnecessary when working on a particular test. + +1) `-test:` - build a particular test project specified by its project file path, +either absolute or relative to `src/tests`. The option can be specified multiple times on the command +line to request building several individual projects; alternatively, a single occurrence of the option +can represent several project paths separated by semicolons. + +**Example**: `src/tests/build.sh -test:JIT/Methodical/divrem/div/i4div_cs_do.csproj;JIT/Methodical/divrem/div/i8div_cs_do.csproj` + +2) `-dir:` - build all test projects within a given directory path, either absolute +or relative to `src/tests`. The option can be specified multiple times on the command line to request +building projects in several folders; alternatively, a single occurrence of this option can represent +several project folders separated by semicolons. + +**Example**: `src/tests/build.sh -dir:JIT/Methodical/Arrays/huge;JIT/Methodical/divrem/div` + +3) `-tree:` - build all test projects within the subtree specified by its root path, +either absolute or relative to `src/tests`. The option can be specified multiple times on the command +line to request building projects in several subtrees; alternatively, a single instance of the option +can represent several project subtree root folder paths separated by semicolons. + +**Example**: `src/tests/build.sh -tree:baseservices/exceptions;JIT/Methodical` + +**Please note** that priority filtering is orthogonal to specifying test subsets; even when you request +building a particular test and the test is Pri1, you need to specify `-priority1` in the command line, +otherwise the test build will get skipped. While somewhat unintuitive, 'fixing' this ad hoc would likely +create more harm than good and we're hoping to ultimately get rid of the test priorities in the long term +anyway. ## Building Individual Tests diff --git a/docs/workflow/testing/coreclr/windows-test-instructions.md b/docs/workflow/testing/coreclr/windows-test-instructions.md index dccf41f..18e22ae 100644 --- a/docs/workflow/testing/coreclr/windows-test-instructions.md +++ b/docs/workflow/testing/coreclr/windows-test-instructions.md @@ -77,6 +77,39 @@ For additional supported parameters use the following: src\tests\build.cmd -? ``` +## Building Test Subsets + +The `src\tests\build.cmd` script supports three options that let you limit the set of tests to build; +when none of these is specified, the entire test tree (under `src\tests`) gets built but that can be +lengthy (especially in `-priority=1` mode) and unnecessary when working on a particular test. + +1) `test ` - build a particular test project specified by its project file path, +either absolute or relative to `src\tests`. The option can be specified multiple times on the command +line to request building several individual projects; alternatively, a single occurrence of the option +can represent several project paths separated by semicolons. + +**Example**: `src\tests\build.cmd test JIT/Methodical/divrem/div/i4div_cs_do.csproj;JIT/Methodical/divrem/div/i8div_cs_do.csproj` + +2) `dir ` - build all test projects within a given directory path, either absolute +or relative to `src\tests`. The option can be specified multiple times on the command line to request +building projects in several folders; alternatively, a single instance of the option +can represent several project folders separated by semicolons. + +**Example**: `src\tests\build.cmd dir JIT/Methodical/Arrays/huge;JIT/Methodical/divrem/div` + +3) `tree ` - build all test projects within the subtree specified by its root path, +either absolute or relative to `src\tests`. The option can be specified multiple times on the command +line to request building projects in several subtrees; alternatively, a single instance of the option +can represent several project subtree root folder paths separated by semicolons. + +**Example**: `src\tests\build.cmd tree baseservices/exceptions;JIT/Methodical` + +**Please note** that priority filtering is orthogonal to specifying test subsets; even when you request +building a particular test and the test is Pri1, you need to specify `-priority=1` in the command line, +otherwise the test build will get skipped. While somewhat unintuitive, 'fixing' this ad hoc would likely +create more harm than good and we're hoping to ultimately get rid of the test priorities in the long term +anyway. + ## Building Individual Tests **Note:** `build.cmd skipmanaged [Any additional flags]` needs to be run at least once if the individual test has native assets. diff --git a/src/tests/Common/dirs.proj b/src/tests/Common/dirs.proj index 6766e6b..68d0ff1 100644 --- a/src/tests/Common/dirs.proj +++ b/src/tests/Common/dirs.proj @@ -20,6 +20,28 @@ + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/tests/build.cmd b/src/tests/build.cmd index 09c10f8..e46ace6 100644 --- a/src/tests/build.cmd +++ b/src/tests/build.cmd @@ -41,6 +41,11 @@ set processedArgs= set __UnprocessedBuildArgs= set __CommonMSBuildArgs= +set __RebuildTests=0 +set __BuildTestProject=%%3B +set __BuildTestDir=%%3B +set __BuildTestTree=%%3B + set __SkipRestorePackages= set __SkipManaged= set __SkipTestWrappers= @@ -95,6 +100,12 @@ if /i "%1" == "skipnative" (set __SkipNative=1&set __CopyNativeProjec if /i "%1" == "skiptestwrappers" (set __SkipTestWrappers=1&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) if /i "%1" == "skipgeneratelayout" (set __SkipGenerateLayout=1&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) +if /i "%1" == "rebuild" (set __RebuildTests=1&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) + +if /i "%1" == "test" (set __BuildTestProject=!__BuildTestProject!%2%%3B&set processedArgs=!processedArgs! %1 %2&shift&shift&goto Arg_Loop) +if /i "%1" == "dir" (set __BuildTestDir=!__BuildTestDir!%2%%3B&set processedArgs=!processedArgs! %1 %2&shift&shift&goto Arg_Loop) +if /i "%1" == "tree" (set __BuildTestTree=!__BuildTestTree!%2%%3B&set processedArgs=!processedArgs! %1 %2&shift&shift&goto Arg_Loop) + if /i "%1" == "copynativeonly" (set __CopyNativeTestBinaries=1&set __SkipNative=1&set __CopyNativeProjectsAfterCombinedTestBuild=false&set __SkipGenerateLayout=1&set __SkipTestWrappers=1&set __SkipCrossgenFramework=1&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) if /i "%1" == "generatelayoutonly" (set __SkipManaged=1&set __SkipNative=1&set __CopyNativeProjectsAfterCombinedTestBuild=false&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) if /i "%1" == "buildtestwrappersonly" (set __SkipNative=1&set __SkipManaged=1&set __BuildTestWrappersOnly=1&set __SkipGenerateLayout=1&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) @@ -156,6 +167,11 @@ set "__TestRootDir=%__RootBinDir%\tests\coreclr" set "__TestBinDir=%__TestRootDir%\%__OSPlatformConfig%" set "__TestIntermediatesDir=%__TestRootDir%\obj\%__OSPlatformConfig%" +if "%__RebuildTests%" == "1" ( + echo Removing tests build dir^: !__TestBinDir! + rmdir /s /q !__TestBinDir! +) + if not defined XunitTestBinBase set XunitTestBinBase=%__TestBinDir%\ set "CORE_ROOT=%XunitTestBinBase%\Tests\Core_Root" @@ -361,6 +377,10 @@ for /l %%G in (1, 1, %__NumberOfTestGroups%) do ( set __MSBuildBuildArgs=!__MSBuildBuildArgs! /p:__SkipPackageRestore=true set __MSBuildBuildArgs=!__MSBuildBuildArgs! !__NativeBinariesLayoutTypeArg! set __MSBuildBuildArgs=!__MSBuildBuildArgs! /p:RuntimeFlavor=!__RuntimeFlavor! + set __MSBuildBuildArgs=!__MSBuildBuildArgs! /p:BuildTestProject=!__BuildTestProject! + set __MSBuildBuildArgs=!__MSBuildBuildArgs! /p:BuildTestDir=!__BuildTestDir! + set __MSBuildBuildArgs=!__MSBuildBuildArgs! /p:BuildTestTree=!__BuildTestTree! + echo Running: msbuild !__MSBuildBuildArgs! !__CommonMSBuildCmdPrefix! !__MSBuildBuildArgs! @@ -559,6 +579,10 @@ echo -priority=^ : specify a set of tests that will be built and run, with p echo 0: Build only priority 0 cases as essential testcases (default) echo 1: Build all tests with priority 0 and 1 echo 666: Build all tests with priority 0, 1 ... 666 +echo test ^: Only build test project ^ ^(relative or absolute project path under src\tests^) +echo dir ^: Build all test projects in the folder ^ ^(relative or absolute folder under src\tests^) +echo tree ^: Build all test projects in the subtree ^ ^(relative or absolute folder under src\tests^) +echo rebuild: Clean up all test artifacts prior to building tests echo allTargets: Build managed tests for all target platforms. echo -verbose: enables detailed file logging for the msbuild tasks into the msbuild log file. exit /b 1 diff --git a/src/tests/build.sh b/src/tests/build.sh index 167cb6c..ac59074 100755 --- a/src/tests/build.sh +++ b/src/tests/build.sh @@ -64,14 +64,6 @@ generate_layout() __ProjectFilesDir="$__TestDir" __TestBinDir="$__TestWorkingDir" - - if [[ "$__RebuildTests" -ne 0 ]]; then - if [[ -d "${__TestBinDir}" ]]; then - echo "Removing tests build dir: ${__TestBinDir}" - rm -rf "$__TestBinDir" - fi - fi - __CMakeBinDir="${__TestBinDir}" if [[ -z "$__TestIntermediateDir" ]]; then @@ -370,6 +362,9 @@ build_MSBuild_projects() buildArgs+=("\"/p:CopyNativeProjectBinaries=${__CopyNativeProjectsAfterCombinedTestBuild}\""); buildArgs+=("/p:__SkipPackageRestore=true"); buildArgs+=("/bl:${__RepoRootDir}/artifacts/log/${__BuildType}/build_managed_tests_${testGroupToBuild}.binlog"); + buildArgs+=("/p:BuildTestProject=${__BuildTestProject}"); + buildArgs+=("/p:BuildTestDir=${__BuildTestDir}"); + buildArgs+=("/p:BuildTestTree=${__BuildTestTree}"); # Disable warnAsError - coreclr issue 19922 nextCommand="\"$__RepoRootDir/eng/common/msbuild.sh\" $__ArcadeScriptArgs --warnAsError false ${buildArgs[@]}" @@ -432,6 +427,10 @@ usage_list+=("-buildtestwrappersonly: only build the test wrappers.") usage_list+=("-copynativeonly: Only copy the native test binaries to the managed output. Do not build the native or managed tests.") usage_list+=("-generatelayoutonly: only pull down dependencies and build coreroot.") +usage_list+=("-test:xxx - only build a single test project"); +usage_list+=("-dir:xxx - build all tests in a given directory"); +usage_list+=("-tree:xxx - build all tests in a given subtree"); + usage_list+=("-crossgen2: Precompiles the framework managed assemblies in coreroot using the Crossgen2 compiler.") usage_list+=("-priority1: include priority=1 tests in the build.") usage_list+=("-allTargets: Build managed tests for all target platforms.") @@ -492,6 +491,24 @@ handle_arguments_local() { __RebuildTests=1 ;; + test*|-test*) + local arg="$1" + local parts=(${arg//:/ }) + __BuildTestProject="$__BuildTestProject${parts[1]}%3B" + ;; + + dir*|-dir*) + local arg="$1" + local parts=(${arg//:/ }) + __BuildTestDir="$__BuildTestDir${parts[1]}%3B" + ;; + + tree*|-tree*) + local arg="$1" + local parts=(${arg//:/ }) + __BuildTestTree="$__BuildTestTree${parts[1]}%3B" + ;; + runtests|-runtests) __RunTests=1 ;; @@ -541,6 +558,9 @@ __DistroRid="" __DoCrossgen2=0 __CompositeBuildMode=0 __TestBuildMode= +__BuildTestProject="%3B" +__BuildTestDir="%3B" +__BuildTestTree="%3B" __DotNetCli="$__RepoRootDir/dotnet.sh" __GenerateLayoutOnly= __IsMSBuildOnNETCoreSupported=0 @@ -611,6 +631,13 @@ if [[ -z "$HOME" ]]; then echo "HOME not defined; setting it to $HOME" fi +if [[ "$__RebuildTests" -ne 0 ]]; then + if [[ -d "${__TestWorkingDir}" ]]; then + echo "Removing tests build dir: ${__TestWorkingDir}" + rm -rf "${__TestWorkingDir}" + fi +fi + if [[ (-z "$__GenerateLayoutOnly") && (-z "$__BuildTestWrappersOnly") && ("$__MonoAot" -eq 0) ]]; then build_Tests elif [[ ! -z "$__BuildTestWrappersOnly" ]]; then -- 2.7.4