From 7d6b73e97f960bc0c8b7432c0fd86934b0eb9f7e Mon Sep 17 00:00:00 2001 From: Egor Chesakov Date: Tue, 17 Nov 2020 10:20:42 -0800 Subject: [PATCH] Use new CoreDisTools package (#44490) R2RDump and GCStress use a version of coredistools produced from https://github.com/dotnet/jitutils --- eng/Versions.props | 1 + eng/coredistools.targets | 15 ++ .../runtimes/android-runtime-and-send-to-helix.yml | 2 +- .../common/templates/runtimes/build-test-job.yml | 2 +- .../common/templates/runtimes/run-test-job.yml | 2 +- .../runtimes/wasm-runtime-and-send-to-helix.yml | 2 +- eng/pipelines/coreclr/templates/build-job.yml | 2 +- .../runtimelab/runtimelab-post-build-steps.yml | 2 +- src/coreclr/src/tools/r2rdump/R2RDump.csproj | 7 +- src/tests/Common/Directory.Build.targets | 14 ++ src/tests/Common/setup-stress-dependencies.cmd | 124 ------------ src/tests/Common/setup-stress-dependencies.sh | 213 --------------------- .../stress_dependencies/stress_dependencies.csproj | 49 ----- src/tests/build.cmd | 33 +--- src/tests/build.sh | 18 -- 15 files changed, 42 insertions(+), 444 deletions(-) create mode 100644 eng/coredistools.targets delete mode 100644 src/tests/Common/setup-stress-dependencies.cmd delete mode 100755 src/tests/Common/setup-stress-dependencies.sh delete mode 100644 src/tests/Common/stress_dependencies/stress_dependencies.csproj diff --git a/eng/Versions.props b/eng/Versions.props index 6cfac23..d7352dd 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -138,6 +138,7 @@ 4.9.4 4.9.4 + 1.0.1-prerelease-00006 16.8.0-release-20201022-02 1.0.0-prerelease.20559.2 1.0.0-prerelease.20559.2 diff --git a/eng/coredistools.targets b/eng/coredistools.targets new file mode 100644 index 0000000..4c99911 --- /dev/null +++ b/eng/coredistools.targets @@ -0,0 +1,15 @@ + + + + + native + true + + + + + $(PkgMicrosoft_NETCore_CoreDisTools)\runtimes\win-$(TargetArchitecture)\native\coredistools.dll + $(PkgMicrosoft_NETCore_CoreDisTools)\runtimes\linux-$(TargetArchitecture)\native\libcoredistools.so + $(PkgMicrosoft_NETCore_CoreDisTools)\runtimes\osx-$(TargetArchitecture)\native\libcoredistools.dylib + + diff --git a/eng/pipelines/common/templates/runtimes/android-runtime-and-send-to-helix.yml b/eng/pipelines/common/templates/runtimes/android-runtime-and-send-to-helix.yml index 606a441..b5390fd 100644 --- a/eng/pipelines/common/templates/runtimes/android-runtime-and-send-to-helix.yml +++ b/eng/pipelines/common/templates/runtimes/android-runtime-and-send-to-helix.yml @@ -25,7 +25,7 @@ parameters: steps: - - script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) /p:LibrariesConfiguration=${{ parameters.buildConfig }} -ci -skipstressdependencies -excludemonofailures os Android x64 $(buildConfigUpper) + - script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) /p:LibrariesConfiguration=${{ parameters.buildConfig }} -ci -excludemonofailures os Android x64 $(buildConfigUpper) displayName: Build Tests # Send tests to Helix diff --git a/eng/pipelines/common/templates/runtimes/build-test-job.yml b/eng/pipelines/common/templates/runtimes/build-test-job.yml index f28ce14..313cb5e 100644 --- a/eng/pipelines/common/templates/runtimes/build-test-job.yml +++ b/eng/pipelines/common/templates/runtimes/build-test-job.yml @@ -120,7 +120,7 @@ jobs: displayName: Disk Usage before Build # Build managed test components - - script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) allTargets skipstressdependencies skipnative skipgeneratelayout skiptestwrappers $(buildConfig) $(archType) $(crossArg) $(priorityArg) ci $(librariesOverrideArg) + - script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) allTargets skipnative skipgeneratelayout skiptestwrappers $(buildConfig) $(archType) $(crossArg) $(priorityArg) ci $(librariesOverrideArg) displayName: Build managed test components - ${{ if in(parameters.osGroup, 'OSX', 'iOS','tvOS') }}: diff --git a/eng/pipelines/common/templates/runtimes/run-test-job.yml b/eng/pipelines/common/templates/runtimes/run-test-job.yml index c3d4481..5fd480b 100644 --- a/eng/pipelines/common/templates/runtimes/run-test-job.yml +++ b/eng/pipelines/common/templates/runtimes/run-test-job.yml @@ -247,7 +247,7 @@ jobs: # and directly unzip them there after download). Unfortunately the logic to copy # the native artifacts to the final test folders is dependent on availability of the # managed test artifacts. - - script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) skipstressdependencies copynativeonly $(crossgenArg) $(buildConfig) $(archType) $(priorityArg) $(librariesOverrideArg) + - script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) copynativeonly $(crossgenArg) $(buildConfig) $(archType) $(priorityArg) $(librariesOverrideArg) displayName: Copy native test components to test output folder diff --git a/eng/pipelines/common/templates/runtimes/wasm-runtime-and-send-to-helix.yml b/eng/pipelines/common/templates/runtimes/wasm-runtime-and-send-to-helix.yml index 79db360..462fe65 100644 --- a/eng/pipelines/common/templates/runtimes/wasm-runtime-and-send-to-helix.yml +++ b/eng/pipelines/common/templates/runtimes/wasm-runtime-and-send-to-helix.yml @@ -31,7 +31,7 @@ parameters: steps: - - script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) /p:LibrariesConfiguration=${{ parameters.buildConfig }} -ci -skipstressdependencies -excludemonofailures os Browser wasm $(buildConfigUpper) + - script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) /p:LibrariesConfiguration=${{ parameters.buildConfig }} -ci -excludemonofailures os Browser wasm $(buildConfigUpper) displayName: Build Tests # Send tests to Helix diff --git a/eng/pipelines/coreclr/templates/build-job.yml b/eng/pipelines/coreclr/templates/build-job.yml index 42b1adf..befa3c1 100644 --- a/eng/pipelines/coreclr/templates/build-job.yml +++ b/eng/pipelines/coreclr/templates/build-job.yml @@ -197,7 +197,7 @@ jobs: # Build native test components - ${{ if ne(parameters.isOfficialBuild, true) }}: - - script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) skipstressdependencies skipmanaged skipgeneratelayout $(buildConfig) $(archType) $(crossArg) $(osArg) $(priorityArg) $(compilerArg) + - script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) skipmanaged skipgeneratelayout $(buildConfig) $(archType) $(crossArg) $(osArg) $(priorityArg) $(compilerArg) displayName: Build native test components # Sign and add entitlements to these MacOS binaries diff --git a/eng/pipelines/runtimelab/runtimelab-post-build-steps.yml b/eng/pipelines/runtimelab/runtimelab-post-build-steps.yml index 124b38f..abccf70 100644 --- a/eng/pipelines/runtimelab/runtimelab-post-build-steps.yml +++ b/eng/pipelines/runtimelab/runtimelab-post-build-steps.yml @@ -14,7 +14,7 @@ parameters: steps: # Build coreclr native test output - ${{ if eq(parameters.uploadRuntimeTests, true) }}: - - script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) skipstressdependencies skipmanaged skipgeneratelayout $(buildConfigUpper) ${{ parameters.archType }} + - script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) skipmanaged skipgeneratelayout $(buildConfigUpper) ${{ parameters.archType }} displayName: Build native test components # Copy all build output into artifacts staging directory diff --git a/src/coreclr/src/tools/r2rdump/R2RDump.csproj b/src/coreclr/src/tools/r2rdump/R2RDump.csproj index 46b8e0c..7d1a73b 100644 --- a/src/coreclr/src/tools/r2rdump/R2RDump.csproj +++ b/src/coreclr/src/tools/r2rdump/R2RDump.csproj @@ -15,15 +15,16 @@ $(RuntimeBinDir)/R2RDump + - - 1.0.1-prerelease-00005 - $(SystemCommandLineVersion) + + PreserveNewest + diff --git a/src/tests/Common/Directory.Build.targets b/src/tests/Common/Directory.Build.targets index cf31232..a52f83e 100644 --- a/src/tests/Common/Directory.Build.targets +++ b/src/tests/Common/Directory.Build.targets @@ -10,6 +10,16 @@ + + false + true + true + false + true + + + + @@ -132,6 +142,10 @@ TargetDir="runtime-test/"/> + + + + /outputdir ^ -echo. -exit /b 1 diff --git a/src/tests/Common/setup-stress-dependencies.sh b/src/tests/Common/setup-stress-dependencies.sh deleted file mode 100755 index d54f5c6..0000000 --- a/src/tests/Common/setup-stress-dependencies.sh +++ /dev/null @@ -1,213 +0,0 @@ -#!/usr/bin/env bash -# set -x - -# -# Constants -# -readonly EXIT_CODE_SUCCESS=0 - -# -# This script should be located in coreclr/tests. -# - -function print_usage { - echo '' - echo 'Download coredistools for GC stress testing' - echo '' - echo 'Command line:' - echo '' - echo './setup-stress-dependencies.sh --arch= --outputDir=' - echo '' - echo 'Required arguments:' - echo ' --arch= : Target arch for the build' - echo ' --outputDir= : Directory to install libcoredistools.so' - echo '' -} - -function exit_with_error { - local errorCode=$1 - local errorMsg=$2 - - if [ ! -z "$2" ]; then - echo $2 - fi - - exit $errorCode -} - -function handle_ctrl_c { - exit_with_error 1 'Aborted by Ctrl+C' - } - -# Register the Ctrl-C handler -trap handle_ctrl_c INT - -# Argument variables -libInstallDir= - -# Handle arguments -verbose=0 -for i in "$@" -do - case $i in - -h|--help) - exit $EXIT_CODE_SUCCESS - ;; - -v|--verbose) - verbose=1 - ;; - --arch=*) - __BuildArch=${i#*=} - ;; - --outputDir=*) - libInstallDir=${i#*=} - ;; - *) - echo "Unknown switch: $i" - print_usage - exit $EXIT_CODE_SUCCESS - ;; - esac -done - -if [ -z "$__BuildArch" ]; then - echo "--arch is required." - print_usage - exit_with_error 1 -fi - -if [ -z "$libInstallDir" ]; then - echo "--outputDir is required." - print_usage - exit_with_error 1 -fi - -if [ "$__BuildArch" = "arm64" ] || [ "$__BuildArch" = "arm" ]; then - echo "No runtime dependencies for arm32/arm64" - exit $EXIT_CODE_SUCCESS -fi - -# This script must be located in coreclr/tests. -scriptDir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - -dotnet=$"${scriptDir}"/../../../dotnet.sh -csprojPath="${scriptDir}"/stress_dependencies/stress_dependencies.csproj - -if [ ! -e $dotnetCmd ]; then - exit_with_error 1 'dotnet commandline does not exist:'$dotnetCmd -fi - -# make output directory -if [ ! -e $libInstallDir ]; then - mkdir -p $libInstallDir -fi - -# Use uname to determine what the OS is. -OSName=$(uname -s) -case "$OSName" in - Linux) - __TargetOS=Linux - __HostOS=Linux - ;; - - Darwin) - __TargetOS=OSX - __HostOS=OSX - ;; - - FreeBSD) - __TargetOS=FreeBSD - __HostOS=FreeBSD - ;; - - OpenBSD) - __TargetOS=OpenBSD - __HostOS=OpenBSD - ;; - - NetBSD) - __TargetOS=NetBSD - __HostOS=NetBSD - ;; - - SunOS) - if uname -o 2>&1 | grep -q illumos; then - __TargetOS=illumos - __HostOS=illumos - else - __TargetOS=Solaris - __HostOS=Solaris - fi - ;; - - *) - echo "Unsupported OS $OSName detected, configuring as if for Linux" - __TargetOS=Linux - __HostOS=Linux - ;; -esac - -isPortable=0 - -source "${scriptDir}"/../../../eng/native/init-distro-rid.sh -initDistroRidGlobal "$__TargetOS" x64 "$isPortable" - -# Hack, replace the rid to ubuntu.14.04 which has a valid non-portable -# package. -# -# The CoreDisTools package is currently manually packaged and we only have -# 14.04 and 16.04 packages. Use the oldest package which will work on newer -# platforms. -if [ "$__TargetOS" = "Linux" ]; then - if [ "$__BuildArch" = "x64" ]; then - __DistroRid=ubuntu.14.04-x64 - elif [ "$__BuildArch" = "x86" ]; then - __DistroRid=ubuntu.14.04-x86 - fi -fi - -# Query runtime Id -rid="$__DistroRid" - -echo "Rid to be used: ${rid}" - -if [ -z "$rid" ]; then - exit_with_error 1 "Failed to query runtime Id" -fi - -# Download the package -echo Downloading CoreDisTools package -bash -c -x "$dotnet restore $csprojPath" -if [ $? -ne 0 ] -then - exit_with_error 1 "Failed to restore the package" -fi - -CoreDisToolsPackagePathOutputFile="${scriptDir}/../../../artifacts/obj/coreclr/${__TargetOS}.x64/optdatapath.txt" - -bash -c -x "$dotnet msbuild $csprojPath /t:DumpCoreDisToolsPackagePath /p:CoreDisToolsPackagePathOutputFile=\"$CoreDisToolsPackagePathOutputFile\" /p:RuntimeIdentifier=\"$rid\"" -if [ $? -ne 0 ] -then - exit_with_error 1 "Failed to find the path to CoreDisTools." -fi - -packageDir=$(<"${CoreDisToolsPackagePathOutputFile}") - -# Get library path -libPath="$(find "$packageDir" -path "*$rid*libcoredistools*" -print | head -n 1)" -echo "libPath to be used: ${libPath}" - -if [ ! -e $libPath ] || [ -z "$libPath" ]; then - exit_with_error 1 'Failed to locate the downloaded library' -fi - -# Copy library to output directory -echo 'Copy library:' $libPath '-->' $libInstallDir/ -cp -f $libPath $libInstallDir -if [ $? -ne 0 ] -then - exit_with_error 1 "Failed to copy the library" -fi - -# Return success -exit $EXIT_CODE_SUCCESS diff --git a/src/tests/Common/stress_dependencies/stress_dependencies.csproj b/src/tests/Common/stress_dependencies/stress_dependencies.csproj deleted file mode 100644 index 10d2cd9..0000000 --- a/src/tests/Common/stress_dependencies/stress_dependencies.csproj +++ /dev/null @@ -1,49 +0,0 @@ - - - BuildOnly - netcoreapp2.0 - false - win-x64;ubuntu.14.04-x64;osx.10.10-x64;win-x86;ubuntu.14.04-x86;osx.10.10-x86 - Microsoft.NETCore.CoreDisTools - - - - 1.0.1-prerelease-00005 - - - - - - - - - - - - - - - - - - - - - - $([System.IO.Path]::GetFileName('$(PkgMicrosoft_NETCore_CoreDisTools)')) - - 1.0.1-prerelease-00002 - $(MicrosoftNETCoreCoreDisToolsPackageName.ToLower()) - $([MSBuild]::NormalizePath($(PkgMicrosoft_NETCore_CoreDisTools),../../,runtime.$(RuntimeIdentifier).$(CoreDisToolsPackageNameLowercase),$(CoreDisToolsPackageVersion))) - - - - - - - - - - - - diff --git a/src/tests/build.cmd b/src/tests/build.cmd index 18d5616..1003f48 100644 --- a/src/tests/build.cmd +++ b/src/tests/build.cmd @@ -48,7 +48,6 @@ set __UnprocessedBuildArgs= set __CommonMSBuildArgs= set __SkipRestorePackages= -set __SkipStressDependencies= set __SkipManaged= set __SkipTestWrappers= set __BuildTestWrappersOnly= @@ -96,16 +95,15 @@ if /i "%1" == "checked" (set __BuildType=Checked&set processedArgs if /i "%1" == "ci" (set __ArcadeScriptArgs="-ci"&set __ErrMsgPrefix=##vso[task.logissue type=error]&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) -if /i "%1" == "skipstressdependencies" (set __SkipStressDependencies=1&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) if /i "%1" == "skiprestorepackages" (set __SkipRestorePackages=1&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) if /i "%1" == "skipmanaged" (set __SkipManaged=1&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) if /i "%1" == "skipnative" (set __SkipNative=1&set __CopyNativeProjectsAfterCombinedTestBuild=false&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) if /i "%1" == "skiptestwrappers" (set __SkipTestWrappers=1&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) if /i "%1" == "skipgeneratelayout" (set __SkipGenerateLayout=1&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) -if /i "%1" == "copynativeonly" (set __CopyNativeTestBinaries=1&set __SkipStressDependencies=1&set __SkipNative=1&set __CopyNativeProjectsAfterCombinedTestBuild=false&set __SkipGenerateLayout=1&set __SkipTestWrappers=1&set __SkipCrossgenFramework=1&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) +if /i "%1" == "copynativeonly" (set __CopyNativeTestBinaries=1&set __SkipNative=1&set __CopyNativeProjectsAfterCombinedTestBuild=false&set __SkipGenerateLayout=1&set __SkipTestWrappers=1&set __SkipCrossgenFramework=1&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) if /i "%1" == "generatelayoutonly" (set __SkipManaged=1&set __SkipNative=1&set __CopyNativeProjectsAfterCombinedTestBuild=false&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) -if /i "%1" == "buildtestwrappersonly" (set __SkipNative=1&set __SkipManaged=1&set __BuildTestWrappersOnly=1&set __SkipGenerateLayout=1&set __SkipStressDependencies=1&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) +if /i "%1" == "buildtestwrappersonly" (set __SkipNative=1&set __SkipManaged=1&set __BuildTestWrappersOnly=1&set __SkipGenerateLayout=1&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) if /i "%1" == "buildagainstpackages" (echo error: Remove /BuildAgainstPackages switch&&exit /b1) if /i "%1" == "crossgen" (set __DoCrossgen=1&set __TestBuildMode=crossgen&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) @@ -186,33 +184,6 @@ set MSBUILDDEBUGPATH=%__MsbuildDebugLogsDir% REM ========================================================================================= REM === -REM === Restore Build Tools -REM === -REM ========================================================================================= - -@if defined _echo @echo on - -set "__ToolsDir=%__ProjectDir%\Tools" - -REM ========================================================================================= -REM === -REM === Resolve runtime dependences -REM === -REM ========================================================================================= - -if defined __SkipStressDependencies goto skipstressdependencies - -call "%__RepoRootDir%\src\tests\Common\setup-stress-dependencies.cmd" /arch %__BuildArch% /outputdir %__BinDir% -if errorlevel 1 ( - echo %__ErrMsgPrefix%%__MsgPrefix%Error: setup-stress-dependencies failed. - exit /b 1 -) - -:skipstressdependencies -@if defined _echo @echo on - -REM ========================================================================================= -REM === REM === Native test build section REM === REM ========================================================================================= diff --git a/src/tests/build.sh b/src/tests/build.sh index cfd6aad..88c7a23 100755 --- a/src/tests/build.sh +++ b/src/tests/build.sh @@ -132,18 +132,6 @@ generate_layout() build_MSBuild_projects "Tests_Overlay_Managed" "$__RepoRootDir/src/tests/run.proj" "Creating test overlay" "/t:CreateTestOverlay" - if [[ "$__TargetOS" != "OSX" && "$__SkipStressDependencies" == 0 ]]; then - nextCommand="\"${__RepoRootDir}/src/tests/Common/setup-stress-dependencies.sh\" --arch=$__BuildArch --outputDir=$CORE_ROOT" - echo "Resolve runtime dependences via $nextCommand" - eval $nextCommand - - local exitCode="$?" - if [[ "$exitCode" != 0 ]]; then - echo "${__ErrMsgPrefix}${__MsgPrefix}Error: setup-stress-dependencies failed." - exit "$exitCode" - fi - fi - # Precompile framework assemblies with crossgen if required if [[ "$__DoCrossgen" != 0 || "$__DoCrossgen2" != 0 ]]; then chmod +x "$__CrossgenExe" @@ -485,7 +473,6 @@ handle_arguments_local() { ;; copynativeonly|-copynativeonly) - __SkipStressDependencies=1 __SkipNative=1 __SkipManaged=1 __CopyNativeTestBinaries=1 @@ -535,10 +522,6 @@ handle_arguments_local() { __SkipRestorePackages=1 ;; - skipstressdependencies|-skipstressdependencies) - __SkipStressDependencies=1 - ;; - skipgeneratelayout|-skipgeneratelayout) __SkipGenerateLayout=1 ;; @@ -596,7 +579,6 @@ __SkipManaged=0 __SkipNative=0 __SkipRestore="" __SkipRestorePackages=0 -__SkipStressDependencies=0 __SkipCrossgenFramework=0 __SourceDir="$__ProjectDir/src" __UnprocessedBuildArgs= -- 2.7.4