From: Mike McLaughlin Date: Mon, 10 Sep 2018 21:54:26 +0000 (-0700) Subject: Add downloading and testing the latest master runtime. (#69) X-Git-Tag: submit/tizen/20190813.035844~80 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=28bc5e7ab60916db10b2f5bd40de1adcecead01b;p=platform%2Fcore%2Fdotnet%2Fdiagnostics.git Add downloading and testing the latest master runtime. (#69) Add "daily-test" option to scripts/config. --- diff --git a/README.md b/README.md index 8ee703de1..ee5f263f2 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ **Currently under construction** -This repository contains the source code for various .NET Core runtime diagnostic tools. It currently contains SOS, the managed portion of SOS and the lldb SOS plugin. One of goals of this repo is to build SOS and the lldb SOS plugin for the portable Linux platform (Centos 7) and the platforms not supported by the portable build (Centos 6, Alpine, eventually macOS) and to test across various indexes in a very large matrix: OSs/distros (Centos 6/7, Ubuntu, Alpine, Fedora, Debian, RHEL 7.2), architectures (x64, x86, arm, arm64), lldb versions (3.9, 4.0, 5.0, 6.0) and even .NET Core versions (1.1, 2.0.x, 2.1). +This repository contains the source code for various .NET Core runtime diagnostic tools. It currently contains SOS, the managed portion of SOS and the lldb SOS plugin. The goals of this repo is to build SOS and the lldb SOS plugin for the portable (glibc based) Linux platform (Centos 7) and the platforms not supported by the portable (musl based) build (Centos 6, Alpine, and macOS) and to test across various indexes in a very large matrix: OSs/distros (Centos 6/7, Ubuntu, Alpine, Fedora, Debian, RHEL 7.2), architectures (x64, x86, arm, arm64), lldb versions (3.9, 4.0, 5.0, 6.0) and .NET Core versions (1.1, 2.0.x, 2.1). Another goal to make it easier to obtain a version of lldb (currently 3.9) with scripts and documentation for platforms/distros like Centos, Alpine, Fedora, etc. that by default provide really old versions. @@ -39,6 +39,12 @@ BUILD: Repo sucessfully built. BUILD: Product binaries are available at c:\git\diagnostics\artifacts\Debug\bin\Windows_NT.x64 ``` +To build for x86: + +```bat +> build.cmd -architecture x86 +``` + To test the resulting SOS: ```bat diff --git a/eng/Build-Native.cmd b/eng/Build-Native.cmd index 499f24b4f..e9ecd61d8 100644 --- a/eng/Build-Native.cmd +++ b/eng/Build-Native.cmd @@ -49,6 +49,7 @@ set __BuildType=Debug set __BuildOS=Windows_NT set __Build=0 set __Test=0 +set __DailyTest= set __Verbosity=minimal set __TestArgs= @@ -74,6 +75,7 @@ if /i "%1" == "--help" goto Usage if /i "%1" == "-build-native" (set __Build=1&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) if /i "%1" == "-test" (set __Test=1&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) +if /i "%1" == "-daily-test" (set __DailyTest=-DailyTest&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) if /i "%1" == "-configuration" (set __BuildType=%2&set processedArgs=!processedArgs! %1 %2&shift&shift&goto Arg_Loop) if /i "%1" == "-architecture" (set __BuildArch=%2&set processedArgs=!processedArgs! %1 %2&shift&shift&goto Arg_Loop) if /i "%1" == "-verbosity" (set __Verbosity=%2&set processedArgs=!processedArgs! %1 %2&shift&shift&goto Arg_Loop) @@ -327,9 +329,8 @@ echo %__MsgPrefix%Product binaries are available at !__BinDir! :: Test components if %__Test% EQU 1 ( - :: Install the other versions of .NET Core runtime we are going to test on (2.0.9 and 1.1.9) - powershell -ExecutionPolicy ByPass -NoProfile -command "& """%__ProjectDir%\.dotnet\dotnet-install.ps1""" -Version 2.0.9 -Architecture %__BuildArch% -SkipNonVersionedFiles -Runtime dotnet -InstallDir %__ProjectDir%\.dotnet" - powershell -ExecutionPolicy ByPass -NoProfile -command "& """%__ProjectDir%\.dotnet\dotnet-install.ps1""" -Version 1.1.9 -Architecture %__BuildArch% -SkipNonVersionedFiles -Runtime dotnet -InstallDir %__ProjectDir%\.dotnet" + :: Install the other versions of .NET Core runtime we are going to test on + powershell -ExecutionPolicy ByPass -NoProfile -command "& """%__ProjectDir%\eng\install-test-runtimes.ps1""" -DotNetDir %__ProjectDir%\.dotnet -TempDir %__IntermediatesDir% -BuildArch %__BuildArch%" %__DailyTest% :: Run the xunit tests powershell -ExecutionPolicy ByPass -NoProfile -command "& """%__ProjectDir%\eng\common\Build.ps1""" -test -configuration %__BuildType% -verbosity %__Verbosity% %__TestArgs%" @@ -355,6 +356,7 @@ echo. echo.-? -h -help --help: view this message. echo -build-native - build native components echo -test - test components +echo -daily-test - test components for daily build job echo -architecture echo -configuration echo -verbosity diff --git a/eng/Build.cmd b/eng/Build.cmd index 8ef13939c..ae3e464da 100644 --- a/eng/Build.cmd +++ b/eng/Build.cmd @@ -3,8 +3,9 @@ setlocal :: remove the -test and -build-native options and pass it to build-native.cmd set __args="%*" -set __args=%__args:-test=% set __args=%__args:-build-native=% +set __args=%__args:-daily-test=% +set __args=%__args:-test=% if %__args% == "" set __args= :: build managed components diff --git a/eng/build-native.sh b/eng/build-native.sh index c556b47c0..c4e9d4c32 100755 --- a/eng/build-native.sh +++ b/eng/build-native.sh @@ -28,6 +28,7 @@ __CrossBuild=0 __NumProc=1 __Build=0 __Test=0 +__DailyTest= __CI=0 __TestArgs= __UnprocessedBuildArgs= @@ -37,6 +38,7 @@ usage() echo "Usage: $0 [options]" echo "--build-native - build native components" echo "--test - test native components" + echo "--daily-test - test native components for daily build job" echo "--architecture " echo "--configuration " echo "--clangx.y - optional argument to build using clang version x.y" @@ -161,6 +163,10 @@ while :; do __Test=1 ;; + --daily-test) + __DailyTest="--daily-test" + ;; + --ci) __CI=1 __TestArgs="$__TestArgs $1" @@ -463,9 +469,8 @@ fi # Run SOS/lldbplugin tests if [ $__Test == 1 ]; then - # Install the other versions of .NET Core runtime we are going to test on (2.0.9 and 1.1.9) - bash "$__ProjectRoot/.dotnet/dotnet-install.sh" --version 2.0.9 --architecture "$__BuildArch" --skip-non-versioned-files --runtime dotnet --install-dir "$__ProjectRoot/.dotnet" - bash "$__ProjectRoot/.dotnet/dotnet-install.sh" --version 1.1.9 --architecture "$__BuildArch" --skip-non-versioned-files --runtime dotnet --install-dir "$__ProjectRoot/.dotnet" + # Install the other versions of .NET Core runtime we are going to test on + "$__ProjectRoot/eng/install-test-runtimes.sh" --dotnet-directory "$__ProjectRoot/.dotnet" --temp-directory "$__IntermediatesDir" --architecture "$__BuildArch" $__DailyTest if [ "$LLDB_PATH" = "" ]; then export LLDB_PATH="$(which lldb-3.9.1 2> /dev/null)" diff --git a/eng/build.sh b/eng/build.sh index ed8e0e985..013dd3d88 100755 --- a/eng/build.sh +++ b/eng/build.sh @@ -17,7 +17,7 @@ done scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" # remove the options that build.sh can't handle and pass it to build-native.sh -__args="$(echo $@ | sed 's/--test//g;s/--build-native//g;s/--clang[0-9]\.[0-9]//g')" +__args="$(echo $@ | sed 's/--build-native//g;s/--daily-test//g;s/--test//g;s/--clang[0-9]\.[0-9]//g')" # build managed components "$scriptroot/common/build.sh" $__args diff --git a/eng/install-test-runtimes.ps1 b/eng/install-test-runtimes.ps1 new file mode 100644 index 000000000..04d84aed0 --- /dev/null +++ b/eng/install-test-runtimes.ps1 @@ -0,0 +1,54 @@ +[cmdletbinding()] +param( + [string]$DotNetDir, + [string]$TempDir, + [string]$BuildArch, + [switch]$DailyTest, + [string]$Branch="master", + [string]$UncachedFeed="https://dotnetcli.blob.core.windows.net/dotnet" +) + +Set-StrictMode -Version Latest +$ErrorActionPreference="Stop" + +$RuntimeVersion11="1.1.9" +$RuntimeVersion20="2.0.9" +$RuntimeVersion21="2.1.0" +$DailyTestText="true" + +# Install the other versions of .NET Core runtime we are going to test. 1.1.x, 2.0.x, 2.1.x (installed with the CLI) +# and latest. Only install the latest master for daily jobs and leave the RuntimeVersion* config properties blank. +if (!$DailyTest) { + $DailyTestText="false" + . $DotNetDir\dotnet-install.ps1 -Version $RuntimeVersion11 -Architecture $BuildArch -SkipNonVersionedFiles -Runtime dotnet -InstallDir $DotNetDir + . $DotNetDir\dotnet-install.ps1 -Version $RuntimeVersion20 -Architecture $BuildArch -SkipNonVersionedFiles -Runtime dotnet -InstallDir $DotNetDir +} + +. $DotNetDir\dotnet-install.ps1 -Channel $Branch -Version latest -Architecture $BuildArch -SkipNonVersionedFiles -Runtime dotnet -InstallDir $DotNetDir + +# Now download the latest runtime version and create a config file containing it +$VersionFileUrl = "$UncachedFeed/Runtime/$Branch/latest.version" +$VersionFile = Join-Path -Path $TempDir latest.version +$ConfigFile = Join-Path -Path $DotNetDir Debugger.Tests.Versions.txt + +Invoke-WebRequest $VersionFileUrl -OutFile $VersionFile + +if (Test-Path $VersionFile) { + $VersionText = cat $VersionFile + $Data = @($VersionText.Split([char[]]@(), [StringSplitOptions]::RemoveEmptyEntries)); + $RuntimeVersionLatest = $Data[1].Trim() + + Write-Host "Latest version: $RuntimeVersionLatest" + + ' +' + $DailyTestText +' +' + $RuntimeVersion11 + ' +' + $RuntimeVersion20 + ' +' + $RuntimeVersion21 + ' +' + $RuntimeVersionLatest + ' +' | Set-Content $ConfigFile + +} +else { + Write-Host "Could not download latest runtime version file" +} diff --git a/eng/install-test-runtimes.sh b/eng/install-test-runtimes.sh new file mode 100755 index 000000000..f4fc0ce7e --- /dev/null +++ b/eng/install-test-runtimes.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash +# Copyright (c) .NET Foundation and contributors. All rights reserved. +# Licensed under the MIT license. See LICENSE file in the project root for full license information. + +dotnet_dir= +temp_dir= +build_arch= +daily_test=0 +branch="master" +uncached_feed="https://dotnetcli.blob.core.windows.net/dotnet" + +while [ $# -ne 0 ]; do + name=$1 + case $name in + --dotnet-directory) + shift + dotnet_dir=$1 + ;; + --temp-directory) + shift + temp_dir=$1 + ;; + --architecture) + shift + build_arch=$1 + ;; + --branch) + shift + branch=$1 + ;; + --daily-test) + daily_test=1 + ;; + *) + args="$args $1" + ;; + esac + shift +done + +runtime_version_11="1.1.9" +runtime_version_20="2.0.9" +runtime_version_21="2.1.0" +daily_test_text="true" + +# Install the other versions of .NET Core runtime we are going to test. 1.1.x, 2.0.x, 2.1.x (installed with the CLI) +# and latest. Only install the latest master for daily jobs and leave the RuntimeVersion* config properties blank. +if [ $daily_test == 0 ]; then + daily_test_text="false" + bash "$dotnet_dir/dotnet-install.sh" --version "$runtime_version_11" --architecture "$build_arch" --skip-non-versioned-files --runtime dotnet --install-dir "$dotnet_dir" + bash "$dotnet_dir/dotnet-install.sh" --version "$runtime_version_20" --architecture "$build_arch" --skip-non-versioned-files --runtime dotnet --install-dir "$dotnet_dir" +fi + +bash "$dotnet_dir/dotnet-install.sh" --channel $branch --version latest --architecture "$build_arch" --skip-non-versioned-files --runtime dotnet --install-dir "$dotnet_dir" + +# Now download the latest runtime version and create a config file containing it +version_file_url="$uncached_feed/Runtime/$branch/latest.version" +version_file="$temp_dir/latest.version" +config_file="$dotnet_dir/Debugger.Tests.Versions.txt" + +# Use curl if available, otherwise use wget +if command -v curl > /dev/null; then + curl "$version_file_url" -sSL --retry 10 --create-dirs -o "$version_file" +else + wget -q -O "$version_file" "$version_file_url" +fi + +if [ -f "$version_file" ]; then + runtime_version_latest=$(cat $version_file | tail -n 1 | sed 's/\r$//') + + echo "Latest version: $runtime_version_latest" + + echo " +$daily_test_text +$runtime_version_11 +$runtime_version_20 +$runtime_version_21 +$runtime_version_latest +" > $config_file + +else + echo "Could not download latest runtime version file" +fi diff --git a/src/Microsoft.Diagnostic.TestHelpers/TestConfiguration.cs b/src/Microsoft.Diagnostic.TestHelpers/TestConfiguration.cs index a8d41b0ba..68bfca413 100644 --- a/src/Microsoft.Diagnostic.TestHelpers/TestConfiguration.cs +++ b/src/Microsoft.Diagnostic.TestHelpers/TestConfiguration.cs @@ -145,7 +145,11 @@ namespace Microsoft.Diagnostic.TestHelpers { foreach (XAttribute attr in node.Attributes("ConfigFile")) { - string file = Path.Combine(TestConfiguration.BaseDir, attr.Value); + string file = ResolveProperties(currentTemplate, attr.Value).Trim(); + if (!Path.IsPathRooted(file)) + { + file = Path.Combine(TestConfiguration.BaseDir, file); + } templates = ParseConfigFile(file, templates); } } diff --git a/src/SOS/SOS.UnitTests/ConfigFiles/Unix/Debugger.Tests.Config.txt b/src/SOS/SOS.UnitTests/ConfigFiles/Unix/Debugger.Tests.Config.txt index 95b3a6d7c..80ac42af4 100644 --- a/src/SOS/SOS.UnitTests/ConfigFiles/Unix/Debugger.Tests.Config.txt +++ b/src/SOS/SOS.UnitTests/ConfigFiles/Unix/Debugger.Tests.Config.txt @@ -8,10 +8,11 @@ --> + ../../../../.. + ProjectK - ../../../../.. $(RepoRootDir)/src/SOS/SOS.UnitTests/Scripts $(RepoRootDir)/artifacts $(RootBinDir)/$(TargetConfiguration)/bin/$(OS).$(TargetArchitecture) @@ -32,23 +33,29 @@ - + - diff --git a/src/SOS/SOS.UnitTests/ConfigFiles/Windows/Debugger.Tests.Config.txt b/src/SOS/SOS.UnitTests/ConfigFiles/Windows/Debugger.Tests.Config.txt index 13027fde9..ea70a5bf2 100644 --- a/src/SOS/SOS.UnitTests/ConfigFiles/Windows/Debugger.Tests.Config.txt +++ b/src/SOS/SOS.UnitTests/ConfigFiles/Windows/Debugger.Tests.Config.txt @@ -8,12 +8,12 @@ --> + ..\..\..\..\.. + - ..\..\..\..\.. $(RepoRootDir)\src\SOS\SOS.UnitTests\Scripts $(RepoRootDir)\artifacts - $(RootBinDir)\$(TargetConfiguration)\bin\Windows_NT.$(TargetArchitecture) $(RootBinDir)\$(TargetConfiguration)\TestResults\sos.unittests_$(Timestamp) $(RootBinDir)\$(TargetConfiguration)\tmp\dumps @@ -36,21 +36,27 @@ ProjectK - + - $(RepoRootDir)\.dotnet\dotnet.exe