From: Mike McLaughlin Date: Fri, 8 Jun 2018 22:58:14 +0000 (-0700) Subject: Setup for yaml based builds. (#23) X-Git-Tag: submit/tizen/20190813.035844~95 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=6323006afd6c82be8d313cf77894bdc404c528d5;p=platform%2Fcore%2Fdotnet%2Fdiagnostics.git Setup for yaml based builds. (#23) --- diff --git a/.vsts-dotnet-ci.yml b/.vsts-dotnet-ci.yml new file mode 100644 index 000000000..9d6e24563 --- /dev/null +++ b/.vsts-dotnet-ci.yml @@ -0,0 +1,39 @@ +phases: +- template: /eng/build.yml + parameters: + agentOs: Windows_NT + buildReason: IndividualCI + queue: + name: Hosted VS2017 + demands: + - agent.os -equals Windows_NT + parallel: 2 + matrix: + Build_Debug: + _BuildConfig: Debug + _PublishType: none + _SignType: test + Build_Release: + _BuildConfig: Release + _PublishType: none + _SignType: test + +- template: /eng/build.yml + parameters: + agentOs: Linux + buildReason: IndividualCI + dockerImage: microsoft/dotnet-buildtools-prereqs:ubuntu-14.04-cross-0cd4667-20170319080304 + queue: + name: DotNetCore-Linux + demands: + - agent.os -equals Linux + parallel: 2 + matrix: + Build_Debug: + _BuildConfig: Debug + _PublishType: none + _SignType: test + Build_Release: + _BuildConfig: Release + _PublishType: none + _SignType: test diff --git a/.vsts-dotnet.yml b/.vsts-dotnet.yml new file mode 100644 index 000000000..8a0514403 --- /dev/null +++ b/.vsts-dotnet.yml @@ -0,0 +1,39 @@ +phases: +- template: /eng/build.yml + parameters: + agentOs: Windows_NT + buildReason: Internal + queue: + name: DotNetCore-Build + demands: + - agent.os -equals Windows_NT + parallel: 2 + matrix: + Build_Debug: + _BuildConfig: Debug + _PublishType: none + _SignType: test + Build_Release: + _BuildConfig: Release + _PublishType: blob + _SignType: real + +- template: /eng/build.yml + parameters: + agentOs: Linux + buildReason: Internal + dockerImage: microsoft/dotnet-buildtools-prereqs:ubuntu-14.04-cross-0cd4667-20170319080304 + queue: + name: DotNetCore-Test + demands: + - agent.os -equals Linux + parallel: 2 + matrix: + Build_Debug: + _BuildConfig: Debug + _PublishType: none + _SignType: test + Build_Release: + _BuildConfig: Release + _PublishType: none + _SignType: real diff --git a/eng/Build-Native.cmd b/eng/Build-Native.cmd index 960024869..d384909bf 100644 --- a/eng/Build-Native.cmd +++ b/eng/Build-Native.cmd @@ -41,11 +41,16 @@ set ghprbCommentBody= :: Thus, these variables are not simply internal to this script! :: Set the default arguments for build + set __BuildArch=x64 +if /i "%PROCESSOR_ARCHITECTURE%" == "amd64" set __BuildArch=x64 +if /i "%PROCESSOR_ARCHITECTURE%" == "x86" set __BuildArch=x86 set __BuildType=Debug set __BuildOS=Windows_NT set __Build=0 set __Test=0 +set __Verbosity=minimal +set __TestArgs= :: Set the various build properties here so that CMake and MSBuild can pick them up set "__ProjectDir=%~dp0" @@ -57,8 +62,8 @@ set "__PackagesDir=%DotNetRestorePackagesPath%" if [%__PackagesDir%]==[] set "__PackagesDir=%__ProjectDir%\packages" set "__RootBinDir=%__ProjectDir%\artifacts" -REM __UnprocessedBuildArgs are args that we pass to msbuild (e.g. /p:__BuildArch=x64) -set "__args= %*" +:: __UnprocessedBuildArgs are args that we pass to msbuild (e.g. /p:__BuildArch=x64) +set "__args=%*" set processedArgs= set __UnprocessedBuildArgs= @@ -74,12 +79,16 @@ if /i "%1" == "-build" (set __Build=1&set processedArgs=!processed if /i "%1" == "-test" (set __Test=1&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) if /i "%1" == "-configuration" (set __BuildType=%2&set processedArgs=!processedArgs! %1 %2&shift&shift&goto Arg_Loop) if /i "%1" == "-architecture" (set __BuildArch=%2&set processedArgs=!processedArgs! %1 %2&shift&shift&goto Arg_Loop) -rem these options are ignored for a native build -if /i "%1" == "-restore" (set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) +if /i "%1" == "-verbosity" (set __Verbosity=%2&set processedArgs=!processedArgs! %1 %2&shift&shift&goto Arg_Loop) +:: These options are passed on to the common build script when testing +if /i "%1" == "-ci" (set __TestArgs=!__TestArgs! %1&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) +if /i "%1" == "-solution" (set __TestArgs=!__TestArgs! %1 %2&set processedArgs=!processedArgs! %1&shift&shift&goto Arg_Loop) +:: These options are ignored for a native build +if /i "%1" == "-rebuild" (set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) if /i "%1" == "-sign" (set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) +if /i "%1" == "-restore" (set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) if /i "%1" == "-pack" (set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) -if /i "%1" == "-ci" (set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) -if /i "%1" == "-prepareMachine" (set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) +if /i "%1" == "-preparemachine" (set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) if [!processedArgs!]==[] ( set __UnprocessedBuildArgs=%__args% @@ -92,7 +101,7 @@ if [!processedArgs!]==[] ( :ArgsDone -REM Determine if this is a cross-arch build +:: Determine if this is a cross-arch build if /i "%__BuildArch%"=="arm64" ( set __DoCrossArchBuild=1 @@ -104,6 +113,9 @@ if /i "%__BuildArch%"=="arm" ( set __CrossArch=x64 ) +if /i "%__BuildType%"=="debug" set __BuildType=Debug +if /i "%__BuildType%"=="release" set __BuildType=Release + :: Set the remaining variables based upon the determined build configuration set "__BinDir=%__RootBinDir%\bin\%__BuildOS%.%__BuildArch%.%__BuildType%" set "__IntermediatesDir=%__RootBinDir%\obj\%__BuildOS%.%__BuildArch%.%__BuildType%" @@ -157,7 +169,7 @@ REM === REM ========================================================================================= if %__Build% EQU 1 ( - REM Scope environment changes start { + rem Scope environment changes start { setlocal echo %__MsgPrefix%Commencing build of native components for %__BuildOS%.%__BuildArch%.%__BuildType% @@ -166,8 +178,8 @@ if %__Build% EQU 1 ( if not "%__ToolsetDir%" == "" ( set __NativePlatformArgs=-useEnv ) if not "%__ToolsetDir%" == "" ( - rem arm64 builds currently use private toolset which has not been released yet - REM TODO, remove once the toolset is open. + :: arm64 builds currently use private toolset which has not been released yet + :: TODO, remove once the toolset is open. call :PrivateToolSet goto GenVSSolution ) @@ -178,7 +190,7 @@ if %__Build% EQU 1 ( if /i "%__BuildArch%" == "arm" ( set __VCBuildArch=x86_arm - REM Make CMake pick the highest installed version in the 10.0.* range + :: Make CMake pick the highest installed version in the 10.0.* range set ___SDKVersion="-DCMAKE_SYSTEM_VERSION=10.0" ) if /i "%__BuildArch%" == "arm64" ( @@ -228,7 +240,7 @@ if %__Build% EQU 1 ( set __MsbuildWrn=/flp1:WarningsOnly;LogFile=!__BuildWrn! set __MsbuildErr=/flp2:ErrorsOnly;LogFile=!__BuildErr! - msbuild.exe %__IntermediatesDir%\install.vcxproj /v:m !__MsbuildLog! !__MsbuildWrn! !__MsbuildErr! /p:Configuration=%__BuildType% /p:Platform=%__BuildArch% %__UnprocessedBuildArgs% + msbuild.exe %__IntermediatesDir%\install.vcxproj /v:!__Verbosity! !__MsbuildLog! !__MsbuildWrn! !__MsbuildErr! /p:Configuration=%__BuildType% /p:Platform=%__BuildArch% %__UnprocessedBuildArgs% if not !errorlevel! == 0 ( echo %__MsgPrefix%Error: native component build failed. Refer to the build log files for details: @@ -239,7 +251,7 @@ if %__Build% EQU 1 ( ) :SkipNativeBuild - REM } Scope environment changes end + rem } Scope environment changes end endlocal ) @@ -250,7 +262,7 @@ REM === REM ========================================================================================= if /i "%__DoCrossArchBuild%"=="1" ( - REM Scope environment changes start { + rem Scope environment changes start { setlocal echo %__MsgPrefix%Commencing build of cross architecture native components for %__BuildOS%.%__BuildArch%.%__BuildType% @@ -294,7 +306,7 @@ if /i "%__DoCrossArchBuild%"=="1" ( set __MsbuildWrn=/flp1:WarningsOnly;LogFile=!__BuildWrn! set __MsbuildErr=/flp2:ErrorsOnly;LogFile=!__BuildErr! - msbuild.exe %__CrossCompIntermediatesDir%\install.vcxproj /v:m !__MsbuildLog! !__MsbuildWrn! !__MsbuildErr! /p:Configuration=%__BuildType% /p:Platform=%__CrossArch% %__UnprocessedBuildArgs% + msbuild.exe %__CrossCompIntermediatesDir%\install.vcxproj /v:!__Verbosity! !__MsbuildLog! !__MsbuildWrn! !__MsbuildErr! /p:Configuration=%__BuildType% /p:Platform=%__CrossArch% %__UnprocessedBuildArgs% if not !errorlevel! == 0 ( echo %__MsgPrefix%Error: cross-arch components build failed. Refer to the build log files for details: @@ -305,7 +317,7 @@ if /i "%__DoCrossArchBuild%"=="1" ( ) :SkipCrossCompBuild - REM } Scope environment changes end + rem } Scope environment changes end endlocal ) @@ -315,8 +327,14 @@ REM === All builds complete! REM === REM ========================================================================================= -echo %__MsgPrefix%Repo successfully built. Finished at %TIME% +echo %__MsgPrefix%Repo successfully built. Finished at %TIME% echo %__MsgPrefix%Product binaries are available at !__BinDir! + +:: test components +if %__Test% EQU 1 ( + powershell -ExecutionPolicy ByPass -command "& """%__ProjectDir%\eng\common\Build.ps1""" -test -configuration %__BuildType% -verbosity %__Verbosity% %__TestArgs%" + exit /b %ERRORLEVEL +) exit /b 0 REM ========================================================================================= @@ -336,9 +354,10 @@ echo All arguments are optional. The options are: echo. echo.-? -h -help --help: view this message. echo -build - build native components -echo -test - test native components -echo -architechure +echo -test - test components +echo -architecture echo -configuration +echo -verbosity exit /b 1 :PrivateToolSet diff --git a/eng/Build.cmd b/eng/Build.cmd index 741751960..8c433a991 100644 --- a/eng/Build.cmd +++ b/eng/Build.cmd @@ -1,9 +1,14 @@ @echo off +setlocal -rem build/test managed components -powershell -ExecutionPolicy ByPass -command "& """%~dp0common\Build.ps1""" %*" +:: remove the -test option and pass it to build-native.cmd +set "__args=%*" +set "__args=%__args:-test=%" + +:: build managed components +powershell -ExecutionPolicy ByPass -command "& """%~dp0common\Build.ps1""" %__args%" if NOT '%ERRORLEVEL%' == '0' exit /b %ERRORLEVEL% -rem build/test native componments +:: build native componments and test managed/native call %~dp0build-native.cmd %* exit /b %ERRORLEVEL% diff --git a/eng/CIBuild.cmd b/eng/CIBuild.cmd index 94fc7744b..82ecb68a5 100644 --- a/eng/CIBuild.cmd +++ b/eng/CIBuild.cmd @@ -1,3 +1,3 @@ @echo off -call %~dp0Build.cmd -restore -build -test -sign -ci %* +call %~dp0Build.cmd -restore -build -test -ci %* exit /b %ErrorLevel% diff --git a/eng/Versions.props b/eng/Versions.props index 59732b8e6..921d6ace7 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -19,7 +19,8 @@ $(RestoreSources); https://dotnet.myget.org/F/symstore/api/v3/index.json; https://dotnet.myget.org/F/roslyn-tools/api/v3/index.json; - https://dotnet.myget.org/F/dotnet-core/api/v3/index.json + https://dotnet.myget.org/F/dotnet-core/api/v3/index.json; + https://dotnet.myget.org/F/dotnet-buildtools/api/v3/index.json diff --git a/eng/build-native.sh b/eng/build-native.sh index fd120a023..eed0d6063 100755 --- a/eng/build-native.sh +++ b/eng/build-native.sh @@ -13,7 +13,7 @@ while [[ -h "$source" ]]; do # symlink file was located [[ $source != /* ]] && source="$scriptroot/$source" done -__ProjectRoot="$( cd -P "$( dirname "$source" )" && pwd )/.." +__ProjectRoot="$( cd -P "$( dirname "$source" )/.." && pwd )" __BuildOS=Linux __HostOS=Linux @@ -26,9 +26,10 @@ __ClangMajorVersion=0 __ClangMinorVersion=0 __CrossBuild=0 __NumProc=1 -__UnprocessedBuildArgs= __Build=0 __Test=0 +__TestArgs= +__UnprocessedBuildArgs= # resolve python-version to use if [ "$PYTHON" == "" ] ; then @@ -54,13 +55,24 @@ usage() echo "Usage: $0 [options]" echo "--build - build native components" echo "--test - test native components" - echo "--architechure " + echo "--architecture " echo "--configuration " echo "--clangx.y - optional argument to build using clang version x.y" + echo "--verbosity " echo "--help - this help message" exit 1 } + +# args: +# input - $1 +to_lowercase() { + #eval $invocation + + echo "$1" | tr '[:upper:]' '[:lower:]' + return 0 +} + # Argument types supported by this script: # # Build architecture - valid values are: x64, x86, arm, armel, arm64 @@ -145,13 +157,12 @@ case $OSName in ;; esac - while :; do if [ $# -le 0 ]; then break fi - lowerI="$(echo $1 | awk '{print tolower($0)}')" + lowerI="$(to_lowercase "$1")" case $lowerI in -\?|-h|--help) usage @@ -159,22 +170,40 @@ while :; do ;; --build) - __Build=1 - ;; + __Build=1 + ;; --test) - __Test=1 - ;; + __Test=1 + ;; + + # Passed to common build script when testing + --ci) + __TestArgs="$__TestArgs $1" + ;; + + --solution) + __TestArgs="$__TestArgs $1 $2" + ;; + + --verbosity) + __TestArgs="$__TestArgs $1 $2" + shift + ;; + + # Ignored for a native build + --rebuild|--sign|--restore|--pack|--preparemachine) + ;; --configuration) - __BuildType=$2 - shift - ;; + __BuildType="$(to_lowercase "$2")" + shift + ;; - --architechure) - __BuildArch=$2 - shift - ;; + --architecture) + __BuildArch="$(to_lowercase "$2")" + shift + ;; --clang3.5) __ClangMajorVersion=3 @@ -206,6 +235,11 @@ while :; do __ClangMinorVersion=0 ;; + --verbosity) + __TestArgs="$__TestArgs --verbosity $2" + shift + ;; + *) __UnprocessedBuildArgs="$__UnprocessedBuildArgs $1" ;; @@ -214,6 +248,13 @@ while :; do shift done +if [ "$__BuildType" == "release" ]; then + __BuildType=Release +fi +if [ "$__BuildType" == "debug" ]; then + __BuildType=Debug +fi + __RootBinDir=$__ProjectRoot/artifacts __IntermediatesDir="$__RootBinDir/obj/$__BuildOS.$__BuildArch.$__BuildType" __LogFileDir="$__RootBinDir/log/$__BuildOS.$__BuildArch.$__BuildType" @@ -360,14 +401,42 @@ if [ $__Build == 1 ]; then build_native "$__BuildArch" "$__IntermediatesDir" "$__ExtraCmakeArgs" fi -# Run native SOS/lldbplugin tests +# Run SOS/lldbplugin tests if [ $__Test == 1 ]; then + + if [ "$LLDB_PATH" = "" ]; then + export LLDB_PATH="$(which lldb-3.9.1 2> /dev/null)" + if [ "$LLDB_PATH" = "" ]; then + export LLDB_PATH="$(which lldb-3.9 2> /dev/null)" + if [ "$LLDB_PATH" = "" ]; then + export LLDB_PATH="$(which lldb 2> /dev/null)" + fi + fi + fi + + if [ "$GDB_PATH" = "" ]; then + export GDB_PATH="$(which gdb 2> /dev/null)" + fi + + echo "lldb: '$LLDB_PATH' gdb: '$GDB_PATH'" + + # Run xunit SOS tests + "$__ProjectRoot/eng/common/build.sh" --test --configuration "$__BuildType" "$__TestArgs" + if [[ $? != 0 ]]; then + exit 1 + fi + if [ "$__BuildOS" == "OSX" ]; then __Plugin=$__CMakeBinDir/libsosplugin.dylib else __Plugin=$__CMakeBinDir/libsosplugin.so fi - "$__ProjectRoot/src/SOS/tests/testsos.sh" "$__ProjectRoot" "$__Plugin" "$__RootBinDir/$__BuildType/bin" "$__LogFileDir" "$__BuildArch" + + # Run lldb python tests + "$__ProjectRoot/src/SOS/lldbplugin.tests/testsos.sh" "$__ProjectRoot" "$__Plugin" "$__RootBinDir/$__BuildType/bin" "$__LogFileDir" + if [[ $? != 0 ]]; then + exit 1 + fi fi echo "BUILD: Repo sucessfully built." diff --git a/eng/build.sh b/eng/build.sh index af7c27fc3..05374f638 100755 --- a/eng/build.sh +++ b/eng/build.sh @@ -16,43 +16,14 @@ done scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" -# ReadJson [filename] [json key] -# Result: Sets 'readjsonvalue' to the value of the provided json key -# Note: this method may return unexpected results if there are duplicate -# keys in the json -function ReadJson { - local file=$1 - local key=$2 +# remove the --test option and pass it to build-native.sh +__args="$(echo $@ | sed 's/--test//g')" - local unamestr="$(uname)" - local sedextended='-r' - if [[ "$unamestr" == 'Darwin' ]]; then - sedextended='-E' - fi; - - readjsonvalue="$(grep -m 1 "\"$key\"" $file | sed $sedextended 's/^ *//;s/.*: *"//;s/",?//')" - if [[ ! "$readjsonvalue" ]]; then - echo "Error: Cannot find \"$key\" in $file" >&2; - ExitWithExitCode 1 - fi; -} - -# install .NET Core -ReadJson "$scriptroot/../global.json" "version" - -# setting DOTNET_INSTALL_DIR prevents build.sh from installing it -export DOTNET_INSTALL_DIR=$scriptroot/../.dotnet -"$scriptroot/install-dotnet.sh" $DOTNET_INSTALL_DIR $readjsonvalue -if [[ $? != 0 ]]; then - exit 1 -fi - -# build/test managed components -"$scriptroot/common/build.sh" $@ +# build managed components +"$scriptroot/common/build.sh" $__args if [[ $? != 0 ]]; then exit 1 fi -# build/test native components +# build native components and test both "$scriptroot/build-native.sh" $@ - diff --git a/eng/build.yml b/eng/build.yml new file mode 100644 index 000000000..44a295227 --- /dev/null +++ b/eng/build.yml @@ -0,0 +1,103 @@ +# build.yml +# Description: Defines the build phase +# Parameters: +# agentOs: [Windows_NT (default), Linux, OSX] Used in templates to define variables which are OS specific +# dockerImage: If defined, specifies docker image to run build steps in +# matrix: build matrix +# queueName: agent pool name +# enableTelemetry: send telemetry if build is not a PR or CI build + +parameters: + agentOs: Windows_NT + buildReason: IndividualCI + queue: {} + dockerImage: '' + enableTelemetry: true + useEsrpSigning: true + +phases: +- template: /eng/common/templates/phases/base.yml + parameters: + agentOs: ${{ parameters.agentOs }} + buildReason: ${{ parameters.buildReason }} + phaseName: ${{ parameters.agentOs }} + dockerImage: ${{ parameters.dockerImage }} + enableTelemetry: ${{ parameters.enableTelemetry }} + phase: + queue: ${{ parameters.queue }} + + variables: + _HelixType: build/product + ${{ if notIn(parameters.buildReason, 'IndividualCI', 'BatchedCI', 'PullRequest') }}: + _PublishBlobFeedUrl: https://dotnetfeed.blob.core.windows.net/dotnet-tools-internal/index.json + _TeamName: DotNetCore + _UseEsrpSigning: ${{ parameters.useEsrpSigning }} + _HelixSource: official/dotnet/arcade/$(Build.SourceBranch) + _PublishArgs: /p:PB_PublishBlobFeedKey=$(dotnetfeed-storage-access-key-1) + /p:PB_PublishBlobFeedUrl=$(_PublishBlobFeedUrl) + /p:PB_PublishType=$(_PublishType) + ${{ if in(parameters.buildReason, 'IndividualCI', 'BatchedCI', 'PullRequest') }}: + _HelixSource: pr/dotnet/arcade/$(Build.SourceBranch) + _PublishArgs: '' + + steps: + - ${{ if notIn(parameters.buildReason, 'IndividualCI', 'BatchedCI', 'PullRequest') }}: + - ${{ if eq(parameters.agentOs, 'Windows_NT') }}: + - task: ms-vseng.MicroBuildTasks.30666190-6959-11e5-9f96-f56098202fef.MicroBuildSigningPlugin@1 + displayName: Install MicroBuild plugin + inputs: + signType: $(_SignType) + zipSources: false + esrpSigning: $(_UseEsrpSigning) + env: + TeamName: $(_TeamName) + continueOnError: false + condition: and(succeeded(), in(variables._SignType, 'real', 'test')) + - task: AzureKeyVault@1 + inputs: + azureSubscription: 'DotNet-Engineering-Services_KeyVault' + KeyVaultName: EngKeyVault + SecretsFilter: 'dotnetfeed-storage-access-key-1' + condition: succeeded() + + - ${{ if eq(parameters.agentOs, 'Windows_NT') }}: + - script: eng\cibuild.cmd + -configuration $(_BuildConfig) + -prepareMachine + /p:SignType=$(_SignType) + $(_PublishArgs) + name: Build_Publish + displayName: Build / Publish + env: + OfficialBuildId: $(BUILD.BUILDNUMBER) + condition: succeeded() + - ${{ if ne(parameters.agentOs, 'Windows_NT') }}: + - script: eng/cibuild.sh + --configuration $(_BuildConfig) + --prepareMachine + /p:SignType=$(_SignType) + $(_PublishArgs) + name: Build_Publish + displayName: Build / Publish + env: + OfficialBuildId: $(BUILD.BUILDNUMBER) + condition: succeeded() + + - ${{ if notIn(parameters.buildReason, 'IndividualCI', 'BatchedCI', 'PullRequest') }}: + - task: CopyFiles@2 + displayName: Gather Logs + inputs: + SourceFolder: '$(Build.SourcesDirectory)/artifacts' + Contents: '**/*log' + TargetFolder: '$(Build.StagingDirectory)/BuildLogs' + continueOnError: true + condition: succeededOrFailed() + - task: PublishBuildArtifacts@1 + displayName: Publish Logs to VSTS + inputs: + PathtoPublish: '$(Build.StagingDirectory)/BuildLogs' + ArtifactName: $(_agentOs)_$(Agent.JobName) + PublishLocation: Container + continueOnError: true + condition: succeededOrFailed() + diff --git a/eng/cibuild.sh b/eng/cibuild.sh index 39fbbbba8..856eeac27 100755 --- a/eng/cibuild.sh +++ b/eng/cibuild.sh @@ -15,4 +15,4 @@ while [[ -h $source ]]; do done scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" -"$scriptroot/build.sh" --restore --build --test --sign --ci $@ +"$scriptroot/build.sh" --restore --build --test --ci $@ diff --git a/eng/common/build.ps1 b/eng/common/build.ps1 index 7fe12a5bd..a30255639 100644 --- a/eng/common/build.ps1 +++ b/eng/common/build.ps1 @@ -1,5 +1,6 @@ [CmdletBinding(PositionalBinding=$false)] Param( + [string] $architecture = "", [string] $configuration = "Debug", [string] $solution = "", [string] $verbosity = "minimal", @@ -97,7 +98,7 @@ function InstallDotNetCli([string] $dotnetRoot) { Invoke-WebRequest "https://dot.net/v1/dotnet-install.ps1" -OutFile $installScript } - & $installScript -Version $GlobalJson.sdk.version -InstallDir $dotnetRoot + & $installScript -Version $GlobalJson.sdk.version -InstallDir $dotnetRoot -Architecture $architecture if ($lastExitCode -ne 0) { Write-Host "Failed to install dotnet cli (exit code '$lastExitCode')." -ForegroundColor Red exit $lastExitCode diff --git a/eng/common/build.sh b/eng/common/build.sh index f17bacb48..e60c8aed1 100755 --- a/eng/common/build.sh +++ b/eng/common/build.sh @@ -10,11 +10,12 @@ while [[ -h "$source" ]]; do # symlink file was located [[ $source != /* ]] && source="$scriptroot/$source" done -scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" +repo_root="$( cd -P "$( dirname "$source" )/../.." && pwd )" build=false ci=false configuration='Debug' +architecture='' help=false pack=false prepare_machine=false @@ -26,19 +27,6 @@ test=false verbosity='minimal' properties='' -repo_root="$scriptroot/../.." -artifacts_dir="$repo_root/artifacts" -artifacts_configuration_dir="$artifacts_dir/$configuration" -toolset_dir="$artifacts_dir/toolset" -log_dir="$artifacts_configuration_dir/log" -log="$log_dir/Build.binlog" -toolset_restore_log="$log_dir/ToolsetRestore.binlog" -temp_dir="$artifacts_configuration_dir/tmp" - -global_json_file="$repo_root/global.json" -build_driver="" -toolset_build_proj="" - while (($# > 0)); do lowerI="$(echo $1 | awk '{print tolower($0)}')" case $lowerI in @@ -54,6 +42,10 @@ while (($# > 0)); do configuration=$2 shift 2 ;; + --architecture) + architecture=$2 + shift 2 + ;; --help) echo "Common settings:" echo " --configuration Build configuration Debug, Release" @@ -115,6 +107,18 @@ while (($# > 0)); do esac done +artifacts_dir="$repo_root/artifacts" +artifacts_configuration_dir="$artifacts_dir/$configuration" +toolset_dir="$artifacts_dir/toolset" +log_dir="$artifacts_configuration_dir/log" +log="$log_dir/Build.binlog" +toolset_restore_log="$log_dir/ToolsetRestore.binlog" +temp_dir="$artifacts_configuration_dir/tmp" + +global_json_file="$repo_root/global.json" +build_driver="" +toolset_build_proj="" + # ReadJson [filename] [json key] # Result: Sets 'readjsonvalue' to the value of the provided json key # Note: this method may return unexpected results if there are duplicate @@ -184,7 +188,7 @@ function InstallDotNetCli { fi fi - bash "$dotnet_install_script" --version $dotnet_sdk_version --install-dir $dotnet_root + bash "$dotnet_install_script" --version $dotnet_sdk_version --install-dir $dotnet_root --architecture $architecture local lastexitcode=$? if [[ $lastexitcode != 0 ]]; then @@ -286,4 +290,4 @@ function Main { ExitWithExitCode $? } -Main \ No newline at end of file +Main diff --git a/eng/common/telemetry/build/end.ps1 b/eng/common/telemetry/build/end.ps1 new file mode 100644 index 000000000..2a441daa7 --- /dev/null +++ b/eng/common/telemetry/build/end.ps1 @@ -0,0 +1,22 @@ +[CmdletBinding()] +param() + +Set-StrictMode -Version 2.0 +$ErrorActionPreference = "Stop" + +if (($env:Agent_JobStatus -eq 'Succeeded') -or ($env:Agent_JobStatus -eq 'PartiallySucceeded')) { + $ErrorCount = 0 +} else { + $ErrorCount = 1 +} +$WarningCount = 0 + +try { + Invoke-RestMethod -Uri "https://helix.dot.net/api/2018-03-14/telemetry/job/build/$env:Helix_WorkItemId/finish?errorCount=$ErrorCount&warningCount=$WarningCount" -Method Post -ContentType "application/json" -Body "" ` + -Headers @{ 'X-Helix-Job-Token'=$env:Helix_JobToken } +} +catch { + Write-Error $_ + Write-Error $_.Exception + exit 1 +} diff --git a/eng/common/telemetry/build/end.sh b/eng/common/telemetry/build/end.sh new file mode 100644 index 000000000..1b65a2477 --- /dev/null +++ b/eng/common/telemetry/build/end.sh @@ -0,0 +1,39 @@ +source="${BASH_SOURCE[0]}" +# resolve $source until the file is no longer a symlink +while [[ -h "$source" ]]; do + scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + source="$(readlink "$source")" + # if $source was a relative symlink, we need to resolve it relative to the path where the + # symlink file was located + [[ $source != /* ]] && source="$scriptroot/$source" +done +scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + +if [ "$AGENT_JOBSTATUS" = "Succeeded" ] || [ "$AGENT_JOBSTATUS" = "PartiallySucceeded" ]; then + errorCount=0 +else + errorCount=1 +fi +warningCount=0 + +curlResult=` +/bin/bash $scriptroot/../curl.sh \ + -H 'Content-Type: application/json' \ + -H "X-Helix-Job-Token: $Helix_JobToken" \ + -H 'Content-Length: 0' \ + -X POST -G "https://helix.dot.net/api/2018-03-14/telemetry/job/build/$Helix_WorkItemId/finish" \ + --data-urlencode "errorCount=$errorCount" \ + --data-urlencode "warningCount=$warningCount" +` +curlStatus=$? + +if [ $curlStatus -ne 0 ]; then + echo "Failed to Send Build Finish information" + echo $curlResult + if /bin/bash "$scriptroot/../../is-vsts.sh"; then + echo "##vso[task.logissue type=error;sourcepath=telemetry/build/end.sh;code=1;]Failed to Send Build Finish information: $curlResult" + fi + exit 1 +fi + +exit 0 diff --git a/eng/common/telemetry/build/start.ps1 b/eng/common/telemetry/build/start.ps1 new file mode 100644 index 000000000..e9c99e45d --- /dev/null +++ b/eng/common/telemetry/build/start.ps1 @@ -0,0 +1,23 @@ +[CmdletBinding()] +param( + [string]$BuildUri +) + +Set-StrictMode -Version 2.0 +$ErrorActionPreference = "Stop" + + +try { + $workItemId = Invoke-RestMethod -Uri "https://helix.dot.net/api/2018-03-14/telemetry/job/build?buildUri=$([Net.WebUtility]::UrlEncode($BuildUri))" -Method Post -ContentType "application/json" -Body "" ` + -Headers @{ 'X-Helix-Job-Token'=$env:Helix_JobToken } + + $env:Helix_WorkItemId = $workItemId + if (& "$PSScriptRoot/../../is-vsts.ps1") { + Write-Host "##vso[task.setvariable variable=Helix_WorkItemId]$env:Helix_WorkItemId" + } +} +catch { + Write-Error $_ + Write-Error $_.Exception + exit 1 +} diff --git a/eng/common/telemetry/build/start.sh b/eng/common/telemetry/build/start.sh new file mode 100644 index 000000000..363a7f39d --- /dev/null +++ b/eng/common/telemetry/build/start.sh @@ -0,0 +1,52 @@ +source="${BASH_SOURCE[0]}" +# resolve $source until the file is no longer a symlink +while [[ -h "$source" ]]; do + scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + source="$(readlink "$source")" + # if $source was a relative symlink, we need to resolve it relative to the path where the + # symlink file was located + [[ $source != /* ]] && source="$scriptroot/$source" +done +scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + +while (($# > 0)); do + lowerI="$(echo $1 | awk '{print tolower($0)}')" + case $lowerI in + --build-uri) + buildUri=$2 + shift 2 + ;; + *) + echo "Unknown Arg '$1'" + exit 1 + ;; + esac +done + + +curlResult=` +/bin/bash $scriptroot/../curl.sh \ + -H 'Content-Type: application/json' \ + -H "X-Helix-Job-Token: $Helix_JobToken" \ + -H 'Content-Length: 0' \ + -X POST -G "https://helix.dot.net/api/2018-03-14/telemetry/job/build" \ + --data-urlencode "buildUri=$buildUri" +` +curlStatus=$? + +if [ $curlStatus -ne 0 ]; then + echo "Failed to Send Build Start information" + echo $curlResult + if /bin/bash "$scriptroot/../../is-vsts.sh"; then + echo "##vso[task.logissue type=error;sourcepath=telemetry/build/start.sh;code=1;]Failed to Send Build Start information: $curlResult" + fi + exit 1 +fi + +export Helix_WorkItemId=`echo $curlResult | xargs echo` # Strip Quotes + +if /bin/bash "$scriptroot/../../is-vsts.sh"; then + echo "##vso[task.setvariable variable=Helix_WorkItemId]$Helix_WorkItemId" +else + echo "export Helix_WorkItemId=$Helix_WorkItemId" +fi diff --git a/eng/common/telemetry/curl.sh b/eng/common/telemetry/curl.sh new file mode 100644 index 000000000..aa607b6b8 --- /dev/null +++ b/eng/common/telemetry/curl.sh @@ -0,0 +1,16 @@ +#runs curl and exits with exit code when http server errors happen +res=`mktemp` +httpCode=$(curl --silent --output $res --write-out "%{http_code}" "$@") +curlCode=$? + +if [ ! $curlCode ]; then + exit $curlCode +fi + +cat $res + +if [ $httpCode -gt 299 ] || [ $httpCode -lt 200 ]; then + exit 1 +else + exit 0 +fi diff --git a/eng/common/telemetry/start-job.ps1 b/eng/common/telemetry/start-job.ps1 new file mode 100644 index 000000000..46028954f --- /dev/null +++ b/eng/common/telemetry/start-job.ps1 @@ -0,0 +1,39 @@ +[CmdletBinding()] +param( + [string]$QueueId, + [string]$Source, + [string]$Type, + [string]$Build, + [string]$Attempt, + [hashtable]$Properties +) + +Set-StrictMode -Version 2.0 +$ErrorActionPreference = "Stop" + + +$jobInfo = [pscustomobject]@{ + QueueId=$QueueId; + Source=$Source; + Type=$Type; + Build=$Build; + Properties=[pscustomobject]$Properties; +} + +$jobInfoJson = $jobInfo | ConvertTo-Json + +try { + Write-Verbose "Job Info: $jobInfoJson" + $jobToken = Invoke-RestMethod -Uri "https://helix.dot.net/api/2018-03-14/telemetry/job?access_token=$($env:HelixApiAccessToken)" -Method Post -ContentType "application/json" -Body $jobInfoJson + + $env:Helix_JobToken = $jobToken + if (& "$PSScriptRoot/../is-vsts.ps1") { + Write-Host "##vso[task.setvariable variable=Helix_JobToken;issecret=true;]$env:Helix_JobToken" + } +} +catch { + Write-Error $_ + Write-Error $_.Exception + exit 1 +} + diff --git a/eng/common/telemetry/start-job.sh b/eng/common/telemetry/start-job.sh new file mode 100644 index 000000000..732c28c23 --- /dev/null +++ b/eng/common/telemetry/start-job.sh @@ -0,0 +1,86 @@ +source="${BASH_SOURCE[0]}" +# resolve $source until the file is no longer a symlink +while [[ -h "$source" ]]; do + scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + source="$(readlink "$source")" + # if $source was a relative symlink, we need to resolve it relative to the path where the + # symlink file was located + [[ $source != /* ]] && source="$scriptroot/$source" +done +scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + +properties=() +while (($# > 0)); do + lowerI="$(echo $1 | awk '{print tolower($0)}')" + case $lowerI in + --queue-id) + queueId=$2 + shift 2 + ;; + --source) + source=$2 + shift 2 + ;; + --type) + type=$2 + shift 2 + ;; + --build) + build=$2 + shift 2 + ;; + --attempt) + attempt=$2 + shift 2 + ;; + -p) + properties+=($2) + shift 2 + ;; + *) + echo "Unknown Arg '$1'" + exit 1 + ;; + esac +done + +jobInfo=`mktemp` + +cat > $jobInfo < - - simpleNode(config.OS, 'latest') { - - stage ('Checkout Source') { - checkout scm - } - - stage ('Build/Test') { - - if (os == "Windows_NT") { - bat ".\\eng\\CIBuild.cmd -configuration ${config.Configuration} -prepareMachine" - } else { - sh "./eng/cibuild.sh --configuration ${config.Configuration} --architechure ${config.Architechure} --prepareMachine" - } - } - - stage ('Archive artifacts') { - def resultFilePattern = "**/artifacts/${config.Configuration}/TestResults/*.xml" - Utilities.addXUnitDotNETResults(job, resultFilePattern, skipIfNoTestFiles: false) - - def filesToArchive = "**/artifacts/${config.Configuration}/**" - archiveArtifacts allowEmptyArchive: true, artifacts: filesToArchive - } - } -}