From 71e7649255bd9236ef3f8cf92bdf3605b6065656 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Wed, 6 Nov 2019 11:07:21 -0800 Subject: [PATCH] Backport pipeline changes (dotnet/core-setup#8780) * Backport changes to get CI working in runtime repository. * /root/coresetup->/root/runtime * Delete broken RPM symlink that might still be around if the RPM packaging job failed. * Add back old Documentation folder excludes with a more specific include for the manpages folder. * Try other variable format for template path. * Treat subsetArg as a parameter with a default value instead of as a variable. * Pass subset arg to template. * Try other variable format. * Try another variant. * Switch back variable format now that I got a different (seemingly actionable) error. * Fix passing the subset arg. Commit migrated from https://github.com/dotnet/core-setup/commit/9e6766a796d47c3678adf6e0c9bfa3e8b070be75 --- eng/pipelines/installer/azure-pipelines.yml | 58 ++++++++++++++++------ eng/pipelines/installer/jobs/bash-build.yml | 26 +++++++--- eng/pipelines/installer/jobs/osx-build.yml | 2 +- .../installer/jobs/steps/build-linux-package.yml | 7 +-- .../installer/jobs/steps/upload-job-artifacts.yml | 7 +++ eng/pipelines/installer/jobs/windows-build.yml | 2 +- eng/pipelines/installer/stages/publish.yml | 2 +- src/installer/corehost/build.sh | 2 +- src/installer/signing/Directory.Build.props | 6 +-- 9 files changed, 79 insertions(+), 33 deletions(-) diff --git a/eng/pipelines/installer/azure-pipelines.yml b/eng/pipelines/installer/azure-pipelines.yml index fcf3876..ed0e0cc 100644 --- a/eng/pipelines/installer/azure-pipelines.yml +++ b/eng/pipelines/installer/azure-pipelines.yml @@ -1,12 +1,20 @@ trigger: - batch: true branches: include: - master - release/3.* paths: + include: + - '*' + - docs/installer/manpages/* + - Documentation/manpages/* exclude: - Documentation/* + - src/coreclr/* + - src/libraries/* + - eng/pipelines/coreclr/* + - eng/pipelines/libraries/* + - docs/* - README.md - CONTRIBUTING.md - LICENSE.TXT @@ -18,8 +26,17 @@ pr: - master - release/3.* paths: + include: + - '*' + - docs/installer/manpages/* + - Documentation/manpages/* exclude: - Documentation/* + - src/coreclr/* + - src/libraries/* + - eng/pipelines/coreclr/* + - eng/pipelines/libraries/* + - docs/* - README.md - CONTRIBUTING.md - LICENSE.TXT @@ -53,11 +70,22 @@ variables: - name: SignType value: $[ coalesce(variables.OfficialSignType, 'real') ] + - ${{ if contains(variables['Build.DefinitionName'], 'runtime') }}: + - name: pipelinesPath + value: /eng/pipelines/installer + - name: buildScriptFileName + value: installer + - ${{ if not(contains(variables['Build.DefinitionName'], 'runtime')) }}: + - name: pipelinesPath + value: /eng/pipelines + - name: buildScriptFileName + value: build + stages: - stage: Build jobs: # -------- Build Bash legs (Linux and FreeBSD) -------- - - template: /eng/pipelines/jobs/bash-build.yml + - template: ${{ variables['pipelinesPath'] }}/jobs/bash-build.yml parameters: crossBuild: true name: Linux_Arm @@ -67,7 +95,7 @@ stages: skipTests: true targetArchitecture: arm - - template: /eng/pipelines/jobs/bash-build.yml + - template: ${{ variables['pipelinesPath'] }}/jobs/bash-build.yml parameters: crossBuild: true name: Linux_Arm64 @@ -79,7 +107,7 @@ stages: # # Tizen build only for PR build # - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: - # - template: /eng/pipelines/jobs/bash-build.yml + # - template: ${{ variables['pipelinesPath'] }}/jobs/bash-build.yml # parameters: # additionalMSBuildArgs: /p:OverridePackageSource=https:%2F%2Ftizen.myget.org/F/dotnet-core/api/v3/index.json /p:OutputRid=tizen.5.0.0-armel # additionalRunArgs: -e ROOTFS_DIR=/crossrootfs/armel.tizen.build @@ -91,7 +119,7 @@ stages: # skipTests: true # targetArchitecture: armel - - template: /eng/pipelines/jobs/bash-build.yml + - template: ${{ variables['pipelinesPath'] }}/jobs/bash-build.yml parameters: additionalMSBuildArgs: /p:OutputRid=linux-musl-arm64 crossBuild: true @@ -102,7 +130,7 @@ stages: skipTests: true targetArchitecture: arm64 - - template: /eng/pipelines/jobs/bash-build.yml + - template: ${{ variables['pipelinesPath'] }}/jobs/bash-build.yml parameters: additionalMSBuildArgs: /p:OutputRid=linux-musl-x64 name: Linux_x64_Alpine39 @@ -110,7 +138,7 @@ stages: portableBuild: false targetArchitecture: x64 - - template: /eng/pipelines/jobs/bash-build.yml + - template: ${{ variables['pipelinesPath'] }}/jobs/bash-build.yml parameters: name: Linux_x64_glibc dockerImage: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7-50f0d02-20190918214028 @@ -129,7 +157,7 @@ stages: portableBuild: true targetArchitecture: x64 - - template: /eng/pipelines/jobs/bash-build.yml + - template: ${{ variables['pipelinesPath'] }}/jobs/bash-build.yml parameters: additionalMSBuildArgs: /p:OutputRid=rhel.6-x64 name: Linux_x64_Rhel6 @@ -138,34 +166,34 @@ stages: targetArchitecture: x64 # -------- Build OSX (macOS) leg -------- - - template: /eng/pipelines/jobs/osx-build.yml + - template: ${{ variables['pipelinesPath'] }}/jobs/osx-build.yml parameters: name: OSX # -------- Build Windows legs -------- # Windows Arm - - template: /eng/pipelines/jobs/windows-build.yml + - template: ${{ variables['pipelinesPath'] }}/jobs/windows-build.yml parameters: name: Windows_Arm skipTests: true targetArchitecture: arm # Windows Arm64 - - template: /eng/pipelines/jobs/windows-build.yml + - template: ${{ variables['pipelinesPath'] }}/jobs/windows-build.yml parameters: name: Windows_Arm64 skipTests: true targetArchitecture: arm64 # Windows x64 - - template: /eng/pipelines/jobs/windows-build.yml + - template: ${{ variables['pipelinesPath'] }}/jobs/windows-build.yml parameters: name: Windows_x64 publishRidAgnosticPackages: true targetArchitecture: x64 # Windows x86 - - template: /eng/pipelines/jobs/windows-build.yml + - template: ${{ variables['pipelinesPath'] }}/jobs/windows-build.yml parameters: name: Windows_x86 targetArchitecture: x86 @@ -176,7 +204,7 @@ stages: dependsOn: Build jobs: # Prep artifacts: sign them and upload pipeline artifacts expected by stages-based publishing. - - template: /eng/pipelines/jobs/prepare-signed-artifacts.yml + - template: ${{ variables['pipelinesPath'] }}/jobs/prepare-signed-artifacts.yml parameters: PublishRidAgnosticPackagesFromJobName: Windows_x64 # Publish to Build Asset Registry in order to generate the ReleaseConfigs artifact. @@ -188,7 +216,7 @@ stages: name: NetCoreInternal-Pool queue: buildpool.windows.10.amd64.vs2017 - - template: /eng/pipelines/stages/publish.yml + - template: ${{ variables['pipelinesPath'] }}/stages/publish.yml parameters: # Publish channel configuration. The publish template wraps Arcade publishing and adds some # extras that aren't implemented in Arcade yet. diff --git a/eng/pipelines/installer/jobs/bash-build.yml b/eng/pipelines/installer/jobs/bash-build.yml index b6a6aa6..5a12623 100644 --- a/eng/pipelines/installer/jobs/bash-build.yml +++ b/eng/pipelines/installer/jobs/bash-build.yml @@ -40,8 +40,8 @@ jobs: ${{ if ne(parameters.name, 'FreeBSD_x64')}}: RunArguments: >- docker run --privileged --rm - -v "$(Build.SourcesDirectory):/root/coresetup" - -w="/root/coresetup" + -v "$(Build.SourcesDirectory):/root/runtime" + -w="/root/runtime" $(PreserveNuGetAuthDockerArgs) ${{ parameters.additionalRunArgs }} ${{ parameters.dockerImage }} @@ -50,12 +50,15 @@ jobs: RunArguments: export DotNetBootstrapCliTarPath=/dotnet-sdk-freebsd-x64.tar && ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - BuildScript: ./eng/install-nuget-credprovider-then-build.sh - MSBuildScript: /root/coresetup/eng/install-nuget-credprovider-then-msbuild.sh + ${{ if not(contains(variables['Build.DefinitionName'], 'runtime')) }}: + BuildScript: ./eng/install-nuget-credprovider-then-build.sh + ${{ if contains(variables['Build.DefinitionName'], 'runtime') }}: + BuildScript: ./eng/install-nuget-credprovider-then-build.sh --subsetCategory installer + MSBuildScript: /root/runtime/eng/install-nuget-credprovider-then-msbuild.sh ${{ if not(and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'))) }}: - BuildScript: ./build.sh - MSBuildScript: /root/coresetup/eng/common/msbuild.sh + BuildScript: ./$(buildScriptFileName).sh + MSBuildScript: /root/runtime/eng/common/msbuild.sh CommonMSBuildArgs: >- /p:Configuration=$(_BuildConfig) @@ -89,11 +92,16 @@ jobs: DockerRunMSBuild: >- docker run - -v $(Build.SourcesDirectory):/root/coresetup + -v $(Build.SourcesDirectory):/root/runtime -v $(Build.StagingDirectory)/sharedFrameworkPublish/:/root/sharedFrameworkPublish/ - -w=/root/coresetup + -w=/root/runtime $(PreserveNuGetAuthDockerArgs) + ${{ if contains(variables['Build.DefinitionName'], 'runtime') }}: + installersSubsetArg: --subset Installers + ${{ if not(contains(variables['Build.DefinitionName'], 'runtime')) }}: + installersSubsetArg: /p:Subset=Installer + steps: - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: @@ -139,6 +147,7 @@ jobs: distroRid: ${{ packageBuild.imageRid }} image: ${{ packageBuild.image }} packageStepDescription: Runtime Deps, Runtime, Framework Packs installers + subsetArg: $(installersSubsetArg) - ${{ each rid in packageBuild.rids }}: # Build distro-dependent packages. @@ -149,6 +158,7 @@ jobs: outputRidArg: /p:OutputRid=${{ rid }}-${{ parameters.targetArchitecture }} packageStepDescription: Runtime Deps installers packagingArgs: /p:BuildDistroIndependentInstallers=false + subsetArg: $(installersSubsetArg) # Files may be owned by root because builds don't set user ID. The next AzDO step runs 'find' in # the source tree, which fails due to permissions in the 'NetCore*-Int-Pool' queues. This step diff --git a/eng/pipelines/installer/jobs/osx-build.yml b/eng/pipelines/installer/jobs/osx-build.yml index 28479a9..74554f1 100644 --- a/eng/pipelines/installer/jobs/osx-build.yml +++ b/eng/pipelines/installer/jobs/osx-build.yml @@ -26,7 +26,7 @@ jobs: - task: NuGetAuthenticate@0 - script: >- - $(Build.SourcesDirectory)/build.sh --ci --test + $(Build.SourcesDirectory)/$(buildScriptFileName).sh --ci --test /p:OfficialBuildId=$(OfficialBuildId) /p:StripSymbols=true $(CommonMSBuildArgs) diff --git a/eng/pipelines/installer/jobs/steps/build-linux-package.yml b/eng/pipelines/installer/jobs/steps/build-linux-package.yml index 9bfac53..e9e342b 100644 --- a/eng/pipelines/installer/jobs/steps/build-linux-package.yml +++ b/eng/pipelines/installer/jobs/steps/build-linux-package.yml @@ -5,6 +5,7 @@ parameters: outputRidArg: '' packageStepDescription: null packagingArgs: '' + subsetArg: '' steps: - ${{ if eq(parameters.buildTraversalBuildDependencies, true) }}: @@ -13,7 +14,7 @@ steps: df -h $(DockerRunMSBuild) ${{ parameters.image }} $(MSBuildScript) \ --ci \ - /root/coresetup/tools-local/tasks/installer.tasks/installer.tasks.csproj \ + /root/runtime/tools-local/tasks/installer.tasks/installer.tasks.csproj \ /t:Restore /t:Build /t:CreateHostMachineInfoFile \ $(CommonMSBuildArgs) \ /bl:msbuild.${{ parameters.distroRid }}.traversaldependencies.binlog @@ -24,12 +25,12 @@ steps: df -h $(DockerRunMSBuild) ${{ parameters.image }} $(BuildScript) \ --ci \ - /p:Subset=Installer \ + ${{ parameters.subsetArg }} \ /p:UsePrebuiltPortableBinariesForInstallers=true \ $(PackagePortableBitsArgs) \ /p:GenerateProjectInstallers=true \ ${{ parameters.packagingArgs }} \ $(CommonMSBuildArgs) \ ${{ parameters.outputRidArg }} \ - /bl:msbuild.${{ parameters.distroRid }}.installers.binlog + /bl:artifacts/log/$(_BuildConfig)/msbuild.${{ parameters.distroRid }}.installers.binlog displayName: Package ${{ parameters.packageStepDescription }} - ${{ parameters.distroRid }} diff --git a/eng/pipelines/installer/jobs/steps/upload-job-artifacts.yml b/eng/pipelines/installer/jobs/steps/upload-job-artifacts.yml index 8b9e42f..d02de12 100644 --- a/eng/pipelines/installer/jobs/steps/upload-job-artifacts.yml +++ b/eng/pipelines/installer/jobs/steps/upload-job-artifacts.yml @@ -35,6 +35,13 @@ steps: continueOnError: true condition: always() +- task: DeleteFiles@1 + displayName: Delete broken symlinks. + inputs: + SourceFolder: '$(Build.SourcesDirectory)' + Contents: artifacts/obj/rhel.7-x64.Debug/packages/dotnet-host/5.0.0/rpmLayoutDirectory/package_root/usr/bin/dotnet + condition: succeededOrFailed() + - task: CopyFiles@2 displayName: Prepare BuildLogs staging directory inputs: diff --git a/eng/pipelines/installer/jobs/windows-build.yml b/eng/pipelines/installer/jobs/windows-build.yml index 47c9c30..57ff8c7 100644 --- a/eng/pipelines/installer/jobs/windows-build.yml +++ b/eng/pipelines/installer/jobs/windows-build.yml @@ -58,7 +58,7 @@ jobs: displayName: Clear NuGet http cache (if exists) - script: >- - build.cmd -ci -test + $(buildScriptFileName).cmd -ci -test $(CommonMSBuildArgs) $(MsbuildSigningArguments) displayName: Build diff --git a/eng/pipelines/installer/stages/publish.yml b/eng/pipelines/installer/stages/publish.yml index ceb6434..ed7eb21 100644 --- a/eng/pipelines/installer/stages/publish.yml +++ b/eng/pipelines/installer/stages/publish.yml @@ -40,7 +40,7 @@ stages: variables: - template: /eng/common/templates/post-build/common-variables.yml jobs: - - template: /eng/pipelines/jobs/run-publish-project.yml + - template: $(pipelinesPath)/jobs/run-publish-project.yml parameters: projectName: publish-final dependency: ${{ dependency }} diff --git a/src/installer/corehost/build.sh b/src/installer/corehost/build.sh index bf5b98c..8443e4a 100755 --- a/src/installer/corehost/build.sh +++ b/src/installer/corehost/build.sh @@ -268,7 +268,7 @@ if [ $__CrossBuild == 1 ]; then fi export TARGET_BUILD_ARCH=$__build_arch_lowcase export __DistroRid=$__rid_plat - cmake "$DIR" -G "Unix Makefiles" $__cmake_defines -DCLI_CMAKE_HOST_VER=$__host_ver -DCLI_CMAKE_COMMON_HOST_VER=$__apphost_ver -DCLI_CMAKE_HOST_FXR_VER=$__fxr_ver -DCLI_CMAKE_HOST_POLICY_VER=$__policy_ver -DCLI_CMAKE_PKG_RID=$__base_rid -DCLI_CMAKE_COMMIT_HASH=$__commit_hash -DCMAKE_INSTALL_PREFIX=$__cmake_bin_prefix -DCMAKE_TOOLCHAIN_FILE=$DIR/../../cross/toolchain.cmake + cmake "$DIR" -G "Unix Makefiles" $__cmake_defines -DCLI_CMAKE_HOST_VER=$__host_ver -DCLI_CMAKE_COMMON_HOST_VER=$__apphost_ver -DCLI_CMAKE_HOST_FXR_VER=$__fxr_ver -DCLI_CMAKE_HOST_POLICY_VER=$__policy_ver -DCLI_CMAKE_PKG_RID=$__base_rid -DCLI_CMAKE_COMMIT_HASH=$__commit_hash -DCMAKE_INSTALL_PREFIX=$__cmake_bin_prefix -DCMAKE_TOOLCHAIN_FILE=$RootRepo/cross/toolchain.cmake else cmake "$DIR" -G "Unix Makefiles" $__cmake_defines -DCLI_CMAKE_HOST_VER=$__host_ver -DCLI_CMAKE_COMMON_HOST_VER=$__apphost_ver -DCLI_CMAKE_HOST_FXR_VER=$__fxr_ver -DCLI_CMAKE_HOST_POLICY_VER=$__policy_ver -DCLI_CMAKE_PKG_RID=$__base_rid -DCLI_CMAKE_COMMIT_HASH=$__commit_hash -DCMAKE_INSTALL_PREFIX=$__cmake_bin_prefix fi diff --git a/src/installer/signing/Directory.Build.props b/src/installer/signing/Directory.Build.props index 0054dfb..b73a942 100644 --- a/src/installer/signing/Directory.Build.props +++ b/src/installer/signing/Directory.Build.props @@ -4,9 +4,9 @@ $(NETCoreAppFramework) + true - - true + + true - -- 2.7.4