From ab0ab29d45bddfc20535dd3ac80155178af8b6d1 Mon Sep 17 00:00:00 2001 From: dotnet-maestro-bot Date: Tue, 20 Aug 2019 16:22:22 -0700 Subject: [PATCH] Port stage work for 3.0 (#26137) (#26285) * Add support for servicing (post-build.yml) --- eng/Publishing.props | 44 ++++++++++++ eng/SignCheckExclusionsFile.txt | 14 ++++ eng/SymbolPublishingExclusionsFile.txt | 4 ++ eng/build-job.yml | 37 ++++------ eng/finalize-publish.yml | 16 ++--- eng/pipelines/internal.yml | 127 ++++++++++++++++++--------------- eng/test-job.yml | 19 ++--- eng/xplat-job.yml | 13 ++-- src/publishwitharcade.proj | 73 ------------------- 9 files changed, 171 insertions(+), 176 deletions(-) create mode 100644 eng/Publishing.props create mode 100644 eng/SignCheckExclusionsFile.txt create mode 100644 eng/SymbolPublishingExclusionsFile.txt delete mode 100644 src/publishwitharcade.proj diff --git a/eng/Publishing.props b/eng/Publishing.props new file mode 100644 index 0000000..b9e551b --- /dev/null +++ b/eng/Publishing.props @@ -0,0 +1,44 @@ + + + + + + CollectCoreClrArtifacts;$(PublishDependsOnTargets) + + + + + $([MSBuild]::NormalizeDirectory('$(BinDir)', 'PDB')) + + + $(OSIdentifier) + + $(BuildArch) + + + + + + + + + + + + + + + + + + + + + + NonShipping=true + + + + \ No newline at end of file diff --git a/eng/SignCheckExclusionsFile.txt b/eng/SignCheckExclusionsFile.txt new file mode 100644 index 0000000..30f34bd --- /dev/null +++ b/eng/SignCheckExclusionsFile.txt @@ -0,0 +1,14 @@ +;; File passed to the SignCheck tool for exclusions + +;; Skip verification on all NuGet packages +*nupkg;; + +;; The IL version of SPCL is never signed. +runtimes/*/il/System.Private.CoreLib.dll;*nupkg; + +;; +;; Platforms that don't support signing. +;; +*;*osx*nupkg; +*;*linux*nupkg; +*;*rhel*nupkg; diff --git a/eng/SymbolPublishingExclusionsFile.txt b/eng/SymbolPublishingExclusionsFile.txt new file mode 100644 index 0000000..3e1a081 --- /dev/null +++ b/eng/SymbolPublishingExclusionsFile.txt @@ -0,0 +1,4 @@ +tools/x86_arm/mscordaccore.dll +tools/x86_arm/mscordbi.dll +tools/x64_arm64/mscordaccore.dll +tools/x64_arm64/mscordbi.dll \ No newline at end of file diff --git a/eng/build-job.yml b/eng/build-job.yml index 40d4ae9..1b0cf8b 100644 --- a/eng/build-job.yml +++ b/eng/build-job.yml @@ -20,7 +20,7 @@ jobs: # Compute job name from template parameters name: ${{ format('build_{0}_{1}_{2}', parameters.osIdentifier, parameters.archType, parameters.buildConfig) }} - displayName: ${{ format('Build {0} {1} {2}', parameters.osIdentifier, parameters.archType, parameters.buildConfig) }} + displayName: ${{ format('{0} {1} {2}', parameters.osIdentifier, parameters.archType, parameters.buildConfig) }} # FreeBSD builds are disabled in the public project because we # don't have any FreeBSD agents in the public pool. @@ -131,23 +131,24 @@ jobs: # Sign on Windows - ${{ if and(ne(variables['System.TeamProject'], 'public'), ne(variables['Build.Reason'], 'PullRequest'), eq(parameters.osGroup, 'Windows_NT')) }}: - - script: powershell eng\common\build.ps1 -ci -sign -restore -configuration:$(buildConfig) -warnaserror:0 /p:ArcadeBuild=true /p:OfficialBuild=true /p:BuildOS=$(osGroup) /p:BuildArch=$(archType) /p:BuildType=$(buildConfigUpper) /p:DotNetSignType=%_SignType% -projects $(Build.SourcesDirectory)\eng\empty.csproj + - powershell: eng\common\build.ps1 -ci -sign -restore -configuration:$(buildConfig) -warnaserror:0 /p:ArcadeBuild=true /p:OfficialBuild=true /p:BuildOS=$(osGroup) /p:BuildArch=$(archType) /p:BuildType=$(_BuildConfig) /p:DotNetSignType=$env:_SignType -projects $(Build.SourcesDirectory)\eng\empty.csproj displayName: Sign Binaries - task: PublishBuildArtifacts@1 - displayName: Publish Signing Logs to VSTS + displayName: Publish Signing Logs inputs: PathtoPublish: '$(Build.SourcesDirectory)/artifacts/' PublishLocation: Container - ArtifactName: $(Agent.Os)_$(Agent.JobName)_$(archType) + ArtifactName: ${{ format('SignLogs_{0}_{1}_{2}', parameters.osIdentifier, parameters.archType, parameters.buildConfig) }} continueOnError: true condition: always() + # Publish product output directory for consumption by tests. - task: PublishBuildArtifacts@1 displayName: Publish product build inputs: - pathtoPublish: $(Build.SourcesDirectory)/bin/Product/$(osGroup).$(archType).$(buildConfigUpper) - artifactName: ${{ format('{0}_{1}_{2}_build', parameters.osIdentifier, parameters.archType, parameters.buildConfig) }} + pathtoPublish: $(Build.SourcesDirectory)/bin/Product/$(osGroup).$(archType).$(_BuildConfig) + artifactName: ${{ format('BinDir_{0}_{1}_{2}', parameters.osIdentifier, parameters.archType, parameters.buildConfig) }} # Get key vault secrets for publishing - ${{ if and(ne(variables['System.TeamProject'], 'public'), ne(variables['Build.Reason'], 'PullRequest')) }}: @@ -159,22 +160,16 @@ jobs: # Build packages - ${{ if ne(parameters.osGroup, 'Windows_NT') }}: - - script: ./build-packages.sh -BuildArch=$(archType) -BuildType=$(buildConfigUpper) $(crossPackagesArg) $(officialBuildIdArg) $(portableBuildArg) -ci + - script: ./build-packages.sh -BuildArch=$(archType) -BuildType=$(_BuildConfig) $(crossPackagesArg) $(officialBuildIdArg) $(portableBuildArg) -ci displayName: Build packages - ${{ if eq(parameters.osGroup, 'Windows_NT') }}: - - script: build-packages.cmd -BuildArch=$(archType) -BuildType=$(buildConfigUpper) $(officialBuildIdArg) -ci + - script: build-packages.cmd -BuildArch=$(archType) -BuildType=$(_BuildConfig) $(officialBuildIdArg) -ci displayName: Build packages # Publish official build - ${{ if and(ne(variables['System.TeamProject'], 'public'), ne(variables['Build.Reason'], 'PullRequest')) }}: - ${{ if ne(parameters.osGroup, 'Windows_NT') }}: - - script: ./eng/common/build.sh /p:DotNetPublishToBlobFeed=true --ci --restore --projects $(Build.SourcesDirectory)/eng/empty.csproj - displayName: Restore blob feed tasks - ${{ if eq(parameters.osGroup, 'FreeBSD') }}: - env: - # Arcade uses this SDK instead of trying to restore one. - DotNetCoreSdkDir: /usr/local/dotnet - - script: ./eng/common/msbuild.sh --ci src/publishwitharcade.proj /p:__BuildType=$(buildConfigUpper) /p:__BuildArch=$(archType) /p:__BuildOS=$(osGroup) /p:OSIdentifier=$(osIdentifier) /p:AzureFeedUrl=$(dotnetfeedUrl) /p:AzureAccountKey=$(dotnetfeedPAT) + - script: ./eng/common/build.sh --ci --restore --publish --configuration $(_BuildConfig) /p:DotNetPublishUsingPipelines=true /p:DotNetPublishToBlobFeed=true /p:DotNetPublishBlobFeedUrl=$(dotnetfeedUrl) /p:DotNetPublishBlobFeedKey=$(dotnetfeedPAT) /p:__BuildType=$(_BuildConfig) /p:__BuildArch=$(archType) /p:__BuildOS=$(osGroup) /p:OSIdentifier=$(osIdentifier) /bl:"$(Build.SourcesDirectory)/bin/Logs/publish-pkgs.binlog" --projects $(Build.SourcesDirectory)/eng/empty.csproj displayName: Publish packages to blob feed env: # TODO: remove NUGET_PACKAGES once https://github.com/dotnet/arcade/issues/1578 is fixed @@ -184,25 +179,17 @@ jobs: DotNetCoreSdkDir: /usr/local/dotnet - ${{ if eq(parameters.osGroup, 'Windows_NT') }}: # TODO: pass publish feed url and access token in from the internal pipeline - - powershell: eng\common\build.ps1 /p:DotNetPublishToBlobFeed=true -ci -restore -projects $(Build.SourcesDirectory)\eng\empty.csproj - displayName: Restore blob feed tasks - - powershell: eng\common\msbuild.ps1 -ci src\publishwitharcade.proj /p:__BuildType=$(buildConfigUpper) /p:__BuildArch=$(archType) /p:OSIdentifier=$(osIdentifier) /p:AzureFeedUrl=$(dotnetfeedUrl) /p:AzureAccountKey=$(dotnetfeedPAT) + - powershell: eng\common\build.ps1 -ci -restore -publish -configuration $(_BuildConfig) /p:DotNetPublishUsingPipelines=true /p:DotNetPublishToBlobFeed=true /p:DotNetPublishBlobFeedUrl=$(dotnetfeedUrl) /p:DotNetPublishBlobFeedKey=$(dotnetfeedPAT) /p:__BuildType=$(_BuildConfig) /p:__BuildArch=$(archType) /p:__BuildOS=$(osGroup) /p:OSIdentifier=$(osIdentifier) /bl:"$(Build.SourcesDirectory)\bin\Logs\publish-pkgs.binlog" -projects $(Build.SourcesDirectory)\eng\empty.csproj displayName: Publish packages to blob feed env: # TODO: remove NUGET_PACKAGES once https://github.com/dotnet/arcade/issues/1578 is fixed NUGET_PACKAGES: $(Build.SourcesDirectory)\.packages - # Save nuget packages in pipeline for update dotnet/versions - - task: PublishPipelineArtifact@0 - displayName: Save packages as pipeline artifact - inputs: - artifactName: ${{ format('build_{0}_{1}_{2}_nuget', parameters.osIdentifier, parameters.archType, parameters.buildConfig) }} - targetPath: $(Build.SourcesDirectory)/bin/Product/$(osGroup).$(archType).$(buildConfigUpper)/.nuget/pkg # Publish Logs - task: PublishBuildArtifacts@1 displayName: Publish Logs inputs: pathtoPublish: $(Build.SourcesDirectory)/bin/Logs - artifactName: ${{ format('build_{0}_{1}_{2}_Logs', parameters.osIdentifier, parameters.archType, parameters.buildConfig) }} + artifactName: ${{ format('BuildLogs_{0}_{1}_{2}', parameters.osIdentifier, parameters.archType, parameters.buildConfig) }} continueOnError: true condition: always() diff --git a/eng/finalize-publish.yml b/eng/finalize-publish.yml index 76d697d..2590052 100644 --- a/eng/finalize-publish.yml +++ b/eng/finalize-publish.yml @@ -19,9 +19,10 @@ jobs: name: NetCoreInternal-Pool queue: BuildPool.Windows.10.Amd64.VS2017 dependsOn: ${{ parameters.dependsOn }} + publishUsingPipelines: true # Update dotnet/versions - job: Finalize_Publish_Versions - displayName: Finalize_Publish_Versions + displayName: Finalize Publish Versions dependsOn: - Asset_Registry_Publish pool: @@ -30,13 +31,12 @@ jobs: - group: DotNet-Versions-Publish steps: # Download nuget packages - # Use parameters.dependsOn to determine the set of nuget packages we are publishing - - ${{ each build_id in parameters.dependsOn }}: - - task: DownloadPipelineArtifact@0 - displayName: Download ${{ build_id }} nuget packages - inputs: - artifactName: ${{ build_id }}_nuget - targetPath: ${{ parameters.artifactsDir }}/nuget + - task: DownloadBuildArtifacts@0 + displayName: Download nuget package artifacts + inputs: + buildType: current + artifactName: PackageArtifacts + downloadPath: ${{ parameters.artifactsDir }}/nuget - powershell: | $prefix = "refs/heads/" diff --git a/eng/pipelines/internal.yml b/eng/pipelines/internal.yml index 38bf9f1..cb41f05 100644 --- a/eng/pipelines/internal.yml +++ b/eng/pipelines/internal.yml @@ -20,65 +20,78 @@ schedules: - master always: true -jobs: -# -# Release builds -# -- template: /eng/platform-matrix.yml - parameters: - jobTemplate: build-job.yml - buildConfig: release - platformGroup: all - jobParameters: - # Publishing packages to blob feeds sometimes takes a long time - # due to waiting for an exclusive lock on the feed. - # See https://github.com/dotnet/arcade/blob/master/Documentation/CorePackages/AsyncPublishing.md - timeoutInMinutes: 120 +# See mappings in https://github.com/dotnet/arcade/blob/055bb2951c3107189551ab912b4e5550928b5afb/src/Microsoft.DotNet.Arcade.Sdk/tools/SdkTasks/PublishToPackageFeed.proj#L33-L48 +variables: +- name: _DotNetArtifactsCategory + value: CORECLR -# -# Release test builds -# -- template: /eng/platform-matrix.yml - parameters: - jobTemplate: test-job.yml - buildConfig: release - platformGroup: all - helixQueueGroup: all - jobParameters: - testGroup: outerloop +stages: + - stage: build + jobs: + # + # Release builds + # + - template: /eng/platform-matrix.yml + parameters: + jobTemplate: build-job.yml + buildConfig: release + platformGroup: all + jobParameters: + # Publishing packages to blob feeds sometimes takes a long time + # due to waiting for an exclusive lock on the feed. + # See https://github.com/dotnet/arcade/blob/master/Documentation/CorePackages/AsyncPublishing.md + timeoutInMinutes: 120 -# -# ReadyToRun test jobs -# -- template: /eng/platform-matrix.yml - parameters: - jobTemplate: test-job.yml - buildConfig: release - platformGroup: all - helixQueueGroup: all - jobParameters: - testGroup: outerloop - readyToRun: true - displayNameArgs: R2R + # + # Release test builds + # + - template: /eng/platform-matrix.yml + parameters: + jobTemplate: test-job.yml + buildConfig: release + platformGroup: all + helixQueueGroup: all + jobParameters: + testGroup: outerloop -# Publish build information to Build Assets Registry + # + # ReadyToRun test builds + # + - template: /eng/platform-matrix.yml + parameters: + jobTemplate: test-job.yml + buildConfig: release + platformGroup: all + helixQueueGroup: all + jobParameters: + testGroup: outerloop + readyToRun: true + displayNameArgs: R2R -# This job gathers build assets from the pipeline (from each official -# product build job), and publishes them to the build assets -# registry. Its dependencies should be updated to include all of the -# official builds if we add more platform/arch combinations. + # + # Publish build information to Build Assets Registry + # + # This job gathers build assets from the pipeline (from each official + # product build job), and publishes them to the build assets + # registry. Its dependencies should be updated to include all of the + # official builds if we add more platform/arch combinations. + - template: /eng/finalize-publish.yml + parameters: + dependsOn: + - build_Linux_arm_release + - build_Linux_arm64_release + - build_Linux_musl_x64_release + - build_Linux_musl_arm64_release + - build_Linux_rhel6_x64_release + - build_Linux_x64_release + - build_OSX_x64_release + - build_Windows_NT_x64_release + - build_Windows_NT_x86_release + - build_Windows_NT_arm_release + - build_Windows_NT_arm64_release -- template: /eng/finalize-publish.yml - parameters: - dependsOn: - - build_Linux_arm_release - - build_Linux_arm64_release - - build_Linux_musl_x64_release - - build_Linux_musl_arm64_release - - build_Linux_rhel6_x64_release - - build_Linux_x64_release - - build_OSX_x64_release - - build_Windows_NT_x64_release - - build_Windows_NT_x86_release - - build_Windows_NT_arm_release - - build_Windows_NT_arm64_release + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - template: /eng/common/templates/post-build/post-build.yml + parameters: + # Symbol validation is not entirely reliable as of yet, so should be turned off until https://github.com/dotnet/arcade/issues/2871 is resolved. + enableSymbolValidation: false diff --git a/eng/test-job.yml b/eng/test-job.yml index fe065d1..bc4e7aa 100644 --- a/eng/test-job.yml +++ b/eng/test-job.yml @@ -27,6 +27,10 @@ jobs: osIdentifier: ${{ parameters.osIdentifier }} helixType: 'build/tests/' + # Test jobs should continue on error for internal builds + ${{ if eq(variables['System.TeamProject'], 'internal') }}: + continueOnError: true + # Compute job name from template parameters ${{ if and(eq(parameters.testGroup, 'innerloop'), eq(parameters.displayNameArgs, '')) }}: name: ${{ format('test_{0}_{1}_{2}_{3}', 'p0', parameters.osIdentifier, parameters.archType, parameters.buildConfig) }} @@ -118,13 +122,13 @@ jobs: displayName: Install native dependencies - # Download product build + # Download product binaries directory - task: DownloadBuildArtifacts@0 displayName: Download product build inputs: buildType: current downloadType: single - artifactName: ${{ format('{0}_{1}_{2}_build', parameters.osIdentifier, parameters.archType, parameters.buildConfig) }} + artifactName: ${{ format('BinDir_{0}_{1}_{2}', parameters.osIdentifier, parameters.archType, parameters.buildConfig) }} downloadPath: $(System.ArtifactsDirectory) @@ -132,7 +136,7 @@ jobs: - task: CopyFiles@2 displayName: Populate Product directory inputs: - sourceFolder: $(System.ArtifactsDirectory)/${{ format('{0}_{1}_{2}_build', parameters.osIdentifier, parameters.archType, parameters.buildConfig) }} + sourceFolder: $(System.ArtifactsDirectory)/${{ format('BinDir_{0}_{1}_{2}', parameters.osIdentifier, parameters.archType, parameters.buildConfig) }} contents: '**' targetFolder: $(Build.SourcesDirectory)/bin/Product/$(osGroup).$(archType).$(buildConfigUpper) @@ -323,19 +327,18 @@ jobs: - forcerelocs - gcstress15 - # Publish Logs - task: PublishBuildArtifacts@1 displayName: Publish Logs inputs: pathtoPublish: $(Build.SourcesDirectory)/bin/Logs ${{ if and(eq(parameters.corefxTests, true), eq(parameters.readyToRun, true)) }}: - artifactName: ${{ format('test{0}_{1}_{2}_{3}_{4}_Logs', '_r2r_corefx', parameters.osIdentifier, parameters.archType, parameters.buildConfig, parameters.testGroup) }} + artifactName: ${{ format('TestLogs_r2r_corefx_{0}_{1}_{2}_{3}', parameters.osIdentifier, parameters.archType, parameters.buildConfig, parameters.testGroup) }} ${{ if and(eq(parameters.corefxTests, true), ne(parameters.readyToRun, true)) }}: - artifactName: ${{ format('test{0}_{1}_{2}_{3}_{4}_Logs', '_corefx', parameters.osIdentifier, parameters.archType, parameters.buildConfig, parameters.testGroup) }} + artifactName: ${{ format('TestLogs_corefx_{0}_{1}_{2}_{3}', parameters.osIdentifier, parameters.archType, parameters.buildConfig, parameters.testGroup) }} ${{ if and(ne(parameters.corefxTests, true), eq(parameters.readyToRun, true)) }}: - artifactName: ${{ format('test{0}_{1}_{2}_{3}_{4}_Logs', '_r2r', parameters.osIdentifier, parameters.archType, parameters.buildConfig, parameters.testGroup) }} + artifactName: ${{ format('TestLogs_r2r_{0}_{1}_{2}_{3}', parameters.osIdentifier, parameters.archType, parameters.buildConfig, parameters.testGroup) }} ${{ if and(ne(parameters.corefxTests, true), ne(parameters.readyToRun, true)) }}: - artifactName: ${{ format('test{0}_{1}_{2}_{3}_{4}_Logs', '', parameters.osIdentifier, parameters.archType, parameters.buildConfig, parameters.testGroup) }} + artifactName: ${{ format('TestLogs_{0}_{1}_{2}_{3}', parameters.osIdentifier, parameters.archType, parameters.buildConfig, parameters.testGroup) }} continueOnError: true condition: always() diff --git a/eng/xplat-job.yml b/eng/xplat-job.yml index e418bc6..6beda4e 100644 --- a/eng/xplat-job.yml +++ b/eng/xplat-job.yml @@ -4,16 +4,17 @@ parameters: osGroup: '' osIdentifier: '' name: '' - displayName: '' helixType: '(unspecified)' - condition: '' - dependsOn: '' container: '' - timeoutInMinutes: '' crossrootfsDir: '' - enableMicrobuild: '' # arcade-specific parameters + condition: '' + continueOnError: false + dependsOn: '' + displayName: '' + timeoutInMinutes: '' + enableMicrobuild: '' gatherAssetManifests: false variables: {} ## any extra variables to add to the defaults defined below @@ -28,6 +29,7 @@ jobs: condition: ${{ parameters.condition }} dependsOn: ${{ parameters.dependsOn }} timeoutInMinutes: ${{ parameters.timeoutInMinutes }} + continueOnError: ${{ parameters.continueOnError }} # Send telemetry for all builds enableTelemetry: true @@ -35,6 +37,7 @@ jobs: helixType: ${{ parameters.helixType }} enableMicrobuild: ${{ parameters.enableMicrobuild }} + enablePublishUsingPipelines: true pool: diff --git a/src/publishwitharcade.proj b/src/publishwitharcade.proj deleted file mode 100644 index b88b54f..0000000 --- a/src/publishwitharcade.proj +++ /dev/null @@ -1,73 +0,0 @@ - - - - - true - - - - - - - - - net462 - - - - - - - - - - - - - - - - - - - - - NonShipping=true - - - - - - - - - - - - - $(ArtifactsLogDir)AssetManifest\$(OSIdentifier)-$(BuildArch) - - <_TemporaryBuildRepositoryUri>$(BUILD_REPOSITORY_URI) - <_TemporaryBuildRepositoryUri Condition="'$(BUILD_REPOSITORY_URI)' == 'https://dnceng.visualstudio.com/internal/_git/dotnet-coreclr'" >https://dnceng@dev.azure.com/dnceng/internal/_git/dotnet-coreclr - - - - - - - - - -- 2.7.4