From ce3239a9c4f72a3802ef92f75b61d10ba624a416 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Tom=C3=A1=C5=A1=20Rylek?= Date: Mon, 4 Nov 2019 02:57:31 -0800 Subject: [PATCH] YAML consolidation changes part 2 (dotnet/coreclr#27649) After finally getting a more or less green run in the runtime repo I have prepared another batch of changes aimed at simplifying the oncoming repo integration: 1) Use relative paths for script references; 2) Add new variable coreClrRepoRootDir with the appended path separator so that we don't need to fork that everywhere; 3) Actually unify the first pair of build-test executions that has become identical after the change; 4) Fixed several previously unseen repo root-relative paths; 5) Fixed missing propagation of testGroup in build-job; 6) Fixed run logs to include R2R in the name (Matt Mitchell pointed out the other day this prevents us from analyzing both sets of logs); 7) Move job.yml to eng to enable using relative template paths; 8) Delete unused jobs.yml. 9) Move performance.yml to eng as run-performance-test.yml to derive it from xplat-pipeline-job and avoid duplicating variables. Thanks Tomas Commit migrated from https://github.com/dotnet/coreclr/commit/818b35978a1681690988b2c9c3f0b6a034d91e82 --- eng/build-job.yml | 29 +-- eng/build-test-job.yml | 18 +- eng/checkout-job.yml | 4 +- eng/crossgen-comparison-job.yml | 2 +- eng/job.yml | 217 +++++++++++++++++++++ eng/perf-job.yml | 40 +--- eng/pipelines/coreclr/ci.yml | 18 +- eng/pipelines/coreclr/corefx-jitstress.yml | 6 +- .../coreclr/corefx-jitstress2-jitstressregs.yml | 6 +- eng/pipelines/coreclr/corefx-jitstressregs.yml | 6 +- eng/pipelines/coreclr/corefx.yml | 6 +- eng/pipelines/coreclr/gc-longrunning.yml | 8 +- eng/pipelines/coreclr/gc-simulator.yml | 6 +- eng/pipelines/coreclr/gcstress-extra.yml | 6 +- eng/pipelines/coreclr/gcstress0x3-gcstress0xc.yml | 6 +- eng/pipelines/coreclr/internal.yml | 12 +- eng/pipelines/coreclr/jitstress-isas-arm.yml | 6 +- eng/pipelines/coreclr/jitstress-isas-x86.yml | 8 +- eng/pipelines/coreclr/jitstress.yml | 6 +- eng/pipelines/coreclr/jitstress2-jitstressregs.yml | 6 +- eng/pipelines/coreclr/jitstressregs-x86.yml | 6 +- eng/pipelines/coreclr/jitstressregs.yml | 8 +- eng/pipelines/coreclr/perf.yml | 6 +- eng/pipelines/coreclr/pr.yml | 24 +-- eng/pipelines/coreclr/r2r-extra.yml | 6 +- eng/pipelines/coreclr/r2r.yml | 6 +- eng/pipelines/coreclr/runincontext.yml | 6 +- eng/run-performance-job.yml | 102 ++++++++++ eng/run-test-job.yml | 42 ++-- eng/send-to-helix-step.yml | 9 +- eng/xplat-job.yml | 4 +- eng/xplat-pipeline-job.yml | 65 +++--- 32 files changed, 500 insertions(+), 200 deletions(-) create mode 100644 eng/job.yml create mode 100644 eng/run-performance-job.yml diff --git a/eng/build-job.yml b/eng/build-job.yml index f0fbebe..0a3a6b2 100644 --- a/eng/build-job.yml +++ b/eng/build-job.yml @@ -19,6 +19,7 @@ jobs: archType: ${{ parameters.archType }} osGroup: ${{ parameters.osGroup }} osSubgroup: ${{ parameters.osSubgroup }} + testGroup: ${{ parameters.testGroup }} helixType: 'build/product/' enableMicrobuild: true stagedBuild: ${{ parameters.stagedBuild }} @@ -97,11 +98,11 @@ jobs: # and FreeBSD builds use a build agent with dependencies # preinstalled, so we only need this step for OSX and Windows. - ${{ if eq(parameters.osGroup, 'OSX') }}: - - script: sh eng/install-native-dependencies.sh $(osGroup) + - script: sh $(coreClrRepoRoot)/eng/install-native-dependencies.sh $(osGroup) displayName: Install native dependencies - ${{ if eq(parameters.osGroup, 'Windows_NT') }}: # Necessary to install python - - script: eng\common\init-tools-native.cmd -InstallDirectory $(Build.SourcesDirectory)\native-tools -Force + - script: $(Build.SourcesDirectory)\eng\common\init-tools-native.cmd -InstallDirectory $(Build.SourcesDirectory)\native-tools -Force displayName: Install native dependencies # Install internal tools on official builds @@ -126,18 +127,18 @@ jobs: # Build - ${{ if ne(parameters.osGroup, 'Windows_NT') }}: - - script: $(coreClrRepoRoot)/build$(scriptExt) $(buildConfig) $(archType) $(crossArg) -ci -skipnuget $(clangArg) $(stripSymbolsArg) $(officialBuildIdArg) + - script: $(coreClrRepoRootDir)build$(scriptExt) $(buildConfig) $(archType) $(crossArg) -ci -skipnuget $(clangArg) $(stripSymbolsArg) $(officialBuildIdArg) displayName: Build product - ${{ if eq(parameters.osGroup, 'Windows_NT') }}: - - script: set __TestIntermediateDir=int&&$(coreClrRepoRoot)\build$(scriptExt) $(buildConfig) $(archType) -ci -skiptests -skipbuildpackages $(officialBuildIdArg) $(ibcOptimizeArg) $(enforcePgoArg) + - script: set __TestIntermediateDir=int&&$(coreClrRepoRootDir)build$(scriptExt) $(buildConfig) $(archType) -ci -skiptests -skipbuildpackages $(officialBuildIdArg) $(ibcOptimizeArg) $(enforcePgoArg) displayName: Build product # Build native test components - ${{ if ne(parameters.osGroup, 'Windows_NT') }}: - - script: $(coreClrRepoRoot)/build-test$(scriptExt) skipmanaged $(buildConfig) $(archType) $(crossArg) $(priorityArg) $(clangArg) + - script: $(coreClrRepoRootDir)build-test$(scriptExt) skipmanaged $(buildConfig) $(archType) $(crossArg) $(priorityArg) $(clangArg) displayName: Build native test components - ${{ if eq(parameters.osGroup, 'Windows_NT') }}: - - script: $(coreClrRepoRoot)\build-test$(scriptExt) skipmanaged $(buildConfig) $(archType) $(priorityArg) + - script: $(coreClrRepoRootDir)build-test$(scriptExt) skipmanaged $(buildConfig) $(archType) $(priorityArg) displayName: Build native test components # Sign on Windows @@ -155,7 +156,7 @@ jobs: condition: always() # Publish product output directory for consumption by tests. - - template: /eng/upload-artifact-step.yml + - template: upload-artifact-step.yml parameters: rootFolder: $(buildProductRootFolderPath) includeRootFolder: false @@ -166,7 +167,7 @@ jobs: displayName: 'product build' # Publish test native components for consumption by test execution. - - template: /eng/upload-artifact-step.yml + - template: upload-artifact-step.yml parameters: rootFolder: $(testNativeRootFolderPath) includeRootFolder: false @@ -177,7 +178,7 @@ jobs: displayName: 'native test components' # Publish test build root for consumption by test execution. - - template: /eng/upload-artifact-step.yml + - template: upload-artifact-step.yml parameters: rootFolder: $(testBuildRootFolderPath) includeRootFolder: false @@ -197,16 +198,16 @@ jobs: # Build packages - ${{ if ne(parameters.osGroup, 'Windows_NT') }}: - - script: $(coreClrRepoRoot)/build-packages$(scriptExt) -BuildArch=$(archType) -BuildType=$(_BuildConfig) $(crossPackagesArg) $(officialBuildIdArg) $(portableBuildArg) -ci + - script: $(coreClrRepoRootDir)build-packages$(scriptExt) -BuildArch=$(archType) -BuildType=$(_BuildConfig) $(crossPackagesArg) $(officialBuildIdArg) $(portableBuildArg) -ci displayName: Build packages - ${{ if eq(parameters.osGroup, 'Windows_NT') }}: - - script: $(coreClrRepoRoot)\build-packages$(scriptExt) -BuildArch=$(archType) -BuildType=$(_BuildConfig) $(officialBuildIdArg) -ci + - script: $(coreClrRepoRootDir)build-packages$(scriptExt) -BuildArch=$(archType) -BuildType=$(_BuildConfig) $(officialBuildIdArg) -ci displayName: Build packages # Publish official build - ${{ if eq(parameters.publishToBlobFeed, 'true') }}: - ${{ if ne(parameters.osGroup, 'Windows_NT') }}: - - script: ./eng/common/build.sh --ci --restore --publish --configuration $(_BuildConfig) /p:DotNetPublishUsingPipelines=true /p:DotNetPublishToBlobFeed=true /p:DotNetPublishBlobFeedUrl=$(dotnetfeedUrl) /p:DotNetPublishBlobFeedKey=$(dotnetfeedPAT) /p:__BuildType=$(_BuildConfig) /p:__BuildArch=$(archType) /p:__BuildOS=$(osGroup) /p:OSIdentifier=$(osGroup)$(osSubgroup) /bl:"$(Build.SourcesDirectory)/bin/Logs/publish-pkgs.binlog" --projects $(Build.SourcesDirectory)/eng/empty.csproj + - script: $(Build.SourcesDirectory)/eng/common/build.sh --ci --restore --publish --configuration $(_BuildConfig) /p:DotNetPublishUsingPipelines=true /p:DotNetPublishToBlobFeed=true /p:DotNetPublishBlobFeedUrl=$(dotnetfeedUrl) /p:DotNetPublishBlobFeedKey=$(dotnetfeedPAT) /p:__BuildType=$(_BuildConfig) /p:__BuildArch=$(archType) /p:__BuildOS=$(osGroup) /p:OSIdentifier=$(osGroup)$(osSubgroup) /bl:"$(Build.SourcesDirectory)/bin/Logs/publish-pkgs.binlog" --projects $(Build.SourcesDirectory)/eng/empty.csproj displayName: Publish packages to blob feed env: # TODO: remove NUGET_PACKAGES once https://github.com/dotnet/arcade/issues/1578 is fixed @@ -226,7 +227,7 @@ jobs: - task: PublishBuildArtifacts@1 displayName: Publish Logs inputs: - pathtoPublish: $(Build.SourcesDirectory)/bin/Logs - artifactName: ${{ format('BuildLogs_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} + pathtoPublish: $(coreClrRepoRoot)/bin/Logs + artifactName: 'BuildLogs_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' continueOnError: true condition: always() diff --git a/eng/build-test-job.yml b/eng/build-test-job.yml index 6ae36ab..e9aa683 100644 --- a/eng/build-test-job.yml +++ b/eng/build-test-job.yml @@ -63,16 +63,16 @@ jobs: # Install test build dependencies - ${{ if eq(parameters.osGroup, 'OSX') }}: - - script: sh eng/install-native-dependencies.sh $(osGroup) + - script: sh $(coreClrRepoRootDir)eng/install-native-dependencies.sh $(osGroup) displayName: Install native dependencies - ${{ if eq(parameters.osGroup, 'Windows_NT') }}: # Necessary to install correct cmake version - - script: eng\common\init-tools-native.cmd -InstallDirectory $(Build.SourcesDirectory)\native-tools -Force + - script: $(coreClrRepoRootDir)eng\common\init-tools-native.cmd -InstallDirectory $(Build.SourcesDirectory)\native-tools -Force displayName: Install native dependencies # Download product binaries directory - - template: /eng/download-artifact-step.yml + - template: download-artifact-step.yml parameters: downloadFolder: $(binTestsPath)/tmp unpackFolder: $(buildProductRootFolderPath) @@ -83,15 +83,15 @@ jobs: # Build managed test components - ${{ if ne(parameters.osGroup, 'Windows_NT') }}: - - script: $(coreClrRepoRoot)/build-test$(scriptExt) skipnative skipgeneratelayout $(buildConfig) $(archType) $(crossArg) $(priorityArg) ci + - script: $(coreClrRepoRootDir)build-test$(scriptExt) skipnative skipgeneratelayout $(buildConfig) $(archType) $(crossArg) $(priorityArg) ci displayName: Build managed test components - ${{ if eq(parameters.osGroup, 'Windows_NT') }}: - - script: $(coreClrRepoRoot)\build-test$(scriptExt) skipnative skipgeneratelayout $(buildConfig) $(archType) $(priorityArg) ci + - script: $(coreClrRepoRootDir)build-test$(scriptExt) skipnative skipgeneratelayout $(buildConfig) $(archType) $(priorityArg) ci displayName: Build managed test components # Zip and publish managed test components - - template: /eng/upload-artifact-step.yml + - template: upload-artifact-step.yml parameters: rootFolder: $(testRootFolderPath) includeRootFolder: false @@ -104,7 +104,7 @@ jobs: # Publish .packages/microsoft.net.sdk.il needed for traversing # test projects during the copynativeonly command in run test job. - - template: /eng/upload-artifact-step.yml + - template: upload-artifact-step.yml parameters: rootFolder: $(microsoftNetSdkIlFolderPath) includeRootFolder: false @@ -119,7 +119,7 @@ jobs: - task: PublishBuildArtifacts@1 displayName: Publish Logs inputs: - pathtoPublish: $(Build.SourcesDirectory)/bin/Logs - artifactName: ${{ format('TestBuildLogs_{0}{1}_{2}_{3}_{4}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig, parameters.testGroup) }} + pathtoPublish: $(coreClrRepoRoot)/bin/Logs + artifactName: 'TestBuildLogs_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)_${{ parameters.testGroup }}' continueOnError: true condition: always() diff --git a/eng/checkout-job.yml b/eng/checkout-job.yml index f737a44..a9aca6b 100644 --- a/eng/checkout-job.yml +++ b/eng/checkout-job.yml @@ -24,7 +24,7 @@ jobs: fetchDepth: 5 ### Zip up downloaded repo and publish to Helix - - template: /eng/upload-artifact-step.yml + - template: upload-artifact-step.yml parameters: displayName: 'GIT repository (Windows)' rootFolder: $(Build.SourcesDirectory) @@ -53,7 +53,7 @@ jobs: fetchDepth: 5 ### Zip up downloaded repo and publish to Helix - - template: /eng/upload-artifact-step.yml + - template: upload-artifact-step.yml parameters: displayName: 'GIT repository (Unix)' rootFolder: $(Build.SourcesDirectory) diff --git a/eng/crossgen-comparison-job.yml b/eng/crossgen-comparison-job.yml index 2df891a..a7528cf 100644 --- a/eng/crossgen-comparison-job.yml +++ b/eng/crossgen-comparison-job.yml @@ -62,7 +62,7 @@ jobs: steps: # Download product build - - template: /eng/download-artifact-step.yml + - template: download-artifact-step.yml parameters: downloadFolder: $(binTestsPath)/tmp unpackFolder: $(buildProductRootFolderPath) diff --git a/eng/job.yml b/eng/job.yml new file mode 100644 index 0000000..757eb87 --- /dev/null +++ b/eng/job.yml @@ -0,0 +1,217 @@ +# Internal resources (telemetry, microbuild) can only be accessed from non-public projects, +# and some (Microbuild) should only be applied to non-PR cases for internal builds. + +parameters: +# Job schema parameters - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job + cancelTimeoutInMinutes: '' + condition: '' + container: '' + continueOnError: false + dependsOn: '' + displayName: '' + pool: '' + steps: [] + strategy: '' + timeoutInMinutes: '' + variables: [] + workspace: '' + +# Job base template specific parameters + # See schema documentation - https://github.com/dotnet/arcade/blob/master/Documentation/AzureDevOps/TemplateSchema.md + artifacts: '' + enableMicrobuild: false + enablePublishBuildArtifacts: false + enablePublishBuildAssets: false + enablePublishTestResults: false + enablePublishUsingPipelines: false + name: '' + preSteps: [] + runAsPublic: false + +jobs: +- job: ${{ parameters.name }} + + ${{ if ne(parameters.cancelTimeoutInMinutes, '') }}: + cancelTimeoutInMinutes: ${{ parameters.cancelTimeoutInMinutes }} + + ${{ if ne(parameters.condition, '') }}: + condition: ${{ parameters.condition }} + + ${{ if ne(parameters.container, '') }}: + container: ${{ parameters.container }} + + ${{ if ne(parameters.continueOnError, '') }}: + continueOnError: ${{ parameters.continueOnError }} + + ${{ if ne(parameters.dependsOn, '') }}: + dependsOn: ${{ parameters.dependsOn }} + + ${{ if ne(parameters.displayName, '') }}: + displayName: ${{ parameters.displayName }} + + ${{ if ne(parameters.pool, '') }}: + pool: ${{ parameters.pool }} + + ${{ if ne(parameters.strategy, '') }}: + strategy: ${{ parameters.strategy }} + + ${{ if ne(parameters.timeoutInMinutes, '') }}: + timeoutInMinutes: ${{ parameters.timeoutInMinutes }} + + variables: + - ${{ if ne(parameters.enableTelemetry, 'false') }}: + - name: DOTNET_CLI_TELEMETRY_PROFILE + value: '$(Build.Repository.Uri)' + - ${{ each variable in parameters.variables }}: + # handle name-value variable syntax + # example: + # - name: [key] + # value: [value] + - ${{ if ne(variable.name, '') }}: + - name: ${{ variable.name }} + value: ${{ variable.value }} + + # handle variable groups + - ${{ if ne(variable.group, '') }}: + - group: ${{ variable.group }} + + # handle key-value variable syntax. + # example: + # - [key]: [value] + - ${{ if and(eq(variable.name, ''), eq(variable.group, '')) }}: + - ${{ each pair in variable }}: + - name: ${{ pair.key }} + value: ${{ pair.value }} + + # DotNet-HelixApi-Access provides 'HelixApiAccessToken' for internal builds + - ${{ if and(eq(parameters.enableTelemetry, 'true'), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - group: DotNet-HelixApi-Access + + ${{ if ne(parameters.workspace, '') }}: + workspace: ${{ parameters.workspace }} + + steps: + - ${{ if ne(parameters.preSteps, '') }}: + - ${{ each preStep in parameters.preSteps }}: + - ${{ preStep }} + + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - ${{ if eq(parameters.enableMicrobuild, 'true') }}: + - task: MicroBuildSigningPlugin@2 + displayName: Install MicroBuild plugin + inputs: + signType: $(_SignType) + zipSources: false + feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json + env: + TeamName: $(_TeamName) + continueOnError: ${{ parameters.continueOnError }} + condition: and(succeeded(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT')) + + - task: NuGetAuthenticate@0 + + - ${{ if or(eq(parameters.artifacts.download, 'true'), ne(parameters.artifacts.download, '')) }}: + - task: DownloadPipelineArtifact@2 + inputs: + buildType: current + artifactName: ${{ coalesce(parameters.artifacts.download.name, 'Artifacts_$(Agent.OS)_$(_BuildConfig)') }} + targetPath: ${{ coalesce(parameters.artifacts.download.path, 'artifacts') }} + itemPattern: ${{ coalesce(parameters.artifacts.download.pattern, '**') }} + + - ${{ each step in parameters.steps }}: + - ${{ if ne(step, '') }}: + - ${{ step }} + + - ${{ if eq(parameters.enableMicrobuild, 'true') }}: + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - task: MicroBuildCleanup@1 + displayName: Execute Microbuild cleanup tasks + condition: and(always(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT')) + continueOnError: ${{ parameters.continueOnError }} + env: + TeamName: $(_TeamName) + + - ${{ if ne(parameters.artifacts.publish, '') }}: + - ${{ if or(eq(parameters.artifacts.publish.artifacts, 'true'), ne(parameters.artifacts.publish.artifacts, '')) }}: + - task: CopyFiles@2 + displayName: Gather binaries for publish to artifacts + inputs: + SourceFolder: 'artifacts/bin' + Contents: '**' + TargetFolder: '$(Build.ArtifactStagingDirectory)/artifacts/bin' + - task: CopyFiles@2 + displayName: Gather packages for publish to artifacts + inputs: + SourceFolder: 'artifacts/packages' + Contents: '**' + TargetFolder: '$(Build.ArtifactStagingDirectory)/artifacts/packages' + - task: PublishBuildArtifacts@1 + displayName: Publish pipeline artifacts + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)/artifacts' + PublishLocation: Container + ArtifactName: ${{ coalesce(parameters.artifacts.publish.artifacts.name , 'Artifacts_$(Agent.Os)_$(_BuildConfig)') }} + continueOnError: true + condition: always() + - ${{ if or(eq(parameters.artifacts.publish.logs, 'true'), ne(parameters.artifacts.publish.logs, '')) }}: + - publish: artifacts/log + artifact: ${{ coalesce(parameters.artifacts.publish.logs.name, 'Logs_Build_$(Agent.Os)_$(_BuildConfig)') }} + displayName: Publish logs + continueOnError: true + condition: always() + - ${{ if or(eq(parameters.artifacts.publish.manifests, 'true'), ne(parameters.artifacts.publish.manifests, '')) }}: + - ${{ if and(ne(parameters.enablePublishUsingPipelines, 'true'), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - task: CopyFiles@2 + displayName: Gather Asset Manifests + inputs: + SourceFolder: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/AssetManifest' + TargetFolder: '$(Build.ArtifactStagingDirectory)/AssetManifests' + continueOnError: ${{ parameters.continueOnError }} + condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true')) + + - task: PublishBuildArtifacts@1 + displayName: Push Asset Manifests + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)/AssetManifests' + PublishLocation: Container + ArtifactName: AssetManifests + continueOnError: ${{ parameters.continueOnError }} + condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true')) + + - ${{ if ne(parameters.enablePublishBuildArtifacts, 'false') }}: + - task: PublishBuildArtifacts@1 + displayName: Publish Logs + inputs: + PathtoPublish: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)' + PublishLocation: Container + ArtifactName: ${{ coalesce(parameters.enablePublishBuildArtifacts.artifactName, '$(Agent.Os)_$(Agent.JobName)' ) }} + continueOnError: true + condition: always() + + - ${{ if eq(parameters.enablePublishTestResults, 'true') }}: + - task: PublishTestResults@2 + displayName: Publish Test Results + inputs: + testResultsFormat: 'xUnit' + testResultsFiles: '*.xml' + searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' + continueOnError: true + condition: always() + + - ${{ if and(eq(parameters.enablePublishBuildAssets, true), ne(parameters.enablePublishUsingPipelines, 'true'), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - task: CopyFiles@2 + displayName: Gather Asset Manifests + inputs: + SourceFolder: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/AssetManifest' + TargetFolder: '$(Build.StagingDirectory)/AssetManifests' + continueOnError: ${{ parameters.continueOnError }} + condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true')) + + - task: PublishBuildArtifacts@1 + displayName: Push Asset Manifests + inputs: + PathtoPublish: '$(Build.StagingDirectory)/AssetManifests' + PublishLocation: Container + ArtifactName: AssetManifests + continueOnError: ${{ parameters.continueOnError }} + condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true')) diff --git a/eng/perf-job.yml b/eng/perf-job.yml index 90d5d98..27e3e57 100644 --- a/eng/perf-job.yml +++ b/eng/perf-job.yml @@ -12,7 +12,7 @@ parameters: ### buildConfig and archType. jobs: -- template: /eng/common/templates/job/performance.yml +- template: run-performance-job.yml parameters: # Compute job name from template parameters jobName: ${{ format('perfbuild_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} @@ -26,7 +26,10 @@ jobs: ${{ if eq(parameters.osGroup, 'Windows_NT') }}: name: NetCoreInternal-Pool queue: BuildPool.Windows.10.Amd64.VS2019 + buildConfig: ${{ parameters.buildConfig }} + archType: ${{ parameters.archType }} osGroup: ${{ parameters.osGroup }} + osSubgroup: ${{ parameters.osSubgroup }} # Test job depends on the corresponding build job dependsOn: ${{ format('build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} @@ -35,39 +38,6 @@ jobs: ${{ if ne(parameters.osGroup, 'Windows_NT') }}: extraSetupParameters: --corerootdirectory $(Build.SourcesDirectory)/bin/tests/${{ parameters.osGroup }}.${{ parameters.archType }}.Release/Tests/Core_Root --architecture ${{ parameters.archType }} - variables: - - name: binTestsPath - value: $(Build.SourcesDirectory)/bin/tests - - - name: buildConfig - value: ${{ parameters.buildConfig }} - - - ${{ if eq(parameters.buildConfig, 'checked') }}: - - name: buildConfigUpper - value: 'Checked' - - - ${{ if eq(parameters.buildConfig, 'debug') }}: - - name: buildConfigUpper - value: 'Debug' - - - ${{ if eq(parameters.buildConfig, 'release') }}: - - name: buildConfigUpper - value: 'Release' - - - name: buildProductArtifactName - value: 'BinDir_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }}' - - - name: buildProductRootFolderPath - value: $(Build.SourcesDirectory)/bin/Product/${{ parameters.osGroup }}.${{ parameters.archType }}.$(buildConfigUpper) - - - name: archiveExtension - value: '.zip' - - - ${{ if ne(parameters.osGroup, 'Windows_NT') }}: - - name: archiveExtension - value: '.tar.gz' - - frameworks: - ${{ parameters.framework }} steps: @@ -75,7 +45,7 @@ jobs: # Download product binaries directory - - template: /eng/download-artifact-step.yml + - template: download-artifact-step.yml parameters: downloadFolder: $(binTestsPath)/tmp unpackFolder: $(buildProductRootFolderPath) diff --git a/eng/pipelines/coreclr/ci.yml b/eng/pipelines/coreclr/ci.yml index 76d32d0..db2659a 100644 --- a/eng/pipelines/coreclr/ci.yml +++ b/eng/pipelines/coreclr/ci.yml @@ -7,12 +7,12 @@ jobs: # # Checkout repository # -- template: /eng/checkout-job.yml +- template: ../checkout-job.yml # # Debug builds # -- template: /eng/platform-matrix.yml +- template: ../platform-matrix.yml parameters: jobTemplate: build-job.yml buildConfig: debug @@ -32,7 +32,7 @@ jobs: # # Checked builds # -- template: /eng/platform-matrix.yml +- template: ../platform-matrix.yml parameters: jobTemplate: build-job.yml buildConfig: checked @@ -43,7 +43,7 @@ jobs: # # Release builds # -- template: /eng/platform-matrix.yml +- template: ../platform-matrix.yml parameters: jobTemplate: build-job.yml buildConfig: release @@ -59,7 +59,7 @@ jobs: # # Checked test builds # -- template: /eng/platform-matrix.yml +- template: ../platform-matrix.yml parameters: jobTemplate: build-test-job.yml buildConfig: checked @@ -77,7 +77,7 @@ jobs: # # Checked JIT test runs # -- template: /eng/platform-matrix.yml +- template: ../platform-matrix.yml parameters: jobTemplate: run-test-job.yml buildConfig: checked @@ -89,7 +89,7 @@ jobs: # # Checked R2R test runs # -- template: /eng/platform-matrix.yml +- template: ../platform-matrix.yml parameters: jobTemplate: run-test-job.yml buildConfig: checked @@ -113,7 +113,7 @@ jobs: # # Crossgen-comparison jobs # -- template: /eng/platform-matrix.yml +- template: ../platform-matrix.yml parameters: jobTemplate: crossgen-comparison-job.yml buildConfig: release @@ -124,7 +124,7 @@ jobs: # # Formatting # -- template: /eng/platform-matrix.yml +- template: ../platform-matrix.yml parameters: jobTemplate: format-job.yml platforms: diff --git a/eng/pipelines/coreclr/corefx-jitstress.yml b/eng/pipelines/coreclr/corefx-jitstress.yml index 5594b2d..1ddcf42 100644 --- a/eng/pipelines/coreclr/corefx-jitstress.yml +++ b/eng/pipelines/coreclr/corefx-jitstress.yml @@ -14,9 +14,9 @@ jobs: # # Checkout repository # -- template: /eng/checkout-job.yml +- template: ../checkout-job.yml -- template: /eng/platform-matrix.yml +- template: ../platform-matrix.yml parameters: jobTemplate: build-job.yml buildConfig: checked @@ -31,7 +31,7 @@ jobs: jobParameters: testGroup: jitstress -- template: /eng/platform-matrix.yml +- template: ../platform-matrix.yml parameters: jobTemplate: run-test-job.yml buildConfig: checked diff --git a/eng/pipelines/coreclr/corefx-jitstress2-jitstressregs.yml b/eng/pipelines/coreclr/corefx-jitstress2-jitstressregs.yml index 4aa014a..103c233 100644 --- a/eng/pipelines/coreclr/corefx-jitstress2-jitstressregs.yml +++ b/eng/pipelines/coreclr/corefx-jitstress2-jitstressregs.yml @@ -14,9 +14,9 @@ jobs: # # Checkout repository # -- template: /eng/checkout-job.yml +- template: ../checkout-job.yml -- template: /eng/platform-matrix.yml +- template: ../platform-matrix.yml parameters: jobTemplate: build-job.yml buildConfig: checked @@ -31,7 +31,7 @@ jobs: jobParameters: testGroup: jitstress -- template: /eng/platform-matrix.yml +- template: ../platform-matrix.yml parameters: jobTemplate: run-test-job.yml buildConfig: checked diff --git a/eng/pipelines/coreclr/corefx-jitstressregs.yml b/eng/pipelines/coreclr/corefx-jitstressregs.yml index 8674396..900d730 100644 --- a/eng/pipelines/coreclr/corefx-jitstressregs.yml +++ b/eng/pipelines/coreclr/corefx-jitstressregs.yml @@ -14,9 +14,9 @@ jobs: # # Checkout repository # -- template: /eng/checkout-job.yml +- template: ../checkout-job.yml -- template: /eng/platform-matrix.yml +- template: ../platform-matrix.yml parameters: jobTemplate: build-job.yml buildConfig: checked @@ -31,7 +31,7 @@ jobs: jobParameters: testGroup: jitstressregs -- template: /eng/platform-matrix.yml +- template: ../platform-matrix.yml parameters: jobTemplate: run-test-job.yml buildConfig: checked diff --git a/eng/pipelines/coreclr/corefx.yml b/eng/pipelines/coreclr/corefx.yml index 972085b..e49107b 100644 --- a/eng/pipelines/coreclr/corefx.yml +++ b/eng/pipelines/coreclr/corefx.yml @@ -14,9 +14,9 @@ jobs: # # Checkout repository # -- template: /eng/checkout-job.yml +- template: ../checkout-job.yml -- template: /eng/platform-matrix.yml +- template: ../platform-matrix.yml parameters: jobTemplate: build-job.yml buildConfig: checked @@ -31,7 +31,7 @@ jobs: jobParameters: testGroup: outerloop -- template: /eng/platform-matrix.yml +- template: ../platform-matrix.yml parameters: jobTemplate: run-test-job.yml buildConfig: checked diff --git a/eng/pipelines/coreclr/gc-longrunning.yml b/eng/pipelines/coreclr/gc-longrunning.yml index 5dcf733..b3cde07 100644 --- a/eng/pipelines/coreclr/gc-longrunning.yml +++ b/eng/pipelines/coreclr/gc-longrunning.yml @@ -14,9 +14,9 @@ jobs: # # Checkout repository # -- template: /eng/checkout-job.yml +- template: ../checkout-job.yml -- template: /eng/platform-matrix.yml +- template: ../platform-matrix.yml parameters: jobTemplate: build-job.yml buildConfig: release @@ -28,7 +28,7 @@ jobs: jobParameters: testGroup: gc-longrunning -- template: /eng/platform-matrix.yml +- template: ../platform-matrix.yml parameters: jobTemplate: build-test-job.yml buildConfig: release @@ -41,7 +41,7 @@ jobs: jobParameters: testGroup: gc-longrunning -- template: /eng/platform-matrix.yml +- template: ../platform-matrix.yml parameters: jobTemplate: run-test-job.yml buildConfig: release diff --git a/eng/pipelines/coreclr/gc-simulator.yml b/eng/pipelines/coreclr/gc-simulator.yml index 23fa3ee..cfcdfc5 100644 --- a/eng/pipelines/coreclr/gc-simulator.yml +++ b/eng/pipelines/coreclr/gc-simulator.yml @@ -14,9 +14,9 @@ jobs: # # Checkout repository # -- template: /eng/checkout-job.yml +- template: ../checkout-job.yml -- template: /eng/platform-matrix.yml +- template: ../platform-matrix.yml parameters: jobTemplate: build-job.yml buildConfig: release @@ -29,7 +29,7 @@ jobs: jobParameters: testGroup: gc-simulator -- template: /eng/platform-matrix.yml +- template: ../platform-matrix.yml parameters: jobTemplate: test-job.yml buildConfig: release diff --git a/eng/pipelines/coreclr/gcstress-extra.yml b/eng/pipelines/coreclr/gcstress-extra.yml index 6cd09d3..4c86cdf 100644 --- a/eng/pipelines/coreclr/gcstress-extra.yml +++ b/eng/pipelines/coreclr/gcstress-extra.yml @@ -14,9 +14,9 @@ jobs: # # Checkout repository # -- template: /eng/checkout-job.yml +- template: ../checkout-job.yml -- template: /eng/platform-matrix.yml +- template: ../platform-matrix.yml parameters: jobTemplate: build-job.yml buildConfig: checked @@ -24,7 +24,7 @@ jobs: jobParameters: testGroup: gcstress-extra -- template: /eng/platform-matrix.yml +- template: ../platform-matrix.yml parameters: jobTemplate: test-job.yml buildConfig: checked diff --git a/eng/pipelines/coreclr/gcstress0x3-gcstress0xc.yml b/eng/pipelines/coreclr/gcstress0x3-gcstress0xc.yml index 24c8411..82f5c9e 100644 --- a/eng/pipelines/coreclr/gcstress0x3-gcstress0xc.yml +++ b/eng/pipelines/coreclr/gcstress0x3-gcstress0xc.yml @@ -14,9 +14,9 @@ jobs: # # Checkout repository # -- template: /eng/checkout-job.yml +- template: ../checkout-job.yml -- template: /eng/platform-matrix.yml +- template: ../platform-matrix.yml parameters: jobTemplate: build-job.yml buildConfig: checked @@ -24,7 +24,7 @@ jobs: jobParameters: testGroup: gcstress0x3-gcstress0xc -- template: /eng/platform-matrix.yml +- template: ../platform-matrix.yml parameters: jobTemplate: test-job.yml buildConfig: checked diff --git a/eng/pipelines/coreclr/internal.yml b/eng/pipelines/coreclr/internal.yml index fd11a77..a847b8b 100644 --- a/eng/pipelines/coreclr/internal.yml +++ b/eng/pipelines/coreclr/internal.yml @@ -33,7 +33,7 @@ stages: # # Checkout repository # - - template: /eng/checkout-job.yml + - template: ../checkout-job.yml - stage: build displayName: Build @@ -41,7 +41,7 @@ stages: # # Release builds # - - template: /eng/platform-matrix.yml + - template: ../platform-matrix.yml parameters: jobTemplate: build-job.yml buildConfig: release @@ -63,7 +63,7 @@ stages: # product build job), and publishes them to the build assets # registry. Its dependencies should be updated to include all of the # official builds if we add more platform/arch combinations. - - template: /eng/finalize-publish.yml + - template: ../finalize-publish.yml parameters: dependsOn: - build_Linux_arm_release @@ -91,7 +91,7 @@ stages: # # Release test builds # - - template: /eng/platform-matrix.yml + - template: ../platform-matrix.yml parameters: jobTemplate: build-test-job.yml buildConfig: release @@ -112,7 +112,7 @@ stages: # # Release JIT test executions # - - template: /eng/platform-matrix.yml + - template: ../platform-matrix.yml parameters: jobTemplate: run-test-job.yml buildConfig: release @@ -126,7 +126,7 @@ stages: # # Release R2R test executions # - - template: /eng/platform-matrix.yml + - template: ../platform-matrix.yml parameters: jobTemplate: run-test-job.yml buildConfig: release diff --git a/eng/pipelines/coreclr/jitstress-isas-arm.yml b/eng/pipelines/coreclr/jitstress-isas-arm.yml index a839c2e..b3140ce 100644 --- a/eng/pipelines/coreclr/jitstress-isas-arm.yml +++ b/eng/pipelines/coreclr/jitstress-isas-arm.yml @@ -14,9 +14,9 @@ jobs: # # Checkout repository # -- template: /eng/checkout-job.yml +- template: ../checkout-job.yml -- template: /eng/platform-matrix.yml +- template: ../platform-matrix.yml parameters: jobTemplate: build-job.yml buildConfig: checked @@ -26,7 +26,7 @@ jobs: jobParameters: testGroup: jitstress-isas-arm -- template: /eng/platform-matrix.yml +- template: ../platform-matrix.yml parameters: jobTemplate: test-job.yml buildConfig: checked diff --git a/eng/pipelines/coreclr/jitstress-isas-x86.yml b/eng/pipelines/coreclr/jitstress-isas-x86.yml index c5ced01..e2765a1 100644 --- a/eng/pipelines/coreclr/jitstress-isas-x86.yml +++ b/eng/pipelines/coreclr/jitstress-isas-x86.yml @@ -14,9 +14,9 @@ jobs: # # Checkout repository # -- template: /eng/checkout-job.yml +- template: ../checkout-job.yml -- template: /eng/platform-matrix.yml +- template: ../platform-matrix.yml parameters: jobTemplate: build-job.yml buildConfig: checked @@ -28,7 +28,7 @@ jobs: jobParameters: testGroup: jitstress-isas-x86 -- template: /eng/platform-matrix.yml +- template: ../platform-matrix.yml parameters: jobTemplate: build-test-job.yml buildConfig: checked @@ -41,7 +41,7 @@ jobs: jobParameters: testGroup: jitstress-isas-x86 -- template: /eng/platform-matrix.yml +- template: ../platform-matrix.yml parameters: jobTemplate: run-test-job.yml buildConfig: checked diff --git a/eng/pipelines/coreclr/jitstress.yml b/eng/pipelines/coreclr/jitstress.yml index 666790d..b7848d7 100644 --- a/eng/pipelines/coreclr/jitstress.yml +++ b/eng/pipelines/coreclr/jitstress.yml @@ -14,9 +14,9 @@ jobs: # # Checkout repository # -- template: /eng/checkout-job.yml +- template: ../checkout-job.yml -- template: /eng/platform-matrix.yml +- template: ../platform-matrix.yml parameters: jobTemplate: build-job.yml buildConfig: checked @@ -24,7 +24,7 @@ jobs: jobParameters: testGroup: jitstress -- template: /eng/platform-matrix.yml +- template: ../platform-matrix.yml parameters: jobTemplate: test-job.yml buildConfig: checked diff --git a/eng/pipelines/coreclr/jitstress2-jitstressregs.yml b/eng/pipelines/coreclr/jitstress2-jitstressregs.yml index 9dd5d4e..da15ecd 100644 --- a/eng/pipelines/coreclr/jitstress2-jitstressregs.yml +++ b/eng/pipelines/coreclr/jitstress2-jitstressregs.yml @@ -14,9 +14,9 @@ jobs: # # Checkout repository # -- template: /eng/checkout-job.yml +- template: ../checkout-job.yml -- template: /eng/platform-matrix.yml +- template: ../platform-matrix.yml parameters: jobTemplate: build-job.yml buildConfig: checked @@ -24,7 +24,7 @@ jobs: jobParameters: testGroup: jitstress2-jitstressregs -- template: /eng/platform-matrix.yml +- template: ../platform-matrix.yml parameters: jobTemplate: test-job.yml buildConfig: checked diff --git a/eng/pipelines/coreclr/jitstressregs-x86.yml b/eng/pipelines/coreclr/jitstressregs-x86.yml index c54d243..5836820 100644 --- a/eng/pipelines/coreclr/jitstressregs-x86.yml +++ b/eng/pipelines/coreclr/jitstressregs-x86.yml @@ -14,9 +14,9 @@ jobs: # # Checkout repository # -- template: /eng/checkout-job.yml +- template: ../checkout-job.yml -- template: /eng/platform-matrix.yml +- template: ../platform-matrix.yml parameters: jobTemplate: build-job.yml buildConfig: checked @@ -27,7 +27,7 @@ jobs: jobParameters: testGroup: jitstressregs-x86 -- template: /eng/platform-matrix.yml +- template: ../platform-matrix.yml parameters: jobTemplate: test-job.yml buildConfig: checked diff --git a/eng/pipelines/coreclr/jitstressregs.yml b/eng/pipelines/coreclr/jitstressregs.yml index 7dc84cd..d90a669 100644 --- a/eng/pipelines/coreclr/jitstressregs.yml +++ b/eng/pipelines/coreclr/jitstressregs.yml @@ -14,9 +14,9 @@ jobs: # # Checkout repository # -- template: /eng/checkout-job.yml +- template: ../checkout-job.yml -- template: /eng/platform-matrix.yml +- template: ../platform-matrix.yml parameters: jobTemplate: build-job.yml buildConfig: checked @@ -24,7 +24,7 @@ jobs: jobParameters: testGroup: jitstressregs -- template: /eng/platform-matrix.yml +- template: ../platform-matrix.yml parameters: jobTemplate: build-test-job.yml buildConfig: checked @@ -33,7 +33,7 @@ jobs: jobParameters: testGroup: jitstressregs -- template: /eng/platform-matrix.yml +- template: ../platform-matrix.yml parameters: jobTemplate: run-test-job.yml buildConfig: checked diff --git a/eng/pipelines/coreclr/perf.yml b/eng/pipelines/coreclr/perf.yml index 7e12ca0..4d7401b 100644 --- a/eng/pipelines/coreclr/perf.yml +++ b/eng/pipelines/coreclr/perf.yml @@ -10,9 +10,9 @@ jobs: # # Checkout repository # -- template: /eng/checkout-job.yml +- template: ../checkout-job.yml -- template: /eng/platform-matrix.yml +- template: ../platform-matrix.yml parameters: jobTemplate: build-job.yml buildConfig: release @@ -23,7 +23,7 @@ jobs: jobParameters: testGroup: perf -- template: /eng/platform-matrix.yml +- template: ../platform-matrix.yml parameters: jobTemplate: perf-job.yml buildConfig: release diff --git a/eng/pipelines/coreclr/pr.yml b/eng/pipelines/coreclr/pr.yml index dbb2e8d..1b2775b 100644 --- a/eng/pipelines/coreclr/pr.yml +++ b/eng/pipelines/coreclr/pr.yml @@ -11,12 +11,12 @@ jobs: # # Checkout repository # -- template: /eng/checkout-job.yml +- template: ../checkout-job.yml # # Debug builds # -- template: /eng/platform-matrix.yml +- template: ../platform-matrix.yml parameters: jobTemplate: build-job.yml buildConfig: debug @@ -29,7 +29,7 @@ jobs: # # Checked builds # -- template: /eng/platform-matrix.yml +- template: ../platform-matrix.yml parameters: jobTemplate: build-job.yml buildConfig: checked @@ -49,7 +49,7 @@ jobs: # # Release builds # -- template: /eng/platform-matrix.yml +- template: ../platform-matrix.yml parameters: jobTemplate: build-job.yml buildConfig: release @@ -67,7 +67,7 @@ jobs: # # Checked test builds # -- template: /eng/platform-matrix.yml +- template: ../platform-matrix.yml parameters: jobTemplate: build-test-job.yml buildConfig: checked @@ -86,7 +86,7 @@ jobs: # # Checked JIT test executions # -- template: /eng/platform-matrix.yml +- template: ../platform-matrix.yml parameters: jobTemplate: run-test-job.yml buildConfig: checked @@ -107,7 +107,7 @@ jobs: # # Checked R2R test executions # -- template: /eng/platform-matrix.yml +- template: ../platform-matrix.yml parameters: jobTemplate: run-test-job.yml buildConfig: checked @@ -125,7 +125,7 @@ jobs: # # CoreFX test runs against CoreCLR # -- template: /eng/platform-matrix.yml +- template: ../platform-matrix.yml parameters: jobTemplate: run-test-job.yml buildConfig: checked @@ -142,7 +142,7 @@ jobs: # # Crossgen-comparison jobs # -- template: /eng/platform-matrix.yml +- template: ../platform-matrix.yml parameters: jobTemplate: crossgen-comparison-job.yml buildConfig: checked @@ -153,7 +153,7 @@ jobs: # # Release test builds # -- template: /eng/platform-matrix.yml +- template: ../platform-matrix.yml parameters: jobTemplate: build-test-job.yml buildConfig: release @@ -166,7 +166,7 @@ jobs: # # Release test builds # -- template: /eng/platform-matrix.yml +- template: ../platform-matrix.yml parameters: jobTemplate: run-test-job.yml buildConfig: release @@ -179,7 +179,7 @@ jobs: # # Formatting # -- template: /eng/platform-matrix.yml +- template: ../platform-matrix.yml parameters: jobTemplate: format-job.yml platforms: diff --git a/eng/pipelines/coreclr/r2r-extra.yml b/eng/pipelines/coreclr/r2r-extra.yml index bdb8c05..ed87c8f 100644 --- a/eng/pipelines/coreclr/r2r-extra.yml +++ b/eng/pipelines/coreclr/r2r-extra.yml @@ -14,9 +14,9 @@ jobs: # # Checkout repository # -- template: /eng/checkout-job.yml +- template: ../checkout-job.yml -- template: /eng/platform-matrix.yml +- template: ../platform-matrix.yml parameters: jobTemplate: build-job.yml buildConfig: checked @@ -24,7 +24,7 @@ jobs: jobParameters: testGroup: r2r-extra -- template: /eng/platform-matrix.yml +- template: ../platform-matrix.yml parameters: jobTemplate: test-job.yml buildConfig: checked diff --git a/eng/pipelines/coreclr/r2r.yml b/eng/pipelines/coreclr/r2r.yml index 198bc34..5920180 100644 --- a/eng/pipelines/coreclr/r2r.yml +++ b/eng/pipelines/coreclr/r2r.yml @@ -6,9 +6,9 @@ jobs: # # Checkout repository # -- template: /eng/checkout-job.yml +- template: ../checkout-job.yml -- template: /eng/platform-matrix.yml +- template: ../platform-matrix.yml parameters: jobTemplate: build-job.yml buildConfig: checked @@ -21,7 +21,7 @@ jobs: jobParameters: testGroup: outerloop -- template: /eng/platform-matrix.yml +- template: ../platform-matrix.yml parameters: jobTemplate: test-job.yml buildConfig: checked diff --git a/eng/pipelines/coreclr/runincontext.yml b/eng/pipelines/coreclr/runincontext.yml index 066fa63..888d07e 100644 --- a/eng/pipelines/coreclr/runincontext.yml +++ b/eng/pipelines/coreclr/runincontext.yml @@ -14,9 +14,9 @@ jobs: # # Checkout repository # -- template: /eng/checkout-job.yml +- template: ../checkout-job.yml -- template: /eng/platform-matrix.yml +- template: ../platform-matrix.yml parameters: jobTemplate: build-job.yml buildConfig: checked @@ -27,7 +27,7 @@ jobs: jobParameters: testGroup: outerloop -- template: /eng/platform-matrix.yml +- template: ../platform-matrix.yml parameters: jobTemplate: test-job.yml buildConfig: checked diff --git a/eng/run-performance-job.yml b/eng/run-performance-job.yml new file mode 100644 index 0000000..2ce63f9 --- /dev/null +++ b/eng/run-performance-job.yml @@ -0,0 +1,102 @@ +parameters: + steps: [] # optional -- any additional steps that need to happen before pulling down the performance repo and sending the performance benchmarks to helix (ie building your repo) + variables: [] # optional -- list of additional variables to send to the template + jobName: '' # required -- job name + displayName: '' # optional -- display name for the job. Will use jobName if not passed + pool: '' # required -- name of the Build pool + container: '' # required -- name of the container + buildConfig: '' # required -- build configuration + archType: '' # required -- targeting CPU architecture + osGroup: '' # required -- operating system for the job + osSubgroup: '' # optional -- operating system subgroup + extraSetupParameters: '' # optional -- extra arguments to pass to the setup script + frameworks: ['netcoreapp3.0'] # optional -- list of frameworks to run against + continueOnError: 'false' # optional -- determines whether to continue the build if the step errors + dependsOn: '' # optional -- dependencies of the job + timeoutInMinutes: 320 # optional -- timeout for the job + enableTelemetry: false # optional -- enable for telemetry + +jobs: +- template: xplat-pipeline-job.yml + parameters: + dependsOn: ${{ parameters.dependsOn }} + buildConfig: ${{ parameters.buildConfig }} + archType: ${{ parameters.archType }} + osGroup: ${{ parameters.osGroup }} + osSubgroup: ${{ parameters.osSubgroup }} + enableTelemetry: ${{ parameters.enableTelemetry }} + enablePublishBuildArtifacts: true + continueOnError: ${{ parameters.continueOnError }} + + jobs: + - job: '${{ parameters.jobName }}' + + ${{ if ne(parameters.displayName, '') }}: + displayName: '${{ parameters.displayName }}' + ${{ if eq(parameters.displayName, '') }}: + displayName: '${{ parameters.jobName }}' + + timeoutInMinutes: ${{ parameters.timeoutInMinutes }} + + variables: + + - ${{ each variable in parameters.variables }}: + - ${{ if ne(variable.name, '') }}: + - name: ${{ variable.name }} + value: ${{ variable.value }} + - ${{ if ne(variable.group, '') }}: + - group: ${{ variable.group }} + + - IsInternal: '' + - HelixApiAccessToken: '' + - HelixPreCommand: '' + + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - ${{ if eq( parameters.osGroup, 'Windows_NT') }}: + - HelixPreCommand: 'set "PERFLAB_UPLOAD_TOKEN=$(PerfCommandUploadToken)"' + - IsInternal: -Internal + - ${{ if ne(parameters.osGroup, 'Windows_NT') }}: + - HelixPreCommand: 'export PERFLAB_UPLOAD_TOKEN="$(PerfCommandUploadTokenLinux)"' + - IsInternal: --internal + + - group: DotNet-HelixApi-Access + - group: dotnet-benchview + + workspace: + clean: all + pool: + ${{ parameters.pool }} + container: ${{ parameters.container }} + strategy: + matrix: + ${{ each framework in parameters.frameworks }}: + ${{ framework }}: + _Framework: ${{ framework }} + steps: + - checkout: self + clean: true + # Run all of the steps to setup repo + - ${{ each step in parameters.steps }}: + - ${{ step }} + - powershell: $(Build.SourcesDirectory)\eng\common\performance\performance-setup.ps1 $(IsInternal) -Framework $(_Framework) ${{ parameters.extraSetupParameters }} + displayName: Performance Setup (Windows) + condition: and(succeeded(), eq(variables['Agent.Os'], 'Windows_NT')) + continueOnError: ${{ parameters.continueOnError }} + - script: $(coreClrRepoRoot)/eng/common/performance/performance-setup.sh $(IsInternal) --framework $(_Framework) ${{ parameters.extraSetupParameters }} + displayName: Performance Setup (Unix) + condition: and(succeeded(), ne(variables['Agent.Os'], 'Windows_NT')) + continueOnError: ${{ parameters.continueOnError }} + - script: $(Python) $(PerformanceDirectory)/scripts/ci_setup.py $(SetupArguments) + displayName: Run ci setup script + # Run perf testing in helix + - template: /eng/common/templates/steps/perf-send-to-helix.yml + parameters: + HelixSource: '$(HelixSourcePrefix)/$(Build.Repository.Name)/$(Build.SourceBranch)' # sources must start with pr/, official/, prodcon/, or agent/ + HelixType: 'test/performance/$(Kind)/$(_Framework)/$(Architecture)' + HelixAccessToken: $(HelixApiAccessToken) + HelixTargetQueues: $(Queue) + HelixPreCommands: $(HelixPreCommand) + Creator: $(Creator) + WorkItemTimeout: 4:00 # 4 hours + WorkItemDirectory: '$(WorkItemDirectory)' # WorkItemDirectory can not be empty, so we send it some docs to keep it happy + CorrelationPayloadDirectory: '$(PayloadDirectory)' # it gets checked out to a folder with shorter path than WorkItemDirectory so we can avoid file name too long exceptions \ No newline at end of file diff --git a/eng/run-test-job.yml b/eng/run-test-job.yml index 6c20b7a..964baf9 100644 --- a/eng/run-test-job.yml +++ b/eng/run-test-job.yml @@ -68,9 +68,14 @@ jobs: - name: crossgenArg value: '' + - name: LogNamePrefix + value: TestRunLogs + - ${{ if eq(parameters.readyToRun, true) }}: - name: crossgenArg value: 'crossgen' + - name: LogNamePrefix + value: TestRunLogs_R2R - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - group: DotNet-HelixApi-Access @@ -91,7 +96,7 @@ jobs: # Download and unzip managed test artifacts - ${{ if ne(parameters.corefxTests, true) }}: - - template: /eng/download-artifact-step.yml + - template: download-artifact-step.yml parameters: downloadFolder: '$(binTestsPath)/tmp' unpackFolder: '$(testRootFolderPath)' @@ -102,7 +107,7 @@ jobs: # Download and unzip test build tree - ${{ if ne(parameters.corefxTests, true) }}: - - template: /eng/download-artifact-step.yml + - template: download-artifact-step.yml parameters: downloadFolder: '$(binTestsPath)/tmp' unpackFolder: '$(testBuildRootFolderPath)' @@ -113,7 +118,7 @@ jobs: # Download product binaries directory - - template: /eng/download-artifact-step.yml + - template: download-artifact-step.yml parameters: downloadFolder: $(binTestsPath)/tmp unpackFolder: $(buildProductRootFolderPath) @@ -125,7 +130,7 @@ jobs: # Download and unzip the Microsoft.NET.Sdk.IL package needed for traversing # ilproj test projects during copynativeonly. - ${{ if ne(parameters.corefxTests, true) }}: - - template: /eng/download-artifact-step.yml + - template: download-artifact-step.yml parameters: downloadFolder: '$(binTestsPath)/tmp' unpackFolder: '$(microsoftNetSdkIlFolderPath)' @@ -136,7 +141,7 @@ jobs: # Download and unzip native test artifacts - ${{ if ne(parameters.corefxTests, true) }}: - - template: /eng/download-artifact-step.yml + - template: download-artifact-step.yml parameters: downloadFolder: '$(binTestsPath)/tmp' unpackFolder: '$(testNativeRootFolderPath)' @@ -151,41 +156,38 @@ jobs: # the native artifacts to the final test folders is dependent on availability of the # managed test artifacts. - ${{ if ne(parameters.corefxTests, true) }}: - - ${{ if ne(parameters.osGroup, 'Windows_NT') }}: - - script: $(coreClrRepoRoot)/build-test$(scriptExt) copynativeonly $(buildConfig) $(archType) $(priorityArg) - displayName: Copy native test components to test output folder - - ${{ if eq(parameters.osGroup, 'Windows_NT') }}: - - script: $(coreClrRepoRoot)\build-test$(scriptExt) copynativeonly $(buildConfig) $(archType) $(priorityArg) - displayName: Copy native test components to test output folder + - script: $(coreClrRepoRootDir)build-test$(scriptExt) copynativeonly $(buildConfig) $(archType) $(priorityArg) + displayName: Copy native test components to test output folder # Generate test host - ${{ if eq(parameters.corefxTests, true) }}: - ${{ if ne(parameters.osGroup, 'Windows_NT') }}: - - script: ./build-test.sh skipmanaged skipnative $(testhostArg) $(buildConfig) $(archType) $(crossArg) $(priorityArg) + - script: $(coreClrRepoRootDir)build-test$(scriptExt) skipmanaged skipnative $(testhostArg) $(buildConfig) $(archType) $(crossArg) $(priorityArg) displayName: Generate test host - ${{ if eq(parameters.osGroup, 'Windows_NT') }}: - - script: build-test.cmd skipmanaged skipnative $(testhostArg) $(buildConfig) $(archType) $(priorityArg) + - script: $(coreClrRepoRootDir)build-test$(scriptExt) skipmanaged skipnative $(testhostArg) $(buildConfig) $(archType) $(priorityArg) displayName: Generate test host # Crossgen framework assemblies prior to triggering readyToRun execution runs. - ${{ if eq(parameters.readyToRun, true) }}: - ${{ if ne(parameters.osGroup, 'Windows_NT') }}: - - script: ./build-test.sh skipmanaged skipnative $(crossgenArg) $(buildConfig) $(archType) $(crossArg) $(priorityArg) + - script: $(coreClrRepoRootDir)build-test$(scriptExt) skipmanaged skipnative $(crossgenArg) $(buildConfig) $(archType) $(crossArg) $(priorityArg) displayName: Crossgen framework assemblies - ${{ if eq(parameters.osGroup, 'Windows_NT') }}: - - script: build-test.cmd skipmanaged skipnative $(crossgenArg) $(buildConfig) $(archType) $(priorityArg) + - script: $(coreClrRepoRootDir)build-test$(scriptExt) skipmanaged skipnative $(crossgenArg) $(buildConfig) $(archType) $(priorityArg) displayName: Crossgen framework assemblies # Send tests to Helix - - template: /eng/send-to-helix-step.yml + - template: send-to-helix-step.yml parameters: displayName: Send tests to Helix buildConfig: $(buildConfigUpper) archType: ${{ parameters.archType }} osGroup: ${{ parameters.osGroup }} + coreClrRepoRoot: $(coreClrRepoRoot) ${{ if eq(variables['System.TeamProject'], 'public') }}: creator: $(Build.DefinitionName) @@ -253,9 +255,9 @@ jobs: # Choose which tests to send to Helix: CoreFX or CoreCLR. ${{ if eq(parameters.corefxTests, true) }}: - helixProjectArguments: 'eng/helixcorefxtests.proj' + helixProjectArguments: '$(coreClrRepoRoot)/eng/helixcorefxtests.proj' ${{ if ne(parameters.corefxTests, true) }}: - helixProjectArguments: 'tests/src/helixpublishwitharcade.proj' + helixProjectArguments: '$(coreClrRepoRoot)/tests/src/helixpublishwitharcade.proj' ${{ if in(parameters.testGroup, 'innerloop', 'outerloop') }}: scenarios: @@ -376,7 +378,7 @@ jobs: - task: PublishBuildArtifacts@1 displayName: Publish Logs inputs: - pathtoPublish: $(Build.SourcesDirectory)/bin/Logs - artifactName: ${{ format('TestRunLogs_{0}{1}_{2}_{3}_{4}_{5}_{6}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig, parameters.managedTestBuildOsGroup, parameters.managedTestBuildOsSubgroup, parameters.testGroup) }} + pathtoPublish: $(coreClrRepoRoot)/bin/Logs + artifactName: '$(LogNamePrefix)_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)_${{ parameters.testGroup }}' continueOnError: true condition: always() \ No newline at end of file diff --git a/eng/send-to-helix-step.yml b/eng/send-to-helix-step.yml index 04058bc..46cd65c 100644 --- a/eng/send-to-helix-step.yml +++ b/eng/send-to-helix-step.yml @@ -19,16 +19,17 @@ parameters: runInUnloadableContext: '' longRunningGcTests: '' gcSimulatorTests: '' + coreClrRepoRoot: '' steps: - ${{ if eq(parameters.osGroup, 'Windows_NT') }}: # TODO: Remove and consolidate this when we move to arcade via init-tools.cmd. - - powershell: eng\common\build.ps1 /p:DotNetPublishToBlobFeed=true -ci -restore -projects $(Build.SourcesDirectory)\eng\empty.csproj + - powershell: $(Build.SourcesDirectory)\eng\common\build.ps1 /p:DotNetPublishToBlobFeed=true -ci -restore -projects ${{ parameters.coreClrRepoRoot }}\eng\empty.csproj displayName: Restore blob feed tasks ${{ if ne(parameters.condition, '') }}: condition: ${{ parameters.condition }} - - powershell: eng\common\msbuild.ps1 -ci ${{ parameters.helixProjectArguments }} /maxcpucount /bl:$(Build.SourcesDirectory)/bin/Logs/SendToHelix.binlog + - powershell: eng\common\msbuild.ps1 -ci ${{ parameters.helixProjectArguments }} /maxcpucount /bl:${{ parameters.coreClrRepoRoot }}/bin/Logs/SendToHelix.binlog displayName: ${{ parameters.displayName }} ${{ if ne(parameters.condition, '') }}: condition: ${{ parameters.condition }} @@ -57,7 +58,7 @@ steps: - ${{ if ne(parameters.osGroup, 'Windows_NT') }}: # TODO: Remove and consolidate this when we move to arcade via init-tools.sh. - - script: ./eng/common/build.sh /p:DotNetPublishToBlobFeed=true --ci --restore --projects $(Build.SourcesDirectory)/eng/empty.csproj + - script: $(Build.SourcesDirectory)/eng/common/build.sh /p:DotNetPublishToBlobFeed=true --ci --restore --projects ${{ parameters.coreClrRepoRoot }}/eng/empty.csproj displayName: Restore blob feed tasks ${{ if ne(parameters.condition, '') }}: condition: ${{ parameters.condition }} @@ -66,7 +67,7 @@ steps: # Arcade uses this SDK instead of trying to restore one. DotNetCoreSdkDir: /usr/local/dotnet - - script: ./eng/common/msbuild.sh --ci ${{ parameters.helixProjectArguments }} /maxcpucount /bl:$(Build.SourcesDirectory)/bin/Logs/SendToHelix.binlog + - script: $(Build.SourcesDirectory)/eng/common/msbuild.sh --ci ${{ parameters.helixProjectArguments }} /maxcpucount /bl:${{ parameters.coreClrRepoRoot }}/bin/Logs/SendToHelix.binlog displayName: ${{ parameters.displayName }} ${{ if ne(parameters.condition, '') }}: condition: ${{ parameters.condition }} diff --git a/eng/xplat-job.yml b/eng/xplat-job.yml index 6370aee..a4a30c8 100644 --- a/eng/xplat-job.yml +++ b/eng/xplat-job.yml @@ -21,7 +21,7 @@ parameters: variables: {} ## any extra variables to add to the defaults defined below jobs: -- template: /eng/common/templates/job/job.yml +- template: job.yml parameters: name: ${{ parameters.name }} @@ -165,7 +165,7 @@ jobs: clean: true # Download the GIT repository - - template: /eng/download-artifact-step.yml + - template: download-artifact-step.yml parameters: displayName: 'GIT repository' downloadFolder: $(Build.SourcesDirectory)/download/ diff --git a/eng/xplat-pipeline-job.yml b/eng/xplat-pipeline-job.yml index b213b0b..b8d3c70 100644 --- a/eng/xplat-pipeline-job.yml +++ b/eng/xplat-pipeline-job.yml @@ -25,7 +25,7 @@ parameters: variables: {} ## any extra variables to add to the defaults defined below jobs: -- template: /eng/xplat-job.yml +- template: xplat-job.yml parameters: buildConfig: ${{ parameters.buildConfig }} archType: ${{ parameters.archType }} @@ -55,23 +55,50 @@ jobs: - name: testArtifactRootName value: ${{ parameters.managedTestBuildOsGroup }}${{ parameters.managedTestBuildOsSubgroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }} + - ${{ if eq(parameters.osGroup, 'Windows_NT') }}: + - name: archiveExtension + value: '.zip' + - name: archiveType + value: zip + - name: tarCompression + value: '' + - name: scriptExt + value: '.cmd' + - name: dir + value: '\' + + - ${{ if ne(parameters.osGroup, 'Windows_NT') }}: + - name: archiveExtension + value: '.tar.gz' + - name: archiveType + value: tar + - name: tarCompression + value: gz + - name: scriptExt + value: '.sh' + - name: dir + value: '/' + - name: coreClrRepoRoot - value: $(Build.SourcesDirectory) + value: '$(Build.SourcesDirectory)' + + - name: coreClrRepoRootDir + value: '$(coreClrRepoRoot)$(dir)' - name: binTestsPath - value: $(coreClrRepoRoot)/bin/tests + value: '$(coreClrRepoRoot)/bin/tests' - name: testRootFolderPath - value: $(binTestsPath)/$(osGroup).$(archType).$(buildConfigUpper) + value: '$(binTestsPath)/$(osGroup).$(archType).$(buildConfigUpper)' - name: nativeRootFolderPath - value: $(binTestsPath)/$(osGroup).$(archType).$(buildConfigUpper) + value: '$(binTestsPath)/$(osGroup).$(archType).$(buildConfigUpper)' - name: buildProductArtifactName value: 'BinDir_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' - name: buildProductRootFolderPath - value: $(Build.SourcesDirectory)/bin/Product/$(osGroup).$(archType).$(buildConfigUpper) + value: '$(coreClrRepoRoot)/bin/Product/$(osGroup).$(archType).$(buildConfigUpper)' - name: testArtifactName value: Tests_${{ parameters.managedTestBuildOsGroup }}${{ parameters.managedTestBuildOsSubgroup }}_$(archType)_$(buildConfig) @@ -80,40 +107,20 @@ jobs: value: 'NativeTestComponents_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' - name: testNativeRootFolderPath - value: $(binTestsPath)/obj/$(osGroup).$(archType).$(buildConfigUpper) + value: '$(binTestsPath)/obj/$(osGroup).$(archType).$(buildConfigUpper)' - name: testBuildRootFolderPath - value: $(binTestsPath)/$(osGroup).$(archType).$(buildConfigUpper) + value: '$(binTestsPath)/$(osGroup).$(archType).$(buildConfigUpper)' - name: testBuildArtifactName value: 'TestBuild_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' - name: microsoftNetSdkIlFolderPath - value: $(Build.SourcesDirectory)/.packages/microsoft.net.sdk.il + value: '$(Build.SourcesDirectory)/.packages/microsoft.net.sdk.il' - name: microsoftNetSdkIlArtifactName value: 'MicrosoftNetSdkIlPackage_${{ parameters.managedTestBuildOsGroup }}${{ parameters.managedTestBuildOsSubgroup }}_$(archType)_$(buildConfig)' - - ${{ if eq(parameters.osGroup, 'Windows_NT') }}: - - name: archiveExtension - value: '.zip' - - name: archiveType - value: zip - - name: tarCompression - value: '' - - name: scriptExt - value: '.cmd' - - - ${{ if ne(parameters.osGroup, 'Windows_NT') }}: - - name: archiveExtension - value: '.tar.gz' - - name: archiveType - value: tar - - name: tarCompression - value: gz - - name: scriptExt - value: '.sh' - - name: priorityArg value: '' -- 2.7.4