From 746fbba9711ae05e39d515fa60ca4ac0ec72ba80 Mon Sep 17 00:00:00 2001 From: Santiago Fernandez Madero Date: Thu, 16 Jul 2020 15:02:32 -0700 Subject: [PATCH] Move libraries test build to product build (#39402) * Move libraries test build to product build * Publish artifacts needed by coreclr tests * Don't build corelib anymore since it is a P2P from libraries now --- .../common/build-coreclr-and-libraries-job.yml | 6 +- eng/pipelines/coreclr/libraries-gcstress-extra.yml | 17 +---- .../coreclr/libraries-gcstress0x3-gcstress0xc.yml | 17 +---- eng/pipelines/coreclr/libraries-jitstress.yml | 17 +---- .../coreclr/libraries-jitstress2-jitstressregs.yml | 17 +---- eng/pipelines/coreclr/libraries-jitstressregs.yml | 17 +---- eng/pipelines/libraries/base-job.yml | 13 ---- eng/pipelines/libraries/build-job.yml | 37 +++++----- eng/pipelines/libraries/build-test-job.yml | 86 ---------------------- eng/pipelines/libraries/run-test-job.yml | 20 ++++- eng/pipelines/runtime.yml | 29 ++------ 11 files changed, 64 insertions(+), 212 deletions(-) delete mode 100644 eng/pipelines/libraries/build-test-job.yml diff --git a/eng/pipelines/common/build-coreclr-and-libraries-job.yml b/eng/pipelines/common/build-coreclr-and-libraries-job.yml index ddf5d81..439d970 100644 --- a/eng/pipelines/common/build-coreclr-and-libraries-job.yml +++ b/eng/pipelines/common/build-coreclr-and-libraries-job.yml @@ -11,6 +11,8 @@ parameters: stagedBuild: false variables: {} pool: '' + platform: '' + testBuildPlatforms: [] jobs: - template: /eng/pipelines/coreclr/templates/build-job.yml @@ -41,8 +43,8 @@ jobs: testGroup: ${{ parameters.testGroup }} crossrootfsDir: ${{ parameters.crossrootfsDir }} timeoutInminutes: ${{ parameters.timeoutInMinutes }} - signBinaries: ${{ parameters.signBinaries }} - stagedBuild: ${{ parameters.stagedBuild }} variables: ${{ parameters.variables }} pool: ${{ parameters.pool }} liveRuntimeBuildConfig: ${{ parameters.buildConfig }} + platform: ${{ parameters.platform }} + testBuildPlatforms: ${{ parameters.testBuildPlatforms }} diff --git a/eng/pipelines/coreclr/libraries-gcstress-extra.yml b/eng/pipelines/coreclr/libraries-gcstress-extra.yml index 58b4370..4063b0a 100644 --- a/eng/pipelines/coreclr/libraries-gcstress-extra.yml +++ b/eng/pipelines/coreclr/libraries-gcstress-extra.yml @@ -25,20 +25,11 @@ jobs: jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml buildConfig: checked platformGroup: gcstress - -# -# Libraries Test Build - Release innerloop. All libraries are built on x64 and reused on all platforms. -# -- template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/libraries/build-test-job.yml - buildConfig: Release - platforms: - - Linux_x64 - - Windows_NT_x64 jobParameters: - liveRuntimeBuildConfig: checked - testScope: innerloop + # libraries test build platforms + testBuildPlatforms: + - Linux_x64 + - Windows_NT_x64 # # Libraries Test Run using Release libraries, Checked CoreCLR, and stress modes diff --git a/eng/pipelines/coreclr/libraries-gcstress0x3-gcstress0xc.yml b/eng/pipelines/coreclr/libraries-gcstress0x3-gcstress0xc.yml index b11e314..d5df152 100644 --- a/eng/pipelines/coreclr/libraries-gcstress0x3-gcstress0xc.yml +++ b/eng/pipelines/coreclr/libraries-gcstress0x3-gcstress0xc.yml @@ -25,20 +25,11 @@ jobs: jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml buildConfig: checked platformGroup: gcstress - -# -# Libraries Test Build - Release innerloop. All libraries are built on x64 and reused on all platforms. -# -- template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/libraries/build-test-job.yml - buildConfig: Release - platforms: - - Linux_x64 - - Windows_NT_x64 jobParameters: - liveRuntimeBuildConfig: checked - testScope: innerloop + # libraries test build platforms + testBuildPlatforms: + - Linux_x64 + - Windows_NT_x64 # # Libraries Test Run using Release libraries, Checked CoreCLR, and stress modes diff --git a/eng/pipelines/coreclr/libraries-jitstress.yml b/eng/pipelines/coreclr/libraries-jitstress.yml index ff7f25b..7128ab1 100644 --- a/eng/pipelines/coreclr/libraries-jitstress.yml +++ b/eng/pipelines/coreclr/libraries-jitstress.yml @@ -30,20 +30,11 @@ jobs: - Windows_NT_x86 - Windows_NT_x64 - Windows_NT_arm64 - -# -# Libraries Test Build - Release innerloop. All libraries are built on x64 and reused on all platforms. -# -- template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/libraries/build-test-job.yml - buildConfig: Release - platforms: - - Linux_x64 - - Windows_NT_x64 jobParameters: - liveRuntimeBuildConfig: checked - testScope: innerloop + # libraries test build platforms + testBuildPlatforms: + - Linux_x64 + - Windows_NT_x64 # # Libraries Test Run using Release libraries, Checked CoreCLR, and stress modes diff --git a/eng/pipelines/coreclr/libraries-jitstress2-jitstressregs.yml b/eng/pipelines/coreclr/libraries-jitstress2-jitstressregs.yml index 656d139..d1a2a0a 100644 --- a/eng/pipelines/coreclr/libraries-jitstress2-jitstressregs.yml +++ b/eng/pipelines/coreclr/libraries-jitstress2-jitstressregs.yml @@ -30,20 +30,11 @@ jobs: - Windows_NT_x86 - Windows_NT_x64 - Windows_NT_arm64 - -# -# Libraries Test Build - Release innerloop. All libraries are built on x64 and reused on all platforms. -# -- template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/libraries/build-test-job.yml - buildConfig: Release - platforms: - - Linux_x64 - - Windows_NT_x64 jobParameters: - liveRuntimeBuildConfig: checked - testScope: innerloop + # libraries test build platforms + testBuildPlatforms: + - Linux_x64 + - Windows_NT_x64 # # Libraries Test Run using Release libraries, Checked CoreCLR, and stress modes diff --git a/eng/pipelines/coreclr/libraries-jitstressregs.yml b/eng/pipelines/coreclr/libraries-jitstressregs.yml index 2e62420..691cb29 100644 --- a/eng/pipelines/coreclr/libraries-jitstressregs.yml +++ b/eng/pipelines/coreclr/libraries-jitstressregs.yml @@ -30,20 +30,11 @@ jobs: - Windows_NT_x86 - Windows_NT_x64 - Windows_NT_arm64 - -# -# Libraries Test Build - Release innerloop. All libraries are built on x64 and reused on all platforms. -# -- template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/libraries/build-test-job.yml - buildConfig: Release - platforms: - - Linux_x64 - - Windows_NT_x64 jobParameters: - liveRuntimeBuildConfig: checked - testScope: innerloop + # libraries test build platforms + testBuildPlatforms: + - Linux_x64 + - Windows_NT_x64 # # Libraries Test Run using Release libraries, Checked CoreCLR, and stress modes diff --git a/eng/pipelines/libraries/base-job.yml b/eng/pipelines/libraries/base-job.yml index b39f1e3..99e6271 100644 --- a/eng/pipelines/libraries/base-job.yml +++ b/eng/pipelines/libraries/base-job.yml @@ -91,10 +91,6 @@ jobs: - ${{ if ne(parameters.liveRuntimeBuildConfig, '') }}: - _runtimeDownloadPath: '$(Build.SourcesDirectory)/artifacts/transport/${{ parameters.runtimeFlavor }}' - _runtimeConfigurationArg: -rc ${{ parameters.liveRuntimeBuildConfig }} - # Download full product dependencies for mono or test - - ${{ if or(ne(parameters.runtimeFlavor, 'coreclr'), ne(parameters.testScope, '')) }}: - - _runtimeArtifactName: '$(runtimeFlavorName)Product_${{ parameters.runtimeVariant}}_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.liveRuntimeBuildConfig }}' - - _runtimeArtifactsPathArg: ' /p:RuntimeArtifactsPath=$(_runtimeDownloadPath)' - ${{ if eq(parameters.testDisplayName, '') }}: - _testRunNamePrefixSuffix: $(runtimeFlavorName)_${{ parameters.liveRuntimeBuildConfig }} - ${{ if ne(parameters.testDisplayName, '') }}: @@ -125,13 +121,4 @@ jobs: steps: - template: /eng/pipelines/common/clone-checkout-bundle-step.yml - - - ${{ if and(ne(parameters.liveRuntimeBuildConfig, ''), or(ne(parameters.runtimeFlavor, 'coreclr'), ne(parameters.testScope, ''))) }}: - - template: /eng/pipelines/common/download-artifact-step.yml - parameters: - unpackFolder: $(_runtimeDownloadPath) - artifactFileName: '$(_runtimeArtifactName)$(archiveExtension)' - artifactName: '$(_runtimeArtifactName)' - displayName: '$(runtimeFlavorName) build drop' - - ${{ parameters.steps }} diff --git a/eng/pipelines/libraries/build-job.yml b/eng/pipelines/libraries/build-job.yml index e3200bd..f2e8ff2 100644 --- a/eng/pipelines/libraries/build-job.yml +++ b/eng/pipelines/libraries/build-job.yml @@ -8,6 +8,7 @@ parameters: isOfficialBuild: false isOfficialAllConfigurations: false runtimeVariant: '' + platform: '' # When set to a non-empty value (Debug / Release), it determines the runtime's # build configuration to use for building libraries and tests. Setting this @@ -25,6 +26,7 @@ parameters: pool: '' runTests: false testScope: '' + testBuildPlatforms: [] jobs: - template: /eng/pipelines/libraries/base-job.yml @@ -49,24 +51,18 @@ jobs: name: build displayName: 'Build' - ${{ if ne(parameters.liveRuntimeBuildConfig, '') }}: - dependsOn: - # Use full product dependency for non-coreclr and test builds - - ${{ if or(ne(parameters.runtimeFlavor, 'coreclr'), ne(parameters.testScope, '')) }}: - - ${{ format('{0}_{1}_product_build_{2}{3}_{4}_{5}', parameters.runtimeFlavor, parameters.runtimeVariant, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveRuntimeBuildConfig) }} - variables: + - librariesTestsArtifactName: ${{ format('libraries_test_assets_{0}_{1}_{2}', parameters.osGroup, parameters.archType, parameters.buildConfig) }} - _subset: libs - _additionalBuildArguments: '' - ${{ parameters.variables }} - ${{ if eq(parameters.osGroup, 'Browser') }}: - EMSDK_PATH: /usr/local/emscripten - # for coreclr library builds (when not testing) build corelib as well. - - ${{ if and(eq(parameters.runtimeFlavor, 'coreclr'), eq(parameters.testScope, '')) }}: - - _subset: clr.corelib+libs + # Tests only run for 'allConfiguration' and 'net472' build-jobs - - ${{ if eq(parameters.runTests, true) }}: - - _subset: clr.corelib+libs+libs.tests + # If platform is in testBuildPlatforms we build tests as well. + - ${{ if or(eq(parameters.runTests, true), containsValue(parameters.testBuildPlatforms, parameters.platform)) }}: + - _subset: libs+libs.tests - _additionalBuildArguments: /p:ArchiveTests=true - ${{ parameters.variables }} @@ -97,7 +93,7 @@ jobs: displayName: Disk Usage after Build - ${{ if eq(parameters.runTests, false) }}: - - ${{ if ne(parameters.isOfficialBuild, true) }}: + - ${{ if ne(parameters.isOfficialBuild, true) }}: - task: CopyFiles@2 displayName: Prepare testhost folder to publish inputs: @@ -105,12 +101,6 @@ jobs: targetFolder: $(Build.ArtifactStagingDirectory)/artifacts/bin/testhost - task: CopyFiles@2 - displayName: Prepare artifacts toolset folder to publish - inputs: - sourceFolder: $(Build.SourcesDirectory)/artifacts/toolset - targetFolder: $(Build.ArtifactStagingDirectory)/artifacts/toolset - - - task: CopyFiles@2 displayName: Prepare runtime folder to publish inputs: sourceFolder: $(Build.SourcesDirectory)/artifacts/bin/runtime @@ -162,6 +152,17 @@ jobs: tarCompression: $(tarCompression) artifactName: $(librariesBuildArtifactName) displayName: Build Assets + + - ${{ if containsValue(parameters.testBuildPlatforms, parameters.platform) }}: + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/helix + includeRootFolder: true + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: $(librariesTestsArtifactName) + displayName: Test Assets # Save AllConfigurations artifacts using the prepare-signed-artifacts format. The # platform-specific jobs' nupkgs automatically flow through the matching platform-specific diff --git a/eng/pipelines/libraries/build-test-job.yml b/eng/pipelines/libraries/build-test-job.yml deleted file mode 100644 index 0ce715a..0000000 --- a/eng/pipelines/libraries/build-test-job.yml +++ /dev/null @@ -1,86 +0,0 @@ -parameters: - buildConfig: '' - osGroup: '' - osSubgroup: '' - archType: '' - framework: '' - isOfficialBuild: false - liveRuntimeBuildConfig: '' - runtimeFlavor: 'coreclr' - runtimeVariant: '' - timeoutInMinutes: 150 - container: '' - publishTestArtifacs: true - pool: '' - testScope: '' - variables: {} - condition: true - runTests: false - -jobs: - - template: /eng/pipelines/libraries/base-job.yml - parameters: - buildConfig: ${{ parameters.buildConfig }} - osGroup: ${{ parameters.osGroup }} - osSubgroup: ${{ parameters.osSubgroup }} - archType: ${{ parameters.archType }} - framework: ${{ parameters.framework }} - isOfficialBuild: ${{ parameters.isOfficialBuild }} - condition: ${{ parameters.condition }} - liveRuntimeBuildConfig: ${{ parameters.liveRuntimeBuildConfig }} - timeoutInMinutes: ${{ parameters.timeoutInMinutes }} - container: ${{ parameters.container }} - runtimeVariant: ${{ parameters.runtimeVariant }} - pool: ${{ parameters.pool }} - testScope: ${{ parameters.testScope }} - name: test_build - displayName: 'Test Build' - - dependsOn: - - ${{ format('libraries_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - # Libraries Test also depends on Product, now that the libraries build only depends on corelib - - ${{ if ne(parameters.liveRuntimeBuildConfig, '') }}: - - ${{ format('{0}_{1}_product_build_{2}{3}_{4}_{5}', parameters.runtimeFlavor, parameters.runtimeVariant, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveRuntimeBuildConfig) }} - - variables: - - librariesTestsArtifactName: ${{ format('libraries_test_assets_{0}_{1}_{2}', parameters.osGroup, parameters.archType, parameters.buildConfig) }} - - _archiveTestsParameter: /p:ArchiveTests=true - - - ${{ parameters.variables }} - - steps: - - template: /eng/pipelines/common/download-artifact-step.yml - parameters: - displayName: Build Assets - artifactName: $(librariesBuildArtifactName) - artifactFileName: $(librariesBuildArtifactName)$(archiveExtension) - unpackFolder: $(Build.SourcesDirectory)/artifacts - cleanUnpackFolder: false - - - ${{ if in(parameters.osGroup, 'OSX', 'iOS','tvOS') }}: - - script: | - du -sh $(Build.SourcesDirectory)/* - df -h - displayName: Disk Usage before Build - - - script: $(_buildScript) - -subset libs.pretest+libs.tests - $(_buildArguments) - $(_archiveTestsParameter) - displayName: Restore and Build - - - ${{ if in(parameters.osGroup, 'OSX', 'iOS','tvOS') }}: - - script: | - du -sh $(Build.SourcesDirectory)/* - df -h - displayName: Disk Usage after Build - - - template: /eng/pipelines/common/upload-artifact-step.yml - parameters: - rootFolder: $(Build.SourcesDirectory)/artifacts/helix - includeRootFolder: true - archiveType: $(archiveType) - archiveExtension: $(archiveExtension) - tarCompression: $(tarCompression) - artifactName: $(librariesTestsArtifactName) - displayName: Test Assets diff --git a/eng/pipelines/libraries/run-test-job.yml b/eng/pipelines/libraries/run-test-job.yml index 58a3427..3b1d1df 100644 --- a/eng/pipelines/libraries/run-test-job.yml +++ b/eng/pipelines/libraries/run-test-job.yml @@ -52,16 +52,20 @@ jobs: dependsOn: - ${{ if notIn(parameters.framework, 'allConfigurations', 'net472') }}: - ${{ format('libraries_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - - ${{ format('libraries_test_build_{0}_{1}_{2}', parameters.osGroup, parameters.dependsOnTestArchitecture, parameters.dependsOnTestBuildConfiguration) }} - - ${{ if in(parameters.framework, 'allConfigurations', 'net472') }}: - - ${{ format('libraries_build_{0}_{1}{2}_{3}_{4}', parameters.framework, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - - ${{ format('libraries_test_build_{0}_{1}_{2}_{3}', parameters.framework, parameters.osGroup, parameters.dependsOnTestArchitecture, parameters.dependsOnTestBuildConfiguration) }} + # tests are built as part of product build + - ${{ if and(ne(parameters.archType, parameters.dependsOnTestArchitecture), ne(parameters.buildConfig, parameters.dependsOnTestBuildConfiguration)) }}: + - ${{ format('libraries_build_{0}_{1}_{2}', parameters.osGroup, parameters.dependsOnTestArchitecture, parameters.dependsOnTestBuildConfiguration) }} - ${{ if ne(parameters.liveRuntimeBuildConfig, '') }}: - ${{ format('{0}_{1}_product_build_{2}{3}_{4}_{5}', parameters.runtimeFlavor, parameters.runtimeVariant, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveRuntimeBuildConfig) }} variables: - librariesTestsArtifactName: ${{ format('libraries_test_assets_{0}_{1}_{2}', parameters.osGroup, parameters.dependsOnTestArchitecture, parameters.dependsOnTestBuildConfiguration) }} - _archiveTestsParameter: /p:ArchiveTests=true + + - ${{ if ne(parameters.liveRuntimeBuildConfig, '') }}: + - _runtimeArtifactName: '$(runtimeFlavorName)Product_${{ parameters.runtimeVariant}}_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.liveRuntimeBuildConfig }}' + - _runtimeArtifactsPathArg: ' /p:RuntimeArtifactsPath=$(_runtimeDownloadPath)' + - ${{ parameters.variables }} steps: @@ -80,6 +84,14 @@ jobs: artifactName: $(librariesTestsArtifactName) artifactFileName: $(librariesTestsArtifactName)$(archiveExtension) unpackFolder: $(Build.SourcesDirectory)/artifacts + + - ${{ if ne(parameters.liveRuntimeBuildConfig, '') }}: + - template: /eng/pipelines/common/download-artifact-step.yml + parameters: + unpackFolder: $(_runtimeDownloadPath) + artifactFileName: '$(_runtimeArtifactName)$(archiveExtension)' + artifactName: '$(_runtimeArtifactName)' + displayName: '$(runtimeFlavorName) build drop' - ${{ if in(parameters.coreclrTestGroup, 'gcstress0x3-gcstress0xc', 'gcstress-extra') }}: # We need to find and download the GC stress dependencies (namely, coredistools). Put them diff --git a/eng/pipelines/runtime.yml b/eng/pipelines/runtime.yml index 7dae35b..6d70e6c 100644 --- a/eng/pipelines/runtime.yml +++ b/eng/pipelines/runtime.yml @@ -541,6 +541,11 @@ jobs: - Windows_NT_x64 - FreeBSD_x64 jobParameters: + testScope: innerloop + testBuildPlatforms: + - Linux_x64 + - Windows_NT_x64 + - OSX_x64 liveRuntimeBuildConfig: release # @@ -629,30 +634,6 @@ jobs: liveLibrariesBuildConfig: ${{ variables.debugOnPrReleaseOnRolling }} # -# Libraries Test Build -# Only when CoreCLR, Mono or Libraries is changed -# -- template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/libraries/build-test-job.yml - buildConfig: ${{ variables.debugOnPrReleaseOnRolling }} - platforms: - - OSX_x64 - - Linux_x64 - - Windows_NT_x64 - jobParameters: - isOfficialBuild: false - liveRuntimeBuildConfig: release - testScope: innerloop - condition: >- - or( - eq(dependencies.checkout.outputs['SetPathVars_libraries.containsChange'], true), - eq(dependencies.checkout.outputs['SetPathVars_coreclr.containsChange'], true), - eq(dependencies.checkout.outputs['SetPathVars_mono.containsChange'], true), - eq(dependencies.checkout.outputs['SetPathVars_runtimetests.containsChange'], true), - eq(variables['isFullMatrix'], true)) - -# # Crossgen-comparison jobs # Only when CoreCLR is changed # -- 2.7.4