Move libraries test build to product build (#39402)

* Move libraries test build to product build

* Publish artifacts needed by coreclr tests

* Don't build corelib anymore since it is a P2P from libraries now

diff --git a/eng/pipelines/common/build-coreclr-and-libraries-job.yml b/eng/pipelines/common/build-coreclr-and-libraries-job.yml
index ddf5d81..439d970 100644 (file)
--- a/eng/pipelines/common/build-coreclr-and-libraries-job.yml
+++ b/eng/pipelines/common/build-coreclr-and-libraries-job.yml
@@ -11,6 +11,8 @@ parameters:
   stagedBuild: false
   variables: {}
   pool: ''
+  platform: ''
+  testBuildPlatforms: []
 
 jobs:
 - template: /eng/pipelines/coreclr/templates/build-job.yml
@@ -41,8 +43,8 @@ jobs:
     testGroup: ${{ parameters.testGroup }}
     crossrootfsDir: ${{ parameters.crossrootfsDir }}
     timeoutInminutes: ${{ parameters.timeoutInMinutes }}
-    signBinaries: ${{ parameters.signBinaries }}
-    stagedBuild: ${{ parameters.stagedBuild }}
     variables: ${{ parameters.variables }}
     pool: ${{ parameters.pool }}
     liveRuntimeBuildConfig: ${{ parameters.buildConfig }}
+    platform: ${{ parameters.platform }}
+    testBuildPlatforms: ${{ parameters.testBuildPlatforms }}

diff --git a/eng/pipelines/coreclr/libraries-gcstress-extra.yml b/eng/pipelines/coreclr/libraries-gcstress-extra.yml
index 58b4370..4063b0a 100644 (file)
--- a/eng/pipelines/coreclr/libraries-gcstress-extra.yml
+++ b/eng/pipelines/coreclr/libraries-gcstress-extra.yml
@@ -25,20 +25,11 @@ jobs:
     jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml
     buildConfig: checked
     platformGroup: gcstress
-
-#
-# Libraries Test Build - Release innerloop. All libraries are built on x64 and reused on all platforms.
-#
-- template: /eng/pipelines/common/platform-matrix.yml
-  parameters:
-    jobTemplate: /eng/pipelines/libraries/build-test-job.yml
-    buildConfig: Release
-    platforms:
-    - Linux_x64
-    - Windows_NT_x64
     jobParameters:
-      liveRuntimeBuildConfig: checked
-      testScope: innerloop
+      # libraries test build platforms
+      testBuildPlatforms:
+      - Linux_x64
+      - Windows_NT_x64
 
 #
 # Libraries Test Run using Release libraries, Checked CoreCLR, and stress modes

diff --git a/eng/pipelines/coreclr/libraries-gcstress0x3-gcstress0xc.yml b/eng/pipelines/coreclr/libraries-gcstress0x3-gcstress0xc.yml
index b11e314..d5df152 100644 (file)
--- a/eng/pipelines/coreclr/libraries-gcstress0x3-gcstress0xc.yml
+++ b/eng/pipelines/coreclr/libraries-gcstress0x3-gcstress0xc.yml
@@ -25,20 +25,11 @@ jobs:
     jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml
     buildConfig: checked
     platformGroup: gcstress
-
-#
-# Libraries Test Build - Release innerloop. All libraries are built on x64 and reused on all platforms.
-#
-- template: /eng/pipelines/common/platform-matrix.yml
-  parameters:
-    jobTemplate: /eng/pipelines/libraries/build-test-job.yml
-    buildConfig: Release
-    platforms:
-    - Linux_x64
-    - Windows_NT_x64
     jobParameters:
-      liveRuntimeBuildConfig: checked
-      testScope: innerloop
+      # libraries test build platforms
+      testBuildPlatforms:
+      - Linux_x64
+      - Windows_NT_x64
 
 #
 # Libraries Test Run using Release libraries, Checked CoreCLR, and stress modes

diff --git a/eng/pipelines/coreclr/libraries-jitstress.yml b/eng/pipelines/coreclr/libraries-jitstress.yml
index ff7f25b..7128ab1 100644 (file)
--- a/eng/pipelines/coreclr/libraries-jitstress.yml
+++ b/eng/pipelines/coreclr/libraries-jitstress.yml
@@ -30,20 +30,11 @@ jobs:
     - Windows_NT_x86
     - Windows_NT_x64
     - Windows_NT_arm64
-
-#
-# Libraries Test Build - Release innerloop. All libraries are built on x64 and reused on all platforms.
-#
-- template: /eng/pipelines/common/platform-matrix.yml
-  parameters:
-    jobTemplate: /eng/pipelines/libraries/build-test-job.yml
-    buildConfig: Release
-    platforms:
-    - Linux_x64
-    - Windows_NT_x64
     jobParameters:
-      liveRuntimeBuildConfig: checked
-      testScope: innerloop
+      # libraries test build platforms
+      testBuildPlatforms:
+      - Linux_x64
+      - Windows_NT_x64
 
 #
 # Libraries Test Run using Release libraries, Checked CoreCLR, and stress modes

diff --git a/eng/pipelines/coreclr/libraries-jitstress2-jitstressregs.yml b/eng/pipelines/coreclr/libraries-jitstress2-jitstressregs.yml
index 656d139..d1a2a0a 100644 (file)
--- a/eng/pipelines/coreclr/libraries-jitstress2-jitstressregs.yml
+++ b/eng/pipelines/coreclr/libraries-jitstress2-jitstressregs.yml
@@ -30,20 +30,11 @@ jobs:
     - Windows_NT_x86
     - Windows_NT_x64
     - Windows_NT_arm64
-
-#
-# Libraries Test Build - Release innerloop. All libraries are built on x64 and reused on all platforms.
-#
-- template: /eng/pipelines/common/platform-matrix.yml
-  parameters:
-    jobTemplate: /eng/pipelines/libraries/build-test-job.yml
-    buildConfig: Release
-    platforms:
-    - Linux_x64
-    - Windows_NT_x64
     jobParameters:
-      liveRuntimeBuildConfig: checked
-      testScope: innerloop
+      # libraries test build platforms
+      testBuildPlatforms:
+      - Linux_x64
+      - Windows_NT_x64
 
 #
 # Libraries Test Run using Release libraries, Checked CoreCLR, and stress modes

diff --git a/eng/pipelines/coreclr/libraries-jitstressregs.yml b/eng/pipelines/coreclr/libraries-jitstressregs.yml
index 2e62420..691cb29 100644 (file)
--- a/eng/pipelines/coreclr/libraries-jitstressregs.yml
+++ b/eng/pipelines/coreclr/libraries-jitstressregs.yml
@@ -30,20 +30,11 @@ jobs:
     - Windows_NT_x86
     - Windows_NT_x64
     - Windows_NT_arm64
-
-#
-# Libraries Test Build - Release innerloop. All libraries are built on x64 and reused on all platforms.
-#
-- template: /eng/pipelines/common/platform-matrix.yml
-  parameters:
-    jobTemplate: /eng/pipelines/libraries/build-test-job.yml
-    buildConfig: Release
-    platforms:
-    - Linux_x64
-    - Windows_NT_x64
     jobParameters:
-      liveRuntimeBuildConfig: checked
-      testScope: innerloop
+      # libraries test build platforms
+      testBuildPlatforms:
+      - Linux_x64
+      - Windows_NT_x64
 
 #
 # Libraries Test Run using Release libraries, Checked CoreCLR, and stress modes

diff --git a/eng/pipelines/libraries/base-job.yml b/eng/pipelines/libraries/base-job.yml
index b39f1e3..99e6271 100644 (file)
--- a/eng/pipelines/libraries/base-job.yml
+++ b/eng/pipelines/libraries/base-job.yml
@@ -91,10 +91,6 @@ jobs:
         - ${{ if ne(parameters.liveRuntimeBuildConfig, '') }}:
           - _runtimeDownloadPath: '$(Build.SourcesDirectory)/artifacts/transport/${{ parameters.runtimeFlavor }}'
           - _runtimeConfigurationArg: -rc ${{ parameters.liveRuntimeBuildConfig }}
-          # Download full product dependencies for mono or test
-          - ${{ if or(ne(parameters.runtimeFlavor, 'coreclr'), ne(parameters.testScope, '')) }}:
-            - _runtimeArtifactName: '$(runtimeFlavorName)Product_${{ parameters.runtimeVariant}}_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.liveRuntimeBuildConfig }}'
-            - _runtimeArtifactsPathArg: ' /p:RuntimeArtifactsPath=$(_runtimeDownloadPath)'
           - ${{ if eq(parameters.testDisplayName, '') }}:
             - _testRunNamePrefixSuffix: $(runtimeFlavorName)_${{ parameters.liveRuntimeBuildConfig }}
           - ${{ if ne(parameters.testDisplayName, '') }}:
@@ -125,13 +121,4 @@ jobs:
 
       steps:
       - template: /eng/pipelines/common/clone-checkout-bundle-step.yml
-
-      - ${{ if and(ne(parameters.liveRuntimeBuildConfig, ''), or(ne(parameters.runtimeFlavor, 'coreclr'), ne(parameters.testScope, ''))) }}:
-        - template: /eng/pipelines/common/download-artifact-step.yml
-          parameters:
-            unpackFolder: $(_runtimeDownloadPath)
-            artifactFileName: '$(_runtimeArtifactName)$(archiveExtension)'
-            artifactName: '$(_runtimeArtifactName)'
-            displayName: '$(runtimeFlavorName) build drop'
-
       - ${{ parameters.steps }}

diff --git a/eng/pipelines/libraries/build-job.yml b/eng/pipelines/libraries/build-job.yml
index e3200bd..f2e8ff2 100644 (file)
--- a/eng/pipelines/libraries/build-job.yml
+++ b/eng/pipelines/libraries/build-job.yml
@@ -8,6 +8,7 @@ parameters:
   isOfficialBuild: false
   isOfficialAllConfigurations: false
   runtimeVariant: ''
+  platform: ''
 
   # When set to a non-empty value (Debug / Release), it determines the runtime's
   # build configuration to use for building libraries and tests. Setting this
@@ -25,6 +26,7 @@ parameters:
   pool: ''
   runTests: false
   testScope: ''
+  testBuildPlatforms: []
 
 jobs:
   - template: /eng/pipelines/libraries/base-job.yml
@@ -49,24 +51,18 @@ jobs:
       name: build
       displayName: 'Build'
 
-      ${{ if ne(parameters.liveRuntimeBuildConfig, '') }}:
-        dependsOn:
-        # Use full product dependency for non-coreclr and test builds
-        - ${{ if or(ne(parameters.runtimeFlavor, 'coreclr'), ne(parameters.testScope, '')) }}:
-          - ${{ format('{0}_{1}_product_build_{2}{3}_{4}_{5}', parameters.runtimeFlavor, parameters.runtimeVariant, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveRuntimeBuildConfig) }}
-
       variables:
+        - librariesTestsArtifactName: ${{ format('libraries_test_assets_{0}_{1}_{2}', parameters.osGroup, parameters.archType, parameters.buildConfig) }}
         - _subset: libs
         - _additionalBuildArguments: ''
         - ${{ parameters.variables }}
         - ${{ if eq(parameters.osGroup, 'Browser') }}:
           - EMSDK_PATH: /usr/local/emscripten
-        # for coreclr library builds (when not testing) build corelib as well.   
-        - ${{ if and(eq(parameters.runtimeFlavor, 'coreclr'), eq(parameters.testScope, '')) }}:
-          - _subset: clr.corelib+libs
+
         # Tests only run for 'allConfiguration' and 'net472' build-jobs
-        - ${{ if eq(parameters.runTests, true) }}:
-          - _subset: clr.corelib+libs+libs.tests
+        # If platform is in testBuildPlatforms we build tests as well.
+        - ${{ if or(eq(parameters.runTests, true), containsValue(parameters.testBuildPlatforms, parameters.platform)) }}:
+          - _subset: libs+libs.tests
           - _additionalBuildArguments: /p:ArchiveTests=true
 
         - ${{ parameters.variables }}
@@ -97,7 +93,7 @@ jobs:
             displayName: Disk Usage after Build
 
         - ${{ if eq(parameters.runTests, false) }}:
-          - ${{ if ne(parameters.isOfficialBuild, true) }}:            
+          - ${{ if ne(parameters.isOfficialBuild, true) }}:          
             - task: CopyFiles@2
               displayName: Prepare testhost folder to publish
               inputs:
@@ -105,12 +101,6 @@ jobs:
                 targetFolder: $(Build.ArtifactStagingDirectory)/artifacts/bin/testhost
 
             - task: CopyFiles@2
-              displayName: Prepare artifacts toolset folder to publish
-              inputs:
-                sourceFolder: $(Build.SourcesDirectory)/artifacts/toolset
-                targetFolder: $(Build.ArtifactStagingDirectory)/artifacts/toolset
-
-            - task: CopyFiles@2
               displayName: Prepare runtime folder to publish
               inputs:
                 sourceFolder: $(Build.SourcesDirectory)/artifacts/bin/runtime
@@ -162,6 +152,17 @@ jobs:
               tarCompression: $(tarCompression)
               artifactName: $(librariesBuildArtifactName)
               displayName: Build Assets
+          
+          - ${{ if containsValue(parameters.testBuildPlatforms, parameters.platform) }}:
+            - template: /eng/pipelines/common/upload-artifact-step.yml
+              parameters:
+                rootFolder: $(Build.SourcesDirectory)/artifacts/helix
+                includeRootFolder: true
+                archiveType: $(archiveType)
+                archiveExtension: $(archiveExtension)
+                tarCompression: $(tarCompression)
+                artifactName: $(librariesTestsArtifactName)
+                displayName: Test Assets
 
           # Save AllConfigurations artifacts using the prepare-signed-artifacts format. The
           # platform-specific jobs' nupkgs automatically flow through the matching platform-specific

diff --git a/eng/pipelines/libraries/build-test-job.yml b/eng/pipelines/libraries/build-test-job.yml
deleted file mode 100644 (file)
index 0ce715a..0000000
--- a/eng/pipelines/libraries/build-test-job.yml
+++ /dev/null
@@ -1,86 +0,0 @@
-parameters:
-  buildConfig: ''
-  osGroup: ''
-  osSubgroup: ''
-  archType: ''
-  framework: ''
-  isOfficialBuild: false
-  liveRuntimeBuildConfig: ''
-  runtimeFlavor: 'coreclr'
-  runtimeVariant: ''
-  timeoutInMinutes: 150
-  container: ''
-  publishTestArtifacs: true
-  pool: ''
-  testScope: ''
-  variables: {}
-  condition: true
-  runTests: false
-
-jobs:
-  - template: /eng/pipelines/libraries/base-job.yml
-    parameters:
-      buildConfig: ${{ parameters.buildConfig }}
-      osGroup:  ${{ parameters.osGroup }}
-      osSubgroup:  ${{ parameters.osSubgroup }}
-      archType:  ${{ parameters.archType }}
-      framework:  ${{ parameters.framework }}
-      isOfficialBuild: ${{ parameters.isOfficialBuild }}
-      condition: ${{ parameters.condition }}
-      liveRuntimeBuildConfig: ${{ parameters.liveRuntimeBuildConfig }}
-      timeoutInMinutes: ${{ parameters.timeoutInMinutes }}
-      container: ${{ parameters.container }}
-      runtimeVariant: ${{ parameters.runtimeVariant }}
-      pool: ${{ parameters.pool }}
-      testScope: ${{ parameters.testScope }}
-      name: test_build
-      displayName: 'Test Build'
-
-      dependsOn:
-      - ${{ format('libraries_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }}
-      # Libraries Test also depends on Product, now that the libraries build only depends on corelib
-      - ${{ if ne(parameters.liveRuntimeBuildConfig, '') }}:
-        - ${{ format('{0}_{1}_product_build_{2}{3}_{4}_{5}', parameters.runtimeFlavor, parameters.runtimeVariant, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveRuntimeBuildConfig) }}
-
-      variables:
-        - librariesTestsArtifactName: ${{ format('libraries_test_assets_{0}_{1}_{2}', parameters.osGroup, parameters.archType, parameters.buildConfig) }}
-        - _archiveTestsParameter: /p:ArchiveTests=true
-        
-        - ${{ parameters.variables }}
-
-      steps:
-        - template: /eng/pipelines/common/download-artifact-step.yml
-          parameters:
-            displayName: Build Assets
-            artifactName: $(librariesBuildArtifactName)
-            artifactFileName: $(librariesBuildArtifactName)$(archiveExtension)
-            unpackFolder: $(Build.SourcesDirectory)/artifacts
-            cleanUnpackFolder: false
-
-        - ${{ if in(parameters.osGroup, 'OSX', 'iOS','tvOS') }}: 
-          - script: |
-              du -sh $(Build.SourcesDirectory)/*
-              df -h
-            displayName: Disk Usage before Build
-
-        - script: $(_buildScript)
-                -subset libs.pretest+libs.tests
-                $(_buildArguments)
-                $(_archiveTestsParameter)
-          displayName: Restore and Build
-
-        - ${{ if in(parameters.osGroup, 'OSX', 'iOS','tvOS') }}: 
-          - script: |
-              du -sh $(Build.SourcesDirectory)/*
-              df -h
-            displayName: Disk Usage after Build
-
-        - template: /eng/pipelines/common/upload-artifact-step.yml
-          parameters:
-            rootFolder: $(Build.SourcesDirectory)/artifacts/helix
-            includeRootFolder: true
-            archiveType: $(archiveType)
-            archiveExtension: $(archiveExtension)
-            tarCompression: $(tarCompression)
-            artifactName: $(librariesTestsArtifactName)
-            displayName: Test Assets

diff --git a/eng/pipelines/libraries/run-test-job.yml b/eng/pipelines/libraries/run-test-job.yml
index 58a3427..3b1d1df 100644 (file)
--- a/eng/pipelines/libraries/run-test-job.yml
+++ b/eng/pipelines/libraries/run-test-job.yml
@@ -52,16 +52,20 @@ jobs:
       dependsOn:
       - ${{ if notIn(parameters.framework, 'allConfigurations', 'net472') }}:
         - ${{ format('libraries_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }}
-        - ${{ format('libraries_test_build_{0}_{1}_{2}', parameters.osGroup, parameters.dependsOnTestArchitecture, parameters.dependsOnTestBuildConfiguration) }}
-      - ${{ if in(parameters.framework, 'allConfigurations', 'net472') }}:
-        - ${{ format('libraries_build_{0}_{1}{2}_{3}_{4}', parameters.framework, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }}
-        - ${{ format('libraries_test_build_{0}_{1}_{2}_{3}', parameters.framework, parameters.osGroup, parameters.dependsOnTestArchitecture, parameters.dependsOnTestBuildConfiguration) }}
+        # tests are built as part of product build
+        - ${{ if and(ne(parameters.archType, parameters.dependsOnTestArchitecture), ne(parameters.buildConfig, parameters.dependsOnTestBuildConfiguration)) }}:
+          - ${{ format('libraries_build_{0}_{1}_{2}', parameters.osGroup, parameters.dependsOnTestArchitecture, parameters.dependsOnTestBuildConfiguration) }}
       - ${{ if ne(parameters.liveRuntimeBuildConfig, '') }}:
         - ${{ format('{0}_{1}_product_build_{2}{3}_{4}_{5}', parameters.runtimeFlavor, parameters.runtimeVariant, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveRuntimeBuildConfig) }}
 
       variables:
         - librariesTestsArtifactName: ${{ format('libraries_test_assets_{0}_{1}_{2}', parameters.osGroup, parameters.dependsOnTestArchitecture, parameters.dependsOnTestBuildConfiguration) }}
         - _archiveTestsParameter: /p:ArchiveTests=true
+        
+        - ${{ if ne(parameters.liveRuntimeBuildConfig, '') }}:
+          - _runtimeArtifactName: '$(runtimeFlavorName)Product_${{ parameters.runtimeVariant}}_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.liveRuntimeBuildConfig }}'
+          - _runtimeArtifactsPathArg: ' /p:RuntimeArtifactsPath=$(_runtimeDownloadPath)'
+
         - ${{ parameters.variables }}
 
       steps:
@@ -80,6 +84,14 @@ jobs:
             artifactName: $(librariesTestsArtifactName)
             artifactFileName: $(librariesTestsArtifactName)$(archiveExtension)
             unpackFolder: $(Build.SourcesDirectory)/artifacts
+        
+        - ${{ if ne(parameters.liveRuntimeBuildConfig, '') }}:
+          - template: /eng/pipelines/common/download-artifact-step.yml
+            parameters:
+              unpackFolder: $(_runtimeDownloadPath)
+              artifactFileName: '$(_runtimeArtifactName)$(archiveExtension)'
+              artifactName: '$(_runtimeArtifactName)'
+              displayName: '$(runtimeFlavorName) build drop'
 
         - ${{ if in(parameters.coreclrTestGroup, 'gcstress0x3-gcstress0xc', 'gcstress-extra') }}:
           # We need to find and download the GC stress dependencies (namely, coredistools). Put them

diff --git a/eng/pipelines/runtime.yml b/eng/pipelines/runtime.yml
index 7dae35b..6d70e6c 100644 (file)
--- a/eng/pipelines/runtime.yml
+++ b/eng/pipelines/runtime.yml
@@ -541,6 +541,11 @@ jobs:
     - Windows_NT_x64
     - FreeBSD_x64
     jobParameters:
+      testScope: innerloop
+      testBuildPlatforms:
+      - Linux_x64
+      - Windows_NT_x64
+      - OSX_x64
       liveRuntimeBuildConfig: release
 
 #
@@ -629,30 +634,6 @@ jobs:
       liveLibrariesBuildConfig: ${{ variables.debugOnPrReleaseOnRolling }}
 
 #
-# Libraries Test Build
-# Only when CoreCLR, Mono or Libraries is changed
-#
-- template: /eng/pipelines/common/platform-matrix.yml
-  parameters:
-    jobTemplate: /eng/pipelines/libraries/build-test-job.yml
-    buildConfig: ${{ variables.debugOnPrReleaseOnRolling }}
-    platforms:
-    - OSX_x64
-    - Linux_x64
-    - Windows_NT_x64
-    jobParameters:
-      isOfficialBuild: false
-      liveRuntimeBuildConfig: release
-      testScope: innerloop
-      condition: >-
-        or(
-          eq(dependencies.checkout.outputs['SetPathVars_libraries.containsChange'], true),
-          eq(dependencies.checkout.outputs['SetPathVars_coreclr.containsChange'], true),
-          eq(dependencies.checkout.outputs['SetPathVars_mono.containsChange'], true),
-          eq(dependencies.checkout.outputs['SetPathVars_runtimetests.containsChange'], true),
-          eq(variables['isFullMatrix'], true))
-
-#
 # Crossgen-comparison jobs
 # Only when CoreCLR is changed
 #