Preparatory changes for implementing Crossgen2 pipeline (dotnet/coreclr#26908)

1) Modify CoreCLR pipelines to use the zipping / unzipping templates
for manipulating the product build.

2) Pass the crossgen option to product build so that it crossgens the
framework assemblies.

3) After my recent split of native vs. managed test artifact build
build-job and xplat-test-job ended up with quite a few variable
duplications. I have just modified the two jobs
needing the common variables (build-job and
crossgen-comparison-job) to use the more specialized template
xplat-test-job instead of xplat-job they used before.

4) Optimize the pipeline to reuse *nix managed artifacts across
architectures. For Windows we're currently splitting managed
artifacts by bitness which matters for some reason I'll investigate
next.

Thanks

Tomas

Commit migrated from https://github.com/dotnet/coreclr/commit/6ab52d607a3e2a65030b8668a759f717a7bf6ee8

diff --git a/eng/build-job.yml b/eng/build-job.yml
index 44d3cad..f3e92da 100644 (file)
--- a/eng/build-job.yml
+++ b/eng/build-job.yml
@@ -5,6 +5,7 @@ parameters:
   osSubgroup: ''
   container: ''
   testGroup: ''
+  crossgenFramework: false
   crossrootfsDir: ''
   timeoutInMinutes: ''
   signBinaries: false
@@ -13,15 +14,16 @@ parameters:
 
 ### Product build
 jobs:
-- template: xplat-job.yml
+- template: xplat-test-job.yml
   parameters:
     buildConfig: ${{ parameters.buildConfig }}
     archType: ${{ parameters.archType }}
     osGroup: ${{ parameters.osGroup }}
     osSubgroup: ${{ parameters.osSubgroup }}
     helixType: 'build/product/'
-    enableMicrobuild: true
+    testGroup: ${{ parameters.testGroup }}
     stagedBuild: ${{ parameters.stagedBuild }}
+    enableMicrobuild: true
 
     # Compute job name from template parameters
     name: ${{ format('build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }}
@@ -37,10 +39,6 @@ jobs:
 
     gatherAssetManifests: true
     variables:
-    - name: osGroup
-      value: ${{ parameters.osGroup }}
-    - name: osSubgroup
-      value: ${{ parameters.osSubgroup }}
     - name: stripSymbolsArg
       value: ''
     # Strip symbols only on the release build
@@ -88,49 +86,11 @@ jobs:
     - ${{ if and(eq(parameters.buildConfig, 'Release'), and(eq(parameters.osGroup, 'Windows_NT'), not(or(eq(parameters.archType, 'arm64'), eq(parameters.archType, 'arm'))))) }}:
       - name: enforcePgoArg
         value: '-enforcepgo'
-
-    - name: artifactRootName
-      value: $(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)
-
-    - name: testNativeArtifactName
-      value: 'NativeTestComponents_$(artifactRootName)'
-
-    - name: testNativeRootFolderPath
-      value: '$(binTestsPath)/obj/$(osGroup).$(archType).$(buildConfigUpper)'
-
-    - name: testBuildArtifactName
-      value: 'TestBuild_$(artifactRootName)'
-
-    - name: testBuildRootFolderPath
-      value: '$(binTestsPath)/$(osGroup).$(archType).$(buildConfigUpper)'
-
-    - name: binTestsPath
-      value: '$(Build.SourcesDirectory)/bin/tests'
-
-    - name: archiveExtension
-      value: '.tar.gz'
-    - name: archiveType
-      value: tar
-    - name: tarCompression
-      value: gz
-
-    - ${{ if eq(parameters.osGroup, 'Windows_NT') }}:
-      - name: archiveExtension
-        value: '.zip'
-      - name: archiveType
-        value: zip
-      - name: tarCompression
-        value: ''
-
-    - name: priorityArg
+    - name: crossgenArg
       value: ''
-    - ${{ if ne(parameters.testGroup, 'innerloop') }}:
-      - ${{ if ne(parameters.osGroup, 'Windows_NT') }}:
-        - name: priorityArg
-          value: 'priority1'
-      - ${{ if eq(parameters.osGroup, 'Windows_NT') }}:
-        - name: priorityArg
-          value: '-priority=1'
+    - ${{ if eq(parameters.crossgenFramework, true) }}:
+      - name: crossgenArg
+        value: 'crossgen'
 
     steps:
 
@@ -174,12 +134,12 @@ jobs:
       - script: set __TestIntermediateDir=int&&build.cmd $(buildConfig) $(archType) -ci -skiptests -skipbuildpackages $(officialBuildIdArg) $(ibcOptimizeArg) $(enforcePgoArg)
         displayName: Build product
 
-    # Build native test components
+    # Build native test components and optionally crossgen the framework assemblies
     - ${{ if ne(parameters.osGroup, 'Windows_NT') }}:
-      - script: ./build-test.sh skipmanaged $(buildConfig) $(archType) $(crossArg) $(priorityArg) $(clangArg)
+      - script: ./build-test.sh skipmanaged $(crossgenArg) $(buildConfig) $(archType) $(crossArg) $(priorityArg) $(clangArg)
         displayName: Build native test components
     - ${{ if eq(parameters.osGroup, 'Windows_NT') }}:
-      - script: build-test.cmd skipmanaged $(buildConfig) $(archType) $(priorityArg)
+      - script: build-test.cmd skipmanaged $(crossgenArg) $(buildConfig) $(archType) $(priorityArg)
         displayName: Build native test components
 
     # Sign on Windows
@@ -197,11 +157,15 @@ jobs:
         condition: always()
 
     # Publish product output directory for consumption by tests.
-    - task: PublishBuildArtifacts@1
-      displayName: Publish product build
-      inputs:
-        pathtoPublish: $(Build.SourcesDirectory)/bin/Product/$(osGroup).$(archType).$(_BuildConfig)
-        artifactName: ${{ format('BinDir_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }}
+    - template: /eng/upload-artifact-step.yml
+      parameters:
+        rootFolder: '$(productBuildRootFolderPath)'
+        includeRootFolder: false
+        archiveFile: $(Build.StagingDirectory)/$(productBuildArtifactName)$(archiveExtension)
+        archiveType: $(archiveType)
+        tarCompression: $(tarCompression)
+        artifactName: $(productBuildArtifactName)
+        displayName: 'product build'
 
     # Publish test native components for consumption by test execution.
     - template: /eng/upload-artifact-step.yml

diff --git a/eng/build-test-job.yml b/eng/build-test-job.yml
index 4bab58c..a2eedf3 100644 (file)
--- a/eng/build-test-job.yml
+++ b/eng/build-test-job.yml
@@ -30,6 +30,7 @@ jobs:
     osSubgroup: ${{ parameters.osSubgroup }}
     managedTestBuildOsGroup: ${{ parameters.osGroup }}
     managedTestBuildOsSubgroup: ${{ parameters.osSubgroup }}
+    managedTestBuildArchType: ${{ parameters.archType }}
     container: ${{ parameters.container }}
     testGroup: ${{ parameters.testGroup }}
     readyToRun: ${{ parameters.readyToRun }}
@@ -116,22 +117,14 @@ jobs:
 
 
     # Download product binaries directory
-    - task: DownloadBuildArtifacts@0
-      displayName: Download product build
-      inputs:
-        buildType: current
-        downloadType: single
-        artifactName: ${{ format('BinDir_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }}
-        downloadPath: $(System.ArtifactsDirectory)
-
-
-    # Populate Product directory
-    - task: CopyFiles@2
-      displayName: Populate Product directory
-      inputs:
-        sourceFolder: $(System.ArtifactsDirectory)/${{ format('BinDir_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }}
-        contents: '**'
-        targetFolder: $(Build.SourcesDirectory)/bin/Product/$(osGroup).$(archType).$(buildConfigUpper)
+    - template: /eng/download-artifact-step.yml
+      parameters:
+        downloadFolder: '$(productBuildRootFolderPath)/tmp'
+        unpackFolder: '$(productBuildRootFolderPath)'
+        cleanUnpackFolder: false
+        artifactFileName: '$(productBuildArtifactName)$(archiveExtension)'
+        artifactName: '$(productBuildArtifactName)'
+        displayName: 'product build'
 
 
     # Build managed test components

diff --git a/eng/crossgen-comparison-job.yml b/eng/crossgen-comparison-job.yml
index d257c48..a4ef8c3 100644 (file)
--- a/eng/crossgen-comparison-job.yml
+++ b/eng/crossgen-comparison-job.yml
@@ -14,7 +14,7 @@ parameters:
 ### crossgen matches that of native, e.g. arm-hosted-arm-targeting, crossgen.
 
 jobs:
-- template: xplat-job.yml
+- template: xplat-test-job.yml
   parameters:
     buildConfig: ${{ parameters.buildConfig }}
     archType: ${{ parameters.archType }}
@@ -62,22 +62,14 @@ jobs:
     steps:
 
     # Download product build
-    - task: DownloadBuildArtifacts@0
-      displayName: Download product build
-      inputs:
-        buildType: current
-        downloadType: single
-        artifactName: ${{ format('BinDir_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }}
-        downloadPath: $(System.ArtifactsDirectory)
-
-
-    # Populate Product directory
-    - task: CopyFiles@2
-      displayName: Populate Product directory
-      inputs:
-        sourceFolder: $(System.ArtifactsDirectory)/${{ format('BinDir_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }}
-        contents: '**'
-        targetFolder: $(productDirectory)/$(targetFlavor)
+    - template: /eng/download-artifact-step.yml
+      parameters:
+        downloadFolder: '$(productBuildRootFolderPath)/tmp'
+        unpackFolder: '$(productBuildRootFolderPath)'
+        cleanUnpackFolder: false
+        artifactFileName: '$(productBuildArtifactName)$(archiveExtension)'
+        artifactName: '$(productBuildArtifactName)'
+        displayName: 'product build'
 
 
     # Create directories and ensure crossgen is executable

diff --git a/eng/download-git-repository-step.yml b/eng/download-git-repository-step.yml
new file mode 100644 (file)
index 0000000..49984bd
--- /dev/null
+++ b/eng/download-git-repository-step.yml
@@ -0,0 +1,17 @@
+parameters:
+  unix: false
+
+steps:
+  # Download the GIT repository
+  - template: /eng/download-artifact-step.yml
+    parameters:
+      displayName: 'GIT repository'
+      downloadFolder: $(Build.SourcesDirectory)/download/
+      cleanUnpackFolder: false
+      unpackFolder: $(Build.SourcesDirectory)
+      ${{ if ne(parameters.unix, true) }}:
+        artifactFileName: repo_windows.zip
+        artifactName: repo_windows
+      ${{ if eq(parameters.unix, true) }}:
+        artifactFileName: repo_unix.tar.gz
+        artifactName: repo_unix

diff --git a/eng/perf-job.yml b/eng/perf-job.yml
index b22bc82..e94dd02 100644 (file)
--- a/eng/perf-job.yml
+++ b/eng/perf-job.yml
@@ -48,22 +48,14 @@ jobs:
     # Extra steps that will be passed to the performance template and run before sending the job to helix (all of which is done in the template)
 
     # Download product binaries directory
-    - task: DownloadBuildArtifacts@0
-      displayName: Download product build
-      inputs:
-        buildType: current
-        downloadType: single
-        artifactName: ${{ format('BinDir_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }}
-        downloadPath: $(System.ArtifactsDirectory)
-
-
-    # Populate Product directory
-    - task: CopyFiles@2
-      displayName: Populate Product directory
-      inputs:
-        sourceFolder: $(System.ArtifactsDirectory)/${{ format('BinDir_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }}
-        contents: '**'
-        targetFolder: $(Build.SourcesDirectory)/bin/Product/${{ parameters.osGroup }}.${{ parameters.archType }}.Release
+    - template: /eng/download-artifact-step.yml
+      parameters:
+        downloadFolder: '$(productBuildRootFolderPath)/tmp'
+        unpackFolder: '$(productBuildRootFolderPath)'
+        cleanUnpackFolder: false
+        artifactFileName: '$(productBuildArtifactName)$(archiveExtension)'
+        artifactName: '$(productBuildArtifactName)'
+        displayName: 'product build'
 
     # Create Core_Root
     - ${{ if ne(parameters.osGroup, 'Windows_NT') }}:

diff --git a/eng/pipelines/coreclr/ci.yml b/eng/pipelines/coreclr/ci.yml
index 83c3a73..d844e10 100644 (file)
--- a/eng/pipelines/coreclr/ci.yml
+++ b/eng/pipelines/coreclr/ci.yml
@@ -39,6 +39,7 @@ jobs:
     platformGroup: all
     jobParameters:
       testGroup: outerloop
+      crossgenFramework: true
 
 #
 # Release builds

diff --git a/eng/pipelines/coreclr/internal.yml b/eng/pipelines/coreclr/internal.yml
index 718131d..2f01520 100644 (file)
--- a/eng/pipelines/coreclr/internal.yml
+++ b/eng/pipelines/coreclr/internal.yml
@@ -55,6 +55,7 @@ stages:
           signBinaries: true
           publishToBlobFeed: true
           testGroup: outerloop
+          crossgenFramework: true
 
     #
     # Publish build information to Build Assets Registry

diff --git a/eng/pipelines/coreclr/pr.yml b/eng/pipelines/coreclr/pr.yml
index baee188..dae1007 100644 (file)
--- a/eng/pipelines/coreclr/pr.yml
+++ b/eng/pipelines/coreclr/pr.yml
@@ -45,6 +45,7 @@ jobs:
     - Windows_NT_x86
     jobParameters:
       testGroup: innerloop
+      crossgenFramework: true
 
 #
 # Release builds

diff --git a/eng/pipelines/coreclr/r2r-extra.yml b/eng/pipelines/coreclr/r2r-extra.yml
index e9aa81a..5f0082d 100644 (file)
--- a/eng/pipelines/coreclr/r2r-extra.yml
+++ b/eng/pipelines/coreclr/r2r-extra.yml
@@ -23,6 +23,7 @@ jobs:
     platformGroup: gcstress
     jobParameters:
       testGroup: r2r-extra
+      crossgenFramework: true
 
 - template: /eng/platform-matrix.yml
   parameters:

diff --git a/eng/pipelines/coreclr/r2r.yml b/eng/pipelines/coreclr/r2r.yml
index 198bc34..2ba2472 100644 (file)
--- a/eng/pipelines/coreclr/r2r.yml
+++ b/eng/pipelines/coreclr/r2r.yml
@@ -20,6 +20,7 @@ jobs:
     - Windows_NT_x86
     jobParameters:
       testGroup: outerloop
+      crossgenFramework: true
 
 - template: /eng/platform-matrix.yml
   parameters:

diff --git a/eng/platform-matrix-combos.yml b/eng/platform-matrix-combos.yml
index 57f0608..bdf38d2 100644 (file)
--- a/eng/platform-matrix-combos.yml
+++ b/eng/platform-matrix-combos.yml
@@ -43,29 +43,40 @@ jobs:
     windowsArm: ${{ parameters.windowsArm }}
     windowsArm64: ${{ parameters.windowsArm64 }}
 
-    # Determine OS for building X86 managed test artifacts on *nix
-    # Currently no-op as we only run X86 tests on Windows
-    x86ManagedTestBuildOsGroup: ''
+    # Determine OS / arch combo for building managed test artifacts on *nix
 
     # Determine OS for building X64 managed test artifacts on *nix
     ${{ if eq(parameters.osxX64, true) }}:
-      x64ManagedTestBuildOsGroup: 'OSX'
+      unixManagedTestBuildOsGroup: 'OSX'
+      unixManagedTestBuildArchType: 'x64'
     ${{ if and(ne(parameters.osxX64, true), eq(parameters.linuxX64, true)) }}:
-      x64ManagedTestBuildOsGroup: 'Linux'
-    ${{ if and(ne(parameters.osxX64, true), ne(parameters.linuxX64, true), eq(parameters.linuxMuslX64, true)) }}:
-      x64ManagedTestBuildOsGroup: 'Linux'
-      x64ManagedTestBuildOsSubgroup: '_musl'
-    ${{ if and(ne(parameters.osxX64, true), ne(parameters.linuxX64, true), ne(parameters.linuxMuslX64, true), eq(parameters.linuxRhel6X64, true)) }}:
-      x64ManagedTestBuildOsGroup: 'Linux'
-      x64ManagedTestBuildOsSubgroup: '_rhel6'
+      unixManagedTestBuildOsGroup: 'Linux'
+      unixManagedTestBuildArchType: 'x64'
+    ${{ if and(ne(parameters.osxX64, true), ne(parameters.linuxX64, true), eq(parameters.linuxArm, true)) }}:
+      unixManagedTestBuildOsGroup: 'Linux'
+      unixManagedTestBuildArchType: 'arm'
+    ${{ if and(ne(parameters.osxX64, true), ne(parameters.linuxX64, true), ne(parameters.linuxArm, true), eq(parameters.linuxArm64, true)) }}:
+      unixManagedTestBuildOsGroup: 'Linux'
+      unixManagedTestBuildArchType: 'arm64'
+    ${{ if and(ne(parameters.osxX64, true), ne(parameters.linuxX64, true), ne(parameters.linuxArm, true), ne(parameters.linuxArm64, true), eq(parameters.linuxMuslX64, true)) }}:
+      unixManagedTestBuildOsGroup: 'Linux'
+      unixManagedTestBuildOsSubgroup: '_musl'
+      unixManagedTestBuildArchType: 'x64'
+    ${{ if and(ne(parameters.osxX64, true), ne(parameters.linuxX64, true), ne(parameters.linuxArm, true), ne(parameters.linuxArm64, true), ne(parameters.linuxMuslX64, true), eq(parameters.linuxMuslArm64, true)) }}:
+      unixManagedTestBuildOsGroup: 'Linux'
+      unixManagedTestBuildOsSubgroup: '_musl'
+      unixManagedTestBuildArchType: 'arm64'
+    ${{ if and(ne(parameters.osxX64, true), ne(parameters.linuxX64, true), ne(parameters.linuxArm, true), ne(parameters.linuxArm64, true), ne(parameters.linuxMuslX64, true), ne(parameters.linuxMuslArm64, true), eq(parameters.linuxRhel6X64, true)) }}:
+      unixManagedTestBuildOsGroup: 'Linux'
+      unixManagedTestBuildOsSubgroup: '_rhel6'
+      unixManagedTestBuildArchType: 'x64'
 
-    # Determine OS for building ARM managed test artifacts on *nix
-    ${{ if eq(parameters.linuxArm, true) }}:
-      armManagedTestBuildOsGroup: 'Linux'
+    ${{ if eq(parameters.windowsX86, true) }}:
+      windows32bitManagedTestBuildArchType: 'x86'
+    ${{ if and(ne(parameters.windowsX86, true), eq(parameters.windowsArm, true)) }}:
+      windows32bitManagedTestBuildArchType: 'arm'
 
-    # Determine OS for building ARM64 managed test artifacts on *nix
-    ${{ if eq(parameters.linuxArm64, true) }}:
-      arm64ManagedTestBuildOsGroup: 'Linux'
-    ${{ if and(ne(parameters.linuxArm64, true), eq(parameters.linuxMuslArm64, true)) }}:
-      arm64ManagedTestBuildOsGroup: 'Linux'
-      arm64ManagedTestBuildOsSubroup: '_musl'
+    ${{ if eq(parameters.windowsX64, true) }}:
+      windows64bitManagedTestBuildArchType: 'x64'
+    ${{ if and(ne(parameters.windowsX64, true), eq(parameters.windowsArm64, true)) }}:
+      windows64bitManagedTestBuildArchType: 'arm64'

diff --git a/eng/platform-matrix-managed-test-build.yml b/eng/platform-matrix-managed-test-build.yml
index dc1702d..4355a5d 100644 (file)
--- a/eng/platform-matrix-managed-test-build.yml
+++ b/eng/platform-matrix-managed-test-build.yml
@@ -23,14 +23,12 @@ parameters:
   windowsArm: false
   windowsArm64: false
 
-  x86ManagedTestBuildOsGroup: ''
-  x86ManagedTestBuildOsSubgroup: ''
-  x64ManagedTestBuildOsGroup: ''
-  x64ManagedTestBuildOsSubgroup: ''
-  armManagedTestBuildOsGroup: ''
-  armManagedTestBuildOsSubgroup: ''
-  arm64ManagedTestBuildOsGroup: ''
-  arm64ManagedTestBuildOsSubgroup: ''
+  unixManagedTestBuildOsGroup: ''
+  unixManagedTestBuildOsSubgroup: ''
+  unixManagedTestBuildArchType: ''
+  
+  windows32bitManagedTestBuildArchType: ''
+  windows64bitManagedTestBuildArchType: ''
 
 jobs:
 
@@ -43,8 +41,9 @@ jobs:
       buildConfig: ${{ parameters.buildConfig }}
       archType: arm
       osGroup: Linux
-      managedTestBuildOsGroup: ${{ parameters.armManagedTestBuildOsGroup }}
-      managedTestBuildOsSubgroup: ${{ parameters.armManagedTestBuildOsSubgroup }}
+      managedTestBuildOsGroup: ${{ parameters.unixManagedTestBuildOsGroup }}
+      managedTestBuildOsSubgroup: ${{ parameters.unixManagedTestBuildOsSubgroup }}
+      managedTestBuildArchType: ${{ parameters.unixManagedTestBuildArchType }}
       container:
         image: ubuntu-16.04-cross-14.04-23cacb0-20190923200213
         registry: mcr
@@ -67,8 +66,9 @@ jobs:
       buildConfig: ${{ parameters.buildConfig }}
       archType: arm64
       osGroup: Linux
-      managedTestBuildOsGroup: ${{ parameters.arm64ManagedTestBuildOsGroup }}
-      managedTestBuildOsSubgroup: ${{ parameters.arm64ManagedTestBuildOsSubgroup }}
+      managedTestBuildOsGroup: ${{ parameters.unixManagedTestBuildOsGroup }}
+      managedTestBuildOsSubgroup: ${{ parameters.unixManagedTestBuildOsSubgroup }}
+      managedTestBuildArchType: ${{ parameters.unixManagedTestBuildArchType }}
       container:
         image: ubuntu-16.04-cross-arm64-cfdd435-20190923200213
         registry: mcr
@@ -93,8 +93,9 @@ jobs:
       archType: x64
       osGroup: Linux
       osSubgroup: _musl
-      managedTestBuildOsGroup: ${{ parameters.x64ManagedTestBuildOsGroup }}
-      managedTestBuildOsSubgroup: ${{ parameters.x64ManagedTestBuildOsSubgroup }}
+      managedTestBuildOsGroup: ${{ parameters.unixManagedTestBuildOsGroup }}
+      managedTestBuildOsSubgroup: ${{ parameters.unixManagedTestBuildOsSubgroup }}
+      managedTestBuildArchType: ${{ parameters.unixManagedTestBuildArchType }}
       container:
         image: alpine-3.9-WithNode-0fc54a3-20190918214015
         registry: mcr
@@ -116,8 +117,9 @@ jobs:
       archType: arm64
       osGroup: Linux
       osSubgroup: _musl
-      managedTestBuildOsGroup: ${{ parameters.arm64ManagedTestBuildOsGroup }}
-      managedTestBuildOsSubgroup: ${{ parameters.arm64ManagedTestBuildOsSubgroup }}
+      managedTestBuildOsGroup: ${{ parameters.unixManagedTestBuildOsGroup }}
+      managedTestBuildOsSubgroup: ${{ parameters.unixManagedTestBuildOsSubgroup }}
+      managedTestBuildArchType: ${{ parameters.unixManagedTestBuildArchType }}
       container:
         image: ubuntu-16.04-cross-arm64-alpine-406629a-20190923200213
         registry: mcr
@@ -139,8 +141,9 @@ jobs:
       archType: x64
       osGroup: Linux
       osSubgroup: _rhel6
-      managedTestBuildOsGroup: ${{ parameters.x64ManagedTestBuildOsGroup }}
-      managedTestBuildOsSubgroup: ${{ parameters.x64ManagedTestBuildOsSubgroup }}
+      managedTestBuildOsGroup: ${{ parameters.unixManagedTestBuildOsGroup }}
+      managedTestBuildOsSubgroup: ${{ parameters.unixManagedTestBuildOsSubgroup }}
+      managedTestBuildArchType: ${{ parameters.unixManagedTestBuildArchType }}
       container:
         image: centos-6-50f0d02-20190918213956
         registry: mcr
@@ -159,8 +162,9 @@ jobs:
       buildConfig: ${{ parameters.buildConfig }}
       archType: x64
       osGroup: Linux
-      managedTestBuildOsGroup: ${{ parameters.x64ManagedTestBuildOsGroup }}
-      managedTestBuildOsSubgroup: ${{ parameters.x64ManagedTestBuildOsSubgroup }}
+      managedTestBuildOsGroup: ${{ parameters.unixManagedTestBuildOsGroup }}
+      managedTestBuildOsSubgroup: ${{ parameters.unixManagedTestBuildOsSubgroup }}
+      managedTestBuildArchType: ${{ parameters.unixManagedTestBuildArchType }}
       container:
         image: centos-7-50f0d02-20190918214028
         registry: mcr
@@ -207,8 +211,9 @@ jobs:
       buildConfig: ${{ parameters.buildConfig }}
       archType: x64
       osGroup: OSX
-      managedTestBuildOsGroup: ${{ parameters.x64ManagedTestBuildOsGroup }}
-      managedTestBuildOsSubgroup: ${{ parameters.x64ManagedTestBuildOsSubgroup }}
+      managedTestBuildOsGroup: ${{ parameters.unixManagedTestBuildOsGroup }}
+      managedTestBuildOsSubgroup: ${{ parameters.unixManagedTestBuildOsSubgroup }}
+      managedTestBuildArchType: ${{ parameters.unixManagedTestBuildArchType }}
       helixQueues:
       - ${{ if and(eq(variables['System.TeamProject'], 'public'), in(parameters.helixQueueGroup, 'pr', 'ci', 'corefx')) }}:
         - OSX.1013.Amd64.Open
@@ -229,6 +234,8 @@ jobs:
       buildConfig: ${{ parameters.buildConfig }}
       archType: x64
       osGroup: Windows_NT
+      managedTestBuildOsGroup: 'Windows_NT'
+      managedTestBuildArchType: ${{ parameters.windows64bitManagedTestBuildArchType }}
       helixQueues:
       - ${{ if and(eq(variables['System.TeamProject'], 'public'), in(parameters.helixQueueGroup, 'pr', 'ci', 'corefx')) }}:
         - Windows.10.Amd64.Open
@@ -254,6 +261,8 @@ jobs:
       buildConfig: ${{ parameters.buildConfig }}
       archType: x86
       osGroup: Windows_NT
+      managedTestBuildOsGroup: 'Windows_NT'
+      managedTestBuildArchType: ${{ parameters.windows32bitManagedTestBuildArchType }}
       helixQueues:
       - ${{ if and(eq(variables['System.TeamProject'], 'public'), in(parameters.helixQueueGroup, 'pr', 'ci', 'corefx')) }}:
         - Windows.10.Amd64.Open
@@ -277,6 +286,8 @@ jobs:
       buildConfig: ${{ parameters.buildConfig }}
       archType: arm
       osGroup: Windows_NT
+      managedTestBuildOsGroup: 'Windows_NT'
+      managedTestBuildArchType: ${{ parameters.windows32bitManagedTestBuildArchType }}
       helixQueues:
       # NOTE: there are no queues specified for Windows_NT_arm public with helixQueueGroup='pr'. This means that specifying
       # Windows_NT_arm for a PR job causes a build, but no test run. If the test build and test runs were separate jobs,
@@ -296,6 +307,8 @@ jobs:
       buildConfig: ${{ parameters.buildConfig }}
       archType: arm64
       osGroup: Windows_NT
+      managedTestBuildOsGroup: 'Windows_NT'
+      managedTestBuildArchType: ${{ parameters.windows64bitManagedTestBuildArchType }}
       helixQueues:
       # TODO: Consider adding Windows.10.Arm64.Open here if capacity is enough for handling both Windows_NT/arm and Windows_NT/arm64 testing
       - ${{ if eq(variables['System.TeamProject'], 'internal') }}:

diff --git a/eng/run-test-job.yml b/eng/run-test-job.yml
index 726c587..8af84a6 100644 (file)
--- a/eng/run-test-job.yml
+++ b/eng/run-test-job.yml
@@ -5,6 +5,7 @@ parameters:
   osSubgroup: ''
   managedTestBuildOsGroup: ''
   managedTestBuildOsSubgroup: ''
+  managedTestBuildArchType: ''
   container: ''
   testGroup: ''
   readyToRun: false
@@ -29,6 +30,7 @@ jobs:
     osSubgroup: ${{ parameters.osSubgroup }}
     managedTestBuildOsGroup: ${{ parameters.managedTestBuildOsGroup }}
     managedTestBuildOsSubgroup: ${{ parameters.managedTestBuildOsSubgroup }}
+    managedTestBuildArchType: ${{ parameters.managedTestBuildArchType }}
     container: ${{ parameters.container }}
     testGroup: ${{ parameters.testGroup }}
     readyToRun: ${{ parameters.readyToRun }}
@@ -44,7 +46,7 @@ jobs:
     ${{ if and(eq(parameters.testGroup, 'innerloop'), eq(parameters.displayNameArgs, '')) }}:
       name: 'run_test_p0_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }}'
       dependsOn:
-      - 'build_test_p0_${{ parameters.managedTestBuildOsGroup }}${{ parameters.managedTestBuildOsSubgroup }}_${{ parameters.archType }}_${{parameters.buildConfig }}'
+      - 'build_test_p0_${{ parameters.managedTestBuildOsGroup }}${{ parameters.managedTestBuildOsSubgroup }}_${{ parameters.managedTestBuildArchType }}_${{parameters.buildConfig }}'
       - ${{ if ne(parameters.stagedBuild, true) }}:
         - ${{ format('build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }}
       displayName: 'Run Test Pri0 ${{ parameters.osGroup }}${{ parameters.osSubgroup }} ${{ parameters.archType }} ${{ parameters.buildConfig }}'
@@ -52,7 +54,7 @@ jobs:
     ${{ if and(ne(parameters.testGroup, 'innerloop'), eq(parameters.displayNameArgs, '')) }}:
       name: 'run_test_p1_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }}'
       dependsOn:
-      - 'build_test_p1_${{ parameters.managedTestBuildOsGroup }}${{ parameters.managedTestBuildOsSubgroup }}_${{ parameters.archType }}_${{parameters.buildConfig }}'
+      - 'build_test_p1_${{ parameters.managedTestBuildOsGroup }}${{ parameters.managedTestBuildOsSubgroup }}_${{ parameters.managedTestBuildArchType }}_${{parameters.buildConfig }}'
       - ${{ if ne(parameters.stagedBuild, true) }}:
         - ${{ format('build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }}
       displayName: 'Run Test Pri1 ${{ parameters.osGroup }}${{ parameters.osSubgroup }} ${{ parameters.archType }} ${{ parameters.buildConfig }}'
@@ -60,7 +62,7 @@ jobs:
     ${{ if and(eq(parameters.testGroup, 'innerloop'), ne(parameters.displayNameArgs, '')) }}:
       name: 'run_test_p0_${{ parameters.displayNameArgs }}_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }}'
       dependsOn:
-      - 'build_test_p0_${{ parameters.displayNameArgs }}_${{ parameters.managedTestBuildOsGroup }}${{ parameters.managedTestBuildOsSubgroup }}_${{ parameters.archType }}_${{parameters.buildConfig }}'      
+      - 'build_test_p0_${{ parameters.displayNameArgs }}_${{ parameters.managedTestBuildOsGroup }}${{ parameters.managedTestBuildOsSubgroup }}_${{ parameters.managedTestBuildArchType }}_${{parameters.buildConfig }}'      
       - ${{ if ne(parameters.stagedBuild, true) }}:
         - ${{ format('build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }}
       displayName: 'Run Test Pri0 ${{ parameters.displayNameArgs }} ${{ parameters.osGroup }}${{ parameters.osSubgroup }} ${{ parameters.archType }} ${{ parameters.buildConfig }}'
@@ -68,7 +70,7 @@ jobs:
     ${{ if and(ne(parameters.testGroup, 'innerloop'), ne(parameters.displayNameArgs, '')) }}:
       name: 'run_test_p1_${{ parameters.displayNameArgs }}_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }}'
       dependsOn:
-      - 'build_test_p1_${{ parameters.displayNameArgs }}_${{ parameters.managedTestBuildOsGroup }}${{ parameters.managedTestBuildOsSubgroup }}_${{ parameters.archType }}_${{parameters.buildConfig }}'      
+      - 'build_test_p1_${{ parameters.displayNameArgs }}_${{ parameters.managedTestBuildOsGroup }}${{ parameters.managedTestBuildOsSubgroup }}_${{ parameters.managedTestBuildArchType }}_${{parameters.buildConfig }}'      
       - ${{ if ne(parameters.stagedBuild, true) }}:
         - ${{ format('build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }}
       displayName: 'Run Test Pri1 ${{ parameters.displayNameArgs }} ${{ parameters.osGroup }}${{ parameters.osSubgroup }} ${{ parameters.archType }} ${{ parameters.buildConfig }}'
@@ -98,7 +100,7 @@ jobs:
         unpackFolder: '$(testRootFolderPath)'
         artifactFileName: '$(testArtifactName)$(archiveExtension)'
         artifactName: '$(testArtifactName)'
-        displayName: 'managed test artifacts (built on ${{ parameters.managedTestBuildOsGroup }}${{ parameters.managedTestBuildOsSubgroup }})'
+        displayName: 'managed test artifacts (built on ${{ parameters.managedTestBuildOsGroup }}${{ parameters.managedTestBuildOsSubgroup }} ${{ parameters.managedTestBuildArchType }})'
 
 
     # Download and unzip test build tree
@@ -113,22 +115,14 @@ jobs:
 
 
     # Download product binaries directory
-    - task: DownloadBuildArtifacts@0
-      displayName: Download product build
-      inputs:
-        buildType: current
-        downloadType: single
-        artifactName: 'BinDir_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)'
-        downloadPath: '$(System.ArtifactsDirectory)'
-
-
-    # Populate Product directory
-    - task: CopyFiles@2
-      displayName: Populate Product directory
-      inputs:
-        sourceFolder: '$(System.ArtifactsDirectory)/BinDir_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)'
-        contents: '**'
-        targetFolder: '$(Build.SourcesDirectory)/bin/Product/$(osGroup).$(archType).$(buildConfigUpper)'
+    - template: /eng/download-artifact-step.yml
+      parameters:
+        downloadFolder: '$(productBuildRootFolderPath)/tmp'
+        unpackFolder: '$(productBuildRootFolderPath)'
+        cleanUnpackFolder: false
+        artifactFileName: '$(productBuildArtifactName)$(archiveExtension)'
+        artifactName: '$(productBuildArtifactName)'
+        displayName: 'product build'
 
 
     # Download and unzip the Microsoft.NET.Sdk.IL package needed for traversing

diff --git a/eng/test-job.yml b/eng/test-job.yml
index d100a96..6d68ced 100644 (file)
--- a/eng/test-job.yml
+++ b/eng/test-job.yml
@@ -5,6 +5,7 @@ parameters:
   osSubgroup: ''
   managedTestBuildOsGroup: ''
   managedTestBuildOsSubgroup: ''
+  managedTestBuildArchType: ''
   container: ''
   testGroup: ''
   readyToRun: false
@@ -23,16 +24,13 @@ parameters:
 ### buildConfig and archType.
 
 jobs:
-- ${{ if and(eq(parameters.osSubgroup, parameters.managedTestBuildOsSubgroup), or(eq(parameters.osGroup, parameters.managedTestBuildOsGroup), eq(parameters.managedTestBuildOsGroup, ''))) }}:
+- ${{ if and(eq(parameters.archType, parameters.managedTestBuildArchType), eq(parameters.osSubgroup, parameters.managedTestBuildOsSubgroup), eq(parameters.osGroup, parameters.managedTestBuildOsGroup)) }}:
   - template: build-test-job.yml
     parameters:
       buildConfig: ${{ parameters.buildConfig }}
       archType: ${{ parameters.archType }}
-      ${{ if ne(parameters.managedTestBuildOsGroup, '') }}:
-        osGroup: ${{ parameters.managedTestBuildOsGroup }}
-      ${{ if eq(parameters.managedTestBuildOsGroup, '') }}:
-        osGroup: ${{ parameters.osGroup }}
-      osSubgroup: ${{ parameters.managedTestBuildOsSubgroup }}
+      osGroup: ${{ parameters.osGroup }}
+      osSubgroup: ${{ parameters.osSubgroup }}
       container: ${{ parameters.container }}
       testGroup: ${{ parameters.testGroup }}
       readyToRun: ${{ parameters.readyToRun }}
@@ -48,11 +46,9 @@ jobs:
     archType: ${{ parameters.archType }}
     osGroup: ${{ parameters.osGroup }}
     osSubgroup: ${{ parameters.osSubgroup }}
-    ${{ if ne(parameters.managedTestBuildOsGroup, '') }}:
-      managedTestBuildOsGroup: ${{ parameters.managedTestBuildOsGroup }}
-    ${{ if eq(parameters.managedTestBuildOsGroup, '') }}:
-      managedTestBuildOsGroup: ${{ parameters.osGroup }}
+    managedTestBuildOsGroup: ${{ parameters.managedTestBuildOsGroup }}
     managedTestBuildOsSubgroup: ${{ parameters.managedTestBuildOsSubgroup }}
+    managedTestBuildArchType: ${{ parameters.managedTestBuildArchType }}
     container: ${{ parameters.container }}
     testGroup: ${{ parameters.testGroup }}
     readyToRun: ${{ parameters.readyToRun }}

diff --git a/eng/xplat-job.yml b/eng/xplat-job.yml
index 0d8f2ee..0decedf 100644 (file)
--- a/eng/xplat-job.yml
+++ b/eng/xplat-job.yml
@@ -165,17 +165,8 @@ jobs:
       clean: true
 
     # Download the GIT repository
-    - template: /eng/download-artifact-step.yml
+    - template: /eng/download-git-repository-step.yml
       parameters:
-        displayName: 'GIT repository'
-        downloadFolder: $(Build.SourcesDirectory)/download/
-        cleanUnpackFolder: false
-        unpackFolder: $(Build.SourcesDirectory)
-        ${{ if eq(parameters.osGroup, 'Windows_NT') }}:
-          artifactFileName: repo_windows.zip
-          artifactName: repo_windows
-        ${{ if ne(parameters.osGroup, 'Windows_NT') }}:
-          artifactFileName: repo_unix.tar.gz
-          artifactName: repo_unix
+        unix: ${{ ne(parameters.osGroup, 'Windows_NT') }}
 
     - ${{ parameters.steps }}

diff --git a/eng/xplat-test-job.yml b/eng/xplat-test-job.yml
index 15add4b..627ba37 100644 (file)
--- a/eng/xplat-test-job.yml
+++ b/eng/xplat-test-job.yml
@@ -5,6 +5,7 @@ parameters:
   osSubgroup: ''
   managedTestBuildOsGroup: ''
   managedTestBuildOsSubgroup: ''
+  managedTestBuildArchType: ''
   name: ''
   helixType: '(unspecified)'
   container: ''
@@ -48,14 +49,6 @@ jobs:
     gatherAssetManifests: ${{ parameters.gatherAssetManifests }}
   
     variables:
-    - ${{ if ne(parameters.testGroup, '') }}:
-      - name: testArtifactRootName
-        value: ${{ parameters.managedTestBuildOsGroup }}${{ parameters.managedTestBuildOsSubgroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }}_${{ parameters.testGroup }}
-
-    - ${{ if eq(parameters.testGroup, '') }}:
-      - name: testArtifactRootName
-        value: ${{ parameters.managedTestBuildOsGroup }}${{ parameters.managedTestBuildOsSubgroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }}
-
     - name: binTestsPath
       value: $(Build.SourcesDirectory)/bin/tests
 
@@ -65,18 +58,29 @@ jobs:
     - name: nativeRootFolderPath
       value: $(binTestsPath)/$(osGroup).$(archType).$(buildConfigUpper)
 
-    - ${{ if and(eq(parameters.corefxTests, true), eq(parameters.readyToRun, true)) }}:
-      - name: testArtifactName
-        value: Tests_r2r_corefx_$(testArtifactRootName)
-    - ${{ if and(eq(parameters.corefxTests, true), ne(parameters.readyToRun, true)) }}:
-      - name: testArtifactName
-        value: Tests_corefx_$(testArtifactRootName)
-    - ${{ if and(ne(parameters.corefxTests, true), eq(parameters.readyToRun, true)) }}:
-      - name: testArtifactName
-        value: Tests_r2r_$(testArtifactRootName)
-    - ${{ if and(ne(parameters.corefxTests, true), ne(parameters.readyToRun, true)) }}:
+    - name: testArtifactNameR2RPart
+      value: ''
+
+    - ${{ if eq(parameters.readyToRun, true) }}:
+      - name: testArtifactNameR2RPart
+        value: '_r2r'
+
+    - name: testArtifactNameCoreFxPart
+      value: ''
+
+    - ${{ if eq(parameters.corefxTests, true) }}:
+      - name: testArtifactNameCoreFxPart
+        value: '_corefx'
+
+    - name: testArtifactNameTestGroupPart
+      value: ''
+
+    - ${{ if ne(parameters.testGroup, '') }}:
       - name: testArtifactName
-        value: Tests_$(testArtifactRootName)
+        value: _$(testGroup)
+
+    - name: testArtifactName
+      value: Tests$(testArtifactNameR2RPart)$(testArtifactNameCoreFxPart)_${{ parameters.managedTestBuildOsGroup }}${{ parameters.managedTestBuildOsSubgroup }}_${{ parameters.managedTestBuildArchType }}_$(buildConfig)$(testArtifactNameTestGroupPart)
 
     - name: testNativeArtifactName
       value: 'NativeTestComponents_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)'
@@ -92,9 +96,15 @@ jobs:
 
     - name: microsoftNetSdkIlFolderPath
       value: $(Build.SourcesDirectory)/.packages/microsoft.net.sdk.il
-      
+
     - name: microsoftNetSdkIlArtifactName
-      value: 'MicrosoftNetSdkIlPackage_${{ parameters.managedTestBuildOsGroup }}${{ parameters.managedTestBuildOsSubgroup }}_$(archType)_$(buildConfig)'
+      value: 'MicrosoftNetSdkIlPackage_${{ parameters.managedTestBuildOsGroup }}${{ parameters.managedTestBuildOsSubgroup }}_${{ parameters.managedTestBuildArchType }}_$(buildConfig)'
+
+    - name: productBuildArtifactName
+      value: 'BinDir_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)'
+
+    - name: productBuildRootFolderPath
+      value: '$(Build.SourcesDirectory)/bin/Product/$(osGroup).$(archType).$(buildConfigUpper)'
 
     - ${{ if eq(parameters.osGroup, 'Windows_NT') }}:
       - name: archiveExtension