Switch over the remaining CoreCLR pipelines to live-live mode (#813)

I have also found out that my recent change to switch over the JIT
stress pipelines was incomplete - it was building the live libraries
but not actually using it due to a typo in the parameter name. I am
fixing that as part of this change.

Thanks

Tomas

diff --git a/eng/liveBuilds.targets b/eng/liveBuilds.targets
index 0fc0a9a..0def9a9 100644 (file)
--- a/eng/liveBuilds.targets
+++ b/eng/liveBuilds.targets
@@ -57,11 +57,11 @@
   </Target>
 
   <Target Name="ResolveLibrariesFromLocalBuild">
-    <Error Condition="!Exists('$(LibrariesSharedFrameworkRefArtifactsPath)')" Text="The libraries subset category must be built before building this project." />
-    <Error Condition="!Exists('$(LibrariesAllRefArtifactsPath)')" Text="The libraries subset category must be built before building this project." />
-    <Error Condition="!Exists('$(LibrariesSharedFrameworkBinArtifactsPath)')" Text="The libraries subset category must be built before building this project." />
-    <Error Condition="!Exists('$(LibrariesAllBinArtifactsPath)')" Text="The libraries subset category must be built before building this project." />
-    <Error Condition="!Exists('$(LibrariesNativeArtifactsPath)')" Text="The libraries subset category must be built before building this project." />
+    <Error Condition="!Exists('$(LibrariesSharedFrameworkRefArtifactsPath)')" Text="The libraries subset category must be built before building this project. Missing artifacts: $(LibrariesSharedFrameworkRefArtifactsPath)" />
+    <Error Condition="!Exists('$(LibrariesAllRefArtifactsPath)')" Text="The libraries subset category must be built before building this project. Missing artifacts: $(LibrariesAllRefArtifactsPath)" />
+    <Error Condition="!Exists('$(LibrariesSharedFrameworkBinArtifactsPath)')" Text="The libraries subset category must be built before building this project. Missing artifacts: $(LibrariesSharedFrameworkBinArtifactsPath)" />
+    <Error Condition="!Exists('$(LibrariesAllBinArtifactsPath)')" Text="The libraries subset category must be built before building this project. Missing artifacts: $(LibrariesAllBinArtifactsPath)" />
+    <Error Condition="!Exists('$(LibrariesNativeArtifactsPath)')" Text="The libraries subset category must be built before building this project. Missing artifacts: $(LibrariesNativeArtifactsPath)" />
     <ItemGroup>
       <LibrariesRefAssemblies Condition="'$(IncludeOOBLibraries)' != 'true'" Include="$(LibrariesSharedFrameworkRefArtifactsPath)*.dll;$(LibrariesSharedFrameworkRefArtifactsPath)*.pdb" />
       <LibrariesRefAssemblies Condition="'$(IncludeOOBLibraries)' == 'true'" Include="$(LibrariesAllRefArtifactsPath)*.dll;$(LibrariesAllRefArtifactsPath)*.pdb" />

diff --git a/eng/pipelines/coreclr/crossgen2.yml b/eng/pipelines/coreclr/crossgen2.yml
index bca212e..3d1ea01 100644 (file)
--- a/eng/pipelines/coreclr/crossgen2.yml
+++ b/eng/pipelines/coreclr/crossgen2.yml
@@ -36,7 +36,7 @@ jobs:
     - Windows_NT_x64
     jobParameters:
       testGroup: innerloop
-      useLiveLibrariesBuildConfig: Release
+      liveLibrariesBuildConfig: Release
 
 - template: /eng/pipelines/common/platform-matrix.yml
   parameters:
@@ -52,4 +52,4 @@ jobs:
       readyToRun: true
       crossgen2: true
       displayNameArgs: R2R_CG2
-      useLiveLibrariesBuildConfig: Release
+      liveLibrariesBuildConfig: Release

diff --git a/eng/pipelines/coreclr/gc-longrunning.yml b/eng/pipelines/coreclr/gc-longrunning.yml
index 16b7060..5d3ec8f 100644 (file)
--- a/eng/pipelines/coreclr/gc-longrunning.yml
+++ b/eng/pipelines/coreclr/gc-longrunning.yml
@@ -18,7 +18,7 @@ jobs:
 
 - template: /eng/pipelines/common/platform-matrix.yml
   parameters:
-    jobTemplate: /eng/pipelines/coreclr/templates/build-job.yml
+    jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml
     buildConfig: release
     platforms:
     - Linux_x64
@@ -39,6 +39,7 @@ jobs:
     - Windows_NT_arm64
     jobParameters:
       testGroup: gc-longrunning
+      liveLibrariesBuildConfig: Release
 
 - template: /eng/pipelines/common/platform-matrix.yml
   parameters:
@@ -54,3 +55,4 @@ jobs:
     managedOsxBuild: false
     jobParameters:
       testGroup: gc-longrunning
+      liveLibrariesBuildConfig: Release

diff --git a/eng/pipelines/coreclr/gc-simulator.yml b/eng/pipelines/coreclr/gc-simulator.yml
index 806fb76..6e8eeb5 100644 (file)
--- a/eng/pipelines/coreclr/gc-simulator.yml
+++ b/eng/pipelines/coreclr/gc-simulator.yml
@@ -18,7 +18,7 @@ jobs:
 
 - template: /eng/pipelines/common/platform-matrix.yml
   parameters:
-    jobTemplate: /eng/pipelines/coreclr/templates/build-job.yml
+    jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml
     buildConfig: release
     platforms:
     # disable Linux x64 for now untill OOMs are resolved.
@@ -43,3 +43,4 @@ jobs:
     helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml
     jobParameters:
       testGroup: gc-simulator
+      liveLibrariesBuildConfig: Release

diff --git a/eng/pipelines/coreclr/gcstress-extra.yml b/eng/pipelines/coreclr/gcstress-extra.yml
index 577225e..aa99514 100644 (file)
--- a/eng/pipelines/coreclr/gcstress-extra.yml
+++ b/eng/pipelines/coreclr/gcstress-extra.yml
@@ -18,7 +18,7 @@ jobs:
 
 - template: /eng/pipelines/common/platform-matrix.yml
   parameters:
-    jobTemplate: /eng/pipelines/coreclr/templates/build-job.yml
+    jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml
     buildConfig: checked
     platformGroup: gcstress
     managedOsxBuild: false
@@ -35,3 +35,4 @@ jobs:
     managedOsxBuild: false
     jobParameters:
       testGroup: gcstress-extra
+      liveLibrariesBuildConfig: Release

diff --git a/eng/pipelines/coreclr/gcstress0x3-gcstress0xc.yml b/eng/pipelines/coreclr/gcstress0x3-gcstress0xc.yml
index 930d9e5..31bedbe 100644 (file)
--- a/eng/pipelines/coreclr/gcstress0x3-gcstress0xc.yml
+++ b/eng/pipelines/coreclr/gcstress0x3-gcstress0xc.yml
@@ -18,7 +18,7 @@ jobs:
 
 - template: /eng/pipelines/common/platform-matrix.yml
   parameters:
-    jobTemplate: /eng/pipelines/coreclr/templates/build-job.yml
+    jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml
     buildConfig: checked
     platformGroup: gcstress
     jobParameters:
@@ -33,3 +33,4 @@ jobs:
     helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml
     jobParameters:
       testGroup: gcstress0x3-gcstress0xc
+      liveLibrariesBuildConfig: Release

diff --git a/eng/pipelines/coreclr/jitstress-isas-arm.yml b/eng/pipelines/coreclr/jitstress-isas-arm.yml
index b816c07..038c34b 100644 (file)
--- a/eng/pipelines/coreclr/jitstress-isas-arm.yml
+++ b/eng/pipelines/coreclr/jitstress-isas-arm.yml
@@ -37,4 +37,4 @@ jobs:
     helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml
     jobParameters:
       testGroup: jitstress-isas-arm
-      useLiveLibrariesBuildConfig: Release
+      liveLibrariesBuildConfig: Release

diff --git a/eng/pipelines/coreclr/jitstress-isas-x86.yml b/eng/pipelines/coreclr/jitstress-isas-x86.yml
index f5eb3ae..52d977d 100644 (file)
--- a/eng/pipelines/coreclr/jitstress-isas-x86.yml
+++ b/eng/pipelines/coreclr/jitstress-isas-x86.yml
@@ -38,7 +38,7 @@ jobs:
     - Windows_NT_x86
     jobParameters:
       testGroup: jitstress-isas-x86
-      useLiveLibrariesBuildConfig: Release
+      liveLibrariesBuildConfig: Release
 
 - template: /eng/pipelines/common/platform-matrix.yml
   parameters:
@@ -53,4 +53,4 @@ jobs:
     helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml
     jobParameters:
       testGroup: jitstress-isas-x86
-      useLiveLibrariesBuildConfig: Release
+      liveLibrariesBuildConfig: Release

diff --git a/eng/pipelines/coreclr/jitstress.yml b/eng/pipelines/coreclr/jitstress.yml
index 6c57ef8..9cf88af 100644 (file)
--- a/eng/pipelines/coreclr/jitstress.yml
+++ b/eng/pipelines/coreclr/jitstress.yml
@@ -33,4 +33,4 @@ jobs:
     helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml
     jobParameters:
       testGroup: jitstress
-      useLiveLibrariesBuildConfig: Release
+      liveLibrariesBuildConfig: Release

diff --git a/eng/pipelines/coreclr/jitstress2-jitstressregs.yml b/eng/pipelines/coreclr/jitstress2-jitstressregs.yml
index 2fedc83..45af4d3 100644 (file)
--- a/eng/pipelines/coreclr/jitstress2-jitstressregs.yml
+++ b/eng/pipelines/coreclr/jitstress2-jitstressregs.yml
@@ -33,4 +33,4 @@ jobs:
     helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml
     jobParameters:
       testGroup: jitstress2-jitstressregs
-      useLiveLibrariesBuildConfig: Release
+      liveLibrariesBuildConfig: Release

diff --git a/eng/pipelines/coreclr/jitstressregs-x86.yml b/eng/pipelines/coreclr/jitstressregs-x86.yml
index 0972c10..a859cd4 100644 (file)
--- a/eng/pipelines/coreclr/jitstressregs-x86.yml
+++ b/eng/pipelines/coreclr/jitstressregs-x86.yml
@@ -40,4 +40,4 @@ jobs:
     helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml
     jobParameters:
       testGroup: jitstressregs-x86
-      useLiveLibrariesBuildConfig: Release
+      liveLibrariesBuildConfig: Release

diff --git a/eng/pipelines/coreclr/jitstressregs.yml b/eng/pipelines/coreclr/jitstressregs.yml
index 274f2a9..f01e456 100644 (file)
--- a/eng/pipelines/coreclr/jitstressregs.yml
+++ b/eng/pipelines/coreclr/jitstressregs.yml
@@ -31,7 +31,7 @@ jobs:
     platformGroup: all
     jobParameters:
       testGroup: jitstressregs
-      useLiveLibrariesBuildConfig: Release
+      liveLibrariesBuildConfig: Release
 
 - template: /eng/pipelines/common/platform-matrix.yml
   parameters:
@@ -42,4 +42,4 @@ jobs:
     helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml
     jobParameters:
       testGroup: jitstressregs
-      useLiveLibrariesBuildConfig: Release
+      liveLibrariesBuildConfig: Release

diff --git a/eng/pipelines/coreclr/perf.yml b/eng/pipelines/coreclr/perf.yml
index 9f3b8d5..aceeb78 100644 (file)
--- a/eng/pipelines/coreclr/perf.yml
+++ b/eng/pipelines/coreclr/perf.yml
@@ -32,7 +32,7 @@ jobs:
 
 - template: /eng/pipelines/common/platform-matrix.yml
   parameters:
-    jobTemplate: /eng/pipelines/coreclr/templates/build-job.yml
+    jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml
     buildConfig: release
     platforms:
     - Linux_x64
@@ -51,3 +51,4 @@ jobs:
     - Windows_NT_x86
     jobParameters:
       testGroup: perf
+      liveLibrariesBuildConfig: Release

diff --git a/eng/pipelines/coreclr/r2r-extra.yml b/eng/pipelines/coreclr/r2r-extra.yml
index 9e552dd..dc26e64 100644 (file)
--- a/eng/pipelines/coreclr/r2r-extra.yml
+++ b/eng/pipelines/coreclr/r2r-extra.yml
@@ -18,7 +18,7 @@ jobs:
 
 - template: /eng/pipelines/common/platform-matrix.yml
   parameters:
-    jobTemplate: /eng/pipelines/coreclr/templates/build-job.yml
+    jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml
     buildConfig: checked
     platformGroup: gcstress
     jobParameters:
@@ -36,3 +36,4 @@ jobs:
       testGroup: r2r-extra
       readyToRun: true
       displayNameArgs: R2R
+      liveLibrariesBuildConfig: Release

diff --git a/eng/pipelines/coreclr/r2r.yml b/eng/pipelines/coreclr/r2r.yml
index 3bacee5..41d5d9e 100644 (file)
--- a/eng/pipelines/coreclr/r2r.yml
+++ b/eng/pipelines/coreclr/r2r.yml
@@ -10,7 +10,7 @@ jobs:
 
 - template: /eng/pipelines/common/platform-matrix.yml
   parameters:
-    jobTemplate: /eng/pipelines/coreclr/templates/build-job.yml
+    jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml
     buildConfig: checked
     platforms:
     - Linux_arm
@@ -38,3 +38,4 @@ jobs:
       testGroup: outerloop
       readyToRun: true
       displayNameArgs: R2R
+      liveLibrariesBuildConfig: Release

diff --git a/eng/pipelines/coreclr/runincontext.yml b/eng/pipelines/coreclr/runincontext.yml
index 7d621c7..c7fd254 100644 (file)
--- a/eng/pipelines/coreclr/runincontext.yml
+++ b/eng/pipelines/coreclr/runincontext.yml
@@ -18,7 +18,7 @@ jobs:
 
 - template: /eng/pipelines/common/platform-matrix.yml
   parameters:
-    jobTemplate: /eng/pipelines/coreclr/templates/build-job.yml
+    jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml
     buildConfig: checked
     platforms:
     - Linux_x64
@@ -42,3 +42,4 @@ jobs:
       testGroup: outerloop
       runInUnloadableContext: true
       displayNameArgs: RunInContext
+      liveLibrariesBuildConfig: Release

diff --git a/eng/pipelines/coreclr/templates/perf-job.yml b/eng/pipelines/coreclr/templates/perf-job.yml
index cf02e25..0a7a49f 100644 (file)
--- a/eng/pipelines/coreclr/templates/perf-job.yml
+++ b/eng/pipelines/coreclr/templates/perf-job.yml
@@ -5,6 +5,7 @@ parameters:
   osSubgroup: ''
   container: ''
   framework: netcoreapp5.0 # Specify the appropriate framework when running release branches (ie netcoreapp3.0 for release/3.0)
+  liveLibrariesBuildConfig: ''
   variables: {}
   pool: ''
 
@@ -25,7 +26,10 @@ jobs:
     osGroup: ${{ parameters.osGroup }}
     osSubgroup: ${{ parameters.osSubgroup }}
     # Test job depends on the corresponding build job
-    dependsOn: ${{ format('coreclr_product_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }}
+    dependsOn:
+    - ${{ format('coreclr_product_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }}
+    - ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}:
+      - ${{ format('libraries_build_{0}_{1}{2}_{3}_{4}', 'netcoreapp', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveLibrariesBuildConfig) }}
 
     ${{ if eq(parameters.osGroup, 'Windows_NT') }}:
       extraSetupParameters: -CoreRootDirectory $(Build.SourcesDirectory)\artifacts\tests\coreclr\${{ parameters.osGroup }}.${{ parameters.archType }}.Release\Tests\Core_Root -Architecture ${{ parameters.archType }}
@@ -33,12 +37,24 @@ jobs:
       extraSetupParameters: --corerootdirectory $(Build.SourcesDirectory)/artifacts/tests/coreclr/${{ parameters.osGroup }}.${{ parameters.archType }}.Release/Tests/Core_Root --architecture ${{ parameters.archType }}
     
     variables: ${{ parameters.variables }}
+
     frameworks:
       - ${{ parameters.framework }}
     steps:
     # Extra steps that will be passed to the performance template and run before sending the job to helix (all of which is done in the template)
 
 
+    # Optionally download live-built libraries
+    - ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}:
+      - template: /eng/pipelines/common/download-artifact-step.yml
+        parameters:
+          unpackFolder: $(librariesDownloadDir)
+          cleanUnpackFolder: false
+          artifactFileName: '$(librariesBuildArtifactName)$(archiveExtension)'
+          artifactName: '$(librariesBuildArtifactName)'
+          displayName: 'live-built libraries'
+
+
     # Download product binaries directory
     - template: /eng/pipelines/common/download-artifact-step.yml
       parameters:
@@ -49,10 +65,5 @@ jobs:
 
 
     # Create Core_Root
-    - ${{ if ne(parameters.osGroup, 'Windows_NT') }}:
-      - script: $(coreClrRepoRootDir)build-test$(scriptExt) ${{ parameters.buildConfig }} ${{ parameters.archType }} generatelayoutonly
-        displayName: Create Core_Root
-    - ${{ if eq(parameters.osGroup, 'Windows_NT') }}:
-      # TODO: add generatelayoutonly to build-test.cmd.
-      - script: $(coreClrRepoRootDir)build-test$(scriptExt) ${{ parameters.buildConfig }} ${{ parameters.archType }} skipmanaged skipnative
-        displayName: Create Core_Root
\ No newline at end of file
+    - script: $(coreClrRepoRootDir)build-test$(scriptExt) $(buildConfig) $(archType) generatelayoutonly $(librariesOverrideArg)
+      displayName: Create Core_Root

diff --git a/eng/pipelines/coreclr/templates/run-performance-job.yml b/eng/pipelines/coreclr/templates/run-performance-job.yml
index a81c0ae..1ff044d 100644 (file)
--- a/eng/pipelines/coreclr/templates/run-performance-job.yml
+++ b/eng/pipelines/coreclr/templates/run-performance-job.yml
@@ -15,6 +15,7 @@ parameters:
   dependsOn: ''                   # optional -- dependencies of the job
   timeoutInMinutes: 320           # optional -- timeout for the job
   enableTelemetry: false          # optional -- enable for telemetry
+  liveLibrariesBuildConfig: ''    # optional -- live-live libraries configuration to use for the run
 
 jobs:
 - template: xplat-pipeline-job.yml
@@ -24,6 +25,7 @@ jobs:
     archType: ${{ parameters.archType }}
     osGroup: ${{ parameters.osGroup }}
     osSubgroup: ${{ parameters.osSubgroup }}
+    liveLibrariesBuildConfig: ${{ parameters.liveLibrariesBuildConfig }}
     enableTelemetry: ${{ parameters.enableTelemetry }}
     enablePublishBuildArtifacts: true
     continueOnError: ${{ parameters.continueOnError }}

diff --git a/eng/pipelines/coreclr/templates/test-job.yml b/eng/pipelines/coreclr/templates/test-job.yml
index 4180bf9..e167992 100644 (file)
--- a/eng/pipelines/coreclr/templates/test-job.yml
+++ b/eng/pipelines/coreclr/templates/test-job.yml
@@ -12,6 +12,7 @@ parameters:
   crossrootfsDir: ''
   # If true, run the corefx tests instead of the coreclr ones
   corefxTests: false
+  liveLibrariesBuildConfig: ''
   displayNameArgs: ''
   runInUnloadableContext: false
   condition: true
@@ -29,6 +30,7 @@ jobs:
   - template: /eng/pipelines/coreclr/templates/build-test-job.yml
     parameters:
       buildConfig: ${{ parameters.buildConfig }}
+      liveLibrariesBuildConfig: ${{ parameters.liveLibrariesBuildConfig }}
       archType: ${{ parameters.archType }}
       osGroup: ${{ parameters.managedTestBuildOsGroup }}
       osSubgroup: ${{ parameters.managedTestBuildOsSubgroup }}
@@ -43,6 +45,7 @@ jobs:
 - template: /eng/pipelines/coreclr/templates/run-test-job.yml
   parameters:
     buildConfig: ${{ parameters.buildConfig }}
+    liveLibrariesBuildConfig: ${{ parameters.liveLibrariesBuildConfig }}
     archType: ${{ parameters.archType }}
     osGroup: ${{ parameters.osGroup }}
     osSubgroup: ${{ parameters.osSubgroup }}