Use testGroup for defining test jobs (dotnet/coreclr#22608)

This adds a mapping between testGroup string and values of the following parameters:

* scenarios
* priority
* timeoutTotalInMinutes
* timeoutPerTestCollectionInMinutes
* timeoutPerTestInMinutes

and uses testGroup on a level of azure-pipeline.yml to set these values.

Commit migrated from https://github.com/dotnet/coreclr/commit/1216c9751f306033809f9699f6b33b7ce68bc499

diff --git a/eng/pipelines/coreclr/azure-pipelines.yml b/eng/pipelines/coreclr/azure-pipelines.yml
index 259b362..2029f7c 100644 (file)
--- a/eng/pipelines/coreclr/azure-pipelines.yml
+++ b/eng/pipelines/coreclr/azure-pipelines.yml
@@ -92,7 +92,7 @@ jobs:
 # https://github.com/Microsoft/azure-pipelines-yaml/pull/46 for more information
 
 #
-# Debug build (Pull Request)
+# Debug build (Pull request)
 #
 - ${{ if and(eq(variables['System.TeamProject'], 'public'), eq(variables['Build.Reason'], 'PullRequest'), eq(variables['Build.DefinitionName'], 'coreclr-ci')) }}:
   - template: eng/platform-matrix.yml
@@ -110,7 +110,7 @@ jobs:
       buildConfig: checked
 
 #
-# Release build (Pull Request)
+# Release build (Pull request)
 #
 - ${{ if and(eq(variables['System.TeamProject'], 'public'), eq(variables['Build.Reason'], 'PullRequest'), eq(variables['Build.DefinitionName'], 'coreclr-ci')) }}:
   - template: eng/platform-matrix.yml
@@ -119,7 +119,7 @@ jobs:
       buildConfig: release
 
 #
-# Release build (Official Build)
+# Release build (Official build)
 #
 - ${{ if and(eq(variables['System.TeamProject'], 'internal'), ne(variables['Build.Reason'], 'PullRequest')) }}:
   - template: eng/platform-matrix.yml
@@ -136,151 +136,66 @@ jobs:
 # Checked test builds
 #
 
-# Pri0 (Pull Request)
-- ${{ if and(eq(variables['System.TeamProject'], 'public'), eq(variables['Build.Reason'], 'PullRequest'), eq(variables['Build.DefinitionName'], 'coreclr-ci')) }}:
-  - template: eng/platform-matrix.yml
-    parameters:
-      jobTemplate: test-job.yml
-      buildConfig: checked
-      jobParameters:
-        priority: 0
-        scenarios:
-          asString: 'normal,no_tiered_compilation'
-          asArray:
-          - normal
-          - no_tiered_compilation
-        # "Total" includes building tests, waiting for a queue in Helix to become available, and running the tests.
-        # In case test-job.yml gets split into two separate jobs (e.g. build-test-job.yml and run-test.yml)
-        # this number should be adjusted accordingly.
-        timeoutTotalInMinutes: 240
-        # "PerCollection" is time needed for the "biggest" xUnit test collection to complete.
-        # In case xUnit test wrappers get refactored this number should also be adjusted.
-        timeoutPerTestCollectionInMinutes: 30
-        # "PerTest" corresponds to individual test running time (i.e. __TestTimeout).
-        timeoutPerTestInMinutes: 10
-
-# Pri1 (Pull Request, Outerloop)
-- ${{ if and(eq(variables['System.TeamProject'], 'public'), eq(variables['Build.Reason'], 'PullRequest'), eq(variables['Build.DefinitionName'], 'coreclr-outerloop')) }}:
+# Pull request
+- ${{ if and(eq(variables['System.TeamProject'], 'public'), eq(variables['Build.Reason'], 'PullRequest')) }}:
   - template: eng/platform-matrix.yml
     parameters:
       jobTemplate: test-job.yml
       buildConfig: checked
       jobParameters:
-        priority: 1
-        scenarios:
-          asString: 'normal,no_tiered_compilation'
-          asArray:
-          - normal
-          - no_tiered_compilation
-        timeoutTotalInMinutes: 360
-        timeoutPerTestCollectionInMinutes: 60
-        timeoutPerTestInMinutes: 10
-
-# Pri1 (CI)
+        ${{ if eq(variables['Build.DefinitionName'], 'coreclr-ci') }}:
+          testGroup: innerloop
+        ${{ if eq(variables['Build.DefinitionName'], 'coreclr-outerloop') }}:
+          testGroup: outerloop
+        ${{ if eq(variables['Build.DefinitionName'], 'coreclr-outerloop-jitminopts-jitstress1-jitstress2') }}:
+          testGroup: outerloop-jitminopts-jitstress1-jitstress2
+
+# CI
 - ${{ if and(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI')) }}:
   - template: eng/platform-matrix.yml
     parameters:
       jobTemplate: test-job.yml
       buildConfig: checked
       jobParameters:
-        priority: 1
-        scenarios:
-          asString: 'normal,no_tiered_compilation'
-          asArray:
-          - normal
-          - no_tiered_compilation
-        timeoutTotalInMinutes: 360
-        timeoutPerTestCollectionInMinutes: 60
-        timeoutPerTestInMinutes: 10
-
-# Pri1 ReadyToRun (CI)
-- ${{ if and(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI')) }}:
+        testGroup: outerloop
+
   - template: eng/platform-matrix.yml
     parameters:
       jobTemplate: test-job.yml
       buildConfig: checked
       jobParameters:
-        priority: 1
         readyToRun: true
-        scenarios:
-          asString: 'normal,no_tiered_compilation'
-          asArray:
-          - normal
-          - no_tiered_compilation
-        timeoutTotalInMinutes: 360
-        timeoutPerTestCollectionInMinutes: 60
-        timeoutPerTestInMinutes: 10
-
-# Pri1 (Schedule, Manual)
+        testGroup: outerloop
+
+# Schedule or Manual
 - ${{ if and(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'Schedule', 'Manual')) }}:
   - template: eng/platform-matrix.yml
     parameters:
       jobTemplate: test-job.yml
       buildConfig: checked
       jobParameters:
-        priority: 1
-        scenarios:
-          asString: 'normal,no_tiered_compilation,jitstress1,jitstress2,jitstress1_tiered,jitstress2_tiered,jitstressregs1,jitstressregs2,jitstressregs3,jitstressregs4,jitstressregs8,jitstressregs0x10,jitstressregs0x80,jitstressregs0x1000,jitminopts'
-          asArray:
-          - normal
-          - no_tiered_compilation
-          - jitstress1
-          - jitstress2
-          - jitstress1_tiered
-          - jitstress2_tiered
-          - jitstressregs1
-          - jitstressregs2
-          - jitstressregs3
-          - jitstressregs4
-          - jitstressregs8
-          - jitstressregs0x10
-          - jitstressregs0x80
-          - jitstressregs0x1000
-          - jitminopts
-        timeoutTotalInMinutes: 480
-        # TODO: Adjust this number as soon as we have more data on how long it takes to run these jobs in Helix.
-        timeoutPerTestCollectionInMinutes: 120
-        timeoutPerTestInMinutes: 30
-
+        testGroup: outerloop
 
 #
 # Release test builds
 #
 
-# Pri1 (Official Build)
+# Official build
 - ${{ if and(eq(variables['System.TeamProject'], 'internal'), ne(variables['Build.Reason'], 'PullRequest')) }}:
   - template: eng/platform-matrix.yml
     parameters:
       jobTemplate: test-job.yml
       buildConfig: release
       jobParameters:
-        priority: 1
-        scenarios:
-          asString: 'normal,no_tiered_compilation'
-          asArray:
-          - normal
-          - no_tiered_compilation
-        timeoutTotalInMinutes: 360
-        timeoutPerTestCollectionInMinutes: 60
-        timeoutPerTestInMinutes: 10
-
-# Pri1 ReadyToRun (Official Build)
-- ${{ if and(eq(variables['System.TeamProject'], 'internal'), ne(variables['Build.Reason'], 'PullRequest')) }}:
+        testGroup: outerloop
+
   - template: eng/platform-matrix.yml
     parameters:
       jobTemplate: test-job.yml
       buildConfig: release
       jobParameters:
-        priority: 1
+        testGroup: outerloop
         readyToRun: true
-        scenarios:
-          asString: 'normal,no_tiered_compilation'
-          asArray:
-          - normal
-          - no_tiered_compilation
-        timeoutTotalInMinutes: 360
-        timeoutPerTestCollectionInMinutes: 60
-        timeoutPerTestInMinutes: 10
 
 
 # Publish build information to Build Assets Registry

diff --git a/eng/send-to-helix-step.yml b/eng/send-to-helix-step.yml
index 802cc8c..cba652d 100644 (file)
--- a/eng/send-to-helix-step.yml
+++ b/eng/send-to-helix-step.yml
@@ -31,10 +31,10 @@ steps:
       _HelixAccessToken: ${{ parameters.helixAccessToken }}
       _HelixBuild: ${{ parameters.helixBuild }}
       _HelixSource: ${{ parameters.helixSource }}
-      _HelixTargetQueues: ${{ parameters.helixQueues }}
+      _HelixTargetQueues: ${{ parameters.helixQueues.asString }}
       _HelixType: ${{ parameters.helixType }}
       _RunCrossGen: ${{ parameters.runCrossGen }}
-      _Scenarios: ${{ parameters.scenarios }}
+      _Scenarios: ${{ parameters.scenarios.asString }}
       _TimeoutPerTestCollectionInMinutes: ${{ parameters.timeoutPerTestCollectionInMinutes }}
       _TimeoutPerTestInMinutes: ${{ parameters.timeoutPerTestInMinutes }}
       ${{ if eq(parameters.publishTestResults, 'true') }}:
@@ -54,10 +54,10 @@ steps:
       _HelixAccessToken: ${{ parameters.helixAccessToken }}
       _HelixBuild: ${{ parameters.helixBuild }}
       _HelixSource: ${{ parameters.helixSource }}
-      _HelixTargetQueues: ${{ parameters.helixQueues }}
+      _HelixTargetQueues: ${{ parameters.helixQueues.asString }}
       _HelixType: ${{ parameters.helixType }}
       _RunCrossGen: ${{ parameters.runCrossGen }}
-      _Scenarios: ${{ parameters.scenarios }}
+      _Scenarios: ${{ parameters.scenarios.asString }}
       _TimeoutPerTestCollectionInMinutes: ${{ parameters.timeoutPerTestCollectionInMinutes }}
       _TimeoutPerTestInMinutes: ${{ parameters.timeoutPerTestInMinutes }}
       ${{ if eq(parameters.publishTestResults, 'true') }}:

diff --git a/eng/test-job.yml b/eng/test-job.yml
index 483ad08..478a75e 100644 (file)
--- a/eng/test-job.yml
+++ b/eng/test-job.yml
@@ -3,13 +3,9 @@ parameters:
   archType: ''
   osGroup: ''
   osIdentifier: ''
-  priority: 0
+  testGroup: ''
   readyToRun: false
-  scenarios: ''
   helixQueues: ''
-  timeoutTotalInMinutes: ''
-  timeoutPerTestCollectionInMinutes: ''
-  timeoutPerTestInMinutes: ''
   crossrootfsDir: ''
 
 ### Test job
@@ -26,35 +22,40 @@ jobs:
     osIdentifier: ${{ parameters.osIdentifier }}
 
     # Compute job name from template parameters
-    ${{ if eq(parameters.readyToRun, false) }}:
-      name: ${{ format('testbuild_pri{0}_{1}_{2}_{3}', parameters.priority, parameters.osIdentifier, parameters.archType, parameters.buildConfig) }}
-      displayName: ${{ format('Test Pri{0} {1} {2} {3}', parameters.priority, parameters.osIdentifier, parameters.archType, parameters.buildConfig) }}
-    ${{ if eq(parameters.readyToRun, true) }}:
-      name: ${{ format('testbuild_pri{0}_r2r_{1}_{2}_{3}', parameters.priority, parameters.osIdentifier, parameters.archType, parameters.buildConfig) }}
-      displayName: ${{ format('Test Pri{0} R2R {1} {2} {3}', parameters.priority, parameters.osIdentifier, parameters.archType, parameters.buildConfig) }}
+    ${{ if and(eq(parameters.testGroup, 'innerloop'), eq(parameters.readyToRun, false)) }}:
+      name: ${{ format('testbuild_pri0_{0}_{1}_{2}', parameters.osIdentifier, parameters.archType, parameters.buildConfig) }}
+      displayName: ${{ format('Test Pri0 {0} {1} {2}', parameters.osIdentifier, parameters.archType, parameters.buildConfig) }}
+    ${{ if and(eq(parameters.testGroup, 'innerloop'), eq(parameters.readyToRun, true)) }}:
+      name: ${{ format('testbuild_pri0_r2r_{0}_{1}_{2}', parameters.osIdentifier, parameters.archType, parameters.buildConfig) }}
+      displayName: ${{ format('Test Pri0 R2R {0} {1} {2}', parameters.osIdentifier, parameters.archType, parameters.buildConfig) }}
+
+    ${{ if and(ne(parameters.testGroup, 'innerloop'), eq(parameters.readyToRun, false)) }}:
+      name: ${{ format('testbuild_pri1_{0}_{1}_{2}', parameters.osIdentifier, parameters.archType, parameters.buildConfig) }}
+      displayName: ${{ format('Test Pri1 {0} {1} {2}', parameters.osIdentifier, parameters.archType, parameters.buildConfig) }}
+    ${{ if and(ne(parameters.testGroup, 'innerloop'), eq(parameters.readyToRun, true)) }}:
+      name: ${{ format('testbuild_pri1_r2r_{0}_{1}_{2}', parameters.osIdentifier, parameters.archType, parameters.buildConfig) }}
+      displayName: ${{ format('Test Pri1 R2R {0} {1} {2}', parameters.osIdentifier, parameters.archType, parameters.buildConfig) }}
 
     crossrootfsDir: ${{ parameters.crossrootfsDir }}
 
     variables:
     - group: DotNet-HelixApi-Access
     # Map template parameters to command line arguments
-    - ${{ if eq(parameters.priority, '1') }}:
+    - name: priorityArg
+      value: ''
+    - ${{ if ne(parameters.testGroup, 'innerloop') }}:
       - ${{ if ne(parameters.osGroup, 'Windows_NT') }}:
         - name: priorityArg
           value: 'priority1'
       - ${{ if eq(parameters.osGroup, 'Windows_NT') }}:
         - name: priorityArg
           value: '-priority=1'
-    - ${{ if eq(parameters.priority, '0') }}:
-      - name: priorityArg
-        value: ''
 
+    - name: crossgenArg
+      value: ''
     - ${{ if eq(parameters.readyToRun, true) }}:
       - name: crossgenArg
         value: 'crossgen'
-    - ${{ if eq(parameters.readyToRun, false) }}:
-      - name: crossgenArg
-        value: ''
 
     - name: clangArg
       value: ''
@@ -83,7 +84,15 @@ jobs:
     # Note that the containers are resources defined in azure-pipelines.yml
     containerName: ${{ parameters.containerName }}
 
-    timeoutInMinutes: ${{ parameters.timeoutTotalInMinutes }}
+    # "Total" means building tests, waiting for a queue in Helix to become available, and running the tests.
+    # In case test-job.yml gets split into two separate jobs (e.g. build-test-job.yml and run-test.yml)
+    # this number should be adjusted accordingly.
+    ${{ if eq(parameters.testGroup, 'innerloop') }}:
+      timeoutInMinutes: 240
+    ${{ if eq(parameters.testGroup, 'outerloop') }}:
+      timeoutInMinutes: 360
+    ${{ if eq(parameters.testGroup, 'outerloop-jitminopts-jitstress1-jitstress2') }}:
+      timeoutInMinutes: 480
 
     steps:
 
@@ -139,18 +148,30 @@ jobs:
           helixSource: ci/dotnet/coreclr/$(Build.SourceBranch)
 
         ${{ if eq(parameters.readyToRun, false) }}:
-          helixType: ${{ format('test/functional/cli/pri{0}', parameters.priority) }}
+          helixType: 'test/functional/cli/'
         ${{ if eq(parameters.readyToRun, true) }}:
-          helixType: ${{ format('test/functional/r2r/cli/pri{0}', parameters.priority) }}
+          helixType: 'test/functional/r2r/cli/'
 
-        helixQueues: ${{ parameters.helixQueues.asString }}
+        helixQueues: ${{ parameters.helixQueues }}
 
         ${{ if eq(parameters.helixQueues.asString, '') }}:
           condition: false
 
         publishTestResults: true
-        timeoutPerTestCollectionInMinutes: ${{ parameters.timeoutPerTestCollectionInMinutes }}
-        timeoutPerTestInMinutes: ${{ parameters.timeoutPerTestInMinutes }}
+
+        ${{ if eq(parameters.testGroup, 'innerloop') }}:
+          # "PerCollection" is time needed for the "biggest" xUnit test collection to complete.
+          # In case xUnit test wrappers get refactored this number should also be adjusted.
+          timeoutPerTestCollectionInMinutes: 30
+          # "PerTest" corresponds to individual test running time (i.e. __TestTimeout).
+          timeoutPerTestInMinutes: 10
+        ${{ if eq(parameters.testGroup, 'outerloop') }}:
+          timeoutPerTestCollectionInMinutes: 60
+          timeoutPerTestInMinutes: 10
+        ${{ if eq(parameters.testGroup, 'outerloop-jitminopts-jitstress1-jitstress2') }}:
+          # TODO: Adjust this number as soon as we have more data on how long it takes to run these jobs in Helix.
+          timeoutPerTestCollectionInMinutes: 120
+          timeoutPerTestInMinutes: 30
 
         runCrossGen: ${{ parameters.readyToRun }}
 
@@ -159,4 +180,18 @@ jobs:
           # DotNet-HelixApi-Access variable group
           helixAccessToken: $(HelixApiAccessToken)
 
-        scenarios: ${{ parameters.scenarios.asString }}
+        ${{ if in(parameters.testGroup, 'innerloop', 'outerloop') }}:
+          scenarios:
+            asString: 'normal,no_tiered_compilation'
+            asArray:
+            - normal
+            - no_tiered_compilation
+        ${{ if eq(parameters.testGroup, 'outerloop-jitminopts-jitstress1-jitstress2') }}:
+          scenarios:
+            asString: 'jitminopts,jitstress1,jitstress1_tiered,jitstress2,jitstress2_tiered'
+            asArray:
+            - jitminopts
+            - jitstress1
+            - jitstress1_tiered
+            - jitstress2
+            - jitstress2_tiered