Allow run Outerloop tests only (dotnet/corefx#37766)

* Allow run Outerloop tests only

* Rename to testScope

* Manual darc update from build '20190517.7'

Commit migrated from https://github.com/dotnet/corefx/commit/963480a7a8f8cf59e3be8f9bfe6ba87623d04088

diff --git a/eng/pipelines/libraries/corefx-base.yml b/eng/pipelines/libraries/corefx-base.yml
index ee8ef95..0cb15e7 100644 (file)
--- a/eng/pipelines/libraries/corefx-base.yml
+++ b/eng/pipelines/libraries/corefx-base.yml
@@ -22,8 +22,8 @@ parameters:
   #     _framework: (netcoreapp, netfx, uap, etc).
   #     _helixQueues: Windows.Amd64 (Only needed if submitToHelix -> true.) -- Queues should be separated by + if multiple.
 
-  # Required: as part of the variables object, the following values need to be passed:
-  #     _outerloop: true | false
+  # Required: empty | innerloop | outerloop | all
+  testScope: ''
 
   # Optional: _publishTests -> Boolean -> Publish test assets to blob storage if true.
   # Default: false
@@ -75,6 +75,10 @@ jobs:
         - _msbuildCommonParameters: ''
         - _archiveTestsParameter: ''
         - _finalFrameworkArg: -framework $(_framework)
+        - _testScopeArg: ''
+
+        - ${{ if ne(parameters.testScope, '') }}:
+          - _testScopeArg: -testScope ${{ parameters.testScope }}
 
         - ${{ if ne(job._jobFramework, '')}}:
           - _finalFrameworkArg: ${{ job._jobFramework }}
@@ -148,7 +152,7 @@ jobs:
                     -build
                     -buildtests
                     $(_commonArguments)
-                    /p:OuterLoop=$(_outerloop)
+                    $(_testScopeArg)
                     $(_msbuildCommonParameters)
               displayName: Build Sources and Tests
 
@@ -195,7 +199,7 @@ jobs:
                 helixQueues: $(_helixQueues)
                 msbuildScript: $(_msbuildCommand)
                 framework: $(_framework)
-                outerloop: $(_outerloop)
+                testScope: ${{ parameters.testScope }}
 
                 ${{ if eq(parameters.isOfficialBuild, 'true') }}:
                   officialBuildId: $(Build.BuildNumber)

diff --git a/eng/pipelines/libraries/freebsd.yml b/eng/pipelines/libraries/freebsd.yml
index dc66aa4..5c18391 100644 (file)
--- a/eng/pipelines/libraries/freebsd.yml
+++ b/eng/pipelines/libraries/freebsd.yml
@@ -6,14 +6,15 @@ parameters:
   # Required: value to specify if the full test matrix should be tested
   #   Default: false
   fullMatrix: false
-  # Required: value to specify if the build is comming from an outerloop pipeline.
-  #   Default: false
-  isOuterloopBuild: false
+  # Optional: value to scope the tests.
+  #   Default: empty
+  testScope: ''
 
 jobs:
 - template: corefx-base.yml
   parameters:
     isOfficialBuild: ${{ parameters.isOfficialBuild }}
+    testScope: ${{ parameters.testScope }}
     targetOS: FreeBSD
     jobs:
 
@@ -31,6 +32,3 @@ jobs:
 
       buildScriptPrefix: $(_buildScriptPrefix)
       submitToHelix: false
-
-      variables:
-        - _outerloop: ${{ parameters.isOuterloopBuild }}

diff --git a/eng/pipelines/libraries/helix.yml b/eng/pipelines/libraries/helix.yml
index 68073f8..b1f0c74 100644 (file)
--- a/eng/pipelines/libraries/helix.yml
+++ b/eng/pipelines/libraries/helix.yml
@@ -8,7 +8,7 @@ parameters:
   msbuildScript: ''
   targetOS: ''
   officialBuildId: ''
-  outerloop: '' # true | false
+  testScope: '' # empty | innerloop | outerloop | all
   condition: always()
 
 steps:
@@ -18,7 +18,7 @@ steps:
             /p:ArchGroup=${{ parameters.archGroup }}
             /p:ConfigurationGroup=${{ parameters.configuration }}
             /p:OSGroup=${{ parameters.targetOS }}
-            /p:OuterLoop=${{ parameters.outerloop }}
+            /p:testScope=${{ parameters.testScope }}
             /p:TargetGroup=${{ parameters.framework }}
             /p:HelixTargetQueues=${{ parameters.helixQueues }}
             /p:HelixBuild=$(Build.BuildNumber)

diff --git a/eng/pipelines/libraries/linux.yml b/eng/pipelines/libraries/linux.yml
index 0e42c2d..7197b64 100644 (file)
--- a/eng/pipelines/libraries/linux.yml
+++ b/eng/pipelines/libraries/linux.yml
@@ -6,14 +6,15 @@ parameters:
   # Required: value to specify if the full test matrix should be tested
   #   Default: false
   fullMatrix: false
-  # Required: value to specify if the build is comming from an outerloop pipeline.
-  #   Default: false
-  isOuterloopBuild: false
+  # Optional: value to scope the tests.
+  #   Default: empty
+  testScope: ''
 
 jobs:
 - template: corefx-base.yml
   parameters:
     isOfficialBuild: ${{ parameters.isOfficialBuild }}
+    testScope: ${{ parameters.testScope }}
     targetOS: Linux
     jobs:
 
@@ -50,7 +51,7 @@ jobs:
              _buildScriptPrefix: ''
              _buildExtraArguments: /p:RuntimeOS=linux-musl
 
-          ${{ if eq(parameters.isOuterloopBuild, 'true') }}:
+          ${{ if or(eq(parameters.testScope, 'outerloop'), eq(parameters.testScope, 'all')) }}:
             x64_Debug:
               _BuildConfig: Debug
               _architecture: x64
@@ -106,7 +107,6 @@ jobs:
       submitToHelix: true
 
       variables:
-        - _outerloop: ${{ parameters.isOuterloopBuild }}
         - linuxArm64Queues: \(Ubuntu.1604.Arm64.Open\)Ubuntu.1604.Arm64.Docker.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-16.04-helix-arm64v8-b049512-20190321153539
 
         - ${{ if eq(parameters.fullMatrix, 'false') }}:
@@ -121,7 +121,7 @@ jobs:
 
     # Legs without helix testing
     # Only run this leg in PRs.
-    - ${{ if and(eq(parameters.isOfficialBuild, 'false'), eq(parameters.isOuterloopBuild, 'false')) }}:
+    - ${{ if and(eq(parameters.isOfficialBuild, 'false'), and(ne(parameters.testScope, 'outerloop'), ne(parameters.testScope, 'all'))) }}:
       - job: LinuxNoTest
         displayName: Linux
         strategy:
@@ -149,6 +149,3 @@ jobs:
         buildExtraArguments: $(_buildExtraArguments)
         buildScriptPrefix: $(_buildScriptPrefix)
         submitToHelix: false
-
-        variables:
-          - _outerloop: ${{ parameters.isOuterloopBuild }}

diff --git a/eng/pipelines/libraries/macos.yml b/eng/pipelines/libraries/macos.yml
index d88b0de..779b9a2 100644 (file)
--- a/eng/pipelines/libraries/macos.yml
+++ b/eng/pipelines/libraries/macos.yml
@@ -6,14 +6,15 @@ parameters:
   # Required: value to specify if the full test matrix should be tested
   #   Default: false
   fullMatrix: false
-  # Required: value to specify if the build is comming from an outerloop pipeline.
-  #   Default: false
-  isOuterloopBuild: false
+  # Optional: value to scope the tests.
+  #   Default: empty
+  testScope: ''
 
 jobs:
 - template: corefx-base.yml
   parameters:
     isOfficialBuild: ${{ parameters.isOfficialBuild }}
+    testScope: ${{ parameters.testScope }}
     targetOS: OSX
     jobs:
 
@@ -51,8 +52,6 @@ jobs:
       submitToHelix: true
 
       variables:
-        - _outerloop: ${{ parameters.isOuterloopBuild }}
-        
         - ${{ if eq(parameters.isOfficialBuild, 'false') }}:
           - macOSQueues: OSX.1012.Amd64.Open+OSX.1013.Amd64.Open
 

diff --git a/eng/pipelines/libraries/outerloop.yml b/eng/pipelines/libraries/outerloop.yml
index edc82b2..3dc88a7 100644 (file)
--- a/eng/pipelines/libraries/outerloop.yml
+++ b/eng/pipelines/libraries/outerloop.yml
@@ -27,7 +27,7 @@ jobs:
       parameters:
         isOfficialBuild: ${{ and(ne(variables['System.TeamProject'], 'public'), notIn(variables['Build.Reason'], 'PullRequest')) }}
         fullMatrix: ${{ notIn(variables['Build.Reason'], 'PullRequest') }}
-        isOuterloopBuild: true
+        testScope: outerloop
 
   # Linux outerloop legs
   - ${{ if or(endsWith(variables['Build.DefinitionName'], 'linux'), endsWith(variables['Build.DefinitionName'], 'outerloop')) }}:
@@ -35,7 +35,7 @@ jobs:
       parameters:
         isOfficialBuild: ${{ and(ne(variables['System.TeamProject'], 'public'), notIn(variables['Build.Reason'], 'PullRequest')) }}
         fullMatrix: ${{ notIn(variables['Build.Reason'], 'PullRequest') }}
-        isOuterloopBuild: true
+        testScope: outerloop
 
   # MacOS outerloop legs
   - ${{ if or(endsWith(variables['Build.DefinitionName'], 'osx'), endsWith(variables['Build.DefinitionName'], 'outerloop')) }}:
@@ -43,4 +43,4 @@ jobs:
       parameters:
         isOfficialBuild: ${{ and(ne(variables['System.TeamProject'], 'public'), notIn(variables['Build.Reason'], 'PullRequest')) }}
         fullMatrix: ${{ notIn(variables['Build.Reason'], 'PullRequest') }}
-        isOuterloopBuild: true
\ No newline at end of file
+        testScope: outerloop
\ No newline at end of file

diff --git a/eng/pipelines/libraries/redhat6.yml b/eng/pipelines/libraries/redhat6.yml
index c7ba361..af7f717 100644 (file)
--- a/eng/pipelines/libraries/redhat6.yml
+++ b/eng/pipelines/libraries/redhat6.yml
@@ -6,14 +6,15 @@ parameters:
   # Required: value to specify if the full test matrix should be tested
   #   Default: false
   fullMatrix: false
-  # Required: value to specify if the build is comming from an outerloop pipeline.
-  #   Default: false
-  isOuterloopBuild: false
+  # Optional: value to scope the tests.
+  #   Default: empty
+  testScope: ''
 
 jobs:
 - template: corefx-base.yml
   parameters:
     isOfficialBuild: ${{ parameters.isOfficialBuild }}
+    testScope: ${{ parameters.testScope }}
     targetOS: Linux
     jobs:
 
@@ -36,5 +37,4 @@ jobs:
       submitToHelix: true
 
       variables:
-        - _outerloop: ${{ parameters.isOuterloopBuild }}
         - redhatHelixQueue: RedHat.6.Amd64.Open

diff --git a/eng/pipelines/libraries/windows.yml b/eng/pipelines/libraries/windows.yml
index 8be4808..30cffd3 100644 (file)
--- a/eng/pipelines/libraries/windows.yml
+++ b/eng/pipelines/libraries/windows.yml
@@ -6,15 +6,16 @@ parameters:
   # Required: value to specify if the full test matrix should be tested
   #   Default: false
   fullMatrix: false
-  # Required: value to specify if the build is comming from an outerloop pipeline.
-  #   Default: false
-  isOuterloopBuild: false
+  # Optional: value to scope the tests.
+  #   Default: empty
+  testScope: ''
 
 jobs:
 
 - template: corefx-base.yml
   parameters:
     isOfficialBuild: ${{ parameters.isOfficialBuild }}
+    testScope: ${{ parameters.testScope }}
     targetOS: Windows_NT
     jobs:
 
@@ -51,7 +52,7 @@ jobs:
               _helixQueues: $(uapNetfxQueues)
 
           # Run arm on the matrix or in Outerloop
-          ${{ if or(eq(parameters.fullMatrix, 'true'), eq(parameters.isOuterloopBuild, 'true')) }}:
+          ${{ if or(eq(parameters.fullMatrix, 'true'), or(eq(parameters.testScope, 'outerloop'), eq(parameters.testScope, 'all'))) }}:
             arm64_Release:
               _BuildConfig: Release
               _architecture: arm64
@@ -108,8 +109,6 @@ jobs:
       buildExtraArguments: /p:RuntimeOS=win10
 
       variables:
-        - _outerloop: ${{ parameters.isOuterloopBuild }}
-
         - nanoQueues: "`(Windows.Nano.1803.Amd64.Open`)windows.10.amd64.serverrs4.open@mcr.microsoft.com/dotnet-buildtools/prereqs:nanoserver-1803-helix-amd64-05227e1-20190509225944"
         - uapNetfxQueues: Windows.10.Amd64.ClientRS5.Open
         - windowsArmQueue: Windows.10.Arm64.Open
@@ -121,7 +120,7 @@ jobs:
           - netcoreappWindowsQueues: Windows.7.Amd64.Open+Windows.81.Amd64.Open+Windows.10.Amd64.ClientRS4.Open+Windows.10.Amd64.ServerRS5.Open          
 
     # There is no point of running legs without outerloop tests, when in an outerloop build.
-    - ${{ if eq(parameters.isOuterloopBuild, 'false') }}:
+    - ${{ if and(ne(parameters.testScope, 'outerloop'), ne(parameters.testScope, 'all')) }}:
       # Packaging all configurations
       - job: AllConfigurations
         displayName: Packaging All Configurations
@@ -150,7 +149,6 @@ jobs:
         buildExtraArguments: /p:RuntimeOS=win10
 
         variables:
-          - _outerloop: false
           - allConfigurationsQueues: Windows.10.Amd64.ClientRS5.Open
 
         _jobFramework: -allConfigurations
@@ -194,6 +192,3 @@ jobs:
 
         submitToHelix: false
         buildExtraArguments: /p:RuntimeOS=win10
-
-        variables:
-          - _outerloop: ${{ parameters.isOuterloopBuild }}

diff --git a/eng/sendtohelix.proj b/eng/sendtohelix.proj
index 8117525..54ab043 100644 (file)
--- a/eng/sendtohelix.proj
+++ b/eng/sendtohelix.proj
@@ -2,7 +2,6 @@
   <PropertyGroup>
     <!-- Set helix source -->
     <HelixSourcePrefix>pr/</HelixSourcePrefix>
-    <HelixSourcePrefix Condition="'$(OfficialBuildId)' != ''">official/</HelixSourcePrefix>
     <HelixSource Condition="'$(HelixSource)' == ''">$(HelixSourcePrefix)dotnet/corefx</HelixSource>
     <HelixSource Condition="'$(BUILD_SOURCEBRANCH)' != ''">$(HelixSource)/$(BUILD_SOURCEBRANCH)</HelixSource>
     
@@ -37,11 +36,9 @@
 
   <PropertyGroup Condition="'$(HelixType)' == ''">
     <!-- For PRs we want helixtype to be the same for all frameworks except package testing-->
-    <HelixType>test/functional/cli/</HelixType>
-    <HelixType Condition="'$(Outerloop)' == 'true' AND '$(OfficialBuildId)' == ''">$(HelixType)/outerloop/</HelixType>
+    <TestFilter Condition="'$(TestFilter)' == ''">innerloop</TestFilter>
+    <HelixType>test/functional/cli/$(TestFilter)/</HelixType>
     <HelixType Condition="'$(TargetGroup)' == 'AllConfigurations'">test/functional/packaging/</HelixType>
-    <HelixType Condition="'$(TargetGroup)' == 'netfx' AND '$(OfficialBuildId)' != ''">test/functional/desktop/cli/</HelixType>
-    <HelixType Condition="'$(TargetGroup)' == 'uap' AND '$(OfficialBuildId)' != ''">test/functional/uwp/</HelixType>
   </PropertyGroup>
 
   <PropertyGroup Condition="'$(HelixCommand)' == '' and '$(TargetGroup)' == 'AllConfigurations'">