Running and visualizing tests in AzureDevOps (Part 1) (#21502)

* Enable Azure Pipelines reporter so the test results can be seen at one place (i.e. "Tests" tab) - TestRunNamePrefix should encode all the needed information for a test run to be identifiable.

* Scenarios property should use comma (not semi-colon) to work correctly on non-Windows.

* Specify timeout to test jobs (the numbers are taken from netci.groovy) - we would adjust them as soon as we get this running in each PR.

* Specify timeout for Helix work items (the number is taken from buildpipeline) - same as above.

* Specify helixType and helixRepo properties to the job and use _HelixType and _HelixSource variables in MSBuild project instead of defining them manually.

*  Limit Pri0 Checked test jobs to running against pull requests. This also should be limited in the future to "default" build definition name - so when we have PR workload selection feature working we can distinguish between default PR validation and user triggered jobs.

* Remove "Prepare tests for helix" step - instead of archiving Core_Root and tests folders ourselves - directly specify them as correlation payload and work item payload directories and let the archiving be handled by Helix Sdk.

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index 92de971..8957e23 100644 (file)
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -110,27 +110,28 @@ jobs:
 # Checked test builds
 #
 
-# Pri0 (CI)
-- ${{ if eq(variables['System.TeamProject'], 'public') }}:
+# Pri0 (PullRequest)
+- ${{ if and(eq(variables['System.TeamProject'], 'public'), eq(variables['Build.Reason'], 'PullRequest')) }}:
   - template: eng/platform-matrix.yml
     parameters:
       jobTemplate: test-job.yml
       buildConfig: checked
       jobParameters:
         priority: 0
+        timeoutInMinutes: 120
 
 # Pri1 (CI)
-- ${{ if eq(variables['System.TeamProject'], 'public') }}:
+- ${{ if and(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI')) }}:
   - template: eng/platform-matrix.yml
     parameters:
       jobTemplate: test-job.yml
       buildConfig: checked
       jobParameters:
         priority: 1
-        scenarios: 'normal;jitstress2'
+        timeoutInMinutes: 240
 
 # Pri1 crossgen (CI)
-- ${{ if eq(variables['System.TeamProject'], 'public') }}:
+- ${{ if and(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI')) }}:
   - template: eng/platform-matrix.yml
     parameters:
       jobTemplate: test-job.yml
@@ -138,14 +139,25 @@ jobs:
       jobParameters:
         priority: 1
         crossgen: true
-        scenarios: 'normal;jitstress2'
+        timeoutInMinutes: 240
+
+# Pri1 (Manual)
+- ${{ if and(eq(variables['System.TeamProject'], 'public'), eq(variables['Build.Reason'], 'Manual')) }}:
+  - template: eng/platform-matrix.yml
+    parameters:
+      jobTemplate: test-job.yml
+      buildConfig: checked
+      jobParameters:
+        priority: 1
+        scenarios: 'normal,jitstress1,jitstress2,gcstress0x3,gcstress0xc'
+        timeoutInMinutes: 300
 
 #
 # Release test builds (Official Build)
 #
 
 # Pri1
-- ${{ if ne(variables['System.TeamProject'], 'public') }}:
+- ${{ if and(ne(variables['System.TeamProject'], 'public'), ne(variables['Build.Reason'], 'PullRequest')) }}:
   - template: eng/platform-matrix.yml
     parameters:
       jobTemplate: test-job.yml
@@ -154,7 +166,7 @@ jobs:
         priority: 1
 
 # Pri1 crossgen (Official Build)
-- ${{ if eq(variables['System.TeamProject'], 'public') }}:
+- ${{ if and(ne(variables['System.TeamProject'], 'public'), ne(variables['Build.Reason'], 'PullRequest')) }}:
   - template: eng/platform-matrix.yml
     parameters:
       jobTemplate: test-job.yml

diff --git a/eng/build-job.yml b/eng/build-job.yml
index 3a5425f..98710b4 100644 (file)
--- a/eng/build-job.yml
+++ b/eng/build-job.yml
@@ -4,6 +4,7 @@ parameters:
   osGroup: ''
   osGroupName: ''
   containerName: ''
+  crossrootfsDir: ''
 
 ### Product build
 jobs:
@@ -21,6 +22,7 @@ jobs:
     # Run all steps in the container.
     # Note that the containers are resources defined in azure-pipelines.yml
     containerName: ${{ parameters.containerName }}
+    crossrootfsDir: ${{ parameters.crossrootfsDir }}
 
     steps:
 
@@ -36,14 +38,8 @@ jobs:
         displayName: Install native dependencies
 
     # Build
-    - ${{ if and(and(ne(parameters.archType, 'arm'), ne(parameters.archType, 'arm64')), ne(parameters.osGroup, 'Windows_NT')) }}:
-      - script: ./build.sh $(buildConfig) $(archType) -skiptests
-        displayName: Build product
-    - ${{ if and(ne(parameters.osGroup, 'Windows_NT'), eq(parameters.archType, 'arm')) }}:
-      - script: ROOTFS_DIR=$(rootfsDir) CAC_ROOTFS_DIR=$(cacRootfsDir) ./build.sh $(buildConfig) $(archType) -cross -skiptests
-        displayName: Build product
-    - ${{ if and(ne(parameters.osGroup, 'Windows_NT'), eq(parameters.archType, 'arm64')) }}:
-      - script: ROOTFS_DIR=$(rootfsDir) ./build.sh $(buildConfig) $(archType) -cross -skiptests
+    - ${{ if ne(parameters.osGroup, 'Windows_NT') }}:
+      - script: ./build.sh $(buildConfig) $(archType) $(crossArg) -skiptests 
         displayName: Build product
     - ${{ if eq(parameters.osGroup, 'Windows_NT') }}:
       # TODO: IBCOptimize? EnforcePGO? pass an OfficialBuildId? SignType? file logging parameters?

diff --git a/eng/platform-matrix.yml b/eng/platform-matrix.yml
index 6d84bbb..647bd9c 100644 (file)
--- a/eng/platform-matrix.yml
+++ b/eng/platform-matrix.yml
@@ -17,6 +17,7 @@ jobs:
     osGroupName: Linux
     containerName: ubuntu_1404_arm_cross_build_image
     helixQueuesPublic: 'Ubuntu.1404.Arm32.Open'
+    crossrootfsDir: '/crossrootfs/arm'
     # Currently we don't have Linux/arm32 internal Helix queues
     ${{ insert }}: ${{ parameters.jobParameters }}
 
@@ -28,8 +29,11 @@ jobs:
     osGroup: Linux
     osGroupName: Linux
     containerName: ubuntu_1604_arm64_cross_build_image
-    helixQueuesPublic: 'Debian.9.Arm64.Open,Ubuntu.1604.Arm64.Open,Ubuntu.1804.Arm64.Open'
-    helixQueuesInternal: 'Debian.9.Arm64,Ubuntu.1604.Arm64,Ubuntu.1804.Arm64'
+    # TODO: enable Debian.9.Arm64.Open and Debian.9.Arm64 queues
+    # when https://github.com/dotnet/core-eng/issues/4805 is resolved
+    helixQueuesPublic: 'Ubuntu.1604.Arm64.Open,Ubuntu.1804.Arm64.Open'
+    helixQueuesInternal: 'Ubuntu.1604.Arm64,Ubuntu.1804.Arm64'
+    crossrootfsDir: '/crossrootfs/arm64'
     ${{ insert }}: ${{ parameters.jobParameters }}
 
 # Linux musl
@@ -40,8 +44,7 @@ jobs:
     osGroup: Linux
     osGroupName: Linux_musl
     containerName: musl_x64_build_image
-    helixQueuesPublic: 'Alpine.3.Arm64.Open'
-    helixQueuesInternal: 'Alpine.3.Arm64'
+    # TODO: add Alpine.Amd64 queues
     ${{ insert }}: ${{ parameters.jobParameters }}
 
 # RHEL 6
@@ -77,8 +80,8 @@ jobs:
     osGroup: Linux
     osGroupName: Linux
     containerName: ubuntu_1604_x64_build_image
-    helixQueuesPublic: 'Debian.8.Amd64.Open,Ubuntu.1604.Amd64.Open,Ubuntu.1804.Arm64.Open'
-    helixQueuesInternal: 'Debian.8.Amd64,Ubuntu.1604.Amd64,Ubuntu.1804.Arm64'
+    helixQueuesPublic: 'Debian.8.Amd64.Open,Ubuntu.1604.Amd64.Open,Ubuntu.1804.Amd64.Open'
+    helixQueuesInternal: 'Debian.8.Amd64,Ubuntu.1604.Amd64,Ubuntu.1804.Amd64'
     ${{ insert }}: ${{ parameters.jobParameters }}
 
 # macOS x64

diff --git a/eng/test-job.yml b/eng/test-job.yml
index 72b37f6..218f8ab 100644 (file)
--- a/eng/test-job.yml
+++ b/eng/test-job.yml
@@ -8,6 +8,8 @@ parameters:
   scenarios: ''
   helixQueuesPublic: ''
   helixQueuesInternal: ''
+  timeoutInMinutes: ''
+  crossrootfsDir: ''
 
 ### Test job
 
@@ -25,11 +27,18 @@ jobs:
     # Compute job name from template parameters
     ${{ if eq(parameters.crossgen, 'false') }}:
         name: ${{ format('testbuild_pri{0}_{1}_{2}_{3}', parameters.priority, parameters.osGroupName, parameters.archType, parameters.buildConfig) }}
-        displayName: ${{ format('Test pri{0} {1} {2} {3}', parameters.priority, parameters.osGroupName, parameters.archType, parameters.buildConfig) }}
+        displayName: ${{ format('Test Pri{0} {1} {2} {3}', parameters.priority, parameters.osGroupName, parameters.archType, parameters.buildConfig) }}
     ${{ if eq(parameters.crossgen, 'true') }}:
         name: ${{ format('testbuild_pri{0}_r2r_{1}_{2}_{3}', parameters.priority, parameters.osGroupName, parameters.archType, parameters.buildConfig) }}
         displayName: ${{ format('Test Pri{0} R2R {1} {2} {3}', parameters.priority, parameters.osGroupName, parameters.archType, parameters.buildConfig) }}
 
+    ${{ if eq(parameters.crossgen, 'false') }}:
+        helixType: ${{ format('test/functional/cli/pri{0}', parameters.priority) }}
+    ${{ if eq(parameters.crossgen, 'true') }}:
+        helixType: ${{ format('test/functional/r2r/cli/pri{0}', parameters.priority) }}
+
+    crossrootfsDir: ${{ parameters.crossrootfsDir }}
+
     variables:
       # Map template parameters to command line arguments
       - ${{ if eq(parameters.priority, '1') }}:
@@ -61,6 +70,8 @@ jobs:
       - ${{ if ne(variables['System.TeamProject'], 'public') }}:
         - name: helixTargetQueuesArg
           value: ${{ format('/p:HelixTargetQueues=\"{0}\"', parameters.helixQueuesInternal) }}
+      - name: commonMSBuildArgs
+        value: ${{ format('/maxcpucount /p:__BuildOS={0} /p:__BuildArch={1} /p:__BuildType={2}', parameters.osGroup, parameters.archType, parameters.buildConfig) }}
 
     # TODO: Enable crossgen in build-test.sh. It currently doesn't
     # accept a crossgen arg, so disable the macos/linux crossgen test
@@ -75,6 +86,8 @@ jobs:
     # Note that the containers are resources defined in azure-pipelines.yml
     containerName: ${{ parameters.containerName }}
 
+    timeoutInMinutes: ${{ parameters.timeoutInMinutes }}
+
     steps:
 
     # Install test build dependencies
@@ -99,29 +112,18 @@ jobs:
 
       
     # Build tests
-    - ${{ if or(eq(parameters.osGroup, 'Linux'), eq(parameters.osGroup, 'OSX')) }}:
-        # TODO: enable crossgen in build-test.sh
-      - script: ./build-test.sh $(buildConfig) $(archType) $(priorityArg) $(crossgenArg)
+    # TODO: enable crossgen in build-test.sh
+    - ${{ if ne(parameters.osGroup, 'Windows_NT') }}:
+      - script: ./build-test.sh $(buildConfig) $(archType) $(crossArg) $(priorityArg) $(crossgenArg)
         displayName: Build tests
     - ${{ if eq(parameters.osGroup, 'Windows_NT') }}:
       - script: build-test.cmd $(buildConfig) $(archType) $(priorityArg) $(crossgenArg)
         displayName: Build tests
 
 
-    # Prepare tests for helix
-    - ${{ if or(eq(parameters.osGroup, 'Linux'), eq(parameters.osGroup, 'OSX')) }}:
-      - script: ./Tools/dotnetcli/dotnet msbuild tests/helixprep.proj /p:CORE_ROOT=$(Build.SourcesDirectory)/bin/tests/$(osGroup).$(archType).$(buildConfigUpper)/Tests/Core_Root /p:__BuildType=$(buildConfig) /p:__BuildArch=$(archType) /p:UsePython=true
-        # TODO: remove UsePython argument once we've removed generated wrappers in helixprep.proj
-        displayName: Prepare test archives for Helix
-    - ${{ if eq(parameters.osGroup, 'Windows_NT') }}:
-        # TODO: remove UsePython argument once we've removed generated wrappers in helixprep.proj
-      - script: .\Tools\dotnetcli\dotnet.exe msbuild tests\helixprep.proj /p:CORE_ROOT=$(Build.SourcesDirectory)\bin\tests\Windows_NT.$(archType).$(buildConfigUpper)\tests\core_root /p:__BuildType=$(buildConfig) /p:__BuildArch=$(archType) /p:UsePython=true
-        displayName: Prepare test archives for Helix
-
-
     # Send tests to helix
     - ${{ if or(eq(parameters.osGroup, 'Linux'), eq(parameters.osGroup, 'OSX')) }}:
-      - script: ./Tools/dotnetcli/dotnet msbuild tests/helixpublishwitharcade.proj /maxcpucount $(scenariosArg) $(helixTargetQueuesArg)
+      - script: ./Tools/dotnetcli/dotnet msbuild tests/helixpublishwitharcade.proj $(commonMSBuildArgs) $(scenariosArg) $(helixTargetQueuesArg)
         displayName: Send test jobs to Helix
         env:
           ${{ if eq(variables['System.TeamProject'], 'internal') }}:
@@ -130,8 +132,9 @@ jobs:
           ${{ if eq(variables['System.TeamProject'], 'public') }}:
             # Access token variable for public project
             HelixAccessToken: $(BotAccount-dotnet-github-anon-kaonashi-bot-helix-token)
+          SYSTEM_ACCESSTOKEN: $(System.AccessToken)
     - ${{ if eq(parameters.osGroup, 'Windows_NT') }}:
-      - script: .\Tools\dotnetcli\dotnet msbuild tests\helixpublishwitharcade.proj /maxcpucount $(scenariosArg) $(helixTargetQueuesArg)
+      - script: .\Tools\dotnetcli\dotnet msbuild tests\helixpublishwitharcade.proj $(commonMSBuildArgs) $(scenariosArg) $(helixTargetQueuesArg)
         displayName: Send test jobs to Helix
         env:
           ${{ if eq(variables['System.TeamProject'], 'internal') }}:
@@ -140,3 +143,4 @@ jobs:
           ${{ if eq(variables['System.TeamProject'], 'public') }}:
             # Access token variable for public project
             HelixAccessToken: $(BotAccount-dotnet-github-anon-kaonashi-bot-helix-token)
+          SYSTEM_ACCESSTOKEN: $(System.AccessToken)

diff --git a/eng/xplat-job.yml b/eng/xplat-job.yml
index d9d45d1..db8cf3b 100644 (file)
--- a/eng/xplat-job.yml
+++ b/eng/xplat-job.yml
@@ -8,6 +8,9 @@ parameters:
   condition: ''
   dependsOn: ''
   containerName: ''
+  timeoutInMinutes: ''
+  helixType: ''
+  crossrootfsDir: ''
   variables: {} ## any extra variables to add to the defaults defined below
 
 jobs:
@@ -18,8 +21,11 @@ jobs:
     displayName: ${{ parameters.displayName }}
 
     condition: ${{ parameters.condition }}
-
     dependsOn: ${{ parameters.dependsOn }}
+    timeoutInMinutes: ${{ parameters.timeoutInMinutes }}
+
+    helixRepo: 'dotnet/coreclr'
+    helixType: ${{ parameters.helixType }}
 
     pool:
       ${{ if and(eq(parameters.osGroup, 'Linux'), eq(variables['System.TeamProject'], 'public')) }}:
@@ -34,7 +40,6 @@ jobs:
         name: dotnet-internal-temp
       ${{ if and(eq(parameters.osGroup, 'Windows_NT'), eq(variables['System.TeamProject'], 'public')) }}:
         name: dotnet-external-temp
-      timeoutInMinutes: 240
 
     workspace:
       clean: all
@@ -74,17 +79,15 @@ jobs:
       - name: osGroupName
         value: ${{ parameters.osGroupName }}
       
-      # Crossbuild specific variables.
-      - ${{ if eq(parameters.archType, 'arm') }}:
-          - name: rootfsDir
-            value: /crossrootfs/arm
-
-          - name: cacRootfsDir
-            value: /crossrootfs/x86
-
-      - ${{ if eq(parameters.archType, 'arm64') }}:
-          - name: rootfsDir
-            value: /crossrootfs/arm64
+      - ${{ if ne(parameters.crossrootfsDir, '') }}:
+          - name: crossArg
+            value: 'cross'
+          # This is only required for cross builds.
+          - name: ROOTFS_DIR
+            value: ${{ parameters.crossrootfsDir }}
+      - ${{ if eq(parameters.crossrootfsDir, '') }}:
+          - name: crossArg
+            value: ''
 
       - ${{ each variable in parameters.variables }}:
         - ${{insert}}: ${{ variable }}

diff --git a/tests/helixpublishwitharcade.proj b/tests/helixpublishwitharcade.proj
index d479969..df0724a 100644 (file)
--- a/tests/helixpublishwitharcade.proj
+++ b/tests/helixpublishwitharcade.proj
@@ -7,25 +7,38 @@
   <Import Project="..\dir.props" />
 
   <PropertyGroup>
-    <!-- TODO: pick appropriate helix source and type. -->
-    <HelixSource>pr/coreclr/master</HelixSource>
-    <HelixType>test/stuff</HelixType>
+    <HelixSource>$(_HelixSource)</HelixSource>
+    <HelixType>$(_HelixType)</HelixType>
     <HelixBuild>$(BUILD_BUILDNUMBER)</HelixBuild>
 
-    <EnableXUnitReporter>true</EnableXUnitReporter>
+    <EnableAzurePipelinesReporter Condition=" '$(EnableAzurePipelinesReporter)' != 'false' ">true</EnableAzurePipelinesReporter>
     <WaitForWorkItemCompletion>true</WaitForWorkItemCompletion>
-    <SourceDirectory>$(MSBuildProjectDirectory)/..</SourceDirectory>
+    <TimeoutInSeconds Condition=" '$(TimeoutInSeconds)' == '' ">1000</TimeoutInSeconds>
+    <CoreRootDirectory>$(TestWorkingDir)\Tests\Core_Root</CoreRootDirectory>
+  </PropertyGroup>
+
+  <PropertyGroup Condition=" '$(TargetsWindows)' == 'true' ">
+    <HelixPythonPath>%HELIX_PYTHONPATH%</HelixPythonPath>
+    <RunnerScript>%HELIX_CORRELATION_PAYLOAD%\runtest_helix.py</RunnerScript>
+  </PropertyGroup>
+
+  <PropertyGroup Condition=" '$(TargetsWindows)' != 'true' ">
+    <HelixPythonPath>$HELIX_PYTHONPATH</HelixPythonPath>
+    <RunnerScript>$HELIX_CORRELATION_PAYLOAD/runtest_helix.py</RunnerScript>
   </PropertyGroup>
 
   <ItemGroup>
-    <HelixCorrelationPayload Include="$(SourceDirectory)\bin\tests\*\archive\Core_Root\*.zip">
-      <PayloadArchive>%(Identity)</PayloadArchive>
-    </HelixCorrelationPayload>
+    <HelixCorrelationPayload Include="$(CoreRootDirectory)" />
+    <XUnitWrapperDlls Include="$(TestWorkingDir)\**\*.XUnitWrapper.dll" />
   </ItemGroup>
 
+  <Target Name="CopyRunnerScript" BeforeTargets="SubmitTestsToHelix">
+    <Copy SourceFiles="$(MSBuildThisFileDirectory)\runtest_helix.py" DestinationFolder="$(CoreRootDirectory)" />
+  </Target>
+
   <Target Name="SubmitTestsToHelix">
     <ItemGroup>
-      <Scenarios Include="$(Scenarios)" />
+      <Scenarios Include="$(Scenarios.Split(','))" />
       <ProjectsToBuild Include="$(MSBuildProjectFile)">
         <Properties>Scenario=%(Scenarios.Identity)</Properties>
       </ProjectsToBuild>
@@ -39,8 +52,7 @@
     <MSBuild Projects="@(ProjectsToBuild)" Targets="Test" BuildInParallel="$(BuildInParallel)" StopOnFirstFailure="false" UnloadProjectsOnCompletion="true" />
   </Target>
 
-  <Target Name="BuildHelixWorkItem"
-          BeforeTargets="Test">
+  <Target Name="BuildHelixWorkItem" BeforeTargets="Test">
     <PropertyGroup>
       <!-- The "normal" scenario is just a way to include the default
            (empty) scenario when specifying multiple scenarios at
@@ -48,18 +60,17 @@
            job names will not have a scenario prefix and the runtest
            script doesn't have to define a "normal" scenario. -->
       <Scenario Condition=" '$(Scenario)' == 'normal' "></Scenario>
-      <ScenarioPrefix Condition=" '$(Scenario)' == '' " />
-      <ScenarioPrefix Condition=" '$(Scenario)' != '' ">$(Scenario) </ScenarioPrefix>
+      <TestRunNamePrefix Condition=" '$(Scenario)' == '' ">$(BuildOS) $(BuildArch) $(BuildType) @ </TestRunNamePrefix>
+      <TestRunNamePrefix Condition=" '$(Scenario)' != '' ">$(BuildOS) $(BuildArch) $(BuildType) $(Scenario) @ </TestRunNamePrefix>
     </PropertyGroup>
     <ItemGroup>
-      <TestZipFiles Include="$(SourceDirectory)\bin\tests\*\archive\tests\*.zip" />
-      <HelixWorkItem Include="@(TestZipFiles->'$(ScenarioPrefix)%(FileName)')" >
-        <PayloadArchive>%(Identity)</PayloadArchive>
-        <Command Condition=" '$(Scenario)' == '' ">python runtest_helix.py -wrapper %(FileName).dll</Command>
-        <Command Condition=" '$(Scenario)' != '' ">python runtest_helix.py -scenario $(Scenario) -wrapper %(FileName).dll</Command>
+      <HelixWorkItem Include="@(XUnitWrapperDlls->'%(FileName)'->Replace('.XUnitWrapper', ''))" >
+        <PayloadDirectory>%(RootDir)%(Directory)</PayloadDirectory>
+        <Command Condition=" '$(Scenario)' == '' ">$(HelixPythonPath) $(RunnerScript) -wrapper %(FileName)%(Extension)</Command>
+        <Command Condition=" '$(Scenario)' != '' ">$(HelixPythonPath) $(RunnerScript) -wrapper %(FileName)%(Extension) -scenario $(Scenario)</Command>
+        <Timeout>$([System.TimeSpan]::FromSeconds($(TimeoutInSeconds)))</Timeout>
       </HelixWorkItem>
     </ItemGroup>
-
   </Target>
 
 </Project>