* Enable Azure Pipelines reporter so the test results can be seen at one place (i.e. "Tests" tab) - TestRunNamePrefix should encode all the needed information for a test run to be identifiable.
* Scenarios property should use comma (not semi-colon) to work correctly on non-Windows.
* Specify timeout to test jobs (the numbers are taken from netci.groovy) - we would adjust them as soon as we get this running in each PR.
* Specify timeout for Helix work items (the number is taken from buildpipeline) - same as above.
* Specify helixType and helixRepo properties to the job and use _HelixType and _HelixSource variables in MSBuild project instead of defining them manually.
* Limit Pri0 Checked test jobs to running against pull requests. This also should be limited in the future to "default" build definition name - so when we have PR workload selection feature working we can distinguish between default PR validation and user triggered jobs.
* Remove "Prepare tests for helix" step - instead of archiving Core_Root and tests folders ourselves - directly specify them as correlation payload and work item payload directories and let the archiving be handled by Helix Sdk.
# Checked test builds
#
-# Pri0 (CI)
-- ${{ if eq(variables['System.TeamProject'], 'public') }}:
+# Pri0 (PullRequest)
+- ${{ if and(eq(variables['System.TeamProject'], 'public'), eq(variables['Build.Reason'], 'PullRequest')) }}:
- template: eng/platform-matrix.yml
parameters:
jobTemplate: test-job.yml
buildConfig: checked
jobParameters:
priority: 0
+ timeoutInMinutes: 120
# Pri1 (CI)
-- ${{ if eq(variables['System.TeamProject'], 'public') }}:
+- ${{ if and(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI')) }}:
- template: eng/platform-matrix.yml
parameters:
jobTemplate: test-job.yml
buildConfig: checked
jobParameters:
priority: 1
- scenarios: 'normal;jitstress2'
+ timeoutInMinutes: 240
# Pri1 crossgen (CI)
-- ${{ if eq(variables['System.TeamProject'], 'public') }}:
+- ${{ if and(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI')) }}:
- template: eng/platform-matrix.yml
parameters:
jobTemplate: test-job.yml
jobParameters:
priority: 1
crossgen: true
- scenarios: 'normal;jitstress2'
+ timeoutInMinutes: 240
+
+# Pri1 (Manual)
+- ${{ if and(eq(variables['System.TeamProject'], 'public'), eq(variables['Build.Reason'], 'Manual')) }}:
+ - template: eng/platform-matrix.yml
+ parameters:
+ jobTemplate: test-job.yml
+ buildConfig: checked
+ jobParameters:
+ priority: 1
+ scenarios: 'normal,jitstress1,jitstress2,gcstress0x3,gcstress0xc'
+ timeoutInMinutes: 300
#
# Release test builds (Official Build)
#
# Pri1
-- ${{ if ne(variables['System.TeamProject'], 'public') }}:
+- ${{ if and(ne(variables['System.TeamProject'], 'public'), ne(variables['Build.Reason'], 'PullRequest')) }}:
- template: eng/platform-matrix.yml
parameters:
jobTemplate: test-job.yml
priority: 1
# Pri1 crossgen (Official Build)
-- ${{ if eq(variables['System.TeamProject'], 'public') }}:
+- ${{ if and(ne(variables['System.TeamProject'], 'public'), ne(variables['Build.Reason'], 'PullRequest')) }}:
- template: eng/platform-matrix.yml
parameters:
jobTemplate: test-job.yml
osGroup: ''
osGroupName: ''
containerName: ''
+ crossrootfsDir: ''
### Product build
jobs:
# Run all steps in the container.
# Note that the containers are resources defined in azure-pipelines.yml
containerName: ${{ parameters.containerName }}
+ crossrootfsDir: ${{ parameters.crossrootfsDir }}
steps:
displayName: Install native dependencies
# Build
- - ${{ if and(and(ne(parameters.archType, 'arm'), ne(parameters.archType, 'arm64')), ne(parameters.osGroup, 'Windows_NT')) }}:
- - script: ./build.sh $(buildConfig) $(archType) -skiptests
- displayName: Build product
- - ${{ if and(ne(parameters.osGroup, 'Windows_NT'), eq(parameters.archType, 'arm')) }}:
- - script: ROOTFS_DIR=$(rootfsDir) CAC_ROOTFS_DIR=$(cacRootfsDir) ./build.sh $(buildConfig) $(archType) -cross -skiptests
- displayName: Build product
- - ${{ if and(ne(parameters.osGroup, 'Windows_NT'), eq(parameters.archType, 'arm64')) }}:
- - script: ROOTFS_DIR=$(rootfsDir) ./build.sh $(buildConfig) $(archType) -cross -skiptests
+ - ${{ if ne(parameters.osGroup, 'Windows_NT') }}:
+ - script: ./build.sh $(buildConfig) $(archType) $(crossArg) -skiptests
displayName: Build product
- ${{ if eq(parameters.osGroup, 'Windows_NT') }}:
# TODO: IBCOptimize? EnforcePGO? pass an OfficialBuildId? SignType? file logging parameters?
osGroupName: Linux
containerName: ubuntu_1404_arm_cross_build_image
helixQueuesPublic: 'Ubuntu.1404.Arm32.Open'
+ crossrootfsDir: '/crossrootfs/arm'
# Currently we don't have Linux/arm32 internal Helix queues
${{ insert }}: ${{ parameters.jobParameters }}
osGroup: Linux
osGroupName: Linux
containerName: ubuntu_1604_arm64_cross_build_image
- helixQueuesPublic: 'Debian.9.Arm64.Open,Ubuntu.1604.Arm64.Open,Ubuntu.1804.Arm64.Open'
- helixQueuesInternal: 'Debian.9.Arm64,Ubuntu.1604.Arm64,Ubuntu.1804.Arm64'
+ # TODO: enable Debian.9.Arm64.Open and Debian.9.Arm64 queues
+ # when https://github.com/dotnet/core-eng/issues/4805 is resolved
+ helixQueuesPublic: 'Ubuntu.1604.Arm64.Open,Ubuntu.1804.Arm64.Open'
+ helixQueuesInternal: 'Ubuntu.1604.Arm64,Ubuntu.1804.Arm64'
+ crossrootfsDir: '/crossrootfs/arm64'
${{ insert }}: ${{ parameters.jobParameters }}
# Linux musl
osGroup: Linux
osGroupName: Linux_musl
containerName: musl_x64_build_image
- helixQueuesPublic: 'Alpine.3.Arm64.Open'
- helixQueuesInternal: 'Alpine.3.Arm64'
+ # TODO: add Alpine.Amd64 queues
${{ insert }}: ${{ parameters.jobParameters }}
# RHEL 6
osGroup: Linux
osGroupName: Linux
containerName: ubuntu_1604_x64_build_image
- helixQueuesPublic: 'Debian.8.Amd64.Open,Ubuntu.1604.Amd64.Open,Ubuntu.1804.Arm64.Open'
- helixQueuesInternal: 'Debian.8.Amd64,Ubuntu.1604.Amd64,Ubuntu.1804.Arm64'
+ helixQueuesPublic: 'Debian.8.Amd64.Open,Ubuntu.1604.Amd64.Open,Ubuntu.1804.Amd64.Open'
+ helixQueuesInternal: 'Debian.8.Amd64,Ubuntu.1604.Amd64,Ubuntu.1804.Amd64'
${{ insert }}: ${{ parameters.jobParameters }}
# macOS x64
scenarios: ''
helixQueuesPublic: ''
helixQueuesInternal: ''
+ timeoutInMinutes: ''
+ crossrootfsDir: ''
### Test job
# Compute job name from template parameters
${{ if eq(parameters.crossgen, 'false') }}:
name: ${{ format('testbuild_pri{0}_{1}_{2}_{3}', parameters.priority, parameters.osGroupName, parameters.archType, parameters.buildConfig) }}
- displayName: ${{ format('Test pri{0} {1} {2} {3}', parameters.priority, parameters.osGroupName, parameters.archType, parameters.buildConfig) }}
+ displayName: ${{ format('Test Pri{0} {1} {2} {3}', parameters.priority, parameters.osGroupName, parameters.archType, parameters.buildConfig) }}
${{ if eq(parameters.crossgen, 'true') }}:
name: ${{ format('testbuild_pri{0}_r2r_{1}_{2}_{3}', parameters.priority, parameters.osGroupName, parameters.archType, parameters.buildConfig) }}
displayName: ${{ format('Test Pri{0} R2R {1} {2} {3}', parameters.priority, parameters.osGroupName, parameters.archType, parameters.buildConfig) }}
+ ${{ if eq(parameters.crossgen, 'false') }}:
+ helixType: ${{ format('test/functional/cli/pri{0}', parameters.priority) }}
+ ${{ if eq(parameters.crossgen, 'true') }}:
+ helixType: ${{ format('test/functional/r2r/cli/pri{0}', parameters.priority) }}
+
+ crossrootfsDir: ${{ parameters.crossrootfsDir }}
+
variables:
# Map template parameters to command line arguments
- ${{ if eq(parameters.priority, '1') }}:
- ${{ if ne(variables['System.TeamProject'], 'public') }}:
- name: helixTargetQueuesArg
value: ${{ format('/p:HelixTargetQueues=\"{0}\"', parameters.helixQueuesInternal) }}
+ - name: commonMSBuildArgs
+ value: ${{ format('/maxcpucount /p:__BuildOS={0} /p:__BuildArch={1} /p:__BuildType={2}', parameters.osGroup, parameters.archType, parameters.buildConfig) }}
# TODO: Enable crossgen in build-test.sh. It currently doesn't
# accept a crossgen arg, so disable the macos/linux crossgen test
# Note that the containers are resources defined in azure-pipelines.yml
containerName: ${{ parameters.containerName }}
+ timeoutInMinutes: ${{ parameters.timeoutInMinutes }}
+
steps:
# Install test build dependencies
# Build tests
- - ${{ if or(eq(parameters.osGroup, 'Linux'), eq(parameters.osGroup, 'OSX')) }}:
- # TODO: enable crossgen in build-test.sh
- - script: ./build-test.sh $(buildConfig) $(archType) $(priorityArg) $(crossgenArg)
+ # TODO: enable crossgen in build-test.sh
+ - ${{ if ne(parameters.osGroup, 'Windows_NT') }}:
+ - script: ./build-test.sh $(buildConfig) $(archType) $(crossArg) $(priorityArg) $(crossgenArg)
displayName: Build tests
- ${{ if eq(parameters.osGroup, 'Windows_NT') }}:
- script: build-test.cmd $(buildConfig) $(archType) $(priorityArg) $(crossgenArg)
displayName: Build tests
- # Prepare tests for helix
- - ${{ if or(eq(parameters.osGroup, 'Linux'), eq(parameters.osGroup, 'OSX')) }}:
- - script: ./Tools/dotnetcli/dotnet msbuild tests/helixprep.proj /p:CORE_ROOT=$(Build.SourcesDirectory)/bin/tests/$(osGroup).$(archType).$(buildConfigUpper)/Tests/Core_Root /p:__BuildType=$(buildConfig) /p:__BuildArch=$(archType) /p:UsePython=true
- # TODO: remove UsePython argument once we've removed generated wrappers in helixprep.proj
- displayName: Prepare test archives for Helix
- - ${{ if eq(parameters.osGroup, 'Windows_NT') }}:
- # TODO: remove UsePython argument once we've removed generated wrappers in helixprep.proj
- - script: .\Tools\dotnetcli\dotnet.exe msbuild tests\helixprep.proj /p:CORE_ROOT=$(Build.SourcesDirectory)\bin\tests\Windows_NT.$(archType).$(buildConfigUpper)\tests\core_root /p:__BuildType=$(buildConfig) /p:__BuildArch=$(archType) /p:UsePython=true
- displayName: Prepare test archives for Helix
-
-
# Send tests to helix
- ${{ if or(eq(parameters.osGroup, 'Linux'), eq(parameters.osGroup, 'OSX')) }}:
- - script: ./Tools/dotnetcli/dotnet msbuild tests/helixpublishwitharcade.proj /maxcpucount $(scenariosArg) $(helixTargetQueuesArg)
+ - script: ./Tools/dotnetcli/dotnet msbuild tests/helixpublishwitharcade.proj $(commonMSBuildArgs) $(scenariosArg) $(helixTargetQueuesArg)
displayName: Send test jobs to Helix
env:
${{ if eq(variables['System.TeamProject'], 'internal') }}:
${{ if eq(variables['System.TeamProject'], 'public') }}:
# Access token variable for public project
HelixAccessToken: $(BotAccount-dotnet-github-anon-kaonashi-bot-helix-token)
+ SYSTEM_ACCESSTOKEN: $(System.AccessToken)
- ${{ if eq(parameters.osGroup, 'Windows_NT') }}:
- - script: .\Tools\dotnetcli\dotnet msbuild tests\helixpublishwitharcade.proj /maxcpucount $(scenariosArg) $(helixTargetQueuesArg)
+ - script: .\Tools\dotnetcli\dotnet msbuild tests\helixpublishwitharcade.proj $(commonMSBuildArgs) $(scenariosArg) $(helixTargetQueuesArg)
displayName: Send test jobs to Helix
env:
${{ if eq(variables['System.TeamProject'], 'internal') }}:
${{ if eq(variables['System.TeamProject'], 'public') }}:
# Access token variable for public project
HelixAccessToken: $(BotAccount-dotnet-github-anon-kaonashi-bot-helix-token)
+ SYSTEM_ACCESSTOKEN: $(System.AccessToken)
condition: ''
dependsOn: ''
containerName: ''
+ timeoutInMinutes: ''
+ helixType: ''
+ crossrootfsDir: ''
variables: {} ## any extra variables to add to the defaults defined below
jobs:
displayName: ${{ parameters.displayName }}
condition: ${{ parameters.condition }}
-
dependsOn: ${{ parameters.dependsOn }}
+ timeoutInMinutes: ${{ parameters.timeoutInMinutes }}
+
+ helixRepo: 'dotnet/coreclr'
+ helixType: ${{ parameters.helixType }}
pool:
${{ if and(eq(parameters.osGroup, 'Linux'), eq(variables['System.TeamProject'], 'public')) }}:
name: dotnet-internal-temp
${{ if and(eq(parameters.osGroup, 'Windows_NT'), eq(variables['System.TeamProject'], 'public')) }}:
name: dotnet-external-temp
- timeoutInMinutes: 240
workspace:
clean: all
- name: osGroupName
value: ${{ parameters.osGroupName }}
- # Crossbuild specific variables.
- - ${{ if eq(parameters.archType, 'arm') }}:
- - name: rootfsDir
- value: /crossrootfs/arm
-
- - name: cacRootfsDir
- value: /crossrootfs/x86
-
- - ${{ if eq(parameters.archType, 'arm64') }}:
- - name: rootfsDir
- value: /crossrootfs/arm64
+ - ${{ if ne(parameters.crossrootfsDir, '') }}:
+ - name: crossArg
+ value: 'cross'
+ # This is only required for cross builds.
+ - name: ROOTFS_DIR
+ value: ${{ parameters.crossrootfsDir }}
+ - ${{ if eq(parameters.crossrootfsDir, '') }}:
+ - name: crossArg
+ value: ''
- ${{ each variable in parameters.variables }}:
- ${{insert}}: ${{ variable }}
<Import Project="..\dir.props" />
<PropertyGroup>
- <!-- TODO: pick appropriate helix source and type. -->
- <HelixSource>pr/coreclr/master</HelixSource>
- <HelixType>test/stuff</HelixType>
+ <HelixSource>$(_HelixSource)</HelixSource>
+ <HelixType>$(_HelixType)</HelixType>
<HelixBuild>$(BUILD_BUILDNUMBER)</HelixBuild>
- <EnableXUnitReporter>true</EnableXUnitReporter>
+ <EnableAzurePipelinesReporter Condition=" '$(EnableAzurePipelinesReporter)' != 'false' ">true</EnableAzurePipelinesReporter>
<WaitForWorkItemCompletion>true</WaitForWorkItemCompletion>
- <SourceDirectory>$(MSBuildProjectDirectory)/..</SourceDirectory>
+ <TimeoutInSeconds Condition=" '$(TimeoutInSeconds)' == '' ">1000</TimeoutInSeconds>
+ <CoreRootDirectory>$(TestWorkingDir)\Tests\Core_Root</CoreRootDirectory>
+ </PropertyGroup>
+
+ <PropertyGroup Condition=" '$(TargetsWindows)' == 'true' ">
+ <HelixPythonPath>%HELIX_PYTHONPATH%</HelixPythonPath>
+ <RunnerScript>%HELIX_CORRELATION_PAYLOAD%\runtest_helix.py</RunnerScript>
+ </PropertyGroup>
+
+ <PropertyGroup Condition=" '$(TargetsWindows)' != 'true' ">
+ <HelixPythonPath>$HELIX_PYTHONPATH</HelixPythonPath>
+ <RunnerScript>$HELIX_CORRELATION_PAYLOAD/runtest_helix.py</RunnerScript>
</PropertyGroup>
<ItemGroup>
- <HelixCorrelationPayload Include="$(SourceDirectory)\bin\tests\*\archive\Core_Root\*.zip">
- <PayloadArchive>%(Identity)</PayloadArchive>
- </HelixCorrelationPayload>
+ <HelixCorrelationPayload Include="$(CoreRootDirectory)" />
+ <XUnitWrapperDlls Include="$(TestWorkingDir)\**\*.XUnitWrapper.dll" />
</ItemGroup>
+ <Target Name="CopyRunnerScript" BeforeTargets="SubmitTestsToHelix">
+ <Copy SourceFiles="$(MSBuildThisFileDirectory)\runtest_helix.py" DestinationFolder="$(CoreRootDirectory)" />
+ </Target>
+
<Target Name="SubmitTestsToHelix">
<ItemGroup>
- <Scenarios Include="$(Scenarios)" />
+ <Scenarios Include="$(Scenarios.Split(','))" />
<ProjectsToBuild Include="$(MSBuildProjectFile)">
<Properties>Scenario=%(Scenarios.Identity)</Properties>
</ProjectsToBuild>
<MSBuild Projects="@(ProjectsToBuild)" Targets="Test" BuildInParallel="$(BuildInParallel)" StopOnFirstFailure="false" UnloadProjectsOnCompletion="true" />
</Target>
- <Target Name="BuildHelixWorkItem"
- BeforeTargets="Test">
+ <Target Name="BuildHelixWorkItem" BeforeTargets="Test">
<PropertyGroup>
<!-- The "normal" scenario is just a way to include the default
(empty) scenario when specifying multiple scenarios at
job names will not have a scenario prefix and the runtest
script doesn't have to define a "normal" scenario. -->
<Scenario Condition=" '$(Scenario)' == 'normal' "></Scenario>
- <ScenarioPrefix Condition=" '$(Scenario)' == '' " />
- <ScenarioPrefix Condition=" '$(Scenario)' != '' ">$(Scenario) </ScenarioPrefix>
+ <TestRunNamePrefix Condition=" '$(Scenario)' == '' ">$(BuildOS) $(BuildArch) $(BuildType) @ </TestRunNamePrefix>
+ <TestRunNamePrefix Condition=" '$(Scenario)' != '' ">$(BuildOS) $(BuildArch) $(BuildType) $(Scenario) @ </TestRunNamePrefix>
</PropertyGroup>
<ItemGroup>
- <TestZipFiles Include="$(SourceDirectory)\bin\tests\*\archive\tests\*.zip" />
- <HelixWorkItem Include="@(TestZipFiles->'$(ScenarioPrefix)%(FileName)')" >
- <PayloadArchive>%(Identity)</PayloadArchive>
- <Command Condition=" '$(Scenario)' == '' ">python runtest_helix.py -wrapper %(FileName).dll</Command>
- <Command Condition=" '$(Scenario)' != '' ">python runtest_helix.py -scenario $(Scenario) -wrapper %(FileName).dll</Command>
+ <HelixWorkItem Include="@(XUnitWrapperDlls->'%(FileName)'->Replace('.XUnitWrapper', ''))" >
+ <PayloadDirectory>%(RootDir)%(Directory)</PayloadDirectory>
+ <Command Condition=" '$(Scenario)' == '' ">$(HelixPythonPath) $(RunnerScript) -wrapper %(FileName)%(Extension)</Command>
+ <Command Condition=" '$(Scenario)' != '' ">$(HelixPythonPath) $(RunnerScript) -wrapper %(FileName)%(Extension) -scenario $(Scenario)</Command>
+ <Timeout>$([System.TimeSpan]::FromSeconds($(TimeoutInSeconds)))</Timeout>
</HelixWorkItem>
</ItemGroup>
-
</Target>
</Project>