-<Project InitialTargets="BuildHelixWorkItems" Sdk="Microsoft.DotNet.Helix.Sdk">
+<Project InitialTargets="BuildHelixWorkItems" DefaultTargets="RunInParallelForEachScenario">
+
+ <!-- This project uses the helix SDK, documented at
+ https://github.com/dotnet/arcade/tree/master/src/Microsoft.DotNet.Helix/Sdk,
+ to send test jobs to helix. -->
+
+ <Import Sdk="Microsoft.DotNet.Helix.Sdk" Project="Sdk.props" Condition=" '$(UsesHelixSdk)' == 'true' " />
+
+ <!-- Import Microsoft.DotNet.Build.Tasks.Feed.targets for `ParseBuildManifest` -->
+ <Import Project="..\dir.props" Condition=" '$(NuGetPackageRoot)'=='' "/>
<Import Project="$(NuGetPackageRoot)microsoft.dotnet.build.tasks.feed\$(MicrosoftDotNetBuildTasksFeedVersion)\build\Microsoft.DotNet.Build.Tasks.Feed.targets" />
- <PropertyGroup>
- <!-- Set the TargetFramework just to make the SDK happy -->
- <TargetFramework>netstandard2.0</TargetFramework>
- </PropertyGroup>
+ <!-- This project is copies much logic from helixpublishwitharcade.proj, used to send coreclr tests to Helix. -->
+
+ <!--
+ "RunInParallelForEachScenario" is the "root" target for this Project. It invokes other targets to set up the
+ information needed to submit a job to Helix. It does so by "recursively" invoking this project file with
+ different properties and targets to run, as well as invoking MSBuild on other project files. In particular,
+ it first creates the "correlation payload directory", which is the directory of files used by all Helix
+ submissions. Then, it recursively invokes this Project once per scenario (e.g., normal Pri-1 run, JitStress=2
+ run, JitStressRegs=8 run, etc.), creating a new set of Helix work items for each scenario. For the CoreFX
+ tests, we create one Helix work item for each CoreFX test assembly; there is currently no concept of
+ grouping multiple assemblies into a single Helix work item to, for instance, consolidate multiple short-running
+ assemblies together. (Note that this grouping is done for the CoreCLR tests in helixpublishwitharcade.proj.)
+ -->
+ <Target Name="RunInParallelForEachScenario">
+ <PropertyGroup>
+ <!-- This specifies what properties are needed to be passed down as global properties to a child project. -->
+
+ <_PropertiesToPass>
+ __BuildArch=$(__BuildArch);
+ __BuildOS=$(__BuildOS);
+ __BuildType=$(__BuildType);
+ Creator=$(_Creator);
+ HelixAccessToken=$(_HelixAccessToken);
+ HelixBuild=$(_HelixBuild);
+ HelixSource=$(_HelixSource);
+ HelixTargetQueues=$(_HelixTargetQueues);
+ HelixType=$(_HelixType);
+ PublishTestResults=$(_PublishTestResults);
+ RunCrossGen=$(_RunCrossGen);
+ TimeoutPerTestCollectionInMinutes=$(_TimeoutPerTestCollectionInMinutes)
+ </_PropertiesToPass>
+ </PropertyGroup>
+
+ <Error Condition="'$(_Scenarios)' == ''" Text="_Scenarios not set" />
+
+ <MSBuild Projects="$(MSBuildProjectFile)" Targets="PrepareCorrelationPayloadDirectory" Properties="Scenarios=$(_Scenarios)" />
+
+ <ItemGroup>
+ <_Scenarios Include="$(_Scenarios.Split(','))" />
+
+ <!-- MSBuild creates a new instance of the project for each %(_Scenarios.Identity) and can build them in parallel. -->
+ <_ProjectsToBuild Include="$(MSBuildProjectFile)">
+ <AdditionalProperties>$(_PropertiesToPass);Scenario=%(_Scenarios.Identity)</AdditionalProperties>
+ </_ProjectsToBuild>
+ </ItemGroup>
+
+ <PropertyGroup>
+ <_BuildInParallel>false</_BuildInParallel>
+ <_BuildInParallel Condition=" '@(_ProjectsToBuild->Count())' > '1' ">true</_BuildInParallel>
+ </PropertyGroup>
+
+ <!-- Invoke MSBuild on this project file once for each Scenario (because of the "batching" defined in "_ProjectsToBuild").
+ Set "UsesHelixSdk=true" to indicate we want to invoke Helix on this invocation: create the Helix work items and start
+ the jobs. This is done by invoking the "Test" Helix target.
+ -->
+ <MSBuild Projects="@(_ProjectsToBuild)" Targets="Test" BuildInParallel="$(_BuildInParallel)" StopOnFirstFailure="false" Properties="UsesHelixSdk=true" />
+ </Target>
+
+ <!-- Define a set of properties that are input to the Project, and that must be passed down to child processes.
+ (See "_PropertiesToPass" in target "RunInParallelForEachScenario".) If something can be computed, such as
+ properties needed by Helix, they should probably be set in target "BuildHelixWorkItems" instead.
+ -->
<PropertyGroup>
<Creator>$(_Creator)</Creator>
<HelixSource>$(_HelixSource)</HelixSource>
<HelixTargetQueues>$(_HelixTargetQueues)</HelixTargetQueues>
<HelixType>$(_HelixType)</HelixType>
- <HelixArchitecture>$(__BuildArch)</HelixArchitecture>
- <HelixConfiguration>$(__BuildType)</HelixConfiguration>
<!--
TODO: ProjectDir, RootBinDir, TestWorkingDir, and TargetsWindows are global properties set in dir.props, remove the property assignment here when we port to arcade.
<TestWorkingDir Condition="'$(__TestWorkingDir)'==''">$(RootBinDir)tests\$(__BuildOS).$(__BuildArch).$(__BuildType)\</TestWorkingDir>
<TargetsWindows Condition="'$(__BuildOS)' == 'Windows_NT'">true</TargetsWindows>
+ <!-- We have a single, universal exclusion file for all CoreFX test exclusions, for CoreFX tests run
+ in the CoreCLR repo. Note in particular that we don't have a unique exclusion file for each test
+ assembly, or for each processor architecture, or for each operating system, or for each stress
+ scenario type. This means that we generally exclude more tests than is strictly necessary for
+ any particular job. However, this mechanism is super simple; we don't need to manage fine-
+ grained exclusions; and there really shouldn't be any (or many) exclusions anyway.
+ -->
+ <CoreFXTestExclusionFile>$(ProjectDir)tests\CoreFX\CoreFX.issues.rsp</CoreFXTestExclusionFile>
+
<TestHostRootPath>$(TestWorkingDir)testhost\</TestHostRootPath>
<TestArchiveRuntimeRoot>$(TestWorkingDir)helix\</TestArchiveRuntimeRoot>
- <TestArchiveRuntimeFile>$(TestArchiveRuntimeRoot)testhost-runtime.zip</TestArchiveRuntimeFile>
- <HelixCorrelationPayload></HelixCorrelationPayload>
- <EnableAzurePipelinesReporter>$(_PublishTestResults)</EnableAzurePipelinesReporter>
- <EnableAzurePipelinesReporter Condition=" '$(EnableAzurePipelinesReporter)' == '' ">false</EnableAzurePipelinesReporter>
- <EnableXUnitReporter>true</EnableXUnitReporter>
- <FailOnMissionControlTestFailure>true</FailOnMissionControlTestFailure>
- <FailOnWorkItemFailure>true</FailOnWorkItemFailure>
-
- <TimeoutInSeconds Condition="'$(TimeoutInSeconds)' == ''">600</TimeoutInSeconds>
- <CommandTimeoutSpan>$([System.TimeSpan]::FromSeconds($(TimeoutInSeconds)))</CommandTimeoutSpan>
- <MaxRetryCount Condition="'$(MaxRetryCount)' == ''">4</MaxRetryCount>
- <WaitForWorkItemCompletion>true</WaitForWorkItemCompletion>
+ <TestAssetBlobFeedUrl>https://dotnetfeed.blob.core.windows.net/dotnet-core</TestAssetBlobFeedUrl>
</PropertyGroup>
<PropertyGroup>
<!--
- For windows we need to use call, since the command is going to be called from a bat script created by Helix
- and we exit /b at the end of RunTests.cmd, Helix runs some other commands after ours within the bat script,
- if we don't use call, then we cause the parent script to exit, and anything after will not be executed.
- The arguments passed in to the run script in order are the runtime directory, the dotnet root directory (for
- helix submissions same as the runtime directory) and the global tools directory.
+ For Windows, we need to use "call", since the command is going to be called from a batch script created by Helix.
+ We "exit /b" at the end of RunTests.cmd. Helix runs some other commands after ours within the batch script,
+ so if we don't use call, then we cause the parent script to exit, and anything after will not be executed.
+ The arguments passed in to the run script in are the runtime directory (-r) and the exclusion response file
+ passed to xunit (rsp-file).
-->
<HelixCommand Condition="'$(TargetsWindows)' == 'true'">call RunTests.cmd -r %HELIX_CORRELATION_PAYLOAD% --rsp-file %HELIX_CORRELATION_PAYLOAD%\CoreFX.issues.rsp </HelixCommand>
<HelixCommand Condition="'$(TargetsWindows)' != 'true'">./RunTests.sh -r $HELIX_CORRELATION_PAYLOAD --rsp-file $HELIX_CORRELATION_PAYLOAD/CoreFX.issues.rsp </HelixCommand>
</PropertyGroup>
- <ItemGroup>
- <TestArchiveRuntimeInputs Include="$(TestHostRootPath)**/*" />
- </ItemGroup>
-
+ <!--
+ Copy the CoreFX test exclusion response file (which is passed to the xunit console runner) to someplace we can
+ access it during the test run. We currently just copy it to the root of the generated .NET Core testhost
+ directory, which we use as the root of the correlation directory.
+ -->
<Target Name="CopyRSPFile">
<Copy
- SourceFiles="$(ProjectDir)\tests\CoreFX\CoreFX.issues.rsp"
+ SourceFiles="$(CoreFXTestExclusionFile)"
DestinationFolder="$(TestHostRootPath)" />
+
+ <Message Importance="High" Text="Copied $(CoreFXTestExclusionFile) into $(TestHostRootPath)" />
</Target>
- <Target Name="CompressRuntimeDirectory"
- Inputs="@(TestArchiveRuntimeInputs)"
- Outputs="$(TestArchiveRuntimeFile)"
- DependsOnTargets="CopyRSPFile" >
+ <PropertyGroup>
+ <!-- Set the name of the scenario file. Note that this is only used in invocations where $(Scenario) is set. -->
+ <TestEnvFileName Condition=" '$(TargetsWindows)' == 'true' ">SetStressModes_$(Scenario).cmd</TestEnvFileName>
+ <TestEnvFileName Condition=" '$(TargetsWindows)' != 'true' ">SetStressModes_$(Scenario).sh</TestEnvFileName>
+ </PropertyGroup>
- <MakeDir Directories="$(TestArchiveRuntimeRoot)" />
+ <Target Name="CreateTestEnvFile">
+ <!-- This target creates one __TestEnv file for the single $(Scenario). -->
- <ZipDirectory
- SourceDirectory="$(TestHostRootPath)"
- DestinationFile="$(TestArchiveRuntimeFile)"
- Overwrite="true" />
+ <PropertyGroup>
+ <TestEnvFilePath>$(TestHostRootPath)$(TestEnvFileName)</TestEnvFilePath>
+ </PropertyGroup>
- <Message Importance="High" Text="Zipped correlation payload into $(TestArchiveRuntimeFile)" />
+ <ItemGroup>
+ <_ProjectsToBuild Include="..\tests\testenvironment.proj">
+ <Properties>Scenario=$(Scenario);TestEnvFileName=$(TestEnvFilePath);TargetsWindows=$(TargetsWindows)</Properties>
+ </_ProjectsToBuild>
+ </ItemGroup>
+
+ <MSBuild Projects="@(_ProjectsToBuild)" Targets="CreateTestEnvFile" StopOnFirstFailure="true" />
+
+ <Message Importance="High" Text="Created $(TestEnvFilePath) for scenario $(Scenario)" />
+ <Error Condition="!Exists('$(TestEnvFilePath)')" Text="File $(TestEnvFilePath) not found!" />
</Target>
- <PropertyGroup>
- <TestAssetBlobFeedUrl>https://dotnetfeed.blob.core.windows.net/dotnet-core</TestAssetBlobFeedUrl>
- </PropertyGroup>
+ <Target Name="CreateAllScenarioTestEnvFiles">
+ <!-- This target creates one __TestEnv file for each of the scenarios in the $(Scenarios) comma-separated list. -->
- <Target Name="GetTestAssetManifest"
- Returns="@(TestAssetBlobInfos)">
+ <ItemGroup>
+ <_Scenario Include="$(Scenarios.Split(','))" />
+ <_ProjectsToBuild Include="$(MSBuildProjectFile)">
+ <AdditionalProperties>Scenario=%(_Scenario.Identity)</AdditionalProperties>
+ </_ProjectsToBuild>
+ </ItemGroup>
+
+ <MSBuild Projects="@(_ProjectsToBuild)" Targets="CreateTestEnvFile" StopOnFirstFailure="true" />
+ </Target>
+
+ <!--
+ Collect all the tasks needed to be run once, to prepare the Helix correlation payload directory.
+ This just causes its dependent targets to be run.
+ -->
+ <Target Name="PrepareCorrelationPayloadDirectory"
+ DependsOnTargets="CopyRSPFile;CreateAllScenarioTestEnvFiles" >
+ </Target>
+ <!--
+ Parse the test asset manifest at, e.g.,
+ https://dotnetfeed.blob.core.windows.net/dotnet-core/corefx-tests/4.6.0-preview6.19264.9/Linux.arm64/netcoreapp/corefx-test-assets.xml
+ to figure out the tests to run. This manifest is created by the corefx official build, for use in the coreclr repo.
+
+ This creates a "TestAssetBlobInfos" item group, which is used to create the Helix work items.
+
+ NOTE: we are running this once for each Scenario, meaning we are reading and parsing it once per scenario. Ideally, we would
+ do this once and pass it down to the "BuildHelixWorkItems" task later. That's a little difficult given all the re-invocations
+ of this Project we do. It doesn't seem too expensive anyway.
+ -->
+ <Target Name="GetTestAssetManifest" Condition=" '$(UsesHelixSdk)' == 'true' " >
<PropertyGroup>
<_TargetGroup>netcoreapp</_TargetGroup>
<_AssetManifestPath>$(TestAssetBlobFeedUrl)/corefx-tests/$(MicrosoftPrivateCoreFxNETCoreAppVersion)/$(__BuildOS).$(__BuildArch)/$(_TargetGroup)/corefx-test-assets.xml</_AssetManifestPath>
</ParseBuildManifest>
</Target>
- <Target Name="BuildHelixWorkItems"
- Inputs="$(TestArchiveRuntimeFile);@(TestAssetBlobInfos)"
- Outputs="@(HelixCorrelationPayload);@(HelixWorkItem)"
- DependsOnTargets="CompressRuntimeDirectory;GetTestAssetManifest">
+ <!--
+ The following item and property groups are setting things needed by Helix, to describe the jobs
+ we are creating. Ideally, we would put them inside the "BuildHelixWorkItems" task, which is what
+ we run to create the Helix correlation payload and work item groups. However, the Helix SDK
+ references some of these in "InitialTargets", and we don't have control over the order of
+ InitialTargets running, so we need to make these "global". All the "global" item and property
+ groups will be evaluated before the Helix InitialTargets are run.
+ -->
- <ItemGroup>
- <HelixCorrelationPayload
- Include="$(TestArchiveRuntimeFile)" />
+ <!-- HelixPreCommands is a set of commands run before the work item command. We use it here to inject
+ setting up the per-scenario environment.
+ -->
+
+ <ItemGroup Condition=" '$(TargetsWindows)' == 'true' ">
+ <HelixPreCommand Include="set __TestEnv=%HELIX_CORRELATION_PAYLOAD%\$(TestEnvFileName)" />
+ <HelixPreCommand Include="type %__TestEnv%" />
+ <HelixPreCommand Include="call %__TestEnv%" />
+ <!-- Display the interesting COMPlus variables that are set in the environment -->
+ <HelixPreCommand Include="set COMPlus" />
+ </ItemGroup>
+
+ <ItemGroup Condition=" '$(TargetsWindows)' != 'true' ">
+ <HelixPreCommand Include="export __TestEnv=$HELIX_CORRELATION_PAYLOAD/$(TestEnvFileName)" />
+ <HelixPreCommand Include="cat $__TestEnv" />
+ <HelixPreCommand Include=". $__TestEnv" /> <!-- Use "." not "source"; some clients appear to run scripts with "sh" not "bash" -->
+ <!-- Display the interesting COMPlus variables that are set in the environment -->
+ <HelixPreCommand Include="printenv | grep COMPlus" />
+ </ItemGroup>
+
+ <PropertyGroup>
+ <HelixPreCommands>@(HelixPreCommand)</HelixPreCommands>
+ </PropertyGroup>
+
+ <PropertyGroup>
+ <HelixArchitecture>$(__BuildArch)</HelixArchitecture>
+
+ <HelixConfiguration Condition=" '$(Scenario)' == 'normal' ">$(BuildType)</HelixConfiguration>
+ <HelixConfiguration Condition=" '$(Scenario)' != 'normal' ">$(BuildType)-$(Scenario)</HelixConfiguration>
+
+ <TestRunNamePrefix>CoreFX </TestRunNamePrefix>
+ <TestRunNamePrefix Condition=" '$(Scenario)' == 'normal' ">$(TestRunNamePrefix)$(BuildOS) $(BuildArch) $(BuildType) @ </TestRunNamePrefix>
+ <TestRunNamePrefix Condition=" '$(Scenario)' != 'normal' ">$(TestRunNamePrefix)$(BuildOS) $(BuildArch) $(BuildType) $(Scenario) @ </TestRunNamePrefix>
+
+ <!-- REVIEW: it seems like this should be referencing "PublishTestResults" instead (without the underscore) -->
+ <EnableAzurePipelinesReporter>$(_PublishTestResults)</EnableAzurePipelinesReporter>
+ <EnableAzurePipelinesReporter Condition=" '$(EnableAzurePipelinesReporter)' == '' ">false</EnableAzurePipelinesReporter>
+
+ <EnableXUnitReporter>true</EnableXUnitReporter>
+ <FailOnMissionControlTestFailure>true</FailOnMissionControlTestFailure>
+ <FailOnWorkItemFailure>true</FailOnWorkItemFailure>
+ <WaitForWorkItemCompletion>true</WaitForWorkItemCompletion>
+
+ <!-- Specify the retry policy for Helix jobs -->
+ <!-- REVIEW: currently don't do any retries
+ <MaxRetryCount Condition="'$(MaxRetryCount)' == ''">4</MaxRetryCount>
+ -->
+ </PropertyGroup>
+
+ <!--
+ Create all the Helix data to start a set of jobs. This is invoked once for each Scenario (see
+ target "RunInParallelForEachScenario"). Create a set of work items, one for each CoreFX test assembly.
+ All will have the same Scenario specific command line. Note that this target is listed in the
+ InitialTargets for this Project. This causes it to be invoked (and the Helix data created,
+ such as the HelixWorkItem item group) before Helix "Test" target is invoked (as a normal target).
+ We conditionalize this (and its dependent targets) with "UsesHelixSdk=true" so they aren't invoked
+ during the other invocations of this Project, such as when "RunInParallelForEachScenario" is run.
+ -->
+ <Target Name="BuildHelixWorkItems" DependsOnTargets="GetTestAssetManifest" Condition=" '$(UsesHelixSdk)' == 'true' " >
+ <Error Condition=" '$(TimeoutPerTestCollectionInMinutes)' == '' " Text="TimeoutPerTestCollectionInMinutes not set" />
+
+ <ItemGroup Condition=" '$(UsesHelixSdk)' == 'true' ">
+ <HelixCorrelationPayload Include="$(TestHostRootPath)" />
<HelixWorkItem Include="@(TestAssetBlobInfos -> '%(FileName)')">
<PayloadUri>$(TestAssetBlobFeedUrl)/%(Identity)</PayloadUri>
<Command>$(HelixCommand)</Command>
- <Timeout>$(CommandTimeoutSpan)</Timeout>
+ <Timeout>$([System.TimeSpan]::FromMinutes($(TimeoutPerTestCollectionInMinutes)))</Timeout>
</HelixWorkItem>
</ItemGroup>
-
- <Message Importance="High" Text="@(HelixWorkItem -> '%(PayloadUri)')" />
</Target>
+
+ <Import Sdk="Microsoft.DotNet.Helix.Sdk" Project="Sdk.targets" Condition=" '$(UsesHelixSdk)' == 'true' " />
+
</Project>
- Linux_x64
- Windows_NT_x64
- Windows_NT_x86
+ ${{ if in(variables['Build.DefinitionName'], 'coreclr-corefx', 'coreclr-corefx-jitstress', 'coreclr-corefx-jitstressregs', 'coreclr-corefx-jitstress2-jitstressregs') }}:
+ platforms:
+ - Linux_x64
+ - Windows_NT_x64
#
# Release builds
# Checked test builds
#
# The test jobs that can be triggered by a PR, manually from ADO and that are scheduled
-- ${{ if and(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest', 'Manual', 'Schedule')) }}:
+# TODO: This long 'in' condition needs to be make data driven, perhaps by adding another layer of template.
+- ${{ if and(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest', 'Manual', 'Schedule'), in(variables['Build.DefinitionName'], 'coreclr-ci', 'coreclr-outerloop', 'coreclr-outerloop-jitstress', 'coreclr-outerloop-jitstress-isas-arm', 'coreclr-outerloop-jitstress-isas-x86', 'coreclr-outerloop-jitstressregs-x86', 'coreclr-outerloop-jitstressregs', 'coreclr-outerloop-jitstress2-jitstressregs', 'coreclr-outerloop-gcstress0x3-gcstress0xc', 'coreclr-outerloop-gcstress-extra', 'coreclr-outerloop-r2r', 'coreclr-outerloop-r2r-extra')) }}:
- template: eng/platform-matrix.yml
parameters:
jobTemplate: test-job.yml
${{ if eq(variables['Build.DefinitionName'], 'coreclr-outerloop') }}:
testGroup: outerloop
${{ if eq(variables['Build.DefinitionName'], 'coreclr-outerloop-jitstress') }}:
- testGroup: outerloop-jitstress
+ testGroup: jitstress
${{ if eq(variables['Build.DefinitionName'], 'coreclr-outerloop-jitstress-isas-arm') }}:
- testGroup: outerloop-jitstress-isas-arm
+ testGroup: jitstress-isas-arm
${{ if eq(variables['Build.DefinitionName'], 'coreclr-outerloop-jitstress-isas-x86') }}:
- testGroup: outerloop-jitstress-isas-x86
+ testGroup: jitstress-isas-x86
${{ if eq(variables['Build.DefinitionName'], 'coreclr-outerloop-jitstressregs-x86') }}:
- testGroup: outerloop-jitstressregs-x86
+ testGroup: jitstressregs-x86
${{ if eq(variables['Build.DefinitionName'], 'coreclr-outerloop-jitstressregs') }}:
- testGroup: outerloop-jitstressregs
+ testGroup: jitstressregs
${{ if eq(variables['Build.DefinitionName'], 'coreclr-outerloop-jitstress2-jitstressregs') }}:
- testGroup: outerloop-jitstress2-jitstressregs
+ testGroup: jitstress2-jitstressregs
${{ if eq(variables['Build.DefinitionName'], 'coreclr-outerloop-gcstress0x3-gcstress0xc') }}:
- testGroup: outerloop-gcstress0x3-gcstress0xc
+ testGroup: gcstress0x3-gcstress0xc
${{ if eq(variables['Build.DefinitionName'], 'coreclr-outerloop-gcstress-extra') }}:
- testGroup: outerloop-gcstress-extra
+ testGroup: gcstress-extra
${{ if eq(variables['Build.DefinitionName'], 'coreclr-outerloop-r2r') }}:
testGroup: outerloop
readyToRun: true
displayNameArgs: R2R
${{ if eq(variables['Build.DefinitionName'], 'coreclr-outerloop-r2r-extra') }}:
- testGroup: outerloop-r2r-extra
+ testGroup: r2r-extra
readyToRun: true
displayNameArgs: R2R
-# The ReadyToRun test jobs that are triggered by default from a PR.
+# ReadyToRun test jobs that are triggered by default from a PR.
+
- ${{ if and(eq(variables['System.TeamProject'], 'public'), eq(variables['Build.Reason'], 'PullRequest'), eq(variables['Build.DefinitionName'], 'coreclr-ci')) }}:
- template: eng/platform-matrix.yml
parameters:
readyToRun: true
testGroup: innerloop
displayNameArgs: R2R
- # The CoreFX runs against CoreCLR
+
+#
+# CoreFX test runs against CoreCLR
+#
+
+# PR innerloop checked build
+
+# TODO: This long 'in' condition needs to be make data driven, perhaps by adding another layer of template.
+- ${{ if and(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest', 'Manual', 'Schedule'), in(variables['Build.DefinitionName'], 'coreclr-ci', 'coreclr-corefx', 'coreclr-corefx-jitstress', 'coreclr-corefx-jitstressregs', 'coreclr-corefx-jitstress2-jitstressregs')) }}:
- template: eng/platform-matrix.yml
parameters:
jobTemplate: test-job.yml
- buildConfig: release
- platforms:
- - Windows_NT_x64
+ buildConfig: checked
+ ${{ if and(eq(variables['Build.Reason'], 'PullRequest'), eq(variables['Build.DefinitionName'], 'coreclr-ci')) }}:
+ platforms:
+ - Linux_x64
+ - Windows_NT_x64
+ ${{ if in(variables['Build.DefinitionName'], 'coreclr-corefx', 'coreclr-corefx-jitstress', 'coreclr-corefx-jitstressregs', 'coreclr-corefx-jitstress2-jitstressregs') }}:
+ platforms:
+ - Linux_x64
+ - Windows_NT_x64
jobParameters:
+ ${{ if eq(variables['Build.DefinitionName'], 'coreclr-ci') }}:
+ testGroup: innerloop
+ ${{ if eq(variables['Build.DefinitionName'], 'coreclr-corefx') }}:
+ testGroup: outerloop
+ ${{ if eq(variables['Build.DefinitionName'], 'coreclr-corefx-jitstress') }}:
+ testGroup: jitstress
+ ${{ if eq(variables['Build.DefinitionName'], 'coreclr-corefx-jitstressregs') }}:
+ testGroup: jitstressregs
+ ${{ if eq(variables['Build.DefinitionName'], 'coreclr-corefx-jitstress2-jitstressregs') }}:
+ testGroup: jitstress2-jitstressregs
corefxTests: true
- testGroup: innerloop
displayNameArgs: CoreFX
-# CI
+# CI (merge) jobs
+
- ${{ if and(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI')) }}:
- template: eng/platform-matrix.yml
parameters:
- Windows_NT_arm
- Windows_NT_arm64
jobParameters:
- readyToRun: true
testGroup: outerloop
+ readyToRun: true
displayNameArgs: R2R
#
helixType: 'build/tests/'
# Compute job name from template parameters
- ${{ if eq(parameters.testGroup, 'innerloop') }}:
- name: ${{ format('testbuild_pri0_{0}_{1}_{2}_{3}', parameters.displayNameArgs, parameters.osIdentifier, parameters.archType, parameters.buildConfig) }}
- displayName: ${{ format('Test Pri0 {0} {1} {2}_{3}', parameters.displayNameArgs, parameters.osIdentifier, parameters.archType, parameters.buildConfig) }}
+ ${{ if and(eq(parameters.testGroup, 'innerloop'), eq(parameters.displayNameArgs, '')) }}:
+ name: ${{ format('test_{0}_{1}_{2}_{3}', 'p0', parameters.osIdentifier, parameters.archType, parameters.buildConfig) }}
+ displayName: ${{ format('Test {0} {1} {2} {3}', 'Pri0', parameters.osIdentifier, parameters.archType, parameters.buildConfig) }}
+
+ ${{ if and(ne(parameters.testGroup, 'innerloop'), eq(parameters.displayNameArgs, '')) }}:
+ name: ${{ format('test_{0}_{1}_{2}_{3}', 'p1', parameters.displayNameArgs, parameters.osIdentifier, parameters.archType, parameters.buildConfig) }}
+ displayName: ${{ format('Test {0} {1} {2} {3}', 'Pri1', parameters.displayNameArgs, parameters.osIdentifier, parameters.archType, parameters.buildConfig) }}
- ${{ if ne(parameters.testGroup, 'innerloop') }}:
- name: ${{ format('testbuild_pri1_{0}_{1}_{2}_{3}', parameters.displayNameArgs, parameters.osIdentifier, parameters.archType, parameters.buildConfig) }}
- displayName: ${{ format('Test Pri1 {0} {1} {2} {3}', parameters.displayNameArgs, parameters.osIdentifier, parameters.archType, parameters.buildConfig) }}
+ ${{ if and(eq(parameters.testGroup, 'innerloop'), ne(parameters.displayNameArgs, '')) }}:
+ name: ${{ format('test_{0}_{1}_{2}_{3}_{4}', 'p0', parameters.displayNameArgs, parameters.osIdentifier, parameters.archType, parameters.buildConfig) }}
+ displayName: ${{ format('Test {0} {1} {2} {3} {4}', 'Pri0', parameters.displayNameArgs, parameters.osIdentifier, parameters.archType, parameters.buildConfig) }}
+
+ ${{ if and(ne(parameters.testGroup, 'innerloop'), ne(parameters.displayNameArgs, '')) }}:
+ name: ${{ format('test_{0}_{1}_{2}_{3}_{4}', 'p1', parameters.displayNameArgs, parameters.osIdentifier, parameters.archType, parameters.buildConfig) }}
+ displayName: ${{ format('Test {0} {1} {2} {3} {4}', 'Pri1', parameters.displayNameArgs, parameters.osIdentifier, parameters.archType, parameters.buildConfig) }}
crossrootfsDir: ${{ parameters.crossrootfsDir }}
variables:
- ${{ if eq(variables['System.TeamProject'], 'internal') }}:
- group: DotNet-HelixApi-Access
+
# Map template parameters to command line arguments
- name: priorityArg
value: ''
# this number should be adjusted accordingly.
${{ if eq(parameters.testGroup, 'innerloop') }}:
timeoutInMinutes: 240
- ${{ if eq(parameters.testGroup, 'outerloop') }}:
+ ${{ if in(parameters.testGroup, 'outerloop') }}:
timeoutInMinutes: 360
- ${{ if in(parameters.testGroup, 'outerloop-jitstress', 'outerloop-jitstress-isas-arm', 'outerloop-jitstressregs-x86', 'outerloop-jitstressregs', 'outerloop-jitstress2-jitstressregs', 'outerloop-gcstress0x3-gcstress0xc') }}:
+ ${{ if in(parameters.testGroup, 'jitstress', 'jitstress-isas-arm', 'jitstressregs-x86', 'jitstressregs', 'jitstress2-jitstressregs', 'gcstress0x3-gcstress0xc') }}:
timeoutInMinutes: 480
- ${{ if in(parameters.testGroup, 'outerloop-jitstress-isas-x86', 'outerloop-gcstress-extra', 'outerloop-r2r-extra') }}:
+ ${{ if in(parameters.testGroup, 'jitstress-isas-x86', 'gcstress-extra', 'r2r-extra') }}:
timeoutInMinutes: 600
steps:
helixBuild: $(Build.BuildNumber)
helixSource: $(_HelixSource)
- ${{ if eq(parameters.readyToRun, false) }}:
- helixType: 'test/functional/cli/'
- ${{ if eq(parameters.readyToRun, true) }}:
+ # REVIEW: not sure why "cli" is part of the names here. Leave it for the ones that already had it,
+ # but don't add it to new ones.
+ ${{ if and(eq(parameters.corefxTests, true), eq(parameters.readyToRun, true)) }}:
+ helixType: 'test/functional/r2r_corefx/'
+ ${{ if and(eq(parameters.corefxTests, true), ne(parameters.readyToRun, true)) }}:
+ helixType: 'test/functional/corefx/'
+ ${{ if and(ne(parameters.corefxTests, true), eq(parameters.readyToRun, true)) }}:
helixType: 'test/functional/r2r/cli/'
+ ${{ if and(ne(parameters.corefxTests, true), ne(parameters.readyToRun, true)) }}:
+ helixType: 'test/functional/cli/'
helixQueues: ${{ parameters.helixQueues }}
publishTestResults: true
- ${{ if eq(parameters.testGroup, 'innerloop') }}:
- # "PerCollection" is time needed for the "biggest" xUnit test collection to complete.
- # In case xUnit test wrappers get refactored this number should also be adjusted.
+ # Set job timeouts
+ #
+ # "timeoutPerTestCollectionInMinutes" is the time needed for the "biggest" xUnit test collection to complete.
+ # In case xUnit test wrappers get refactored this number should also be adjusted.
+ #
+ # "timeoutPerTestInMinutes" corresponds to individual test running time. This is implemented by setting
+ # the __TestTimeout variable, which is later read by the coreclr xunit test wrapper code (the code in the
+ # xunit test dlls that invokes the actual tests). Note this doesn't apply to CoreFX testing.
+
+ ${{ if and(eq(parameters.corefxTests, true), eq(parameters.testGroup, 'innerloop')) }}:
+ timeoutPerTestCollectionInMinutes: 120
+ ${{ if and(ne(parameters.corefxTests, true), eq(parameters.testGroup, 'innerloop')) }}:
timeoutPerTestCollectionInMinutes: 30
- # "PerTest" corresponds to individual test running time (i.e. __TestTimeout).
timeoutPerTestInMinutes: 10
- ${{ if eq(parameters.testGroup, 'outerloop') }}:
+ ${{ if in(parameters.testGroup, 'outerloop') }}:
timeoutPerTestCollectionInMinutes: 120
timeoutPerTestInMinutes: 10
- ${{ if in(parameters.testGroup, 'outerloop-jitstress', 'outerloop-jitstress-isas-arm', 'outerloop-jitstress-isas-x86', 'outerloop-jitstressregs-x86', 'outerloop-jitstressregs', 'outerloop-jitstress2-jitstressregs') }}:
+ ${{ if in(parameters.testGroup, 'jitstress', 'jitstress-isas-arm', 'jitstress-isas-x86', 'jitstressregs-x86', 'jitstressregs', 'jitstress2-jitstressregs' ) }}:
timeoutPerTestCollectionInMinutes: 120
timeoutPerTestInMinutes: 30
- ${{ if in(parameters.testGroup, 'outerloop-gcstress0x3-gcstress0xc') }}:
+ ${{ if in(parameters.testGroup, 'gcstress0x3-gcstress0xc') }}:
timeoutPerTestCollectionInMinutes: 240
timeoutPerTestInMinutes: 60
- ${{ if in(parameters.testGroup, 'outerloop-gcstress-extra', 'outerloop-r2r-extra') }}:
+ ${{ if in(parameters.testGroup, 'gcstress-extra', 'r2r-extra') }}:
timeoutPerTestCollectionInMinutes: 300
timeoutPerTestInMinutes: 90
# DotNet-HelixApi-Access variable group
helixAccessToken: $(HelixApiAccessToken)
- # Sets up the template to run the corefx tests
+ # Choose which tests to send to Helix: CoreFX or CoreCLR.
${{ if eq(parameters.corefxTests, true) }}:
- helixProjectArguments: 'eng/helixcorefxtests.proj /t:Test /p:ArcadeBuild=True'
- ${{ if eq(parameters.corefxTests, false) }}:
+ helixProjectArguments: 'eng/helixcorefxtests.proj'
+ ${{ if ne(parameters.corefxTests, true) }}:
helixProjectArguments: 'tests/helixpublishwitharcade.proj'
${{ if in(parameters.testGroup, 'innerloop', 'outerloop') }}:
scenarios:
- normal
- no_tiered_compilation
- ${{ if eq(parameters.testGroup, 'outerloop-jitstress') }}:
+ ${{ if in(parameters.testGroup, 'jitstress') }}:
scenarios:
- jitminopts
- jitstress1
- jitstress2_tiered
- zapdisable
- tailcallstress
- ${{ if eq(parameters.testGroup, 'outerloop-jitstress-isas-arm') }}:
+ ${{ if in(parameters.testGroup, 'jitstress-isas-arm') }}:
scenarios:
- jitstress_isas_incompletehwintrinsic
- jitstress_isas_nohwintrinsic
- jitstress_isas_nohwintrinsic_nosimd
- jitstress_isas_nosimd
- ${{ if eq(parameters.testGroup, 'outerloop-jitstress-isas-x86') }}:
+ ${{ if in(parameters.testGroup, 'jitstress-isas-x86') }}:
scenarios:
- jitstress_isas_incompletehwintrinsic
- jitstress_isas_nohwintrinsic
- jitstress_isas_x86_nosse41
- jitstress_isas_x86_nosse42
- jitstress_isas_x86_nossse3
- ${{ if eq(parameters.testGroup, 'outerloop-jitstressregs-x86') }}:
+ ${{ if in(parameters.testGroup, 'jitstressregs-x86') }}:
scenarios:
- jitstressregs1_x86_noavx
- jitstressregs2_x86_noavx
- jitstressregs0x10_x86_noavx
- jitstressregs0x80_x86_noavx
- jitstressregs0x1000_x86_noavx
- ${{ if eq(parameters.testGroup, 'outerloop-jitstressregs') }}:
+ ${{ if in(parameters.testGroup, 'jitstressregs' ) }}:
scenarios:
- jitstressregs1
- jitstressregs2
- jitstressregs0x10
- jitstressregs0x80
- jitstressregs0x1000
- ${{ if eq(parameters.testGroup, 'outerloop-jitstress2-jitstressregs') }}:
+ ${{ if in(parameters.testGroup, 'jitstress2-jitstressregs') }}:
scenarios:
- jitstress2_jitstressregs1
- jitstress2_jitstressregs2
- jitstress2_jitstressregs0x10
- jitstress2_jitstressregs0x80
- jitstress2_jitstressregs0x1000
- ${{ if eq(parameters.testGroup, 'outerloop-gcstress0x3-gcstress0xc') }}:
+ ${{ if in(parameters.testGroup, 'gcstress0x3-gcstress0xc') }}:
scenarios:
- gcstress0x3
- gcstress0xc
- ${{ if eq(parameters.testGroup, 'outerloop-gcstress-extra') }}:
+ ${{ if in(parameters.testGroup, 'gcstress-extra') }}:
scenarios:
- heapverify1
- gcstress0xc_zapdisable
- gcstress0xc_jitstress1
- gcstress0xc_jitstress2
- gcstress0xc_jitminopts_heapverify1
- ${{ if eq(parameters.testGroup, 'outerloop-r2r-extra') }}:
+ ${{ if in(parameters.testGroup, 'r2r-extra') }}:
scenarios:
- jitstress1
- jitstress2
displayName: Publish Logs
inputs:
pathtoPublish: $(Build.SourcesDirectory)/bin/Logs
- ${{ if eq(parameters.readyToRun, false) }}:
- artifactName: ${{ format('testbuild_{0}_{1}_{2}_{3}_Logs', parameters.osIdentifier, parameters.archType, parameters.buildConfig, parameters.testGroup) }}
- ${{ if eq(parameters.readyToRun, true) }}:
- artifactName: ${{ format('testbuild_{0}_{1}_{2}_{3}_r2r_Logs', parameters.osIdentifier, parameters.archType, parameters.buildConfig, parameters.testGroup) }}
+ ${{ if and(eq(parameters.corefxTests, true), eq(parameters.readyToRun, true)) }}:
+ artifactName: ${{ format('test{0}_{1}_{2}_{3}_{4}_Logs', '_r2r_corefx', parameters.osIdentifier, parameters.archType, parameters.buildConfig, parameters.testGroup) }}
+ ${{ if and(eq(parameters.corefxTests, true), ne(parameters.readyToRun, true)) }}:
+ artifactName: ${{ format('test{0}_{1}_{2}_{3}_{4}_Logs', '_corefx', parameters.osIdentifier, parameters.archType, parameters.buildConfig, parameters.testGroup) }}
+ ${{ if and(ne(parameters.corefxTests, true), eq(parameters.readyToRun, true)) }}:
+ artifactName: ${{ format('test{0}_{1}_{2}_{3}_{4}_Logs', '_r2r', parameters.osIdentifier, parameters.archType, parameters.buildConfig, parameters.testGroup) }}
+ ${{ if and(ne(parameters.corefxTests, true), ne(parameters.readyToRun, true)) }}:
+ artifactName: ${{ format('test{0}_{1}_{2}_{3}_{4}_Logs', '', parameters.osIdentifier, parameters.archType, parameters.buildConfig, parameters.testGroup) }}
continueOnError: true
condition: always()
-nomethod System.Tests.StringTests.CasingNegativeTest
-nomethod System.Tests.StringTests.CompareNegativeTest
-nomethod System.Tests.StringTests.ToLowerNullCulture
--nomethod System.Tests.StringTests.ToUpperNullCulture
\ No newline at end of file
+-nomethod System.Tests.StringTests.ToUpperNullCulture
+
+#
+# Temporarily disable tests while bringing more CoreFX job configurations online.
+#
+
+-nonamespace System.Net.Http.Functional.Tests
+-nonamespace System.Net.Sockets.Tests
+
+-noclass System.Net.Tests.HttpListenerContextTests
+-noclass System.Net.Tests.HttpListenerAuthenticationTests
+-noclass System.Net.Tests.HttpListenerResponseTests
+-noclass System.Net.Tests.HttpListenerResponseCookiesTests
+
+-nomethod System.Net.Tests.HttpListenerResponseTests.Redirect_Invoke_SetsRedirectionProperties
+-nomethod System.Net.Mail.Tests.SmtpClientTest.TestMailDeliveryAsync
+
+-nomethod System.Net.Security.Tests.LoggingTest.EventSource_EventsRaisedAsExpected
+-nomethod System.Net.Security.Tests.SslStreamSniTest.SslStream_NoSniFromClient_CallbackReturnsNull
+
+# System.Linq.Expressions.Tests: timeout on Linux/x64
+-nonamespace System.Linq.Expressions.Tests
+
+# assert/crashes
+-nomethod System.Tests.RealFormatterTestsBase.TestFormatterSingle_F20
+-noclass System.Tests.RealFormatterTestsBase
+-noclass System.Tests.RealFormatterTests
+-noclass System.Buffers.Text.Tests.RealFormatterTests
+
+# System.Runtime.Serialization.Formatters.Tests: timeouts
+-nonamespace System.Runtime.Serialization.Formatters.Tests
+
+-nomethod Generic.Dictionary.DictionaryConcurrentAccessDetectionTests.DictionaryConcurrentAccessDetection_ReferenceTypeKey
+-nomethod Generic.Dictionary.DictionaryConcurrentAccessDetectionTests.DictionaryConcurrentAccessDetection_ValueTypeKey
+-nomethod MonoTests.System.Runtime.Caching.MemoryCacheTest.Contains
+-nomethod System.Numerics.Tests.ToStringTest.RunRegionSpecificStandardFormatToStringTests
+-nomethod System.Text.RegularExpressions.Tests.RegexMatchTests.Match_ExcessPrefix
+
+# Assert with JITMinOpts=1: https://github.com/dotnet/coreclr/issues/25070
+-nonamespace Microsoft.VisualBasic.Tests
+
+# Assert: https://github.com/dotnet/coreclr/issues/25050
+-nonamespace System.Data.Common.Tests
</_PropertiesToPass>
</PropertyGroup>
+ <Error Condition="'$(_Scenarios)' == ''" Text="_Scenarios not set" />
+
<MSBuild Projects="$(MSBuildProjectFile)" Targets="PrepareCorrelationPayloadDirectory" />
<MSBuild Projects="$(MSBuildProjectFile)" Targets="PreparePayloadsDirectories" Properties="Scenarios=$(_Scenarios)" StopOnFirstFailure="true" />
<Target Name="CreateTestEnvFile" DependsOnTargets="OutputCOMPlusVariables">
<ItemGroup Condition="'$(TargetsWindows)' == 'true'">
- <_TestEnvFileLine Include="@echo off" />
<_TestEnvFileLine Include="@(_COMPlusVariable->'set %(Identity)=%(Value)')" />
</ItemGroup>
projects / platform / upstream / dotnet / runtime.git / commitdiff