From: Bruce Forstall <brucefo@microsoft.com>
Date: Tue, 11 Jun 2019 21:18:08 +0000 (-0700)
Subject: Improve corefx testing (dotnet/coreclr#24962)
X-Git-Tag: submit/tizen/20210909.063632~11030^2~1248
X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=c4a1b3047510039de99eabe2bb14841b88d4d7b2;p=platform%2Fupstream%2Fdotnet%2Fruntime.git

Improve corefx testing (dotnet/coreclr#24962)

CoreFX tests are now used to test CoreCLR on Windows/x64 and Linux/x64.
PR testing includes normal and no-tiered modes. CI testing will include
all JIT stress modes that apply to CoreCLR testing. Many tests have been
disabled to allow the jobs to (mostly) run error-free.

Support for other platforms requires fixes to the testhost generation.


Commit migrated from https://github.com/dotnet/coreclr/commit/96c8a1259f5605f52a4d08d804c5f0cdb59308b2
---

diff --git a/eng/helixcorefxtests.proj b/eng/helixcorefxtests.proj
index 166eafe..980c5c4 100644
--- a/eng/helixcorefxtests.proj
+++ b/eng/helixcorefxtests.proj
@@ -1,10 +1,77 @@
-<Project InitialTargets="BuildHelixWorkItems" Sdk="Microsoft.DotNet.Helix.Sdk">
+<Project InitialTargets="BuildHelixWorkItems" DefaultTargets="RunInParallelForEachScenario">
+
+  <!-- This project uses the helix SDK, documented at
+       https://github.com/dotnet/arcade/tree/master/src/Microsoft.DotNet.Helix/Sdk,
+       to send test jobs to helix. -->
+
+  <Import Sdk="Microsoft.DotNet.Helix.Sdk" Project="Sdk.props" Condition=" '$(UsesHelixSdk)' == 'true' " />
+
+  <!-- Import Microsoft.DotNet.Build.Tasks.Feed.targets for `ParseBuildManifest` -->
+  <Import Project="..\dir.props" Condition=" '$(NuGetPackageRoot)'=='' "/>
   <Import Project="$(NuGetPackageRoot)microsoft.dotnet.build.tasks.feed\$(MicrosoftDotNetBuildTasksFeedVersion)\build\Microsoft.DotNet.Build.Tasks.Feed.targets" />
 
-  <PropertyGroup>
-    <!-- Set the TargetFramework just to make the SDK happy -->
-    <TargetFramework>netstandard2.0</TargetFramework>
-  </PropertyGroup>
+  <!-- This project is copies much logic from helixpublishwitharcade.proj, used to send coreclr tests to Helix. -->
+
+  <!--
+    "RunInParallelForEachScenario" is the "root" target for this Project. It invokes other targets to set up the
+    information needed to submit a job to Helix. It does so by "recursively" invoking this project file with
+    different properties and targets to run, as well as invoking MSBuild on other project files. In particular,
+    it first creates the "correlation payload directory", which is the directory of files used by all Helix
+    submissions. Then, it recursively invokes this Project once per scenario (e.g., normal Pri-1 run, JitStress=2
+    run, JitStressRegs=8 run, etc.), creating a new set of Helix work items for each scenario. For the CoreFX
+    tests, we create one Helix work item for each CoreFX test assembly; there is currently no concept of
+    grouping multiple assemblies into a single Helix work item to, for instance, consolidate multiple short-running
+    assemblies together. (Note that this grouping is done for the CoreCLR tests in helixpublishwitharcade.proj.)
+  -->
+  <Target Name="RunInParallelForEachScenario">
+    <PropertyGroup>
+      <!-- This specifies what properties are needed to be passed down as global properties to a child project. -->
+
+      <_PropertiesToPass>
+        __BuildArch=$(__BuildArch);
+        __BuildOS=$(__BuildOS);
+        __BuildType=$(__BuildType);
+        Creator=$(_Creator);
+        HelixAccessToken=$(_HelixAccessToken);
+        HelixBuild=$(_HelixBuild);
+        HelixSource=$(_HelixSource);
+        HelixTargetQueues=$(_HelixTargetQueues);
+        HelixType=$(_HelixType);
+        PublishTestResults=$(_PublishTestResults);
+        RunCrossGen=$(_RunCrossGen);
+        TimeoutPerTestCollectionInMinutes=$(_TimeoutPerTestCollectionInMinutes)
+      </_PropertiesToPass>
+    </PropertyGroup>
+
+    <Error Condition="'$(_Scenarios)' == ''" Text="_Scenarios not set" />
+
+    <MSBuild Projects="$(MSBuildProjectFile)" Targets="PrepareCorrelationPayloadDirectory" Properties="Scenarios=$(_Scenarios)" />
+
+    <ItemGroup>
+      <_Scenarios Include="$(_Scenarios.Split(','))" />
+
+      <!-- MSBuild creates a new instance of the project for each %(_Scenarios.Identity) and can build them in parallel. -->
+      <_ProjectsToBuild Include="$(MSBuildProjectFile)">
+        <AdditionalProperties>$(_PropertiesToPass);Scenario=%(_Scenarios.Identity)</AdditionalProperties>
+      </_ProjectsToBuild>
+    </ItemGroup>
+
+    <PropertyGroup>
+      <_BuildInParallel>false</_BuildInParallel>
+      <_BuildInParallel Condition=" '@(_ProjectsToBuild->Count())' &gt; '1' ">true</_BuildInParallel>
+    </PropertyGroup>
+
+    <!-- Invoke MSBuild on this project file once for each Scenario (because of the "batching" defined in "_ProjectsToBuild").
+         Set "UsesHelixSdk=true" to indicate we want to invoke Helix on this invocation: create the Helix work items and start
+         the jobs. This is done by invoking the "Test" Helix target.
+    -->
+    <MSBuild Projects="@(_ProjectsToBuild)" Targets="Test" BuildInParallel="$(_BuildInParallel)" StopOnFirstFailure="false" Properties="UsesHelixSdk=true" />
+  </Target>
+
+  <!-- Define a set of properties that are input to the Project, and that must be passed down to child processes.
+      (See "_PropertiesToPass" in target "RunInParallelForEachScenario".) If something can be computed, such as
+      properties needed by Helix, they should probably be set in target "BuildHelixWorkItems" instead.
+  -->
 
   <PropertyGroup>
     <Creator>$(_Creator)</Creator>
@@ -13,8 +80,6 @@
     <HelixSource>$(_HelixSource)</HelixSource>
     <HelixTargetQueues>$(_HelixTargetQueues)</HelixTargetQueues>
     <HelixType>$(_HelixType)</HelixType>
-    <HelixArchitecture>$(__BuildArch)</HelixArchitecture>
-    <HelixConfiguration>$(__BuildType)</HelixConfiguration>
 
     <!--
       TODO: ProjectDir, RootBinDir, TestWorkingDir, and TargetsWindows are global properties set in dir.props, remove the property assignment here when we port to arcade.
@@ -24,67 +89,104 @@
     <TestWorkingDir Condition="'$(__TestWorkingDir)'==''">$(RootBinDir)tests\$(__BuildOS).$(__BuildArch).$(__BuildType)\</TestWorkingDir>
     <TargetsWindows Condition="'$(__BuildOS)' == 'Windows_NT'">true</TargetsWindows>
 
+    <!-- We have a single, universal exclusion file for all CoreFX test exclusions, for CoreFX tests run
+         in the CoreCLR repo. Note in particular that we don't have a unique exclusion file for each test
+         assembly, or for each processor architecture, or for each operating system, or for each stress
+         scenario type. This means that we generally exclude more tests than is strictly necessary for
+         any particular job. However, this mechanism is super simple; we don't need to manage fine-
+         grained exclusions; and there really shouldn't be any (or many) exclusions anyway.
+    -->
+    <CoreFXTestExclusionFile>$(ProjectDir)tests\CoreFX\CoreFX.issues.rsp</CoreFXTestExclusionFile>
+
     <TestHostRootPath>$(TestWorkingDir)testhost\</TestHostRootPath>
     <TestArchiveRuntimeRoot>$(TestWorkingDir)helix\</TestArchiveRuntimeRoot>
-    <TestArchiveRuntimeFile>$(TestArchiveRuntimeRoot)testhost-runtime.zip</TestArchiveRuntimeFile>
-    <HelixCorrelationPayload></HelixCorrelationPayload>
 
-    <EnableAzurePipelinesReporter>$(_PublishTestResults)</EnableAzurePipelinesReporter>
-    <EnableAzurePipelinesReporter Condition=" '$(EnableAzurePipelinesReporter)' == '' ">false</EnableAzurePipelinesReporter>
-    <EnableXUnitReporter>true</EnableXUnitReporter>
-    <FailOnMissionControlTestFailure>true</FailOnMissionControlTestFailure>
-    <FailOnWorkItemFailure>true</FailOnWorkItemFailure>
-
-    <TimeoutInSeconds Condition="'$(TimeoutInSeconds)' == ''">600</TimeoutInSeconds>
-    <CommandTimeoutSpan>$([System.TimeSpan]::FromSeconds($(TimeoutInSeconds)))</CommandTimeoutSpan>
-    <MaxRetryCount Condition="'$(MaxRetryCount)' == ''">4</MaxRetryCount>
-    <WaitForWorkItemCompletion>true</WaitForWorkItemCompletion>
+    <TestAssetBlobFeedUrl>https://dotnetfeed.blob.core.windows.net/dotnet-core</TestAssetBlobFeedUrl>
   </PropertyGroup>
 
   <PropertyGroup>
     <!--
-      For windows we need to use call, since the command is going to be called from a bat script created by Helix
-      and we exit /b at the end of RunTests.cmd, Helix runs some other commands after ours within the bat script,
-      if we don't use call, then we cause the parent script to exit, and anything after will not be executed.
-      The arguments passed in to the run script in order are the runtime directory, the dotnet root directory (for
-      helix submissions same as the runtime directory) and the global tools directory.
+      For Windows, we need to use "call", since the command is going to be called from a batch script created by Helix.
+      We "exit /b" at the end of RunTests.cmd. Helix runs some other commands after ours within the batch script,
+      so if we don't use call, then we cause the parent script to exit, and anything after will not be executed.
+      The arguments passed in to the run script in are the runtime directory (-r) and the exclusion response file
+      passed to xunit (rsp-file).
     -->
     <HelixCommand Condition="'$(TargetsWindows)' == 'true'">call RunTests.cmd -r %HELIX_CORRELATION_PAYLOAD% --rsp-file %HELIX_CORRELATION_PAYLOAD%\CoreFX.issues.rsp </HelixCommand>
     <HelixCommand Condition="'$(TargetsWindows)' != 'true'">./RunTests.sh -r $HELIX_CORRELATION_PAYLOAD --rsp-file $HELIX_CORRELATION_PAYLOAD/CoreFX.issues.rsp </HelixCommand>
   </PropertyGroup>
 
-  <ItemGroup>
-    <TestArchiveRuntimeInputs Include="$(TestHostRootPath)**/*" />
-  </ItemGroup>
-
+  <!--
+    Copy the CoreFX test exclusion response file (which is passed to the xunit console runner) to someplace we can
+    access it during the test run. We currently just copy it to the root of the generated .NET Core testhost
+    directory, which we use as the root of the correlation directory.
+  -->
   <Target Name="CopyRSPFile">
     <Copy
-      SourceFiles="$(ProjectDir)\tests\CoreFX\CoreFX.issues.rsp"
+      SourceFiles="$(CoreFXTestExclusionFile)"
       DestinationFolder="$(TestHostRootPath)" />
+
+    <Message Importance="High" Text="Copied $(CoreFXTestExclusionFile) into $(TestHostRootPath)" />
   </Target>
 
-  <Target Name="CompressRuntimeDirectory"
-          Inputs="@(TestArchiveRuntimeInputs)"
-          Outputs="$(TestArchiveRuntimeFile)"
-          DependsOnTargets="CopyRSPFile" >
+  <PropertyGroup>
+    <!-- Set the name of the scenario file. Note that this is only used in invocations where $(Scenario) is set. -->
+    <TestEnvFileName Condition=" '$(TargetsWindows)' == 'true' ">SetStressModes_$(Scenario).cmd</TestEnvFileName>
+    <TestEnvFileName Condition=" '$(TargetsWindows)' != 'true' ">SetStressModes_$(Scenario).sh</TestEnvFileName>
+  </PropertyGroup>
 
-    <MakeDir Directories="$(TestArchiveRuntimeRoot)" />
+  <Target Name="CreateTestEnvFile">
+    <!-- This target creates one __TestEnv file for the single $(Scenario). -->
 
-    <ZipDirectory
-        SourceDirectory="$(TestHostRootPath)"
-        DestinationFile="$(TestArchiveRuntimeFile)"
-        Overwrite="true" />
+    <PropertyGroup>
+      <TestEnvFilePath>$(TestHostRootPath)$(TestEnvFileName)</TestEnvFilePath>
+    </PropertyGroup>
 
-    <Message Importance="High" Text="Zipped correlation payload into $(TestArchiveRuntimeFile)" />
+    <ItemGroup>
+      <_ProjectsToBuild Include="..\tests\testenvironment.proj">
+        <Properties>Scenario=$(Scenario);TestEnvFileName=$(TestEnvFilePath);TargetsWindows=$(TargetsWindows)</Properties>
+      </_ProjectsToBuild>
+    </ItemGroup>
+
+    <MSBuild Projects="@(_ProjectsToBuild)" Targets="CreateTestEnvFile" StopOnFirstFailure="true" />
+
+    <Message Importance="High" Text="Created $(TestEnvFilePath) for scenario $(Scenario)" />
+    <Error Condition="!Exists('$(TestEnvFilePath)')" Text="File $(TestEnvFilePath) not found!" />
   </Target>
 
-  <PropertyGroup>
-    <TestAssetBlobFeedUrl>https://dotnetfeed.blob.core.windows.net/dotnet-core</TestAssetBlobFeedUrl>
-  </PropertyGroup>
+  <Target Name="CreateAllScenarioTestEnvFiles">
+    <!-- This target creates one __TestEnv file for each of the scenarios in the $(Scenarios) comma-separated list. -->
 
-  <Target Name="GetTestAssetManifest" 
-          Returns="@(TestAssetBlobInfos)">
+    <ItemGroup>
+      <_Scenario Include="$(Scenarios.Split(','))" />
+      <_ProjectsToBuild Include="$(MSBuildProjectFile)">
+        <AdditionalProperties>Scenario=%(_Scenario.Identity)</AdditionalProperties>
+      </_ProjectsToBuild>
+    </ItemGroup>
+
+    <MSBuild Projects="@(_ProjectsToBuild)" Targets="CreateTestEnvFile" StopOnFirstFailure="true" />
+  </Target>
+
+  <!--
+    Collect all the tasks needed to be run once, to prepare the Helix correlation payload directory.
+    This just causes its dependent targets to be run.
+  -->
+  <Target Name="PrepareCorrelationPayloadDirectory"
+          DependsOnTargets="CopyRSPFile;CreateAllScenarioTestEnvFiles" >
+  </Target>
 
+  <!--
+    Parse the test asset manifest at, e.g.,
+        https://dotnetfeed.blob.core.windows.net/dotnet-core/corefx-tests/4.6.0-preview6.19264.9/Linux.arm64/netcoreapp/corefx-test-assets.xml
+    to figure out the tests to run. This manifest is created by the corefx official build, for use in the coreclr repo.
+    
+    This creates a "TestAssetBlobInfos" item group, which is used to create the Helix work items.
+
+    NOTE: we are running this once for each Scenario, meaning we are reading and parsing it once per scenario. Ideally, we would
+    do this once and pass it down to the "BuildHelixWorkItems" task later. That's a little difficult given all the re-invocations
+    of this Project we do. It doesn't seem too expensive anyway.
+  -->
+  <Target Name="GetTestAssetManifest" Condition=" '$(UsesHelixSdk)' == 'true' " >
     <PropertyGroup>
       <_TargetGroup>netcoreapp</_TargetGroup>
       <_AssetManifestPath>$(TestAssetBlobFeedUrl)/corefx-tests/$(MicrosoftPrivateCoreFxNETCoreAppVersion)/$(__BuildOS).$(__BuildArch)/$(_TargetGroup)/corefx-test-assets.xml</_AssetManifestPath>
@@ -95,22 +197,87 @@
     </ParseBuildManifest>
   </Target>
 
-  <Target Name="BuildHelixWorkItems"
-          Inputs="$(TestArchiveRuntimeFile);@(TestAssetBlobInfos)"
-          Outputs="@(HelixCorrelationPayload);@(HelixWorkItem)"
-          DependsOnTargets="CompressRuntimeDirectory;GetTestAssetManifest">
+  <!--
+    The following item and property groups are setting things needed by Helix, to describe the jobs
+    we are creating. Ideally, we would put them inside the "BuildHelixWorkItems" task, which is what
+    we run to create the Helix correlation payload and work item groups. However, the Helix SDK
+    references some of these in "InitialTargets", and we don't have control over the order of
+    InitialTargets running, so we need to make these "global". All the "global" item and property
+    groups will be evaluated before the Helix InitialTargets are run.
+  -->
 
-    <ItemGroup>
-      <HelixCorrelationPayload 
-        Include="$(TestArchiveRuntimeFile)" />
+  <!-- HelixPreCommands is a set of commands run before the work item command. We use it here to inject
+       setting up the per-scenario environment.
+  -->
+
+  <ItemGroup Condition=" '$(TargetsWindows)' == 'true' ">
+    <HelixPreCommand Include="set __TestEnv=%HELIX_CORRELATION_PAYLOAD%\$(TestEnvFileName)" />
+    <HelixPreCommand Include="type %__TestEnv%" />
+    <HelixPreCommand Include="call %__TestEnv%" />
+    <!-- Display the interesting COMPlus variables that are set in the environment -->
+    <HelixPreCommand Include="set COMPlus" />
+  </ItemGroup>
+
+  <ItemGroup Condition=" '$(TargetsWindows)' != 'true' ">
+    <HelixPreCommand Include="export __TestEnv=$HELIX_CORRELATION_PAYLOAD/$(TestEnvFileName)" />
+    <HelixPreCommand Include="cat $__TestEnv" />
+    <HelixPreCommand Include=". $__TestEnv" /> <!-- Use "." not "source"; some clients appear to run scripts with "sh" not "bash" -->
+    <!-- Display the interesting COMPlus variables that are set in the environment -->
+    <HelixPreCommand Include="printenv | grep COMPlus" />
+  </ItemGroup>
+
+  <PropertyGroup>
+    <HelixPreCommands>@(HelixPreCommand)</HelixPreCommands>
+  </PropertyGroup>
+
+  <PropertyGroup>
+    <HelixArchitecture>$(__BuildArch)</HelixArchitecture>
+
+    <HelixConfiguration Condition=" '$(Scenario)' == 'normal' ">$(BuildType)</HelixConfiguration>
+    <HelixConfiguration Condition=" '$(Scenario)' != 'normal' ">$(BuildType)-$(Scenario)</HelixConfiguration>
+
+    <TestRunNamePrefix>CoreFX </TestRunNamePrefix>
+    <TestRunNamePrefix Condition=" '$(Scenario)' == 'normal' ">$(TestRunNamePrefix)$(BuildOS) $(BuildArch) $(BuildType) @ </TestRunNamePrefix>
+    <TestRunNamePrefix Condition=" '$(Scenario)' != 'normal' ">$(TestRunNamePrefix)$(BuildOS) $(BuildArch) $(BuildType) $(Scenario) @ </TestRunNamePrefix>
+
+    <!-- REVIEW: it seems like this should be referencing "PublishTestResults" instead (without the underscore) -->
+    <EnableAzurePipelinesReporter>$(_PublishTestResults)</EnableAzurePipelinesReporter>
+    <EnableAzurePipelinesReporter Condition=" '$(EnableAzurePipelinesReporter)' == '' ">false</EnableAzurePipelinesReporter>
+
+    <EnableXUnitReporter>true</EnableXUnitReporter>
+    <FailOnMissionControlTestFailure>true</FailOnMissionControlTestFailure>
+    <FailOnWorkItemFailure>true</FailOnWorkItemFailure>
+    <WaitForWorkItemCompletion>true</WaitForWorkItemCompletion>
+
+    <!-- Specify the retry policy for Helix jobs -->
+    <!-- REVIEW: currently don't do any retries
+    <MaxRetryCount Condition="'$(MaxRetryCount)' == ''">4</MaxRetryCount>
+    -->
+  </PropertyGroup>
+
+  <!--
+    Create all the Helix data to start a set of jobs. This is invoked once for each Scenario (see
+    target "RunInParallelForEachScenario"). Create a set of work items, one for each CoreFX test assembly.
+    All will have the same Scenario specific command line.  Note that this target is listed in the
+    InitialTargets for this Project. This causes it to be invoked (and the Helix data created,
+    such as the HelixWorkItem item group) before Helix "Test" target is invoked (as a normal target).
+    We conditionalize this (and its dependent targets) with "UsesHelixSdk=true" so they aren't invoked
+    during the other invocations of this Project, such as when "RunInParallelForEachScenario" is run.
+  -->
+  <Target Name="BuildHelixWorkItems" DependsOnTargets="GetTestAssetManifest" Condition=" '$(UsesHelixSdk)' == 'true' " >
+    <Error Condition=" '$(TimeoutPerTestCollectionInMinutes)' == '' " Text="TimeoutPerTestCollectionInMinutes not set" />
+
+    <ItemGroup Condition=" '$(UsesHelixSdk)' == 'true' ">
+      <HelixCorrelationPayload Include="$(TestHostRootPath)" />
 
       <HelixWorkItem Include="@(TestAssetBlobInfos -> '%(FileName)')">
         <PayloadUri>$(TestAssetBlobFeedUrl)/%(Identity)</PayloadUri>
         <Command>$(HelixCommand)</Command>
-        <Timeout>$(CommandTimeoutSpan)</Timeout>
+        <Timeout>$([System.TimeSpan]::FromMinutes($(TimeoutPerTestCollectionInMinutes)))</Timeout>
       </HelixWorkItem>
     </ItemGroup>
-
-    <Message Importance="High" Text="@(HelixWorkItem -> '%(PayloadUri)')" />
   </Target>
+
+  <Import Sdk="Microsoft.DotNet.Helix.Sdk" Project="Sdk.targets" Condition=" '$(UsesHelixSdk)' == 'true' " />
+
 </Project>
diff --git a/eng/pipelines/coreclr/azure-pipelines.yml b/eng/pipelines/coreclr/azure-pipelines.yml
index fb05c8f..0762ed4 100644
--- a/eng/pipelines/coreclr/azure-pipelines.yml
+++ b/eng/pipelines/coreclr/azure-pipelines.yml
@@ -151,6 +151,10 @@ jobs:
         - Linux_x64
         - Windows_NT_x64
         - Windows_NT_x86
+      ${{ if in(variables['Build.DefinitionName'], 'coreclr-corefx', 'coreclr-corefx-jitstress', 'coreclr-corefx-jitstressregs', 'coreclr-corefx-jitstress2-jitstressregs') }}:
+        platforms:
+        - Linux_x64
+        - Windows_NT_x64
 
 #
 # Release builds
@@ -194,7 +198,8 @@ jobs:
 # Checked test builds
 #
 # The test jobs that can be triggered by a PR, manually from ADO and that are scheduled
-- ${{ if and(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest', 'Manual', 'Schedule')) }}:
+# TODO: This long 'in' condition needs to be make data driven, perhaps by adding another layer of template.
+- ${{ if and(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest', 'Manual', 'Schedule'), in(variables['Build.DefinitionName'], 'coreclr-ci', 'coreclr-outerloop', 'coreclr-outerloop-jitstress', 'coreclr-outerloop-jitstress-isas-arm', 'coreclr-outerloop-jitstress-isas-x86', 'coreclr-outerloop-jitstressregs-x86', 'coreclr-outerloop-jitstressregs', 'coreclr-outerloop-jitstress2-jitstressregs', 'coreclr-outerloop-gcstress0x3-gcstress0xc', 'coreclr-outerloop-gcstress-extra', 'coreclr-outerloop-r2r', 'coreclr-outerloop-r2r-extra')) }}:
   - template: eng/platform-matrix.yml
     parameters:
       jobTemplate: test-job.yml
@@ -238,31 +243,32 @@ jobs:
         ${{ if eq(variables['Build.DefinitionName'], 'coreclr-outerloop') }}:
           testGroup: outerloop
         ${{ if eq(variables['Build.DefinitionName'], 'coreclr-outerloop-jitstress') }}:
-          testGroup: outerloop-jitstress
+          testGroup: jitstress
         ${{ if eq(variables['Build.DefinitionName'], 'coreclr-outerloop-jitstress-isas-arm') }}:
-          testGroup: outerloop-jitstress-isas-arm
+          testGroup: jitstress-isas-arm
         ${{ if eq(variables['Build.DefinitionName'], 'coreclr-outerloop-jitstress-isas-x86') }}:
-          testGroup: outerloop-jitstress-isas-x86
+          testGroup: jitstress-isas-x86
         ${{ if eq(variables['Build.DefinitionName'], 'coreclr-outerloop-jitstressregs-x86') }}:
-          testGroup: outerloop-jitstressregs-x86
+          testGroup: jitstressregs-x86
         ${{ if eq(variables['Build.DefinitionName'], 'coreclr-outerloop-jitstressregs') }}:
-          testGroup: outerloop-jitstressregs
+          testGroup: jitstressregs
         ${{ if eq(variables['Build.DefinitionName'], 'coreclr-outerloop-jitstress2-jitstressregs') }}:
-          testGroup: outerloop-jitstress2-jitstressregs
+          testGroup: jitstress2-jitstressregs
         ${{ if eq(variables['Build.DefinitionName'], 'coreclr-outerloop-gcstress0x3-gcstress0xc') }}:
-          testGroup: outerloop-gcstress0x3-gcstress0xc
+          testGroup: gcstress0x3-gcstress0xc
         ${{ if eq(variables['Build.DefinitionName'], 'coreclr-outerloop-gcstress-extra') }}:
-          testGroup: outerloop-gcstress-extra
+          testGroup: gcstress-extra
         ${{ if eq(variables['Build.DefinitionName'], 'coreclr-outerloop-r2r') }}:
           testGroup: outerloop
           readyToRun: true
           displayNameArgs: R2R
         ${{ if eq(variables['Build.DefinitionName'], 'coreclr-outerloop-r2r-extra') }}:
-          testGroup: outerloop-r2r-extra
+          testGroup: r2r-extra
           readyToRun: true
           displayNameArgs: R2R
 
-# The ReadyToRun test jobs that are triggered by default from a PR.
+# ReadyToRun test jobs that are triggered by default from a PR.
+
 - ${{ if and(eq(variables['System.TeamProject'], 'public'), eq(variables['Build.Reason'], 'PullRequest'), eq(variables['Build.DefinitionName'], 'coreclr-ci')) }}:
   - template: eng/platform-matrix.yml
     parameters:
@@ -277,19 +283,43 @@ jobs:
         readyToRun: true
         testGroup: innerloop
         displayNameArgs: R2R
-  # The CoreFX runs against CoreCLR
+
+#
+# CoreFX test runs against CoreCLR
+#
+
+# PR innerloop checked build
+
+# TODO: This long 'in' condition needs to be make data driven, perhaps by adding another layer of template.
+- ${{ if and(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest', 'Manual', 'Schedule'), in(variables['Build.DefinitionName'], 'coreclr-ci', 'coreclr-corefx', 'coreclr-corefx-jitstress', 'coreclr-corefx-jitstressregs', 'coreclr-corefx-jitstress2-jitstressregs')) }}:
   - template: eng/platform-matrix.yml
     parameters:
       jobTemplate: test-job.yml
-      buildConfig: release
-      platforms:
-      - Windows_NT_x64
+      buildConfig: checked
+      ${{ if and(eq(variables['Build.Reason'], 'PullRequest'), eq(variables['Build.DefinitionName'], 'coreclr-ci')) }}:
+        platforms:
+        - Linux_x64
+        - Windows_NT_x64
+      ${{ if in(variables['Build.DefinitionName'], 'coreclr-corefx', 'coreclr-corefx-jitstress', 'coreclr-corefx-jitstressregs', 'coreclr-corefx-jitstress2-jitstressregs') }}:
+        platforms:
+        - Linux_x64
+        - Windows_NT_x64
       jobParameters:
+        ${{ if eq(variables['Build.DefinitionName'], 'coreclr-ci') }}:
+          testGroup: innerloop
+        ${{ if eq(variables['Build.DefinitionName'], 'coreclr-corefx') }}:
+          testGroup: outerloop
+        ${{ if eq(variables['Build.DefinitionName'], 'coreclr-corefx-jitstress') }}:
+          testGroup: jitstress
+        ${{ if eq(variables['Build.DefinitionName'], 'coreclr-corefx-jitstressregs') }}:
+          testGroup: jitstressregs
+        ${{ if eq(variables['Build.DefinitionName'], 'coreclr-corefx-jitstress2-jitstressregs') }}:
+          testGroup: jitstress2-jitstressregs
         corefxTests: true
-        testGroup: innerloop
         displayNameArgs: CoreFX
 
-# CI
+# CI (merge) jobs
+
 - ${{ if and(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI')) }}:
   - template: eng/platform-matrix.yml
     parameters:
@@ -314,8 +344,8 @@ jobs:
       - Windows_NT_arm
       - Windows_NT_arm64
       jobParameters:
-        readyToRun: true
         testGroup: outerloop
+        readyToRun: true
         displayNameArgs: R2R
 
 #
diff --git a/eng/test-job.yml b/eng/test-job.yml
index 010fb61..522ef8b 100644
--- a/eng/test-job.yml
+++ b/eng/test-job.yml
@@ -26,19 +26,28 @@ jobs:
     helixType: 'build/tests/'
 
     # Compute job name from template parameters
-    ${{ if eq(parameters.testGroup, 'innerloop') }}:
-      name: ${{ format('testbuild_pri0_{0}_{1}_{2}_{3}', parameters.displayNameArgs, parameters.osIdentifier, parameters.archType, parameters.buildConfig) }}
-      displayName: ${{ format('Test Pri0 {0} {1} {2}_{3}', parameters.displayNameArgs, parameters.osIdentifier, parameters.archType, parameters.buildConfig) }}
+    ${{ if and(eq(parameters.testGroup, 'innerloop'), eq(parameters.displayNameArgs, '')) }}:
+      name: ${{ format('test_{0}_{1}_{2}_{3}', 'p0', parameters.osIdentifier, parameters.archType, parameters.buildConfig) }}
+      displayName: ${{ format('Test {0} {1} {2} {3}', 'Pri0', parameters.osIdentifier, parameters.archType, parameters.buildConfig) }}
+
+    ${{ if and(ne(parameters.testGroup, 'innerloop'), eq(parameters.displayNameArgs, '')) }}:
+      name: ${{ format('test_{0}_{1}_{2}_{3}', 'p1', parameters.displayNameArgs, parameters.osIdentifier, parameters.archType, parameters.buildConfig) }}
+      displayName: ${{ format('Test {0} {1} {2} {3}', 'Pri1', parameters.displayNameArgs, parameters.osIdentifier, parameters.archType, parameters.buildConfig) }}
 
-    ${{ if ne(parameters.testGroup, 'innerloop') }}:
-      name: ${{ format('testbuild_pri1_{0}_{1}_{2}_{3}', parameters.displayNameArgs, parameters.osIdentifier, parameters.archType, parameters.buildConfig) }}
-      displayName: ${{ format('Test Pri1 {0} {1} {2} {3}', parameters.displayNameArgs, parameters.osIdentifier, parameters.archType, parameters.buildConfig) }}
+    ${{ if and(eq(parameters.testGroup, 'innerloop'), ne(parameters.displayNameArgs, '')) }}:
+      name: ${{ format('test_{0}_{1}_{2}_{3}_{4}', 'p0', parameters.displayNameArgs, parameters.osIdentifier, parameters.archType, parameters.buildConfig) }}
+      displayName: ${{ format('Test {0} {1} {2} {3} {4}', 'Pri0', parameters.displayNameArgs, parameters.osIdentifier, parameters.archType, parameters.buildConfig) }}
+
+    ${{ if and(ne(parameters.testGroup, 'innerloop'), ne(parameters.displayNameArgs, '')) }}:
+      name: ${{ format('test_{0}_{1}_{2}_{3}_{4}', 'p1', parameters.displayNameArgs, parameters.osIdentifier, parameters.archType, parameters.buildConfig) }}
+      displayName: ${{ format('Test {0} {1} {2} {3} {4}', 'Pri1', parameters.displayNameArgs, parameters.osIdentifier, parameters.archType, parameters.buildConfig) }}
 
     crossrootfsDir: ${{ parameters.crossrootfsDir }}
 
     variables:
     - ${{ if eq(variables['System.TeamProject'], 'internal') }}:
       - group: DotNet-HelixApi-Access
+
     # Map template parameters to command line arguments
     - name: priorityArg
       value: ''
@@ -92,11 +101,11 @@ jobs:
     # this number should be adjusted accordingly.
     ${{ if eq(parameters.testGroup, 'innerloop') }}:
       timeoutInMinutes: 240
-    ${{ if eq(parameters.testGroup, 'outerloop') }}:
+    ${{ if in(parameters.testGroup, 'outerloop') }}:
       timeoutInMinutes: 360
-    ${{ if in(parameters.testGroup, 'outerloop-jitstress', 'outerloop-jitstress-isas-arm', 'outerloop-jitstressregs-x86', 'outerloop-jitstressregs', 'outerloop-jitstress2-jitstressregs', 'outerloop-gcstress0x3-gcstress0xc') }}:
+    ${{ if in(parameters.testGroup, 'jitstress', 'jitstress-isas-arm', 'jitstressregs-x86', 'jitstressregs', 'jitstress2-jitstressregs', 'gcstress0x3-gcstress0xc') }}:
       timeoutInMinutes: 480
-    ${{ if in(parameters.testGroup, 'outerloop-jitstress-isas-x86', 'outerloop-gcstress-extra', 'outerloop-r2r-extra') }}:
+    ${{ if in(parameters.testGroup, 'jitstress-isas-x86', 'gcstress-extra', 'r2r-extra') }}:
       timeoutInMinutes: 600
 
     steps:
@@ -149,10 +158,16 @@ jobs:
         helixBuild: $(Build.BuildNumber)
         helixSource: $(_HelixSource)
 
-        ${{ if eq(parameters.readyToRun, false) }}:
-          helixType: 'test/functional/cli/'
-        ${{ if eq(parameters.readyToRun, true) }}:
+        # REVIEW: not sure why "cli" is part of the names here. Leave it for the ones that already had it,
+        # but don't add it to new ones.
+        ${{ if and(eq(parameters.corefxTests, true), eq(parameters.readyToRun, true)) }}:
+          helixType: 'test/functional/r2r_corefx/'
+        ${{ if and(eq(parameters.corefxTests, true), ne(parameters.readyToRun, true)) }}:
+          helixType: 'test/functional/corefx/'
+        ${{ if and(ne(parameters.corefxTests, true), eq(parameters.readyToRun, true)) }}:
           helixType: 'test/functional/r2r/cli/'
+        ${{ if and(ne(parameters.corefxTests, true), ne(parameters.readyToRun, true)) }}:
+          helixType: 'test/functional/cli/'
 
         helixQueues: ${{ parameters.helixQueues }}
 
@@ -162,22 +177,30 @@ jobs:
 
         publishTestResults: true
 
-        ${{ if eq(parameters.testGroup, 'innerloop') }}:
-          # "PerCollection" is time needed for the "biggest" xUnit test collection to complete.
-          # In case xUnit test wrappers get refactored this number should also be adjusted.
+        # Set job timeouts
+        #
+        # "timeoutPerTestCollectionInMinutes" is the time needed for the "biggest" xUnit test collection to complete.
+        # In case xUnit test wrappers get refactored this number should also be adjusted.
+        #
+        # "timeoutPerTestInMinutes" corresponds to individual test running time. This is implemented by setting
+        # the __TestTimeout variable, which is later read by the coreclr xunit test wrapper code (the code in the
+        # xunit test dlls that invokes the actual tests). Note this doesn't apply to CoreFX testing.
+
+        ${{ if and(eq(parameters.corefxTests, true), eq(parameters.testGroup, 'innerloop')) }}:
+          timeoutPerTestCollectionInMinutes: 120
+        ${{ if and(ne(parameters.corefxTests, true), eq(parameters.testGroup, 'innerloop')) }}:
           timeoutPerTestCollectionInMinutes: 30
-          # "PerTest" corresponds to individual test running time (i.e. __TestTimeout).
           timeoutPerTestInMinutes: 10
-        ${{ if eq(parameters.testGroup, 'outerloop') }}:
+        ${{ if in(parameters.testGroup, 'outerloop') }}:
           timeoutPerTestCollectionInMinutes: 120
           timeoutPerTestInMinutes: 10
-        ${{ if in(parameters.testGroup, 'outerloop-jitstress', 'outerloop-jitstress-isas-arm', 'outerloop-jitstress-isas-x86', 'outerloop-jitstressregs-x86', 'outerloop-jitstressregs', 'outerloop-jitstress2-jitstressregs') }}:
+        ${{ if in(parameters.testGroup, 'jitstress', 'jitstress-isas-arm', 'jitstress-isas-x86', 'jitstressregs-x86', 'jitstressregs', 'jitstress2-jitstressregs' ) }}:
           timeoutPerTestCollectionInMinutes: 120
           timeoutPerTestInMinutes: 30
-        ${{ if in(parameters.testGroup, 'outerloop-gcstress0x3-gcstress0xc') }}:
+        ${{ if in(parameters.testGroup, 'gcstress0x3-gcstress0xc') }}:
           timeoutPerTestCollectionInMinutes: 240
           timeoutPerTestInMinutes: 60
-        ${{ if in(parameters.testGroup, 'outerloop-gcstress-extra', 'outerloop-r2r-extra') }}:
+        ${{ if in(parameters.testGroup, 'gcstress-extra', 'r2r-extra') }}:
           timeoutPerTestCollectionInMinutes: 300
           timeoutPerTestInMinutes: 90
 
@@ -188,17 +211,17 @@ jobs:
           # DotNet-HelixApi-Access variable group
           helixAccessToken: $(HelixApiAccessToken)
 
-        # Sets up the template to run the corefx tests
+        # Choose which tests to send to Helix: CoreFX or CoreCLR.
         ${{ if eq(parameters.corefxTests, true) }}:
-          helixProjectArguments: 'eng/helixcorefxtests.proj /t:Test /p:ArcadeBuild=True'
-        ${{ if eq(parameters.corefxTests, false) }}:
+          helixProjectArguments: 'eng/helixcorefxtests.proj'
+        ${{ if ne(parameters.corefxTests, true) }}:
           helixProjectArguments: 'tests/helixpublishwitharcade.proj'
 
         ${{ if in(parameters.testGroup, 'innerloop', 'outerloop') }}:
           scenarios:
           - normal
           - no_tiered_compilation
-        ${{ if eq(parameters.testGroup, 'outerloop-jitstress') }}:
+        ${{ if in(parameters.testGroup, 'jitstress') }}:
           scenarios:
           - jitminopts
           - jitstress1
@@ -207,13 +230,13 @@ jobs:
           - jitstress2_tiered
           - zapdisable
           - tailcallstress
-        ${{ if eq(parameters.testGroup, 'outerloop-jitstress-isas-arm') }}:
+        ${{ if in(parameters.testGroup, 'jitstress-isas-arm') }}:
           scenarios:
           - jitstress_isas_incompletehwintrinsic
           - jitstress_isas_nohwintrinsic
           - jitstress_isas_nohwintrinsic_nosimd
           - jitstress_isas_nosimd
-        ${{ if eq(parameters.testGroup, 'outerloop-jitstress-isas-x86') }}:
+        ${{ if in(parameters.testGroup, 'jitstress-isas-x86') }}:
           scenarios:
           - jitstress_isas_incompletehwintrinsic
           - jitstress_isas_nohwintrinsic
@@ -236,7 +259,7 @@ jobs:
           - jitstress_isas_x86_nosse41
           - jitstress_isas_x86_nosse42
           - jitstress_isas_x86_nossse3
-        ${{ if eq(parameters.testGroup, 'outerloop-jitstressregs-x86') }}:
+        ${{ if in(parameters.testGroup, 'jitstressregs-x86') }}:
           scenarios:
           - jitstressregs1_x86_noavx
           - jitstressregs2_x86_noavx
@@ -246,7 +269,7 @@ jobs:
           - jitstressregs0x10_x86_noavx
           - jitstressregs0x80_x86_noavx
           - jitstressregs0x1000_x86_noavx
-        ${{ if eq(parameters.testGroup, 'outerloop-jitstressregs') }}:
+        ${{ if in(parameters.testGroup, 'jitstressregs' ) }}:
           scenarios:
           - jitstressregs1
           - jitstressregs2
@@ -256,7 +279,7 @@ jobs:
           - jitstressregs0x10
           - jitstressregs0x80
           - jitstressregs0x1000
-        ${{ if eq(parameters.testGroup, 'outerloop-jitstress2-jitstressregs') }}:
+        ${{ if in(parameters.testGroup, 'jitstress2-jitstressregs') }}:
           scenarios:
           - jitstress2_jitstressregs1
           - jitstress2_jitstressregs2
@@ -266,11 +289,11 @@ jobs:
           - jitstress2_jitstressregs0x10
           - jitstress2_jitstressregs0x80
           - jitstress2_jitstressregs0x1000
-        ${{ if eq(parameters.testGroup, 'outerloop-gcstress0x3-gcstress0xc') }}:
+        ${{ if in(parameters.testGroup, 'gcstress0x3-gcstress0xc') }}:
           scenarios:
           - gcstress0x3
           - gcstress0xc
-        ${{ if eq(parameters.testGroup, 'outerloop-gcstress-extra') }}:
+        ${{ if in(parameters.testGroup, 'gcstress-extra') }}:
           scenarios:
           - heapverify1
           - gcstress0xc_zapdisable
@@ -279,7 +302,7 @@ jobs:
           - gcstress0xc_jitstress1
           - gcstress0xc_jitstress2
           - gcstress0xc_jitminopts_heapverify1
-        ${{ if eq(parameters.testGroup, 'outerloop-r2r-extra') }}:
+        ${{ if in(parameters.testGroup, 'r2r-extra') }}:
           scenarios:
           - jitstress1
           - jitstress2
@@ -303,9 +326,13 @@ jobs:
       displayName: Publish Logs
       inputs:
         pathtoPublish: $(Build.SourcesDirectory)/bin/Logs
-        ${{ if eq(parameters.readyToRun, false) }}:
-          artifactName: ${{ format('testbuild_{0}_{1}_{2}_{3}_Logs', parameters.osIdentifier, parameters.archType, parameters.buildConfig, parameters.testGroup) }}
-        ${{ if  eq(parameters.readyToRun, true) }}:
-          artifactName: ${{ format('testbuild_{0}_{1}_{2}_{3}_r2r_Logs', parameters.osIdentifier, parameters.archType, parameters.buildConfig, parameters.testGroup) }}
+        ${{ if and(eq(parameters.corefxTests, true), eq(parameters.readyToRun, true)) }}:
+          artifactName: ${{ format('test{0}_{1}_{2}_{3}_{4}_Logs', '_r2r_corefx', parameters.osIdentifier, parameters.archType, parameters.buildConfig, parameters.testGroup) }}
+        ${{ if and(eq(parameters.corefxTests, true), ne(parameters.readyToRun, true)) }}:
+          artifactName: ${{ format('test{0}_{1}_{2}_{3}_{4}_Logs', '_corefx',     parameters.osIdentifier, parameters.archType, parameters.buildConfig, parameters.testGroup) }}
+        ${{ if and(ne(parameters.corefxTests, true), eq(parameters.readyToRun, true)) }}:
+          artifactName: ${{ format('test{0}_{1}_{2}_{3}_{4}_Logs', '_r2r',        parameters.osIdentifier, parameters.archType, parameters.buildConfig, parameters.testGroup) }}
+        ${{ if and(ne(parameters.corefxTests, true), ne(parameters.readyToRun, true)) }}:
+          artifactName: ${{ format('test{0}_{1}_{2}_{3}_{4}_Logs', '',            parameters.osIdentifier, parameters.archType, parameters.buildConfig, parameters.testGroup) }}
       continueOnError: true
       condition: always()
diff --git a/src/coreclr/tests/CoreFX/CoreFX.issues.rsp b/src/coreclr/tests/CoreFX/CoreFX.issues.rsp
index 743f0d5..632658c 100644
--- a/src/coreclr/tests/CoreFX/CoreFX.issues.rsp
+++ b/src/coreclr/tests/CoreFX/CoreFX.issues.rsp
@@ -70,4 +70,46 @@
 -nomethod System.Tests.StringTests.CasingNegativeTest
 -nomethod System.Tests.StringTests.CompareNegativeTest
 -nomethod System.Tests.StringTests.ToLowerNullCulture
--nomethod System.Tests.StringTests.ToUpperNullCulture
\ No newline at end of file
+-nomethod System.Tests.StringTests.ToUpperNullCulture
+
+#
+# Temporarily disable tests while bringing more CoreFX job configurations online.
+#
+
+-nonamespace System.Net.Http.Functional.Tests
+-nonamespace System.Net.Sockets.Tests
+
+-noclass System.Net.Tests.HttpListenerContextTests
+-noclass System.Net.Tests.HttpListenerAuthenticationTests
+-noclass System.Net.Tests.HttpListenerResponseTests
+-noclass System.Net.Tests.HttpListenerResponseCookiesTests
+
+-nomethod System.Net.Tests.HttpListenerResponseTests.Redirect_Invoke_SetsRedirectionProperties
+-nomethod System.Net.Mail.Tests.SmtpClientTest.TestMailDeliveryAsync
+
+-nomethod System.Net.Security.Tests.LoggingTest.EventSource_EventsRaisedAsExpected
+-nomethod System.Net.Security.Tests.SslStreamSniTest.SslStream_NoSniFromClient_CallbackReturnsNull
+
+# System.Linq.Expressions.Tests: timeout on Linux/x64
+-nonamespace System.Linq.Expressions.Tests
+
+# assert/crashes
+-nomethod System.Tests.RealFormatterTestsBase.TestFormatterSingle_F20
+-noclass System.Tests.RealFormatterTestsBase
+-noclass System.Tests.RealFormatterTests
+-noclass System.Buffers.Text.Tests.RealFormatterTests
+
+# System.Runtime.Serialization.Formatters.Tests: timeouts
+-nonamespace System.Runtime.Serialization.Formatters.Tests
+
+-nomethod Generic.Dictionary.DictionaryConcurrentAccessDetectionTests.DictionaryConcurrentAccessDetection_ReferenceTypeKey
+-nomethod Generic.Dictionary.DictionaryConcurrentAccessDetectionTests.DictionaryConcurrentAccessDetection_ValueTypeKey
+-nomethod MonoTests.System.Runtime.Caching.MemoryCacheTest.Contains
+-nomethod System.Numerics.Tests.ToStringTest.RunRegionSpecificStandardFormatToStringTests
+-nomethod System.Text.RegularExpressions.Tests.RegexMatchTests.Match_ExcessPrefix
+
+# Assert with JITMinOpts=1: https://github.com/dotnet/coreclr/issues/25070
+-nonamespace Microsoft.VisualBasic.Tests
+
+# Assert: https://github.com/dotnet/coreclr/issues/25050
+-nonamespace System.Data.Common.Tests
diff --git a/src/coreclr/tests/helixpublishwitharcade.proj b/src/coreclr/tests/helixpublishwitharcade.proj
index 74d5eb7..c7994c3 100644
--- a/src/coreclr/tests/helixpublishwitharcade.proj
+++ b/src/coreclr/tests/helixpublishwitharcade.proj
@@ -31,6 +31,8 @@
       </_PropertiesToPass>
     </PropertyGroup>
 
+    <Error Condition="'$(_Scenarios)' == ''" Text="_Scenarios not set" />
+
     <MSBuild Projects="$(MSBuildProjectFile)" Targets="PrepareCorrelationPayloadDirectory" />
     <MSBuild Projects="$(MSBuildProjectFile)" Targets="PreparePayloadsDirectories" Properties="Scenarios=$(_Scenarios)" StopOnFirstFailure="true" />
 
diff --git a/src/coreclr/tests/testenvironment.proj b/src/coreclr/tests/testenvironment.proj
index ee6c039..7a0aa63 100644
--- a/src/coreclr/tests/testenvironment.proj
+++ b/src/coreclr/tests/testenvironment.proj
@@ -154,7 +154,6 @@
 
   <Target Name="CreateTestEnvFile" DependsOnTargets="OutputCOMPlusVariables">
     <ItemGroup Condition="'$(TargetsWindows)' == 'true'">
-      <_TestEnvFileLine Include="@echo off" />
       <_TestEnvFileLine Include="@(_COMPlusVariable->'set %(Identity)=%(Value)')"  />
     </ItemGroup>