Add dotnet cli to runtime test Helix jobs (#35426)

* Currently managed tools such as crossgen2 and XUnit are run against the built runtime which is slow on Debug builds, and can obscure errors when the XUnit test harness fails due to an introduced runtime bug.
* Add `xunit.console.runtimeconfig.dev.json` which allows `xunit.console` to use the repo-local dotnet.cmd on dev boxes and the same version installed on the path in Helix.
* When running crossgen2 during test execution, support both local dev and Helix scenario when deciding which dotnet to run. On Helix, simply use `dotnet` which assumes a compatible dotnet runtime is in the path. Locally, tests are run with `runtest.cmd|sh` which sets `__TestDotNetCmd` to the repo-local dotnet script. This preserves the characteristic that no machine-wide 5.0 dotnet runtime must be installed for the runtime tests.
* Update batch scripting to also use `dotnet.cmd|sh` when running Crossgen2 replacing corerun.exe.
* crossgen2's `runtimes` folder is not getting copied to `CORE_ROOT` which causes the runtime host to abort the launch on the Unix CI VMs since crossgen.deps.json refers to files in that subfolder. Adjust the `CORE_ROOT` pruning in `Directory.Build.targets` to include subfolders for the two tools that need it.
* Improve XUnit test boilerplate. Printing `Exception.Message` doesn't include stack trace. Use `ToString()` instead.
* Import notargets sdk in `helixpublicwitharcade.proj`. It doesn't use the official sdk so `BundledNETCoreAppPackageVersion` wasn't set. Import the `Microsoft.Build.NoTargets` sdk so we can find the bundled runtime package version.

diff --git a/eng/Versions.props b/eng/Versions.props
index 3588f19..20e0006 100644 (file)
--- a/eng/Versions.props
+++ b/eng/Versions.props
@@ -63,7 +63,7 @@
     <MicrosoftDotNetGenAPIVersion>5.0.0-beta.20261.9</MicrosoftDotNetGenAPIVersion>
     <MicrosoftDotNetGenFacadesVersion>5.0.0-beta.20261.9</MicrosoftDotNetGenFacadesVersion>
     <MicrosoftDotNetXUnitExtensionsVersion>5.0.0-beta.20261.9</MicrosoftDotNetXUnitExtensionsVersion>
-    <MicrosoftDotNetXUnitConsoleRunnerVersion>2.5.1-beta.20261.9</MicrosoftDotNetXUnitConsoleRunnerVersion>
+    <MicrosoftDotNetXUnitConsoleRunnerVersion>2.5.1-beta.20264.3</MicrosoftDotNetXUnitConsoleRunnerVersion>
     <MicrosoftDotNetBuildTasksPackagingVersion>5.0.0-beta.20261.9</MicrosoftDotNetBuildTasksPackagingVersion>
     <MicrosoftDotNetRemoteExecutorVersion>5.0.0-beta.20261.9</MicrosoftDotNetRemoteExecutorVersion>
     <MicrosoftDotNetVersionToolsTasksVersion>5.0.0-beta.20261.9</MicrosoftDotNetVersionToolsTasksVersion>

diff --git a/src/coreclr/build-test.cmd b/src/coreclr/build-test.cmd
index 9247e81..01a41c0 100644 (file)
--- a/src/coreclr/build-test.cmd
+++ b/src/coreclr/build-test.cmd
@@ -652,15 +652,17 @@ for %%F in ("%CORE_ROOT%\System.*.dll";"%CORE_ROOT%\Microsoft.*.dll";%CORE_ROOT%
     )))))
 )
 
-echo Composite response line^: %__CompositeResponseFile%
-type "%__CompositeResponseFile%"
+if defined __CompositeBuildMode (
+    echo Composite response line^: %__CompositeResponseFile%
+    type "%__CompositeResponseFile%"
+)
 
 if defined __CompositeBuildMode (
-    set __CompositeCommandLine="%CORE_ROOT%\corerun"
+    set __CompositeCommandLine="%__RepoRootDir%\dotnet.cmd"
     set __CompositeCommandLine=!__CompositeCommandLine! "%CORE_ROOT%\crossgen2\crossgen2.dll"
     set __CompositeCommandLine=!__CompositeCommandLine! "@%__CompositeResponseFile%"
     echo Building composite R2R framework^: !__CompositeCommandLine!
-    !__CompositeCommandLine!
+    call !__CompositeCommandLine!
     set __FailedToPrecompile=!ERRORLEVEL!
     copy /Y "!__CompositeOutputDir!\*.*" "!CORE_ROOT!\"
 )
@@ -684,7 +686,7 @@ if /i "%__BuildArch%" == "arm64" ( set __CrossgenExe="%__BinDir%\x64\crossgen.ex
 set __CrossgenExe=%__CrossgenExe%
 
 if defined __DoCrossgen2 (
-    set __CrossgenExe="%CORE_ROOT%\corerun" "%__BinDir%\crossgen2\crossgen2.dll"
+    set __CrossgenExe="%__RepoRootDir%\dotnet.cmd" "%CORE_ROOT%\crossgen2\crossgen2.dll"
 )
 
 REM Intentionally avoid using the .dll extension to prevent
@@ -694,12 +696,14 @@ set __CrossgenCmd=
 
 if defined __DoCrossgen (
     set __CrossgenCmd=!__CrossgenExe! /Platform_Assemblies_Paths "!CORE_ROOT!" /in !AssemblyPath! /out !__CrossgenOutputFile!
+    echo !__CrossgenCmd!
+    !__CrossgenCmd!
 ) else (
     set __CrossgenCmd=!__CrossgenExe! -r:"!CORE_ROOT!\System.*.dll" -r:"!CORE_ROOT!\Microsoft.*.dll" -r:"!CORE_ROOT!\mscorlib.dll" -r:"!CORE_ROOT!\netstandard.dll" -O --inputbubble --out:!__CrossgenOutputFile! !AssemblyPath!
+    echo !__CrossgenCmd!
+    call !__CrossgenCmd!
 )
 
-echo %__CrossgenCmd%
-%__CrossgenCmd%
 set /a __exitCode = !errorlevel!
 
 set /a "%~3+=1"

diff --git a/src/coreclr/build-test.sh b/src/coreclr/build-test.sh
index 4667513..78d294b 100755 (executable)
--- a/src/coreclr/build-test.sh
+++ b/src/coreclr/build-test.sh
@@ -180,8 +180,8 @@ precompile_coreroot_fx()
 
     local totalPrecompiled=0
     local failedToPrecompile=0
-    local compositeCommandLine="$overlayDir/corerun"
-    compositeCommandLine+=" ${__BinDir}/crossgen2/crossgen2.dll"
+    local compositeCommandLine="${__DotNetCli}"
+    compositeCommandLine+=" $__BinDir/crossgen2/crossgen2.dll"
     compositeCommandLine+=" --composite"
     compositeCommandLine+=" -O"
     compositeCommandLine+=" --out:$outputDir/framework-r2r.dll"
@@ -206,7 +206,7 @@ precompile_coreroot_fx()
         fi
 
         if [[ "$__DoCrossgen2" != 0 ]]; then
-            commandLine="$overlayDir/corerun $overlayDir/crossgen2/crossgen2.dll $crossgen2References -O --inputbubble --out $outputDir/$(basename $filename) $filename"
+            commandLine="${__DotNetCli} $overlayDir/crossgen2/crossgen2.dll $crossgen2References -O --inputbubble --out $outputDir/$(basename $filename) $filename"
         fi
 
         echo Precompiling "$filename"

diff --git a/src/coreclr/tests/helixpublishwitharcade.proj b/src/coreclr/tests/helixpublishwitharcade.proj
index c7aa317..6d46021 100644 (file)
--- a/src/coreclr/tests/helixpublishwitharcade.proj
+++ b/src/coreclr/tests/helixpublishwitharcade.proj
@@ -5,7 +5,7 @@
        to send test jobs to helix. -->
 
   <Import Sdk="Microsoft.DotNet.Helix.Sdk" Project="Sdk.props" Condition=" '$(UsesHelixSdk)' == 'true' " />
-  <Import Project="Directory.Build.props" Condition=" '$(UsesHelixSdk)' != 'true' "/>
+  <Import Sdk="Microsoft.Build.NoTargets" Project="Sdk.props" Condition=" '$(UsesHelixSdk)' != 'true' " />
 
   <!-- This target runs once and creates several instances of this project (one for each scenario)
        that will run in parallel. -->
@@ -33,10 +33,14 @@
         RunInUnloadableContext=$(_RunInUnloadableContext);
         TimeoutPerTestCollectionInMinutes=$(_TimeoutPerTestCollectionInMinutes);
         TimeoutPerTestInMinutes=$(_TimeoutPerTestInMinutes);
-        RuntimeMode=$(_RuntimeMode)
+        RuntimeMode=$(_RuntimeMode);
+        BundledNETCoreAppPackageVersion=$(BundledNETCoreAppPackageVersion)
       </_PropertiesToPass>
     </PropertyGroup>
 
+    <Message Text="DotNetCliVersion: $(DotNetCliVersion)" Importance="High" />
+    <Message Text="DotNetCliPackageType: $(DotNetCliPackageType)" Importance="High" />
+    <Message Text="HelixRuntimeRid: $(HelixRuntimeRid)" Importance="High" />
     <Error Condition="'$(_Scenarios)' == ''" Text="_Scenarios not set" />
 
     <MSBuild Projects="$(MSBuildProjectFile)" Targets="PrepareCorrelationPayloadDirectory" />
@@ -59,12 +63,24 @@
     <MSBuild Projects="@(_ProjectsToBuild)" Targets="Test" BuildInParallel="$(_BuildInParallel)" StopOnFirstFailure="false" Properties="UsesHelixSdk=true" />
   </Target>
 
+  <!-- Choose a suitable runtime RID for Helix to restore the dotnet cli -->
+  <PropertyGroup>
+    <HelixRuntimeRid Condition="'$(TargetOS)' == 'Windows_NT'">win-$(TargetArchitecture)</HelixRuntimeRid>
+    <HelixRuntimeRid Condition="'$(TargetOS)' == 'OSX'">osx-$(TargetArchitecture)</HelixRuntimeRid>
+    <HelixRuntimeRid Condition="'$(TargetOS)' == 'Linux' or '$(TargetOS)' == 'Linux_musl'">linux-$(TargetArchitecture)</HelixRuntimeRid>
+  </PropertyGroup>
+
   <PropertyGroup>
     <BinDir>$([MSBuild]::NormalizeDirectory($(TestWorkingDir)))</BinDir>
     <CoreRootDirectory>$(BinDir)Tests\Core_Root\</CoreRootDirectory>
     <PayloadsRootDirectory>$(BinDir)Payloads\</PayloadsRootDirectory>
     <TestEnvFileName Condition=" '$(TargetsWindows)' == 'true' ">SetStressModes_$(Scenario).cmd</TestEnvFileName>
     <TestEnvFileName Condition=" '$(TargetsWindows)' != 'true' ">SetStressModes_$(Scenario).sh</TestEnvFileName>
+
+    <IncludeDotNetCli>true</IncludeDotNetCli>
+    <DotNetCliPackageType>runtime</DotNetCliPackageType>
+    <DotNetCliVersion>$(BundledNETCoreAppPackageVersion)</DotNetCliVersion>
+    <DotNetCliRuntime>$(HelixRuntimeRid)</DotNetCliRuntime>
   </PropertyGroup>
 
   <Import Project="testgrouping.proj" />
@@ -236,12 +252,10 @@
   </PropertyGroup>
 
   <PropertyGroup Condition=" '$(TargetsWindows)' == 'true' ">
-    <CoreRun>%CORE_ROOT%\CoreRun.exe</CoreRun>
     <XUnitRunnerDll>%CORE_ROOT%\xunit.console.dll</XUnitRunnerDll>
   </PropertyGroup>
 
   <PropertyGroup Condition=" '$(TargetsWindows)' != 'true' ">
-    <CoreRun>$CORE_ROOT/corerun</CoreRun>
     <XUnitRunnerDll>$CORE_ROOT/xunit.console.dll</XUnitRunnerDll>
   </PropertyGroup>
 
@@ -263,12 +277,13 @@
 
     <HelixWorkItem Include="@(Payloads->Metadata('PayloadGroup'))">
       <PayloadDirectory>%(PayloadDirectory)</PayloadDirectory>
-      <Command>$(CoreRun) $(XUnitRunnerDll) %(XUnitWrapperDlls) $(XUnitRunnerArgs)</Command>
-      <Command Condition=" '%(TestGroup)' != '' ">$(CoreRun) $(XUnitRunnerDll) %(XUnitWrapperDlls) $(XUnitRunnerArgs) -trait TestGroup=%(TestGroup)</Command>
+      <Command>dotnet $(XUnitRunnerDll) %(XUnitWrapperDlls) $(XUnitRunnerArgs)</Command>
+      <Command Condition=" '%(TestGroup)' != '' ">dotnet $(XUnitRunnerDll) %(XUnitWrapperDlls) $(XUnitRunnerArgs) -trait TestGroup=%(TestGroup)</Command>
       <Timeout Condition=" '$(TimeoutPerTestCollectionInMinutes)' != '' ">$([System.TimeSpan]::FromMinutes($(TimeoutPerTestCollectionInMinutes)))</Timeout>
     </HelixWorkItem>
   </ItemGroup>
 
   <Import Sdk="Microsoft.DotNet.Helix.Sdk" Project="Sdk.targets" Condition=" '$(UsesHelixSdk)' == 'true' " />
+  <Import Sdk="Microsoft.Build.NoTargets" Project="Sdk.targets" Condition=" '$(UsesHelixSdk)' != 'true' " />
 
 </Project>

diff --git a/src/coreclr/tests/runtest.py b/src/coreclr/tests/runtest.py
index 68444d5..e5ef2b0 100755 (executable)
--- a/src/coreclr/tests/runtest.py
+++ b/src/coreclr/tests/runtest.py
@@ -934,6 +934,9 @@ def run_tests(args,
     print("Setting CORE_ROOT=%s" % args.core_root)
     os.environ["CORE_ROOT"] = args.core_root
 
+    # Set __TestDotNetCmd so tests which need to run dotnet can use the repo-local script on dev boxes
+    os.environ["__TestDotNetCmd"] = args.dotnetcli_script_path
+
     # Set test env script path if it is set.
     if test_env_script_path is not None:
         print("Setting __TestEnv=%s" % test_env_script_path)

diff --git a/src/coreclr/tests/src/CLRTest.CrossGen.targets b/src/coreclr/tests/src/CLRTest.CrossGen.targets
index 00c9990..0dcc732 100644 (file)
--- a/src/coreclr/tests/src/CLRTest.CrossGen.targets
+++ b/src/coreclr/tests/src/CLRTest.CrossGen.targets
@@ -83,7 +83,12 @@ if [ ! -z ${RunCrossGen2+x} ]%3B then
     rm $__ResponseFile
 
     __Command=$_DebuggerFullPath
-    __Command+=" $CORE_ROOT/corerun"
+    # Tests run locally need __TestDotNetCmd (set by runtest.py) or a compatible 5.0 dotnet runtime in the path
+    if [ ! -z ${__TestDotNetCmd+x} ] %3B then
+        __Command+=" $__TestDotNetCmd"
+    else
+        __Command+=" dotnet"
+    fi
     __Command+=" $CORE_ROOT/crossgen2/crossgen2.dll"
     __Command+=" @$__ResponseFile"
     __Command+=" $ExtraCrossGen2Args"
@@ -178,7 +183,12 @@ if defined RunCrossGen2 (
     del /Q !__ResponseFile!
 
     set __Command=!_DebuggerFullPath!
-    set __Command=!__Command! "!CORE_ROOT!\CoreRun.exe"
+    REM Tests run locally need __TestDotNetCmd (set by runtest.py) or a compatible 5.0 dotnet runtime in the path
+    if defined __TestDotNetCmd (
+        set __Command=!__Command! "!__TestDotNetCmd!"
+    ) else (
+        set __Command=!__Command! "dotnet"
+    )
     set __Command=!__Command! "!CORE_ROOT!\crossgen2\crossgen2.dll"
     set __Command=!__Command! @"!__ResponseFile!"
     set __Command=!__Command! !ExtraCrossGen2Args!

diff --git a/src/coreclr/tests/src/Common/Directory.Build.targets b/src/coreclr/tests/src/Common/Directory.Build.targets
index c4e3b79..65ff6bd 100644 (file)
--- a/src/coreclr/tests/src/Common/Directory.Build.targets
+++ b/src/coreclr/tests/src/Common/Directory.Build.targets
@@ -58,26 +58,37 @@
       <RunTimeArtifactsIncludeFolders Include="IL/" />
 
       <!-- Used for Crossgen2 R2R tests -->
-      <RunTimeArtifactsIncludeFolders Include="crossgen2/" />
+      <RunTimeArtifactsIncludeFolders Include="crossgen2/">
+        <IncludeSubFolders>True</IncludeSubFolders>
+      </RunTimeArtifactsIncludeFolders>
 
       <!-- Used for capturing symbolic stack traces using Watson -->
       <RunTimeArtifactsIncludeFolders Include="PDB/" />
 
       <!-- Used by the coreroot_determinism test -->
-      <RunTimeArtifactsIncludeFolders Include="R2RTest/" />
+      <RunTimeArtifactsIncludeFolders Include="R2RTest/">
+        <IncludeSubFolders>True</IncludeSubFolders>
+      </RunTimeArtifactsIncludeFolders>
     </ItemGroup>
 
     <ItemGroup>
       <!-- Add binary dependencies to copy-local items -->
       <RunTimeDependencyCopyLocal
+          Condition="'%(RuntimeArtifactsIncludeFolders.IncludeSubFolders)' != 'True'"
           Include="$(CoreCLRArtifactsPath)%(RunTimeArtifactsIncludeFolders.Identity)*"
           Exclude="@(RunTimeArtifactsExcludeFiles -> '$(CoreCLRArtifactsPath)%(Identity)')"
           TargetDir="%(RunTimeArtifactsIncludeFolders.Identity)" />
+
+      <RunTimeDependencyCopyLocal
+          Condition="'%(RuntimeArtifactsIncludeFolders.IncludeSubFolders)' == 'True'"
+          Include="$(CoreCLRArtifactsPath)%(RunTimeArtifactsIncludeFolders.Identity)**/*"
+          Exclude="@(RunTimeArtifactsExcludeFiles -> '$(CoreCLRArtifactsPath)%(Identity)')"
+          TargetDir="%(RunTimeArtifactsIncludeFolders.Identity)" />
     </ItemGroup>
 
     <Copy
       SourceFiles="@(RunTimeDependencyCopyLocal)"
-      DestinationFiles="@(RunTimeDependencyCopyLocal -> '$(CORE_ROOT)/%(TargetDir)%(Filename)%(Extension)')"
+      DestinationFiles="@(RunTimeDependencyCopyLocal -> '$(CORE_ROOT)/%(TargetDir)%(RecursiveDir)%(Filename)%(Extension)')"
       SkipUnchangedFiles="$(SkipCopyUnchangedFiles)"
       OverwriteReadOnlyFiles="$(OverwriteReadOnlyFiles)"
       Retries="$(CopyRetryCount)"

diff --git a/src/coreclr/tests/src/runtest.proj b/src/coreclr/tests/src/runtest.proj
index 562be79..da6ea0d 100644 (file)
--- a/src/coreclr/tests/src/runtest.proj
+++ b/src/coreclr/tests/src/runtest.proj
@@ -291,7 +291,7 @@ namespace $([System.String]::Copy($(Category)).Replace(".","_").Replace("\","").
 
                 if (infraEx != null)
                 {
-                    Assert.True(false, "Test Infrastructure Failure: " + infraEx.Message)%3B
+                    Assert.True(false, "Test Infrastructure Failure: " + infraEx.ToString())%3B
                 }
                 else
                 {
@@ -304,7 +304,7 @@ namespace $([System.String]::Copy($(Category)).Replace(".","_").Replace("\","").
                     catch (Exception ex)
                     {
                         testOutput.Add("Unable to read error file: " + errorFile)%3B
-                        testOutput.Add(ex.Message)%3B
+                        testOutput.Add(ex.ToString())%3B
                     }
 
                     testOutput.Add(string.Empty)%3B
@@ -319,7 +319,7 @@ namespace $([System.String]::Copy($(Category)).Replace(".","_").Replace("\","").
                     catch(Exception ex)
                     {
                         testOutput.Add("Unable to read output file: " + outputFile)%3B
-                        testOutput.Add(ex.Message)%3B
+                        testOutput.Add(ex.ToString())%3B
                     }
 
                     testOutput.Add("To run the test:")%3B