Always run CopyAllNativeProjectReferenceBinaries (#67718)

Previously this target was only called from the recursive traversal
used when globally copying native artifacts for all projects (this
is used in the lab where the native artifacts are produced on
a different machine than the managed test artifacts). In case of
combined (local) test build we weren't running the target, only
its dependencies (ResolveCMakeNativeProjectReference and
ConsolidateNativeProjectReference); this used to be sufficient as
these two targets handle all logic related to copying native
components of a normal project, however I have recently added
special logic for copying merged test wrapper native artifacts
the CopyAllNativeProjectReferenceBinaries target and so we now must
make sure it also always runs.

Mark JIT optimization-sensitive tests as out-of-process and fix
logic around skipping of out-of-process tests on the excluded platforms.

Fix GetFilteredExcludeList to properly report out-of-process tests

Add issues.targets exclusions for compat_i4_u_il_d / compat_i4_u_il_r
on all Mono platforms

Log timestamps in generated wrapper for investigation of timeouts

Log timestamps around Crossgen2 invocations for investigation of timeouts

Suppress noisy stderr spew for deletion of non-existent rsp files

Expand test timeout to 30 minutes

In the merged scenario we're intentionally reducing the number of
work items at the expense of a single work item size. When
Crossgen2 compilation is involved, with hundreds of tests in the
Methodical_*** projects we're hitting the 15 minute test timeout
so I'm bumping it up to 30. I have also suppressed uninteresting
stderr spew from deletion of the pre-existing composite image.

Thanks

Tomas

diff --git a/src/tests/Common/CLRTest.CrossGen.targets b/src/tests/Common/CLRTest.CrossGen.targets
index 606c4a6..30102cc 100644 (file)
--- a/src/tests/Common/CLRTest.CrossGen.targets
+++ b/src/tests/Common/CLRTest.CrossGen.targets
@@ -65,7 +65,7 @@ if [ ! -z ${RunCrossGen2+x} ]%3B then
         cp $CORE_ROOT/lib*.so $CORE_ROOT/lib*.dylib $(scriptPath)
       else
         cp *.dll IL-CG2/
-        rm IL-CG2/composite-r2r.dll
+        rm IL-CG2/composite-r2r.dll 2>/dev/null
       fi
 
       ExtraCrossGen2Args+=" $(CrossGen2TestExtraArguments)"
@@ -77,10 +77,11 @@ if [ ! -z ${RunCrossGen2+x} ]%3B then
       __cg2ExitCode=0
 
       OneFileCrossgen2() {
+        date +%H:%M:%S
         __OutputFile=$1
 
         __ResponseFile="$__OutputFile.rsp"
-        rm $__ResponseFile
+        rm $__ResponseFile 2>/dev/null
 
         __Command=$_DebuggerFullPath
         # Tests run locally need __TestDotNetCmd (set by runtest.py) or a compatible 5.0 dotnet runtime in the path
@@ -114,6 +115,7 @@ if [ ! -z ${RunCrossGen2+x} ]%3B then
         __cg2ExitCode=$?
 
         export COMPlus_GCName COMPlus_GCStress COMPlus_HeapVerify COMPlus_ReadyToRun COMPlus_TC_OnStackReplacement COMPlus_TC_PartialCompilation
+        date +%H:%M:%S
       }
 
       if [ ! -z ${CompositeBuildMode+x} ]%3B then
@@ -183,7 +185,7 @@ if defined RunCrossGen2 (
 
     mkdir IL-CG2
     copy *.dll IL-CG2\
-    del IL-CG2\composite-r2r.dll
+    del IL-CG2\composite-r2r.dll 2>nul
 
     if defined CompositeBuildMode (
         set ExtraCrossGen2Args=!ExtraCrossGen2Args! --composite
@@ -208,8 +210,9 @@ if defined RunCrossGen2 (
 
     goto :DoneCrossgen2Operations
 :CompileOneFileCrossgen2
+    echo %time%
     set __ResponseFile=!__OutputFile!.rsp
-    del /Q !__ResponseFile!
+    del /Q !__ResponseFile! 2>nul
 
     set __Command=!_DebuggerFullPath!
     REM Tests run locally need __TestDotNetCmd (set by runtest.py) or a compatible 5.0 dotnet runtime in the path
@@ -253,6 +256,7 @@ if defined RunCrossGen2 (
     call !__Command!
     endlocal
     set CrossGen2Status=!ERRORLEVEL!
+    echo %time%
     Exit /b 0
 
 :DoneCrossgen2Operations

diff --git a/src/tests/Common/CoreCLRTestLibrary/OutOfProcessTest.cs b/src/tests/Common/CoreCLRTestLibrary/OutOfProcessTest.cs
index 976f542..ad68f35 100644 (file)
--- a/src/tests/Common/CoreCLRTestLibrary/OutOfProcessTest.cs
+++ b/src/tests/Common/CoreCLRTestLibrary/OutOfProcessTest.cs
@@ -1,4 +1,4 @@
-// Licensed to the .NET Foundation under one or more agreements.
+// Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
 using System;
@@ -71,6 +71,12 @@ namespace TestLibrary
                     testExecutable = Path.Combine(baseDir, Path.ChangeExtension(assemblyPath.Replace("\\", "/"), ".sh"));
                 }
 
+                if (!File.Exists(testExecutable))
+                {
+                    // Skip platform-specific test when running on the excluded platform
+                    return;
+                }
+
                 System.IO.Directory.CreateDirectory(outputDir);
 
                 ret = wrapper.RunTest(testExecutable, outputFile, errorFile, Assembly.GetEntryAssembly()!.FullName!, testBinaryBase, outputDir);

diff --git a/src/tests/Common/XUnitWrapperGenerator/ITestInfo.cs b/src/tests/Common/XUnitWrapperGenerator/ITestInfo.cs
index d0db16d..213fc3d 100644 (file)
--- a/src/tests/Common/XUnitWrapperGenerator/ITestInfo.cs
+++ b/src/tests/Common/XUnitWrapperGenerator/ITestInfo.cs
@@ -338,15 +338,15 @@ sealed class WrapperLibraryTestSummaryReporting : ITestReporterWrapper
 
         builder.AppendLine($"System.TimeSpan testStart = stopwatch.Elapsed;");
         builder.AppendLine("try {");
-        builder.AppendLine($"System.Console.WriteLine(\"Running test: {{0}}\", {test.TestNameExpression});");
+        builder.AppendLine($"System.Console.WriteLine(\"{{0:HH:mm:ss.fff}} Running test: {{1}}\", System.DateTime.Now, {test.TestNameExpression});");
         builder.AppendLine($"{_outputRecorderIdentifier}.ResetTestOutput();");
         builder.AppendLine(testExecutionExpression);
         builder.AppendLine($"{_summaryLocalIdentifier}.ReportPassedTest({test.TestNameExpression}, \"{test.ContainingType}\", @\"{test.Method}\", stopwatch.Elapsed - testStart, {_outputRecorderIdentifier}.GetTestOutput());");
-        builder.AppendLine($"System.Console.WriteLine(\"Passed test: {{0}}\", {test.TestNameExpression});");
+        builder.AppendLine($"System.Console.WriteLine(\"{{0:HH:mm:ss.fff}} Passed test: {{1}}\", System.DateTime.Now, {test.TestNameExpression});");
         builder.AppendLine("}");
         builder.AppendLine("catch (System.Exception ex) {");
         builder.AppendLine($"{_summaryLocalIdentifier}.ReportFailedTest({test.TestNameExpression}, \"{test.ContainingType}\", @\"{test.Method}\", stopwatch.Elapsed - testStart, ex, {_outputRecorderIdentifier}.GetTestOutput());");
-        builder.AppendLine($"System.Console.WriteLine(\"Failed test: {{0}}\", {test.TestNameExpression});");
+        builder.AppendLine($"System.Console.WriteLine(\"{{0:HH:mm:ss.fff}} Failed test: {{1}}\", System.DateTime.Now, {test.TestNameExpression});");
         builder.AppendLine("}");
 
         builder.AppendLine("}");

diff --git a/src/tests/Directory.Build.targets b/src/tests/Directory.Build.targets
index a9ddd84..a9caa31 100644 (file)
--- a/src/tests/Directory.Build.targets
+++ b/src/tests/Directory.Build.targets
@@ -217,12 +217,17 @@
   </Target>
 
   <Target Name="CopyAllNativeProjectReferenceBinaries"
-          DependsOnTargets="ResolveCMakeNativeProjectReference;ConsolidateNativeProjectReference">
+          DependsOnTargets="ResolveCMakeNativeProjectReference;ConsolidateNativeProjectReference"
+          BeforeTargets="Build">
 
     <ItemGroup Condition="'$(IsMergedTestRunnerAssembly)' == 'true'">
-      <MergedWrapperReferenceFolders Include="@(ProjectReference->'$([System.IO.Path]::GetFullPath('%(ProjectReference.Identity)/..', '$(OutDir)/..'))/%(ProjectReference.FileName)')" />
+      <OutOfProcessTestMarkers Include="$(OutDir)/../**/*.OutOfProcessTest" />
+      <OutOfProcessTestFolders Include="$([MSBuild]::NormalizePath('$([System.IO.Path]::GetDirectoryName('%(OutOfProcessTestMarkers.FullPath)'))'))" Condition="'@(OutOfProcessTestMarkers)' != ''" />
+      <MergedWrapperReferenceFolders Include="@(ProjectReference->'$([MSBuild]::NormalizePath('$([System.IO.Path]::GetFullPath('%(ProjectReference.Identity)/..', '$(OutDir)/..'))/%(ProjectReference.FileName)'))')" />
+      <!-- Don't copy out-of-process test components to the merged wrapper output folder -->
+      <MergedWrapperReferenceFolders Remove="@(OutOfProcessTestFolders)" />
       <!-- For merged project wrappers, include native libraries in all project references -->
-      <MergedWrapperReferenceFiles Include="%(MergedWrapperReferenceFolders.Identity)/*$(LibSuffix)" />
+      <MergedWrapperReferenceFiles Include="%(MergedWrapperReferenceFolders.Identity)/*$(LibSuffix)" Condition="'@(MergedWrapperReferenceFolders)' != ''" />
     </ItemGroup>
     <Copy SourceFiles="@(MergedWrapperReferenceFiles)"
         DestinationFiles="@(MergedWrapperReferenceFiles->'$(OutDir)/%(FileName)%(Extension)')"

diff --git a/src/tests/JIT/Methodical/Arrays/misc/arrres_il_r.ilproj b/src/tests/JIT/Methodical/Arrays/misc/arrres_il_r.ilproj
index 1020cf9..28aac69 100644 (file)
--- a/src/tests/JIT/Methodical/Arrays/misc/arrres_il_r.ilproj
+++ b/src/tests/JIT/Methodical/Arrays/misc/arrres_il_r.ilproj
@@ -7,6 +7,7 @@
   <PropertyGroup>
     <DebugType>PdbOnly</DebugType>
     <JitOptimizationSensitive>true</JitOptimizationSensitive>
+    <RequiresProcessIsolation>true</RequiresProcessIsolation>
   </PropertyGroup>
   <ItemGroup>
     <Compile Include="arrres.il" />

diff --git a/src/tests/JIT/Methodical/tailcall_v4/hijacking.ilproj b/src/tests/JIT/Methodical/tailcall_v4/hijacking.ilproj
index 2856022..da27e6f 100644 (file)
--- a/src/tests/JIT/Methodical/tailcall_v4/hijacking.ilproj
+++ b/src/tests/JIT/Methodical/tailcall_v4/hijacking.ilproj
@@ -13,6 +13,7 @@
   </PropertyGroup>
   <PropertyGroup>
     <JitOptimizationSensitive>true</JitOptimizationSensitive>
+    <RequiresProcessIsolation>true</RequiresProcessIsolation>
   </PropertyGroup>
   <ItemGroup>
     <Compile Include="hijacking.il" />

diff --git a/src/tests/issues.targets b/src/tests/issues.targets
index caaae13..31c42e2 100644 (file)
--- a/src/tests/issues.targets
+++ b/src/tests/issues.targets
@@ -1868,6 +1868,12 @@
         <ExcludeList Include="$(XunitTestBinBase)/JIT/Methodical/refany/lcs_il_r/**">
             <Issue>https://github.com/dotnet/runtime/issues/34196</Issue>
         </ExcludeList>
+        <ExcludeList Include = "$(XunitTestBinBase)/JIT/Methodical/tailcall/compat_i4_u_il_d/**">
+            <Issue>https://github.com/dotnet/runtime/issues/67756</Issue>
+        </ExcludeList>
+        <ExcludeList Include = "$(XunitTestBinBase)/JIT/Methodical/tailcall/compat_i4_u_il_r/**">
+            <Issue>https://github.com/dotnet/runtime/issues/67756</Issue>
+        </ExcludeList>
         <ExcludeList Include="$(XunitTestBinBase)JIT/Methodical/tailcall_v4/hijacking/**">
             <Issue>needs triage</Issue>
         </ExcludeList>
@@ -4007,6 +4013,8 @@
             <FilteredExcludeList
                 Include="$([System.IO.Path]::GetRelativePath('$(XunitTestBinBase)', '%(ExcludeList.FullPath)'))"
                 Condition="'%(ExcludeList.Extension)' == '.dll'" />
+            <FilteredExcludeList Include="$([System.IO.Path]::ChangeExtension('$([System.IO.Path]::GetRelativePath('$(XunitTestBinBase)', '%(ExcludeList.RootDir)%(ExcludeList.Directory)%(ExcludeList.FileName)'))', '.cmd'))"
+                Condition="'%(ExcludeList.Extension)' == '.OutOfProcessTest'" />
       </ItemGroup>
     </Target>
 </Project>

diff --git a/src/tests/run.py b/src/tests/run.py
index aab137b..b6a1565 100755 (executable)
--- a/src/tests/run.py
+++ b/src/tests/run.py
@@ -844,8 +844,8 @@ def run_tests(args,
     if args.skip_test_run:
         return
 
-    # Set default per-test timeout to 15 minutes (in milliseconds).
-    per_test_timeout = 15*60*1000
+    # Set default per-test timeout to 30 minutes (in milliseconds).
+    per_test_timeout = 30*60*1000
 
     # Setup the environment
     if args.long_gc: