Fix JitBench (#19584)

Fix JitBench

Fixes https://github.com/dotnet/coreclr/issues/19569
- I believe the issue was that JitBench (which was an netstandard1.6 project) was running against a netcoreapp3.0 layout with corerun (layout built by build + recent packages), and XmlDocument is in a different place now
- Copied the unofficial project and replaced the official one
- Switched to netcoreapp3.0 along with other miscellaneous fixes to get it working
- I haven't figured out yet how to get one project that builds with the test build and works with dotnet. TargetFramework would have to be different anyway if running with dotnet runtime 2.1. So the unofficial project is still there for use with dotnet.
- Also retargeted the empty console project and changed to always set ExePath (when using existing setup for instance)

diff --git a/tests/src/performance/Scenario/JitBench/Benchmarks/EmptyConsoleProgramExecution.cs b/tests/src/performance/Scenario/JitBench/Benchmarks/EmptyConsoleProgramExecution.cs
index 6dbb5d9..ba20c13 100644 (file)
--- a/tests/src/performance/Scenario/JitBench/Benchmarks/EmptyConsoleProgramExecution.cs
+++ b/tests/src/performance/Scenario/JitBench/Benchmarks/EmptyConsoleProgramExecution.cs
@@ -11,7 +11,10 @@ namespace JitBench
     {
         private const string ExecutableName = "console.dll";
 
-        public EmptyConsoleProgramExecution() : base("Empty Console Program") { }
+        public EmptyConsoleProgramExecution() : base("Empty Console Program")
+        {
+            ExePath = ExecutableName;
+        }
 
         public override async Task Setup(DotNetInstallation dotNetInstall, string outputDir, bool useExistingSetup, ITestOutputHelper output)
         {
@@ -20,6 +23,7 @@ namespace JitBench
                 using (var setupSection = new IndentedTestOutputHelper("Setup " + Name, output))
                 {
                     await SetupSourceToCompile(outputDir, dotNetInstall.FrameworkDir, useExistingSetup, setupSection);
+                    RetargetProjects(dotNetInstall, GetRootDir(outputDir), new string[] { "console.csproj" });
                     await Publish(dotNetInstall, outputDir, setupSection);
                 }
             }
@@ -77,8 +81,6 @@ namespace JitBench
             publishDir = GetAppPublishDirectory(dotNetInstall, outputDir, tfm);
             if (publishDir == null)
                 throw new DirectoryNotFoundException("Could not find 'publish' directory");
-
-            ExePath = Path.Combine(publishDir, ExecutableName);
             return publishDir;
         }
 

diff --git a/tests/src/performance/Scenario/JitBench/JitBench.csproj b/tests/src/performance/Scenario/JitBench/JitBench.csproj
index 2e384d1..583a3c2 100644 (file)
--- a/tests/src/performance/Scenario/JitBench/JitBench.csproj
+++ b/tests/src/performance/Scenario/JitBench/JitBench.csproj
@@ -1,58 +1,52 @@
 <?xml version="1.0" encoding="utf-8"?>
 <Project ToolsVersion="12.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+
   <Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory), dir.props))\dir.props" />
+
   <PropertyGroup>
-    <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
-    <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
-    <AssemblyName>JitBench</AssemblyName>
-    <SchemaVersion>2.0</SchemaVersion>
-    <ProjectGuid>{507E3CC2-5D95-414D-9F01-2A106FC177DC}</ProjectGuid>
-    <OutputType>exe</OutputType>
-    <AppDesignerFolder>Properties</AppDesignerFolder>
-    <FileAlignment>512</FileAlignment>
-    <ProjectTypeGuids>{786C830F-07A1-408B-BD7F-6EE04809D6DB};{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}</ProjectTypeGuids>
-    <SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\..\</SolutionDir>
-    <NuGetTargetMoniker>.NETStandard,Version=v1.6</NuGetTargetMoniker>
-    <NuGetTargetMonikerShort>netstandard1.6</NuGetTargetMonikerShort>
-  </PropertyGroup>
-  <!-- Default configurations to help VS understand the configurations -->
-  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' " />
-  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' " />
-  <PropertyGroup>
-    <RestoreOutputPath>..\..\obj</RestoreOutputPath>
-    <DebugType>pdbonly</DebugType>
-    <Optimize>true</Optimize>
+    <Platform Condition="'$(Platform)' == ''">AnyCPU</Platform>
+    <Configuration Condition="'$(Configuration)' == ''">Debug</Configuration>
+    <OutputType>Exe</OutputType>
+    <TargetExt>.exe</TargetExt>
+
+    <CLRTestKind>BuildOnly</CLRTestKind>
   </PropertyGroup>
+
   <ItemGroup>
-    <CodeAnalysisDependentAssemblyPaths Condition=" '$(VS100COMNTOOLS)' != '' " Include="$(VS100COMNTOOLS)..\IDE\PrivateAssemblies">
-      <Visible>False</Visible>
-    </CodeAnalysisDependentAssemblyPaths>
-  </ItemGroup>
-  <ItemGroup>
-    <Compile Include="Runner\*.cs" />
-    <Compile Include="Benchmarks\*.cs" />
-    <Compile Include="Utilities\*.cs" />
-    <Compile Include="$(BaseIntermediateOutputPath)AutoGeneratedVersioningConstants.cs" />
+    <PackageReference Include="CommandLineParser">
+      <Version>$(CommandLineParserVersion)</Version>
+    </PackageReference>
+    <PackageReference Include="Microsoft.Diagnostics.Tracing.TraceEvent">
+      <Version>$(MicrosoftDiagnosticsTracingTraceEventPackageVersion)</Version>
+    </PackageReference>
+    <PackageReference Include="xunit.performance.api">
+      <Version>$(XunitPerformanceApiPackageVersion)</Version>
+    </PackageReference>
+    <PackageReference Include="xunit.performance.core">
+      <Version>$(XunitPerformanceApiPackageVersion)</Version>
+    </PackageReference>
+    <PackageReference Include="xunit.performance.execution">
+      <Version>$(XunitPerformanceApiPackageVersion)</Version>
+    </PackageReference>
+    <PackageReference Include="xunit.performance.metrics">
+      <Version>$(XunitPerformanceApiPackageVersion)</Version>
+    </PackageReference>
   </ItemGroup>
+
   <ItemGroup>
-    <Service Include="{82A7F48D-3B50-4B1E-B82E-3ADA8210C358}" />
+    <Compile Include="**\*.cs" />
   </ItemGroup>
-  <Import Project="..\..\performance.targets" />
-  <PropertyGroup>
-    <ProjectAssetsFile>..\..\obj\project.assets.json</ProjectAssetsFile>
-  </PropertyGroup>
 
   <ItemGroup>
     <!-- BaselineMicrosoftNetCoreAppPackageVersion comes from dependencies.props in the root of the coreclr tree -->
     <VersioningConstantsLines Include="namespace JitBench { public static class VersioningConstants { public static string MicrosoftNetCoreAppPackageVersion=&quot;$(BaselineMicrosoftNetCoreAppPackageVersion)&quot;%3B } }" />
+    <Compile Include="$(BaseIntermediateOutputPath)AutoGeneratedVersioningConstants.cs" />
   </ItemGroup>
-
+    
   <Target Name="GenerateVersioningConstantsFile" BeforeTargets="CoreCompile">
-      <WriteLinesToFile
-          File="$(BaseIntermediateOutputPath)AutoGeneratedVersioningConstants.cs"
-          Lines="@(VersioningConstantsLines)"
-          Overwrite="true"
-          Encoding="Unicode"/>
+    <WriteLinesToFile File="$(BaseIntermediateOutputPath)AutoGeneratedVersioningConstants.cs" Lines="@(VersioningConstantsLines)" Overwrite="true" Encoding="Unicode" />
   </Target>
-  
+
+  <Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory), dir.targets))\dir.targets" />
+
 </Project>

diff --git a/tests/src/performance/Scenario/JitBench/Runner/Benchmark.cs b/tests/src/performance/Scenario/JitBench/Runner/Benchmark.cs
index 781ce8a..218a797 100644 (file)
--- a/tests/src/performance/Scenario/JitBench/Runner/Benchmark.cs
+++ b/tests/src/performance/Scenario/JitBench/Runner/Benchmark.cs
@@ -142,7 +142,7 @@ namespace JitBench
                 string[] args = new string[] { "--perf:collect", string.Join("+", run.MetricNames), "--perf:outputdir", run.OutputDir, "--perf:runid", run.BenchviewRunId };
                 using (var harness = new XunitPerformanceHarness(args))
                 {
-                    ProcessStartInfo startInfo = new ProcessStartInfo(run.DotNetInstallation.DotNetExe, ExePath + " " + CommandLineArguments);
+                    ProcessStartInfo startInfo = new ProcessStartInfo(run.DotNetInstallation.DotNetExe, (ExePath + " " + CommandLineArguments).Trim());
                     startInfo.WorkingDirectory = WorkingDirPath;
                     startInfo.RedirectStandardError = true;
                     startInfo.RedirectStandardOutput = true;
@@ -152,7 +152,9 @@ namespace JitBench
                         startInfo.Environment[kv.Key] = kv.Value;
                     }
                     output.WriteLine("XUnitPerfHarness doesn't log env vars it uses to run processes. To workaround, logging them here:");
-                    output.WriteLine(string.Join(", ", extraEnvVars.Select(kv => kv.Key + "=" + kv.Value)));
+                    output.WriteLine($"Environment variables: {string.Join(", ", extraEnvVars.Select(kv => kv.Key + "=" + kv.Value))}");
+                    output.WriteLine($"Working directory: \"{startInfo.WorkingDirectory}\"");
+                    output.WriteLine($"Command line: \"{startInfo.FileName}\" {startInfo.Arguments}");
 
                     BenchmarkRunResult result = new BenchmarkRunResult(this, config);
                     StringBuilder stderr = new StringBuilder();

diff --git a/tests/src/performance/Scenario/JitBench/Directory.Build.props b/tests/src/performance/Scenario/JitBench/unofficial_dotnet/Directory.Build.props
similarity index 100%
rename from tests/src/performance/Scenario/JitBench/Directory.Build.props
rename to tests/src/performance/Scenario/JitBench/unofficial_dotnet/Directory.Build.props

diff --git a/tests/src/performance/Scenario/JitBench/Directory.Build.targets b/tests/src/performance/Scenario/JitBench/unofficial_dotnet/Directory.Build.targets
similarity index 100%
rename from tests/src/performance/Scenario/JitBench/Directory.Build.targets
rename to tests/src/performance/Scenario/JitBench/unofficial_dotnet/Directory.Build.targets