Some JitBench fixes (dotnet/coreclr#19479)

Some JitBench fixes

- Fixed `dotnet run` from the `unofficial_dotnet` folder after `Directory.Build.targets` was added to `coreclr/tests/src`. This file is auto-imported and was adding dependencies and properties for targets that were not necessary. Added files to the root JitBench folder to override.
- Updated JitBench to be able to run against netcoreapp3.0. I haven't changed the default yet, could consider doing so once there are no pending perf issues being looked into.
  - Added a `RetargetProjects()` function to the setup steps to replace `TargetFramework` and `RuntimeFrameworkVersion` values in the relevant `.csproj` files to run against the expected runtime. This reduces some dependency on other repos containing the actual perf projects such that we don't have to update every repo to support environment variables to set those values.
- Disabled tiering for the `MinOpts` config

Commit migrated from https://github.com/dotnet/coreclr/commit/72e9b8d589f114776e3f42058d721058b77133b9

diff --git a/src/coreclr/tests/src/performance/Scenario/JitBench/Benchmarks/BuildHelloWorldBenchmark.cs b/src/coreclr/tests/src/performance/Scenario/JitBench/Benchmarks/BuildHelloWorldBenchmark.cs
index 0215fa3dade2624cffc416f0f10fdc5b69f9cd74..5d26afc2f71875263fab22eed0a99edb9e6b0108 100644 (file)
--- a/src/coreclr/tests/src/performance/Scenario/JitBench/Benchmarks/BuildHelloWorldBenchmark.cs
+++ b/src/coreclr/tests/src/performance/Scenario/JitBench/Benchmarks/BuildHelloWorldBenchmark.cs
@@ -14,11 +14,11 @@ namespace JitBench
         {
             using (var setupSection = new IndentedTestOutputHelper("Setup " + Name, output))
             {
-                await SetupHelloWorldProject(dotNetInstall.DotNetExe, intermediateOutputDir, useExistingSetup, setupSection);
+                await SetupHelloWorldProject(dotNetInstall, intermediateOutputDir, useExistingSetup, setupSection);
             }
         }
 
-        protected async Task SetupHelloWorldProject(string dotNetExePath, string intermediateOutputDir, bool useExistingSetup, ITestOutputHelper output)
+        protected async Task SetupHelloWorldProject(DotNetInstallation dotNetInstall, string intermediateOutputDir, bool useExistingSetup, ITestOutputHelper output)
         {
             string helloWorldProjectDir = Path.Combine(intermediateOutputDir, "helloworld");
             //the 'exePath' gets passed as an argument to dotnet.exe
@@ -37,10 +37,12 @@ namespace JitBench
             {
                 FileTasks.DeleteDirectory(helloWorldProjectDir, output);
                 FileTasks.CreateDirectory(helloWorldProjectDir, output);
-                await new ProcessRunner(dotNetExePath, "new console")
+                await new ProcessRunner(dotNetInstall.DotNetExe, "new console")
                     .WithWorkingDirectory(helloWorldProjectDir)
                     .WithLog(output)
                     .Run();
+
+                RetargetProjects(dotNetInstall, helloWorldProjectDir, new string[] { "helloworld.csproj" });
             }
         }
     }

diff --git a/src/coreclr/tests/src/performance/Scenario/JitBench/Benchmarks/WebAppBenchmarks.cs b/src/coreclr/tests/src/performance/Scenario/JitBench/Benchmarks/WebAppBenchmarks.cs
index 81c37aec7475d26eb69fb409ea757f57cc731967..7e59064ec75ca95ed0f8a49296417426d92a2031 100644 (file)
--- a/src/coreclr/tests/src/performance/Scenario/JitBench/Benchmarks/WebAppBenchmarks.cs
+++ b/src/coreclr/tests/src/performance/Scenario/JitBench/Benchmarks/WebAppBenchmarks.cs
@@ -9,7 +9,12 @@ namespace JitBench
 {
     class MusicStoreBenchmark : WebAppBenchmark
     {
-        public MusicStoreBenchmark() : base("MusicStore", "MusicStore.dll") { }
+        public MusicStoreBenchmark() :
+            base(
+                "MusicStore",
+                "MusicStore.dll",
+                new string[] { Path.Combine("src", "MusicStore", "MusicStore.csproj") })
+        { }
 
         protected override string GetJitBenchRepoRootDir(string outputDir)
         {
@@ -24,7 +29,12 @@ namespace JitBench
 
     class AllReadyBenchmark : WebAppBenchmark
     {
-        public AllReadyBenchmark() : base("AllReady", "AllReady.dll") { }
+        public AllReadyBenchmark() :
+            base(
+                "AllReady",
+                "AllReady.dll",
+                new string[] { Path.Combine("src", "AllReady", "AllReady.csproj") })
+        { }
 
         protected override string GetJitBenchRepoRootDir(string outputDir)
         {
@@ -41,9 +51,12 @@ namespace JitBench
     {
         private static readonly HashSet<int> DefaultExitCodes = new HashSet<int>(new[] { 0 });
 
-        public WebAppBenchmark(string name, string executableName) : base(name)
+        private string[] _projectFileRelativePaths;
+
+        public WebAppBenchmark(string name, string executableName, string[] projectFileRelativePaths) : base(name)
         {
             ExePath = executableName;
+            _projectFileRelativePaths = projectFileRelativePaths;
         }
 
         public override async Task Setup(DotNetInstallation dotNetInstall, string outputDir, bool useExistingSetup, ITestOutputHelper output)
@@ -53,6 +66,7 @@ namespace JitBench
                 using (var setupSection = new IndentedTestOutputHelper("Setup " + Name, output))
                 {
                     await CloneAspNetJitBenchRepo(outputDir, setupSection);
+                    RetargetProjects(dotNetInstall, GetJitBenchRepoRootDir(outputDir), _projectFileRelativePaths);
                     await CreateStore(dotNetInstall, outputDir, setupSection);
                     await Publish(dotNetInstall, outputDir, setupSection);
                 }
@@ -120,7 +134,7 @@ namespace JitBench
             publishDir = GetWebAppPublishDirectory(dotNetInstall, outputDir, tfm);
             if (publishDir == null)
             {
-                throw new DirectoryNotFoundException("Could not find 'publish' directory");
+                throw new DirectoryNotFoundException($"Could not find 'publish' directory: {publishDir}");
             }
             return publishDir;
         }

diff --git a/src/coreclr/tests/src/performance/Scenario/JitBench/Benchmarks/Word2VecBenchmark.cs b/src/coreclr/tests/src/performance/Scenario/JitBench/Benchmarks/Word2VecBenchmark.cs
index 6d6687d5ae1f03165be686422c7efb69a0c916fc..9d72fa67b90816c4c97f1663d86f615269a631ee 100644 (file)
--- a/src/coreclr/tests/src/performance/Scenario/JitBench/Benchmarks/Word2VecBenchmark.cs
+++ b/src/coreclr/tests/src/performance/Scenario/JitBench/Benchmarks/Word2VecBenchmark.cs
@@ -40,6 +40,14 @@ namespace JitBench
                 using (var setupSection = new IndentedTestOutputHelper("Setup " + Name, output))
                 {
                     await CloneWord2VecNetRepo(outputDir, setupSection);
+                    RetargetProjects(
+                        dotNetInstall,
+                        GetWord2VecNetRepoRootDir(outputDir),
+                        new string[]
+                        {
+                            Path.Combine("Word2Vec.Net", "Word2Vec.Net.csproj"),
+                            Path.Combine("Word2VecScenario", "Word2VecScenario.csproj")
+                        });
                     await Publish(dotNetInstall, outputDir, setupSection);
                     await DownloadAndExtractTextCorpus(dotNetInstall, outputDir, setupSection);
                 }
@@ -101,7 +109,7 @@ namespace JitBench
             publishDir = GetWord2VecNetPublishDirectory(dotNetInstall, outputDir, tfm);
             if (publishDir == null)
             {
-                throw new DirectoryNotFoundException("Could not find 'publish' directory");
+                throw new DirectoryNotFoundException($"Could not find 'publish' directory: {publishDir}");
             }
             return publishDir;
         }

diff --git a/src/coreclr/tests/src/performance/Scenario/JitBench/Directory.Build.props b/src/coreclr/tests/src/performance/Scenario/JitBench/Directory.Build.props
new file mode 100644 (file)
index 0000000..ae5b479
--- /dev/null
+++ b/src/coreclr/tests/src/performance/Scenario/JitBench/Directory.Build.props
@@ -0,0 +1,6 @@
+<Project>
+  <!--
+    MSBuild automatically imports this project into any projects in the current or child folders. The projects under this
+    folder directly import what they need, and this file exists just to prevent the auto-import.
+  -->
+</Project>

diff --git a/src/coreclr/tests/src/performance/Scenario/JitBench/Directory.Build.targets b/src/coreclr/tests/src/performance/Scenario/JitBench/Directory.Build.targets
new file mode 100644 (file)
index 0000000..ae5b479
--- /dev/null
+++ b/src/coreclr/tests/src/performance/Scenario/JitBench/Directory.Build.targets
@@ -0,0 +1,6 @@
+<Project>
+  <!--
+    MSBuild automatically imports this project into any projects in the current or child folders. The projects under this
+    folder directly import what they need, and this file exists just to prevent the auto-import.
+  -->
+</Project>

diff --git a/src/coreclr/tests/src/performance/Scenario/JitBench/Runner/Benchmark.cs b/src/coreclr/tests/src/performance/Scenario/JitBench/Runner/Benchmark.cs
index a060bf1d610f358f09ce692fa433b04d575566bc..781ce8a6d9f2645e9b0d917469be2150a330a2ab 100644 (file)
--- a/src/coreclr/tests/src/performance/Scenario/JitBench/Runner/Benchmark.cs
+++ b/src/coreclr/tests/src/performance/Scenario/JitBench/Runner/Benchmark.cs
@@ -6,6 +6,7 @@ using System.Linq;
 using System.Runtime.InteropServices;
 using System.Text;
 using System.Threading.Tasks;
+using System.Xml;
 using Microsoft.Xunit.Performance.Api;
 using Microsoft.Xunit.Performance.Api.Profilers.Etw;
 
@@ -35,6 +36,71 @@ namespace JitBench
 
         public abstract Task Setup(DotNetInstallation dotnetInstall, string intermediateOutputDir, bool useExistingSetup, ITestOutputHelper output);
 
+        protected void RetargetProjects(
+            DotNetInstallation dotNetInstall,
+            string rootDir,
+            IEnumerable<string> projectFileRelativePaths)
+        {
+            if (string.IsNullOrWhiteSpace(rootDir))
+            {
+                throw new ArgumentNullException(rootDir);
+            }
+            if (!Directory.Exists(rootDir))
+            {
+                throw new DirectoryNotFoundException($"Root directory was not found: {rootDir}");
+            }
+
+            foreach (string projectFileRelativePath in projectFileRelativePaths)
+            {
+                string projectFile = Path.Combine(rootDir, projectFileRelativePath);
+                if (!File.Exists(projectFile))
+                {
+                    throw new FileNotFoundException($"Project file was not found: {projectFile}");
+                }
+
+                var doc = new XmlDocument();
+                Encoding docEncoding;
+                using (var fs = new FileStream(projectFile, FileMode.Open, FileAccess.Read, FileShare.Read))
+                using (var sr = new StreamReader(fs))
+                {
+                    docEncoding = sr.CurrentEncoding;
+                    doc.Load(sr);
+                }
+                XmlElement root = doc.DocumentElement;
+
+                // Comment out all existing TargetFramework and RuntimeFrameworkVersion elements
+                foreach (XmlElement e in root.SelectNodes("PropertyGroup/TargetFramework").OfType<XmlElement>())
+                {
+                    e.ParentNode.ReplaceChild(doc.CreateComment(e.OuterXml), e);
+                }
+                foreach (XmlElement e in root.SelectNodes("PropertyGroup/RuntimeFrameworkVersion").OfType<XmlElement>())
+                {
+                    e.ParentNode.ReplaceChild(doc.CreateComment(e.OuterXml), e);
+                }
+
+                // Add TargetFramework and RuntimeFrameworkVersion elements with the requested values to the top
+                {
+                    XmlElement propertyGroupElement = doc.CreateElement("PropertyGroup");
+                    root.PrependChild(propertyGroupElement);
+
+                    XmlElement targetFrameworkElement = doc.CreateElement("TargetFramework");
+                    XmlElement runtimeFrameworkVersionElement = doc.CreateElement("RuntimeFrameworkVersion");
+                    propertyGroupElement.AppendChild(targetFrameworkElement);
+                    propertyGroupElement.AppendChild(runtimeFrameworkVersionElement);
+
+                    targetFrameworkElement.InnerText =
+                        DotNetSetup.GetTargetFrameworkMonikerForFrameworkVersion(dotNetInstall.FrameworkVersion);
+                    runtimeFrameworkVersionElement.InnerText = dotNetInstall.FrameworkVersion;
+                }
+
+                using (var fs = new FileStream(projectFile, FileMode.Truncate, FileAccess.Write, FileShare.Read))
+                using (var sw = new StreamWriter(fs, docEncoding))
+                {
+                    doc.Save(sw);
+                }
+            }
+        }
+
         public virtual Metric[] GetDefaultDisplayMetrics()
         {
             return new Metric[] { Metric.ElapsedTimeMilliseconds };

diff --git a/src/coreclr/tests/src/performance/Scenario/JitBench/Runner/BenchmarkConfiguration.cs b/src/coreclr/tests/src/performance/Scenario/JitBench/Runner/BenchmarkConfiguration.cs
index a72963cec239c0bc34196812371c987d83dfa04c..7043489fa5a5c56c51eaea197166f97217662360 100644 (file)
--- a/src/coreclr/tests/src/performance/Scenario/JitBench/Runner/BenchmarkConfiguration.cs
+++ b/src/coreclr/tests/src/performance/Scenario/JitBench/Runner/BenchmarkConfiguration.cs
@@ -22,7 +22,11 @@ namespace JitBench
 
         public BenchmarkConfiguration WithMinOpts()
         {
-            return WithModifier("Minopts", "COMPLUS_JitMinOpts", "1");
+            BenchmarkConfiguration configuration =
+                WithModifier("Minopts", "COMPLUS_JitMinOpts", "1").
+                WithModifier("Tiering", "COMPLUS_TieredCompilation", "0");
+            configuration.Name = "Minopts";
+            return configuration;
         }
 
         public BenchmarkConfiguration WithNoR2R()

diff --git a/src/coreclr/tests/src/performance/Scenario/JitBench/Utilities/DotNetSetup.cs b/src/coreclr/tests/src/performance/Scenario/JitBench/Utilities/DotNetSetup.cs
index 9f8a0da3c1ea6801f4cf5628b73a71072c451569..1d43c1d8f1bd15c5ddcbaa52b4f3bcbf48a58ab0 100644 (file)
--- a/src/coreclr/tests/src/performance/Scenario/JitBench/Utilities/DotNetSetup.cs
+++ b/src/coreclr/tests/src/performance/Scenario/JitBench/Utilities/DotNetSetup.cs
@@ -235,14 +235,18 @@ namespace JitBench
 
         public static string GetTargetFrameworkMonikerForFrameworkVersion(string runtimeVersion)
         {
-            if(runtimeVersion.StartsWith("2.0"))
+            if (runtimeVersion.StartsWith("3.0"))
             {
-                return "netcoreapp2.0";
+                return "netcoreapp3.0";
             }
-            else if(runtimeVersion.StartsWith("2.1"))
+            else if (runtimeVersion.StartsWith("2.1"))
             {
                 return "netcoreapp2.1";
             }
+            else if (runtimeVersion.StartsWith("2.0"))
+            {
+                return "netcoreapp2.0";
+            }
             else
             {
                 throw new NotSupportedException("Version " + runtimeVersion + " doesn't have a known TFM");
@@ -257,14 +261,18 @@ namespace JitBench
 
         public static string GetCompatibleDefaultSDKVersionForRuntimeTFM(string targetFrameworkMoniker)
         {
-            if (targetFrameworkMoniker == "netcoreapp2.0")
+            if (targetFrameworkMoniker == "netcoreapp3.0")
             {
-                return "2.0.0";
+                return "3.0.100-alpha1-009410";
             }
             else if (targetFrameworkMoniker == "netcoreapp2.1")
             {
                 return "2.2.0-preview1-007558";
             }
+            else if (targetFrameworkMoniker == "netcoreapp2.0")
+            {
+                return "2.0.9";
+            }
             else
             {
                 throw new Exception("No compatible SDK version has been designated for TFM: " + targetFrameworkMoniker);

diff --git a/src/coreclr/tests/src/performance/Scenario/JitBench/unofficial_dotnet/JitBench.csproj b/src/coreclr/tests/src/performance/Scenario/JitBench/unofficial_dotnet/JitBench.csproj
index a8616c21e94b66370a8c0faa7fad1abeceb05e44..6488c3e915487360bd4661ffe8a69cf37e860fb9 100644 (file)
--- a/src/coreclr/tests/src/performance/Scenario/JitBench/unofficial_dotnet/JitBench.csproj
+++ b/src/coreclr/tests/src/performance/Scenario/JitBench/unofficial_dotnet/JitBench.csproj
@@ -1,19 +1,14 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
-  <!--The common test dirs props expects Platform to be empty in order to initialize it and
-      by default this project style appears to set it as "AnyCPU" so we need to clear it -->
-  <PropertyGroup>
-    <Platform></Platform>
-    <Platforms>AnyCPU;x64</Platforms>
-    <Configuration></Configuration>
-  </PropertyGroup>
-
   <Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory), dir.props))\dir.props" />
+
   <PropertyGroup>
+    <Platform Condition="'$(Platform)' == ''">AnyCPU</Platform>
+    <Configuration Condition="'$(Configuration)' == ''">Release</Configuration>
     <OutputType>Exe</OutputType>
     <TargetFramework>netcoreapp2.0</TargetFramework>
     
-    <!-- the common test dirs props pushes all tests to use a more recent NTM unless we explictly opt-out -->
+    <!-- The common test dirs props pushes all tests to use a more recent NTM unless we explictly opt-out -->
     <NuGetTargetMoniker>.NETCoreApp,Version=v2.0</NuGetTargetMoniker>
     <NuGetTargetMonikerShort>netcoreapp2.0</NuGetTargetMonikerShort>
   </PropertyGroup>
@@ -49,11 +44,8 @@
     <Compile Include="$(BaseIntermediateOutputPath)AutoGeneratedVersioningConstants.cs" />
   </ItemGroup>
     
-  <Target Name="GenerateVersioningConstantsFile" BeforeTargets="CoreCompile"> 
-    <WriteLinesToFile File="$(BaseIntermediateOutputPath)AutoGeneratedVersioningConstants.cs" Lines="@(VersioningConstantsLines)" Overwrite="true" Encoding="Unicode" /> 
-   </Target> 
-
-  <!-- The CoreCLR test build system requires a target named RestorePackage in order to do BatchRestore -->
-  <Target Name="RestorePackage" DependsOnTargets="Restore" />
+  <Target Name="GenerateVersioningConstantsFile" BeforeTargets="CoreCompile">
+    <WriteLinesToFile File="$(BaseIntermediateOutputPath)AutoGeneratedVersioningConstants.cs" Lines="@(VersioningConstantsLines)" Overwrite="true" Encoding="Unicode" />
+  </Target>
 
 </Project>