Some JitBench fixes (#19479)

Some JitBench fixes

- Fixed `dotnet run` from the `unofficial_dotnet` folder after `Directory.Build.targets` was added to `coreclr/tests/src`. This file is auto-imported and was adding dependencies and properties for targets that were not necessary. Added files to the root JitBench folder to override.
- Updated JitBench to be able to run against netcoreapp3.0. I haven't changed the default yet, could consider doing so once there are no pending perf issues being looked into.
  - Added a `RetargetProjects()` function to the setup steps to replace `TargetFramework` and `RuntimeFrameworkVersion` values in the relevant `.csproj` files to run against the expected runtime. This reduces some dependency on other repos containing the actual perf projects such that we don't have to update every repo to support environment variables to set those values.
- Disabled tiering for the `MinOpts` config

diff --git a/tests/src/performance/Scenario/JitBench/Benchmarks/BuildHelloWorldBenchmark.cs b/tests/src/performance/Scenario/JitBench/Benchmarks/BuildHelloWorldBenchmark.cs
index 0215fa3..5d26afc 100644 (file)
--- a/tests/src/performance/Scenario/JitBench/Benchmarks/BuildHelloWorldBenchmark.cs
+++ b/tests/src/performance/Scenario/JitBench/Benchmarks/BuildHelloWorldBenchmark.cs
@@ -14,11 +14,11 @@ namespace JitBench
         {
             using (var setupSection = new IndentedTestOutputHelper("Setup " + Name, output))
             {
-                await SetupHelloWorldProject(dotNetInstall.DotNetExe, intermediateOutputDir, useExistingSetup, setupSection);
+                await SetupHelloWorldProject(dotNetInstall, intermediateOutputDir, useExistingSetup, setupSection);
             }
         }
 
-        protected async Task SetupHelloWorldProject(string dotNetExePath, string intermediateOutputDir, bool useExistingSetup, ITestOutputHelper output)
+        protected async Task SetupHelloWorldProject(DotNetInstallation dotNetInstall, string intermediateOutputDir, bool useExistingSetup, ITestOutputHelper output)
         {
             string helloWorldProjectDir = Path.Combine(intermediateOutputDir, "helloworld");
             //the 'exePath' gets passed as an argument to dotnet.exe
@@ -37,10 +37,12 @@ namespace JitBench
             {
                 FileTasks.DeleteDirectory(helloWorldProjectDir, output);
                 FileTasks.CreateDirectory(helloWorldProjectDir, output);
-                await new ProcessRunner(dotNetExePath, "new console")
+                await new ProcessRunner(dotNetInstall.DotNetExe, "new console")
                     .WithWorkingDirectory(helloWorldProjectDir)
                     .WithLog(output)
                     .Run();
+
+                RetargetProjects(dotNetInstall, helloWorldProjectDir, new string[] { "helloworld.csproj" });
             }
         }
     }

diff --git a/tests/src/performance/Scenario/JitBench/Benchmarks/WebAppBenchmarks.cs b/tests/src/performance/Scenario/JitBench/Benchmarks/WebAppBenchmarks.cs
index 81c37ae..7e59064 100644 (file)
--- a/tests/src/performance/Scenario/JitBench/Benchmarks/WebAppBenchmarks.cs
+++ b/tests/src/performance/Scenario/JitBench/Benchmarks/WebAppBenchmarks.cs
@@ -9,7 +9,12 @@ namespace JitBench
 {
     class MusicStoreBenchmark : WebAppBenchmark
     {
-        public MusicStoreBenchmark() : base("MusicStore", "MusicStore.dll") { }
+        public MusicStoreBenchmark() :
+            base(
+                "MusicStore",
+                "MusicStore.dll",
+                new string[] { Path.Combine("src", "MusicStore", "MusicStore.csproj") })
+        { }
 
         protected override string GetJitBenchRepoRootDir(string outputDir)
         {
@@ -24,7 +29,12 @@ namespace JitBench
 
     class AllReadyBenchmark : WebAppBenchmark
     {
-        public AllReadyBenchmark() : base("AllReady", "AllReady.dll") { }
+        public AllReadyBenchmark() :
+            base(
+                "AllReady",
+                "AllReady.dll",
+                new string[] { Path.Combine("src", "AllReady", "AllReady.csproj") })
+        { }
 
         protected override string GetJitBenchRepoRootDir(string outputDir)
         {
@@ -41,9 +51,12 @@ namespace JitBench
     {
         private static readonly HashSet<int> DefaultExitCodes = new HashSet<int>(new[] { 0 });
 
-        public WebAppBenchmark(string name, string executableName) : base(name)
+        private string[] _projectFileRelativePaths;
+
+        public WebAppBenchmark(string name, string executableName, string[] projectFileRelativePaths) : base(name)
         {
             ExePath = executableName;
+            _projectFileRelativePaths = projectFileRelativePaths;
         }
 
         public override async Task Setup(DotNetInstallation dotNetInstall, string outputDir, bool useExistingSetup, ITestOutputHelper output)
@@ -53,6 +66,7 @@ namespace JitBench
                 using (var setupSection = new IndentedTestOutputHelper("Setup " + Name, output))
                 {
                     await CloneAspNetJitBenchRepo(outputDir, setupSection);
+                    RetargetProjects(dotNetInstall, GetJitBenchRepoRootDir(outputDir), _projectFileRelativePaths);
                     await CreateStore(dotNetInstall, outputDir, setupSection);
                     await Publish(dotNetInstall, outputDir, setupSection);
                 }
@@ -120,7 +134,7 @@ namespace JitBench
             publishDir = GetWebAppPublishDirectory(dotNetInstall, outputDir, tfm);
             if (publishDir == null)
             {
-                throw new DirectoryNotFoundException("Could not find 'publish' directory");
+                throw new DirectoryNotFoundException($"Could not find 'publish' directory: {publishDir}");
             }
             return publishDir;
         }

diff --git a/tests/src/performance/Scenario/JitBench/Benchmarks/Word2VecBenchmark.cs b/tests/src/performance/Scenario/JitBench/Benchmarks/Word2VecBenchmark.cs
index 6d6687d..9d72fa6 100644 (file)
--- a/tests/src/performance/Scenario/JitBench/Benchmarks/Word2VecBenchmark.cs
+++ b/tests/src/performance/Scenario/JitBench/Benchmarks/Word2VecBenchmark.cs
@@ -40,6 +40,14 @@ namespace JitBench
                 using (var setupSection = new IndentedTestOutputHelper("Setup " + Name, output))
                 {
                     await CloneWord2VecNetRepo(outputDir, setupSection);
+                    RetargetProjects(
+                        dotNetInstall,
+                        GetWord2VecNetRepoRootDir(outputDir),
+                        new string[]
+                        {
+                            Path.Combine("Word2Vec.Net", "Word2Vec.Net.csproj"),
+                            Path.Combine("Word2VecScenario", "Word2VecScenario.csproj")
+                        });
                     await Publish(dotNetInstall, outputDir, setupSection);
                     await DownloadAndExtractTextCorpus(dotNetInstall, outputDir, setupSection);
                 }
@@ -101,7 +109,7 @@ namespace JitBench
             publishDir = GetWord2VecNetPublishDirectory(dotNetInstall, outputDir, tfm);
             if (publishDir == null)
             {
-                throw new DirectoryNotFoundException("Could not find 'publish' directory");
+                throw new DirectoryNotFoundException($"Could not find 'publish' directory: {publishDir}");
             }
             return publishDir;
         }

diff --git a/tests/src/performance/Scenario/JitBench/Directory.Build.props b/tests/src/performance/Scenario/JitBench/Directory.Build.props
new file mode 100644 (file)
index 0000000..ae5b479
--- /dev/null
+++ b/tests/src/performance/Scenario/JitBench/Directory.Build.props
@@ -0,0 +1,6 @@
+<Project>
+  <!--
+    MSBuild automatically imports this project into any projects in the current or child folders. The projects under this
+    folder directly import what they need, and this file exists just to prevent the auto-import.
+  -->
+</Project>

diff --git a/tests/src/performance/Scenario/JitBench/Directory.Build.targets b/tests/src/performance/Scenario/JitBench/Directory.Build.targets
new file mode 100644 (file)
index 0000000..ae5b479
--- /dev/null
+++ b/tests/src/performance/Scenario/JitBench/Directory.Build.targets
@@ -0,0 +1,6 @@
+<Project>
+  <!--
+    MSBuild automatically imports this project into any projects in the current or child folders. The projects under this
+    folder directly import what they need, and this file exists just to prevent the auto-import.
+  -->
+</Project>

diff --git a/tests/src/performance/Scenario/JitBench/Runner/Benchmark.cs b/tests/src/performance/Scenario/JitBench/Runner/Benchmark.cs
index a060bf1..781ce8a 100644 (file)
--- a/tests/src/performance/Scenario/JitBench/Runner/Benchmark.cs
+++ b/tests/src/performance/Scenario/JitBench/Runner/Benchmark.cs
@@ -6,6 +6,7 @@ using System.Linq;
 using System.Runtime.InteropServices;
 using System.Text;
 using System.Threading.Tasks;
+using System.Xml;
 using Microsoft.Xunit.Performance.Api;
 using Microsoft.Xunit.Performance.Api.Profilers.Etw;
 
@@ -35,6 +36,71 @@ namespace JitBench
 
         public abstract Task Setup(DotNetInstallation dotnetInstall, string intermediateOutputDir, bool useExistingSetup, ITestOutputHelper output);
 
+        protected void RetargetProjects(
+            DotNetInstallation dotNetInstall,
+            string rootDir,
+            IEnumerable<string> projectFileRelativePaths)
+        {
+            if (string.IsNullOrWhiteSpace(rootDir))
+            {
+                throw new ArgumentNullException(rootDir);
+            }
+            if (!Directory.Exists(rootDir))
+            {
+                throw new DirectoryNotFoundException($"Root directory was not found: {rootDir}");
+            }
+
+            foreach (string projectFileRelativePath in projectFileRelativePaths)
+            {
+                string projectFile = Path.Combine(rootDir, projectFileRelativePath);
+                if (!File.Exists(projectFile))
+                {
+                    throw new FileNotFoundException($"Project file was not found: {projectFile}");
+                }
+
+                var doc = new XmlDocument();
+                Encoding docEncoding;
+                using (var fs = new FileStream(projectFile, FileMode.Open, FileAccess.Read, FileShare.Read))
+                using (var sr = new StreamReader(fs))
+                {
+                    docEncoding = sr.CurrentEncoding;
+                    doc.Load(sr);
+                }
+                XmlElement root = doc.DocumentElement;
+
+                // Comment out all existing TargetFramework and RuntimeFrameworkVersion elements
+                foreach (XmlElement e in root.SelectNodes("PropertyGroup/TargetFramework").OfType<XmlElement>())
+                {
+                    e.ParentNode.ReplaceChild(doc.CreateComment(e.OuterXml), e);
+                }
+                foreach (XmlElement e in root.SelectNodes("PropertyGroup/RuntimeFrameworkVersion").OfType<XmlElement>())
+                {
+                    e.ParentNode.ReplaceChild(doc.CreateComment(e.OuterXml), e);
+                }
+
+                // Add TargetFramework and RuntimeFrameworkVersion elements with the requested values to the top
+                {
+                    XmlElement propertyGroupElement = doc.CreateElement("PropertyGroup");
+                    root.PrependChild(propertyGroupElement);
+
+                    XmlElement targetFrameworkElement = doc.CreateElement("TargetFramework");
+                    XmlElement runtimeFrameworkVersionElement = doc.CreateElement("RuntimeFrameworkVersion");
+                    propertyGroupElement.AppendChild(targetFrameworkElement);
+                    propertyGroupElement.AppendChild(runtimeFrameworkVersionElement);
+
+                    targetFrameworkElement.InnerText =
+                        DotNetSetup.GetTargetFrameworkMonikerForFrameworkVersion(dotNetInstall.FrameworkVersion);
+                    runtimeFrameworkVersionElement.InnerText = dotNetInstall.FrameworkVersion;
+                }
+
+                using (var fs = new FileStream(projectFile, FileMode.Truncate, FileAccess.Write, FileShare.Read))
+                using (var sw = new StreamWriter(fs, docEncoding))
+                {
+                    doc.Save(sw);
+                }
+            }
+        }
+
         public virtual Metric[] GetDefaultDisplayMetrics()
         {
             return new Metric[] { Metric.ElapsedTimeMilliseconds };

diff --git a/tests/src/performance/Scenario/JitBench/Runner/BenchmarkConfiguration.cs b/tests/src/performance/Scenario/JitBench/Runner/BenchmarkConfiguration.cs
index a72963c..7043489 100644 (file)
--- a/tests/src/performance/Scenario/JitBench/Runner/BenchmarkConfiguration.cs
+++ b/tests/src/performance/Scenario/JitBench/Runner/BenchmarkConfiguration.cs
@@ -22,7 +22,11 @@ namespace JitBench
 
         public BenchmarkConfiguration WithMinOpts()
         {
-            return WithModifier("Minopts", "COMPLUS_JitMinOpts", "1");
+            BenchmarkConfiguration configuration =
+                WithModifier("Minopts", "COMPLUS_JitMinOpts", "1").
+                WithModifier("Tiering", "COMPLUS_TieredCompilation", "0");
+            configuration.Name = "Minopts";
+            return configuration;
         }
 
         public BenchmarkConfiguration WithNoR2R()

diff --git a/tests/src/performance/Scenario/JitBench/Utilities/DotNetSetup.cs b/tests/src/performance/Scenario/JitBench/Utilities/DotNetSetup.cs
index 9f8a0da..1d43c1d 100644 (file)
--- a/tests/src/performance/Scenario/JitBench/Utilities/DotNetSetup.cs
+++ b/tests/src/performance/Scenario/JitBench/Utilities/DotNetSetup.cs
@@ -235,14 +235,18 @@ namespace JitBench
 
         public static string GetTargetFrameworkMonikerForFrameworkVersion(string runtimeVersion)
         {
-            if(runtimeVersion.StartsWith("2.0"))
+            if (runtimeVersion.StartsWith("3.0"))
             {
-                return "netcoreapp2.0";
+                return "netcoreapp3.0";
             }
-            else if(runtimeVersion.StartsWith("2.1"))
+            else if (runtimeVersion.StartsWith("2.1"))
             {
                 return "netcoreapp2.1";
             }
+            else if (runtimeVersion.StartsWith("2.0"))
+            {
+                return "netcoreapp2.0";
+            }
             else
             {
                 throw new NotSupportedException("Version " + runtimeVersion + " doesn't have a known TFM");
@@ -257,14 +261,18 @@ namespace JitBench
 
         public static string GetCompatibleDefaultSDKVersionForRuntimeTFM(string targetFrameworkMoniker)
         {
-            if (targetFrameworkMoniker == "netcoreapp2.0")
+            if (targetFrameworkMoniker == "netcoreapp3.0")
             {
-                return "2.0.0";
+                return "3.0.100-alpha1-009410";
             }
             else if (targetFrameworkMoniker == "netcoreapp2.1")
             {
                 return "2.2.0-preview1-007558";
             }
+            else if (targetFrameworkMoniker == "netcoreapp2.0")
+            {
+                return "2.0.9";
+            }
             else
             {
                 throw new Exception("No compatible SDK version has been designated for TFM: " + targetFrameworkMoniker);

diff --git a/tests/src/performance/Scenario/JitBench/unofficial_dotnet/JitBench.csproj b/tests/src/performance/Scenario/JitBench/unofficial_dotnet/JitBench.csproj
index a8616c2..6488c3e 100644 (file)
--- a/tests/src/performance/Scenario/JitBench/unofficial_dotnet/JitBench.csproj
+++ b/tests/src/performance/Scenario/JitBench/unofficial_dotnet/JitBench.csproj
@@ -1,19 +1,14 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
-  <!--The common test dirs props expects Platform to be empty in order to initialize it and
-      by default this project style appears to set it as "AnyCPU" so we need to clear it -->
-  <PropertyGroup>
-    <Platform></Platform>
-    <Platforms>AnyCPU;x64</Platforms>
-    <Configuration></Configuration>
-  </PropertyGroup>
-
   <Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory), dir.props))\dir.props" />
+
   <PropertyGroup>
+    <Platform Condition="'$(Platform)' == ''">AnyCPU</Platform>
+    <Configuration Condition="'$(Configuration)' == ''">Release</Configuration>
     <OutputType>Exe</OutputType>
     <TargetFramework>netcoreapp2.0</TargetFramework>
     
-    <!-- the common test dirs props pushes all tests to use a more recent NTM unless we explictly opt-out -->
+    <!-- The common test dirs props pushes all tests to use a more recent NTM unless we explictly opt-out -->
     <NuGetTargetMoniker>.NETCoreApp,Version=v2.0</NuGetTargetMoniker>
     <NuGetTargetMonikerShort>netcoreapp2.0</NuGetTargetMonikerShort>
   </PropertyGroup>
@@ -49,11 +44,8 @@
     <Compile Include="$(BaseIntermediateOutputPath)AutoGeneratedVersioningConstants.cs" />
   </ItemGroup>
     
-  <Target Name="GenerateVersioningConstantsFile" BeforeTargets="CoreCompile"> 
-    <WriteLinesToFile File="$(BaseIntermediateOutputPath)AutoGeneratedVersioningConstants.cs" Lines="@(VersioningConstantsLines)" Overwrite="true" Encoding="Unicode" /> 
-   </Target> 
-
-  <!-- The CoreCLR test build system requires a target named RestorePackage in order to do BatchRestore -->
-  <Target Name="RestorePackage" DependsOnTargets="Restore" />
+  <Target Name="GenerateVersioningConstantsFile" BeforeTargets="CoreCompile">
+    <WriteLinesToFile File="$(BaseIntermediateOutputPath)AutoGeneratedVersioningConstants.cs" Lines="@(VersioningConstantsLines)" Overwrite="true" Encoding="Unicode" />
+  </Target>
 
 </Project>