Updating the xUnit Performance Api with bug fixes to the benchmark scenarios (#14986)

This newer version fixes collecting data on x86 where:
1. The Jit generate methods at runtime and that are not associated with any loaded module
2. Modules are loaded/unloaded multiple times
3. This newer version of the Api added some breaking changes to the Api interface

diff --git a/dependencies.props b/dependencies.props
index c95d3a1..670c9fa 100644 (file)
--- a/dependencies.props
+++ b/dependencies.props
@@ -37,7 +37,7 @@
     <MicrosoftNETCoreRuntimeCoreCLRPackageVersion>2.1.0-preview1-25901-06</MicrosoftNETCoreRuntimeCoreCLRPackageVersion>
     <XunitPackageVersion>2.2.0-beta2-build3300</XunitPackageVersion>
     <XunitConsoleNetcorePackageVersion>1.0.2-prerelease-00177</XunitConsoleNetcorePackageVersion>
-    <XunitPerformanceApiPackageVersion>1.0.0-beta-build0010</XunitPerformanceApiPackageVersion>
+    <XunitPerformanceApiPackageVersion>1.0.0-beta-build0012</XunitPerformanceApiPackageVersion>
     <MicrosoftDiagnosticsTracingTraceEventPackageVersion>1.0.3-alpha-experimental</MicrosoftDiagnosticsTracingTraceEventPackageVersion>
     <VCRuntimeVersion>1.2.0</VCRuntimeVersion>
   </PropertyGroup>

diff --git a/tests/src/performance/Scenario/JitBench/JitBenchHarness.cs b/tests/src/performance/Scenario/JitBench/JitBenchHarness.cs
index 5960524..f628740 100644 (file)
--- a/tests/src/performance/Scenario/JitBench/JitBenchHarness.cs
+++ b/tests/src/performance/Scenario/JitBench/JitBenchHarness.cs
@@ -58,13 +58,14 @@ namespace JitBench
                     scenarioName += " NoNgen";
                 }
 
-                var program = new JitBenchHarness("JitBench");
+                var program = new JitBenchHarness();
                 try
                 {
-                    var scenarioConfiguration = new ScenarioConfiguration(TimeSpan.FromMilliseconds(60000), startInfo) {
+                    var scenarioConfiguration = new ScenarioTestConfiguration(TimeSpan.FromMilliseconds(60000), startInfo) {
                         Iterations = (int)options.Iterations,
                         PreIterationDelegate = program.PreIteration,
                         PostIterationDelegate = program.PostIteration,
+                        Scenario = new ScenarioBenchmark("JitBench"),
                     };
                     var processesOfInterest = new string[] {
                         "dotnet.exe",
@@ -85,8 +86,8 @@ namespace JitBench
                     if (!Directory.Exists(startInfo.WorkingDirectory))
                         throw new DirectoryNotFoundException(startInfo.WorkingDirectory);
 
-                    h.RunScenario(scenarioConfiguration, teardownDelegate: () => {
-                        return program.PostRun("MusicStore", processesOfInterest, modulesOfInterest);
+                    h.RunScenario(scenarioConfiguration, teardownDelegate: (ScenarioBenchmark scenarioBenchmark) => {
+                        program.PostRun(scenarioBenchmark, "MusicStore", processesOfInterest, modulesOfInterest);
                     });
                 }
                 catch
@@ -98,9 +99,8 @@ namespace JitBench
             }
         }
 
-        public JitBenchHarness(string scenarioBenchmarkName)
+        public JitBenchHarness()
         {
-            _scenarioBenchmarkName = scenarioBenchmarkName;
             _stdout = new StringBuilder();
             _stderr = new StringBuilder();
             IterationsData = new List<IterationData>();
@@ -389,7 +389,7 @@ namespace JitBench
         private const string JitBenchTargetFramework = "netcoreapp2.1";
         private const string EnvironmentFileName = "JitBenchEnvironment.txt";
 
-        private void PreIteration(Scenario scenario)
+        private void PreIteration(ScenarioTest scenario)
         {
             PrintHeader("Setting up data standard output/error process handlers.");
 
@@ -476,15 +476,14 @@ namespace JitBench
             _stderr.Clear();
         }
 
-        private ScenarioBenchmark PostRun(
+        private void PostRun(
+            ScenarioBenchmark scenarioBenchmark,
             string scenarioTestModelName,
             IReadOnlyCollection<string> processesOfInterest,
             IReadOnlyCollection<string> modulesOfInterest)
         {
             PrintHeader("Post-Processing scenario data.");
 
-            var scenarioBenchmark = new ScenarioBenchmark(_scenarioBenchmarkName);
-
             foreach (var iter in IterationsData)
             {
                 var scenarioExecutionResult = iter.ScenarioExecutionResult;
@@ -553,12 +552,6 @@ namespace JitBench
                         scenarioBenchmark, iter.ScenarioExecutionResult, processesOfInterest, modulesOfInterest);
                 }
             }
-
-            for (int i = scenarioBenchmark.Tests.Count - 1; i >= 0; i--)
-                if (scenarioBenchmark.Tests[i].Performance.IterationModels.All(iter => iter.Iteration.Count == 0))
-                    scenarioBenchmark.Tests.RemoveAt(i);
-
-            return scenarioBenchmark;
         }
 
         private static ScenarioBenchmark AddEtwData(
@@ -704,7 +697,6 @@ namespace JitBench
 #else
         private const int NumberOfIterations = 11;
 #endif
-        private readonly string _scenarioBenchmarkName;
         private readonly StringBuilder _stdout;
         private readonly StringBuilder _stderr;
 

diff --git a/tests/src/performance/linkbench/linkbench.cs b/tests/src/performance/linkbench/linkbench.cs
index 885546f..37b04ca 100644 (file)
--- a/tests/src/performance/linkbench/linkbench.cs
+++ b/tests/src/performance/linkbench/linkbench.cs
@@ -396,7 +396,10 @@ namespace LinkBench
                 Console.WriteLine($"{string.Join(" ", newArgs)}");
                 using (var h = new XunitPerformanceHarness(newArgs.ToArray()))
                 {
-                    var configuration = new ScenarioConfiguration(new TimeSpan(2000000), emptyCmd);
+                    var configuration = new ScenarioTestConfiguration(new TimeSpan(2000000), emptyCmd)
+                    {
+                        Scenario = new ScenarioBenchmark(CurrentBenchmark.Name) { Namespace = "LinkBench" },
+                    };
                     h.RunScenario(configuration, PostRun);
                 }
             }
@@ -412,16 +415,11 @@ namespace LinkBench
             Console.WriteLine("**********************************************************************");
         }
 
-        private static ScenarioBenchmark PostRun()
+        private static void PostRun(ScenarioBenchmark scenario)
         {
             // The XUnit output doesn't print the benchmark name, so print it now.
             Console.WriteLine("{0}", CurrentBenchmark.Name);
 
-            var scenario = new ScenarioBenchmark(CurrentBenchmark.Name)
-            {
-                Namespace = "LinkBench"
-            };
-
             CurrentBenchmark.Compute();
 
             addMeasurement(ref scenario, "MSIL Unlinked", SizeMetric, CurrentBenchmark.UnlinkedMsilSize);
@@ -430,8 +428,6 @@ namespace LinkBench
             addMeasurement(ref scenario, "Total Uninked", SizeMetric, CurrentBenchmark.UnlinkedDirSize);
             addMeasurement(ref scenario, "Total Linked", SizeMetric, CurrentBenchmark.LinkedDirSize);
             addMeasurement(ref scenario, "Total Reduction", PercMetric, CurrentBenchmark.DirSizeReduction);
-
-            return scenario;
         }
 
         private static void addMeasurement(ref ScenarioBenchmark scenario, string name, MetricModel metric, double value)