Fix jitbench

Addressed 3 issues:
1) coreclr and CoreFx were out of sync -> update dependencies.props
2) Word2Vec fails on x86 sometimes with OutOfMemory -> disabled it there because it appears the behavior is by design
3) CommandLineParser 2.1.1 doesn't restore anymore? -> NuGet was already rolling forward to 2.2.0 but changing it in the source removes the warning when using dotnet.exe to run JitBench

diff --git a/dependencies.props b/dependencies.props
index 0f733af..d53a38d 100644 (file)
--- a/dependencies.props
+++ b/dependencies.props
@@ -39,13 +39,13 @@
     <XunitConsoleNetcorePackageVersion>1.0.2-prerelease-00177</XunitConsoleNetcorePackageVersion>
     <XunitPerformanceApiPackageVersion>1.0.0-beta-build0015</XunitPerformanceApiPackageVersion>
     <MicrosoftDiagnosticsTracingTraceEventPackageVersion>2.0.4</MicrosoftDiagnosticsTracingTraceEventPackageVersion>
-    <CommandLineParserVersion>2.1.1</CommandLineParserVersion>
+    <CommandLineParserVersion>2.2.0</CommandLineParserVersion>
     <VCRuntimeVersion>1.2.0</VCRuntimeVersion>
     
     <!-- Scenario tests install this version of Microsoft.NetCore.App, then patch coreclr binaries via xcopy. At the moment it is
          updated manually whenever breaking changes require it to move forward, but it would be nice if we could update it automatically
          as we do with many of the package versions above -->
-    <BaselineMicrosoftNetCoreAppPackageVersion>2.1.0-preview3-26327-01</BaselineMicrosoftNetCoreAppPackageVersion>
+    <BaselineMicrosoftNetCoreAppPackageVersion>2.1.0-preview3-26416-01</BaselineMicrosoftNetCoreAppPackageVersion>
   </PropertyGroup>
 
   <!-- Package versions used as toolsets -->

diff --git a/tests/src/performance/Scenario/JitBench/Benchmarks/MLBenchmark.cs b/tests/src/performance/Scenario/JitBench/Benchmarks/MLBenchmark.cs
index f9c5dfd..88dd9d9 100644 (file)
--- a/tests/src/performance/Scenario/JitBench/Benchmarks/MLBenchmark.cs
+++ b/tests/src/performance/Scenario/JitBench/Benchmarks/MLBenchmark.cs
@@ -5,6 +5,7 @@ using System.Text.RegularExpressions;
 using System.Threading.Tasks;
 using System.Reflection;
 using Microsoft.Xunit.Performance.Api;
+using System.Runtime.InteropServices;
 
 namespace JitBench
 {
@@ -12,6 +13,20 @@ namespace JitBench
     {
         public Word2VecBenchmark() : base("Word2Vec") { }
 
+        public override bool IsArchitectureSupported(Architecture arch)
+        {
+            //Word2Vec uses large amounts of virtual address space which it may or may not exhaust 
+            //when running on x86. In the case I investigated there was still a few 100MB free,
+            //but it was sufficiently fragmented that the largest block was less than 32MB which
+            //is what the GC wants for a new LOH segment. The GC behavior here is by-design.
+            //Tiered jitting increases the memory usage (more code) and may cause different
+            //fragmentation patterns - arguably either 'by design' or 'known issue that won't change
+            //unless customers tell us its a problem'. I'm OK telling people not to use tiered jitting 
+            //if their app already uses most of the address space on x86, and having an intermitently 
+            //failing test in a perf suite won't give us useful info hence x64 only for this one.
+            return arch == Architecture.X64;
+        }
+
         protected override string ExecutableName => "Word2VecScenario.dll";
 
         protected override string GetWord2VecNetSrcDirectory(string outputDir)

diff --git a/tests/src/performance/Scenario/JitBench/Runner/Benchmark.cs b/tests/src/performance/Scenario/JitBench/Runner/Benchmark.cs
index 13554c8..a060bf1 100644 (file)
--- a/tests/src/performance/Scenario/JitBench/Runner/Benchmark.cs
+++ b/tests/src/performance/Scenario/JitBench/Runner/Benchmark.cs
@@ -3,6 +3,7 @@ using System.Collections.Generic;
 using System.Diagnostics;
 using System.IO;
 using System.Linq;
+using System.Runtime.InteropServices;
 using System.Text;
 using System.Threading.Tasks;
 using Microsoft.Xunit.Performance.Api;
@@ -39,6 +40,14 @@ namespace JitBench
             return new Metric[] { Metric.ElapsedTimeMilliseconds };
         }
 
+        /// <summary>
+        /// Does this benchmark run properly on a given architecture?
+        /// </summary>
+        public virtual bool IsArchitectureSupported(Architecture arch)
+        {
+            return (arch == Architecture.X86 || arch == Architecture.X64);
+        }
+
         BenchmarkRunResult[] MeasureIterations(TestRun run, ITestOutputHelper output)
         {
             List<BenchmarkRunResult> results = new List<BenchmarkRunResult>();

diff --git a/tests/src/performance/Scenario/JitBench/Runner/TestRun.cs b/tests/src/performance/Scenario/JitBench/Runner/TestRun.cs
index 492bcd5..49b6e26 100644 (file)
--- a/tests/src/performance/Scenario/JitBench/Runner/TestRun.cs
+++ b/tests/src/performance/Scenario/JitBench/Runner/TestRun.cs
@@ -129,6 +129,11 @@ namespace JitBench
             await PrepareDotNet(output);
             foreach (Benchmark benchmark in Benchmarks)
             {
+                if(!benchmark.IsArchitectureSupported(Architecture))
+                {
+                    output.WriteLine("Benchmark " + benchmark.Name + " does not support architecture " + Architecture + ". Skipping setup.");
+                    continue;
+                }
                 await benchmark.Setup(DotNetInstallation, OutputDir, UseExistingSetup, output);
             }
         }
@@ -163,6 +168,11 @@ namespace JitBench
             output.WriteLine("");
             foreach (Benchmark benchmark in Benchmarks)
             {
+                if (!benchmark.IsArchitectureSupported(Architecture))
+                {
+                    output.WriteLine("Benchmark " + benchmark.Name + " does not support architecture " + Architecture + ". Skipping run.");
+                    continue;
+                }
                 BenchmarkRunResults.AddRange(benchmark.Run(this, output));
             }
         }