Counter benchmark tests (#475)

* Basic benchmark test added

* don't include benchmarkdotnet artifacts in the git repo

* Some README changes, added gitignore

diff --git a/src/tests/benchmarks/.gitignore b/src/tests/benchmarks/.gitignore
new file mode 100644 (file)
index 0000000..b9bb255
--- /dev/null
+++ b/src/tests/benchmarks/.gitignore
@@ -0,0 +1 @@
+BenchmarkDotNet.artifacts

diff --git a/src/tests/benchmarks/Program.cs b/src/tests/benchmarks/Program.cs
new file mode 100644 (file)
index 0000000..7265ecb
--- /dev/null
+++ b/src/tests/benchmarks/Program.cs
@@ -0,0 +1,120 @@
+using System;
+using System.Reflection;
+using System.Threading;
+using BenchmarkDotNet.Attributes;
+using BenchmarkDotNet.Running;
+
+namespace CounterBenchmarks
+{
+    [CoreJob]
+    [RPlotExporter, RankColumn]
+    public class CounterBenchmarks
+    {
+        private MethodInfo getGenerationSize;
+        private MethodInfo getPercentTimeInGC;
+        private MethodInfo getExceptionCount;
+        private MethodInfo getAssemblyCount;
+
+        // These are pre-allocated objects for passing as parameters private reflection methods to avoid allocation overhead in the actual benchmark.
+        private object[] gen0SizeParam = new object[] { 0 };
+        private object[] gen1SizeParam = new object[] { 1 };
+        private object[] gen2SizeParam = new object[] { 2 };
+        private object[] gen3SizeParam = new object[] { 3 };
+
+        [Params(10000)]
+        public int N;
+
+        [GlobalSetup]
+        public void Setup()
+        {
+            // Use reflection to get all the internal API we use inside the runtime to emit counters.
+            Assembly SPC = typeof(System.Diagnostics.Tracing.EventSource).Assembly;
+            if (SPC == null)
+            {
+                Environment.FailFast("Failed to get System.Private.CoreLib assembly.");
+            }
+
+            Type gcType = SPC.GetType("System.GC");
+            if (gcType == null)
+            {
+                Environment.FailFast("Failed to get System.GC type.");
+            }
+            getGenerationSize = gcType.GetMethod("GetGenerationSize", BindingFlags.Static | BindingFlags.NonPublic);
+            getPercentTimeInGC = gcType.GetMethod("GetLastGCPercentTimeInGC", BindingFlags.Static | BindingFlags.NonPublic);
+
+            Type exceptionType = SPC.GetType("System.Exception");
+            if (exceptionType == null)
+            {
+                Environment.FailFast("Failed to get System.Environment Type.");
+            }
+            getExceptionCount = exceptionType.GetMethod("GetExceptionCount", BindingFlags.Static | BindingFlags.NonPublic);
+
+            Type assemblyType = SPC.GetType("System.Reflection.Assembly");
+            if (assemblyType == null)
+            {
+                Environment.FailFast("Failed to get System.Reflection.Assembly Type.");
+            }
+            getAssemblyCount = assemblyType.GetMethod("GetAssemblyCount", BindingFlags.Static | BindingFlags.NonPublic);
+        }
+
+        [Benchmark]
+        public int Gen0Count() => GC.CollectionCount(0);
+
+        [Benchmark]
+        public int Gen1Count() => GC.CollectionCount(1);
+
+        [Benchmark]
+        public int Gen2Count() => GC.CollectionCount(2);
+
+        [Benchmark]
+        public object Gen0Size() => getGenerationSize.Invoke(null, gen0SizeParam);
+
+        [Benchmark]
+        public object Gen1Size() => getGenerationSize.Invoke(null, gen1SizeParam);
+
+        [Benchmark]
+        public object Gen2Size() => getGenerationSize.Invoke(null, gen2SizeParam);
+
+        [Benchmark]
+        public object LOHSize() => getGenerationSize.Invoke(null, gen3SizeParam);
+
+        [Benchmark]
+        public object TimeInGC() => getPercentTimeInGC.Invoke(null, null);
+
+        [Benchmark]
+        public object AssemblyCount() => getAssemblyCount.Invoke(null, null);
+
+        [Benchmark]
+        public object ExceptionCount() => getExceptionCount.Invoke(null, null);
+
+        [Benchmark]
+        public long Workingset() => Environment.WorkingSet;
+
+        [Benchmark]
+        public long ThreadPoolCount() => ThreadPool.ThreadCount;
+
+        [Benchmark]
+        public long LockContentionCount() => Monitor.LockContentionCount;
+
+        [Benchmark]
+        public long PendingItemCount() => ThreadPool.PendingWorkItemCount;
+
+        [Benchmark]
+        public long CompletedItemCount() => ThreadPool.CompletedWorkItemCount;
+
+        [Benchmark]
+        public long TotalAllocatedBytes() => GC.GetTotalAllocatedBytes();
+
+        [Benchmark]
+        public long ActiveTimerCount() => Timer.ActiveCount;
+
+    }
+
+    public class Program
+    {
+        public static void Main(string[] args)
+        {
+            BenchmarkRunner.Run<CounterBenchmarks>();
+        }
+    }
+}
\ No newline at end of file

diff --git a/src/tests/benchmarks/README.md b/src/tests/benchmarks/README.md
new file mode 100644 (file)
index 0000000..20d2194
--- /dev/null
+++ b/src/tests/benchmarks/README.md
@@ -0,0 +1,36 @@
+# .NET Core Runtime Performance Counter Benchmarks
+
+## Intro 
+
+This part of the repo contains benchmark tests for the internal (or public) APIs that are currently used by CoreCLR to emit these values.
+
+It uses BenchmarkDotNet to measure the raw performance of the underlying APIs to produce runtime counter values. 
+
+An example output looks like this:
+
+```
+|              Method |     N |          Mean |       Error |      StdDev | Rank |
+|-------------------- |------ |--------------:|------------:|------------:|-----:|
+|           Gen0Count | 10000 |      4.199 ns |   0.0302 ns |   0.0283 ns |    3 |
+|           Gen1Count | 10000 |      4.181 ns |   0.0392 ns |   0.0367 ns |    3 |
+|           Gen2Count | 10000 |      4.239 ns |   0.0237 ns |   0.0222 ns |    3 |
+|            Gen0Size | 10000 |    203.872 ns |   1.3477 ns |   1.1254 ns |    8 |
+|            Gen1Size | 10000 |    204.422 ns |   1.1772 ns |   1.1011 ns |    8 |
+|            Gen2Size | 10000 |    202.772 ns |   1.0590 ns |   0.9906 ns |    8 |
+|             LOHSize | 10000 |    202.729 ns |   0.7927 ns |   0.7027 ns |    8 |
+|            TimeInGC | 10000 |    133.375 ns |   0.8509 ns |   0.7959 ns |    6 |
+|       AssemblyCount | 10000 |    131.736 ns |   0.4501 ns |   0.3990 ns |    6 |
+|      ExceptionCount | 10000 |    131.689 ns |   0.7521 ns |   0.7035 ns |    6 |
+|          Workingset | 10000 |    684.784 ns |   3.8186 ns |   2.0261 ns |    9 |
+|     ThreadPoolCount | 10000 |      1.845 ns |   0.0175 ns |   0.0155 ns |    1 |
+| LockContentionCount | 10000 |     66.371 ns |   0.2625 ns |   0.2327 ns |    5 |
+|    PendingItemCount | 10000 |     10.218 ns |   0.1101 ns |   0.1030 ns |    4 |
+|  CompletedItemCount | 10000 |     66.067 ns |   0.3538 ns |   0.3137 ns |    5 |
+| TotalAllocatedBytes | 10000 |      3.636 ns |   0.0185 ns |   0.0173 ns |    2 |
+|    ActiveTimerCount | 10000 |    151.222 ns |   2.1967 ns |   2.0548 ns |    7 |
+```
+
+## How to run
+
+Simply clone the repo, navigate to src\tests\benchmarks\, and run `RunBenchmarks.cmd` (or `RunBenchmarks.sh`).
+

diff --git a/src/tests/benchmarks/RunBenchmarks.cmd b/src/tests/benchmarks/RunBenchmarks.cmd
new file mode 100644 (file)
index 0000000..45f5925
--- /dev/null
+++ b/src/tests/benchmarks/RunBenchmarks.cmd
@@ -0,0 +1 @@
+dotnet run -c release
\ No newline at end of file

diff --git a/src/tests/benchmarks/RunBenchmarks.sh b/src/tests/benchmarks/RunBenchmarks.sh
new file mode 100755 (executable)
index 0000000..45f5925
--- /dev/null
+++ b/src/tests/benchmarks/RunBenchmarks.sh
@@ -0,0 +1 @@
+dotnet run -c release
\ No newline at end of file

diff --git a/src/tests/benchmarks/benchmarks.csproj b/src/tests/benchmarks/benchmarks.csproj
new file mode 100644 (file)
index 0000000..6e70865
--- /dev/null
+++ b/src/tests/benchmarks/benchmarks.csproj
@@ -0,0 +1,12 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>netcoreapp3.0</TargetFramework>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="BenchmarkDotNet" Version="0.11.5" />
+  </ItemGroup>
+
+</Project>