From 1f01e7bcf8d74701201dc48ddcbb9a3f7c76cf82 Mon Sep 17 00:00:00 2001 From: Sung Yoon Whang Date: Wed, 18 Sep 2019 17:32:20 -0700 Subject: [PATCH] Counter benchmark tests (#475) * Basic benchmark test added * don't include benchmarkdotnet artifacts in the git repo * Some README changes, added gitignore --- src/tests/benchmarks/.gitignore | 1 + src/tests/benchmarks/Program.cs | 120 +++++++++++++++++++++++++ src/tests/benchmarks/README.md | 36 ++++++++ src/tests/benchmarks/RunBenchmarks.cmd | 1 + src/tests/benchmarks/RunBenchmarks.sh | 1 + src/tests/benchmarks/benchmarks.csproj | 12 +++ 6 files changed, 171 insertions(+) create mode 100644 src/tests/benchmarks/.gitignore create mode 100644 src/tests/benchmarks/Program.cs create mode 100644 src/tests/benchmarks/README.md create mode 100644 src/tests/benchmarks/RunBenchmarks.cmd create mode 100755 src/tests/benchmarks/RunBenchmarks.sh create mode 100644 src/tests/benchmarks/benchmarks.csproj diff --git a/src/tests/benchmarks/.gitignore b/src/tests/benchmarks/.gitignore new file mode 100644 index 000000000..b9bb25586 --- /dev/null +++ b/src/tests/benchmarks/.gitignore @@ -0,0 +1 @@ +BenchmarkDotNet.artifacts diff --git a/src/tests/benchmarks/Program.cs b/src/tests/benchmarks/Program.cs new file mode 100644 index 000000000..7265ecb98 --- /dev/null +++ b/src/tests/benchmarks/Program.cs @@ -0,0 +1,120 @@ +using System; +using System.Reflection; +using System.Threading; +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Running; + +namespace CounterBenchmarks +{ + [CoreJob] + [RPlotExporter, RankColumn] + public class CounterBenchmarks + { + private MethodInfo getGenerationSize; + private MethodInfo getPercentTimeInGC; + private MethodInfo getExceptionCount; + private MethodInfo getAssemblyCount; + + // These are pre-allocated objects for passing as parameters private reflection methods to avoid allocation overhead in the actual benchmark. + private object[] gen0SizeParam = new object[] { 0 }; + private object[] gen1SizeParam = new object[] { 1 }; + private object[] gen2SizeParam = new object[] { 2 }; + private object[] gen3SizeParam = new object[] { 3 }; + + [Params(10000)] + public int N; + + [GlobalSetup] + public void Setup() + { + // Use reflection to get all the internal API we use inside the runtime to emit counters. + Assembly SPC = typeof(System.Diagnostics.Tracing.EventSource).Assembly; + if (SPC == null) + { + Environment.FailFast("Failed to get System.Private.CoreLib assembly."); + } + + Type gcType = SPC.GetType("System.GC"); + if (gcType == null) + { + Environment.FailFast("Failed to get System.GC type."); + } + getGenerationSize = gcType.GetMethod("GetGenerationSize", BindingFlags.Static | BindingFlags.NonPublic); + getPercentTimeInGC = gcType.GetMethod("GetLastGCPercentTimeInGC", BindingFlags.Static | BindingFlags.NonPublic); + + Type exceptionType = SPC.GetType("System.Exception"); + if (exceptionType == null) + { + Environment.FailFast("Failed to get System.Environment Type."); + } + getExceptionCount = exceptionType.GetMethod("GetExceptionCount", BindingFlags.Static | BindingFlags.NonPublic); + + Type assemblyType = SPC.GetType("System.Reflection.Assembly"); + if (assemblyType == null) + { + Environment.FailFast("Failed to get System.Reflection.Assembly Type."); + } + getAssemblyCount = assemblyType.GetMethod("GetAssemblyCount", BindingFlags.Static | BindingFlags.NonPublic); + } + + [Benchmark] + public int Gen0Count() => GC.CollectionCount(0); + + [Benchmark] + public int Gen1Count() => GC.CollectionCount(1); + + [Benchmark] + public int Gen2Count() => GC.CollectionCount(2); + + [Benchmark] + public object Gen0Size() => getGenerationSize.Invoke(null, gen0SizeParam); + + [Benchmark] + public object Gen1Size() => getGenerationSize.Invoke(null, gen1SizeParam); + + [Benchmark] + public object Gen2Size() => getGenerationSize.Invoke(null, gen2SizeParam); + + [Benchmark] + public object LOHSize() => getGenerationSize.Invoke(null, gen3SizeParam); + + [Benchmark] + public object TimeInGC() => getPercentTimeInGC.Invoke(null, null); + + [Benchmark] + public object AssemblyCount() => getAssemblyCount.Invoke(null, null); + + [Benchmark] + public object ExceptionCount() => getExceptionCount.Invoke(null, null); + + [Benchmark] + public long Workingset() => Environment.WorkingSet; + + [Benchmark] + public long ThreadPoolCount() => ThreadPool.ThreadCount; + + [Benchmark] + public long LockContentionCount() => Monitor.LockContentionCount; + + [Benchmark] + public long PendingItemCount() => ThreadPool.PendingWorkItemCount; + + [Benchmark] + public long CompletedItemCount() => ThreadPool.CompletedWorkItemCount; + + [Benchmark] + public long TotalAllocatedBytes() => GC.GetTotalAllocatedBytes(); + + [Benchmark] + public long ActiveTimerCount() => Timer.ActiveCount; + + } + + public class Program + { + public static void Main(string[] args) + { + BenchmarkRunner.Run(); + } + } +} \ No newline at end of file diff --git a/src/tests/benchmarks/README.md b/src/tests/benchmarks/README.md new file mode 100644 index 000000000..20d2194cf --- /dev/null +++ b/src/tests/benchmarks/README.md @@ -0,0 +1,36 @@ +# .NET Core Runtime Performance Counter Benchmarks + +## Intro + +This part of the repo contains benchmark tests for the internal (or public) APIs that are currently used by CoreCLR to emit these values. + +It uses BenchmarkDotNet to measure the raw performance of the underlying APIs to produce runtime counter values. + +An example output looks like this: + +``` +| Method | N | Mean | Error | StdDev | Rank | +|-------------------- |------ |--------------:|------------:|------------:|-----:| +| Gen0Count | 10000 | 4.199 ns | 0.0302 ns | 0.0283 ns | 3 | +| Gen1Count | 10000 | 4.181 ns | 0.0392 ns | 0.0367 ns | 3 | +| Gen2Count | 10000 | 4.239 ns | 0.0237 ns | 0.0222 ns | 3 | +| Gen0Size | 10000 | 203.872 ns | 1.3477 ns | 1.1254 ns | 8 | +| Gen1Size | 10000 | 204.422 ns | 1.1772 ns | 1.1011 ns | 8 | +| Gen2Size | 10000 | 202.772 ns | 1.0590 ns | 0.9906 ns | 8 | +| LOHSize | 10000 | 202.729 ns | 0.7927 ns | 0.7027 ns | 8 | +| TimeInGC | 10000 | 133.375 ns | 0.8509 ns | 0.7959 ns | 6 | +| AssemblyCount | 10000 | 131.736 ns | 0.4501 ns | 0.3990 ns | 6 | +| ExceptionCount | 10000 | 131.689 ns | 0.7521 ns | 0.7035 ns | 6 | +| Workingset | 10000 | 684.784 ns | 3.8186 ns | 2.0261 ns | 9 | +| ThreadPoolCount | 10000 | 1.845 ns | 0.0175 ns | 0.0155 ns | 1 | +| LockContentionCount | 10000 | 66.371 ns | 0.2625 ns | 0.2327 ns | 5 | +| PendingItemCount | 10000 | 10.218 ns | 0.1101 ns | 0.1030 ns | 4 | +| CompletedItemCount | 10000 | 66.067 ns | 0.3538 ns | 0.3137 ns | 5 | +| TotalAllocatedBytes | 10000 | 3.636 ns | 0.0185 ns | 0.0173 ns | 2 | +| ActiveTimerCount | 10000 | 151.222 ns | 2.1967 ns | 2.0548 ns | 7 | +``` + +## How to run + +Simply clone the repo, navigate to src\tests\benchmarks\, and run `RunBenchmarks.cmd` (or `RunBenchmarks.sh`). + diff --git a/src/tests/benchmarks/RunBenchmarks.cmd b/src/tests/benchmarks/RunBenchmarks.cmd new file mode 100644 index 000000000..45f592539 --- /dev/null +++ b/src/tests/benchmarks/RunBenchmarks.cmd @@ -0,0 +1 @@ +dotnet run -c release \ No newline at end of file diff --git a/src/tests/benchmarks/RunBenchmarks.sh b/src/tests/benchmarks/RunBenchmarks.sh new file mode 100755 index 000000000..45f592539 --- /dev/null +++ b/src/tests/benchmarks/RunBenchmarks.sh @@ -0,0 +1 @@ +dotnet run -c release \ No newline at end of file diff --git a/src/tests/benchmarks/benchmarks.csproj b/src/tests/benchmarks/benchmarks.csproj new file mode 100644 index 000000000..6e7086538 --- /dev/null +++ b/src/tests/benchmarks/benchmarks.csproj @@ -0,0 +1,12 @@ + + + + Exe + netcoreapp3.0 + + + + + + + -- 2.34.1