From: Joseph Tremoulet Date: Thu, 16 Mar 2017 00:43:01 +0000 (-0400) Subject: Update SpanBench test X-Git-Tag: submit/tizen/20210909.063632~11030^2~7660^2 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=02233550ec97bdcb640e168eed294b783df64177;p=platform%2Fupstream%2Fdotnet%2Fruntime.git Update SpanBench test - Re-enable the tests that were disabled in e859c309. - Re-work how tests are invoked from the command line to require less boilerplate - Now each test has a single [Benchmark] entrypoint - That entrypoint invokes its test's single inner-loop by wrapping it in a lambda that it passes to a new helper Invoke method (shared across all tests) which handles the xunit vs. command-line differences.. - Main finds the entrypoints by using reflection to search for the [Benchmark] attributes, so the explicit list of stringified test names is no longer needed and the command line will run the same set of tests that xunit-perf does. - The new SpanAPI tests now get invoked when this tests is run from the command line as well. - Add [NoInlining] to the SpanAPI tests' kernels. - Add some heap writes and conditional writes to prevent deadcode optimization from eliminating tests' kernels (assuming we don't do interprocedural constant propagation or deadcode, or store sinking and final value calculation, or PRE...). - Split the Index SpanAPI tests into one version that uses a loop-invariant index and thus should get hoisted out of the loop, and another version that uses variant indices. Commit migrated from https://github.com/dotnet/coreclr/commit/65226a257f173edc03de3b4b43d001f801d0b379 --- diff --git a/src/coreclr/tests/src/Common/test_dependencies/project.json b/src/coreclr/tests/src/Common/test_dependencies/project.json index 6b74f64..c3564ef 100644 --- a/src/coreclr/tests/src/Common/test_dependencies/project.json +++ b/src/coreclr/tests/src/Common/test_dependencies/project.json @@ -2,6 +2,7 @@ "dependencies": { "Microsoft.NETCore.Platforms": "2.0.0-beta-25118-01", "Microsoft.Private.CoreFx.NETCoreApp": "4.4.0-beta-25118-01", + "System.Memory": "4.4.0-beta-25118-01", "System.Runtime.CompilerServices.Unsafe": "4.4.0-beta-25118-01" }, "frameworks": { diff --git a/src/coreclr/tests/src/JIT/Performance/CodeQuality/Span/SpanBench.cs b/src/coreclr/tests/src/JIT/Performance/CodeQuality/Span/SpanBench.cs index 2a1fe1e..07e5030 100644 --- a/src/coreclr/tests/src/JIT/Performance/CodeQuality/Span/SpanBench.cs +++ b/src/coreclr/tests/src/JIT/Performance/CodeQuality/Span/SpanBench.cs @@ -27,13 +27,15 @@ namespace Span const int FillAllIterations = 1; const int BaseIterations = 1; #else + // Appropriately-scaled iteration counts for the various benchmarks const int BubbleSortIterations = 100; const int QuickSortIterations = 1000; const int FillAllIterations = 100000; const int BaseIterations = 10000000; #endif - const int Size = 1024; + // Default length for arrays of mock input data + const int DefaultLength = 1024; // Helpers #region Helpers @@ -44,41 +46,60 @@ namespace Span public T[] C0; } - /*[MethodImpl(MethodImplOptions.NoInlining)] - static void TestFillAllSpan(Span span) + // Copying the result of a computation to Sink.Instance is a way + // to prevent the jit from considering the computation dead and removing it. + private sealed class Sink { - for (int i = 0; i < span.Length; ++i) { - span[i] = unchecked((byte)i); - } + public T Data; + public static Sink Instance = new Sink(); } - [MethodImpl(MethodImplOptions.NoInlining)] - static void TestFillAllArray(byte[] data) - { - for (int i = 0; i < data.Length; ++i) { - data[i] = unchecked((byte)i); - } - } + // Use statics to smuggle some information from Main to Invoke when running tests + // from the command line. + static bool IsXunitInvocation = true; // xunit-perf leaves this true; command line Main sets to false + static int CommandLineInnerIterationCount = 0; // used to communicate iteration count from BenchmarkAttribute + // (xunit-perf exposes the same in static property Benchmark.InnerIterationCount) + static bool DoWarmUp; // Main sets this when calling a new benchmark routine - [MethodImpl(MethodImplOptions.NoInlining)] - static void TestFillAllReverseSpan(Span span) + + // Invoke routine to abstract away the difference between running under xunit-perf vs running from the + // command line. Inner loop to be measured is taken as an Action, and invoked passing the number + // of iterations that the inner loop should execute. + static void Invoke(Action innerLoop, string nameFormat, params object[] nameArgs) { - for (int i = span.Length; --i >= 0;) { - span[i] = unchecked((byte)i); + if (IsXunitInvocation) + { + foreach (var iteration in Benchmark.Iterations) + using (iteration.StartMeasurement()) + innerLoop((int)Benchmark.InnerIterationCount); } - } + else + { + if (DoWarmUp) + { + // Run some warm-up iterations before measuring + innerLoop(CommandLineInnerIterationCount); + // Clear the flag since we're now warmed up (caller will + // reset it before calling new code) + DoWarmUp = false; + } - [MethodImpl(MethodImplOptions.NoInlining)] - static void TestFillAllReverseArray(byte[] data) - { - for (int i = data.Length; --i >= 0;) { - data[i] = unchecked((byte)i); + // Now do the timed run of the inner loop. + Stopwatch sw = Stopwatch.StartNew(); + innerLoop(CommandLineInnerIterationCount); + sw.Stop(); + + // Print result. + string name = String.Format(nameFormat, nameArgs); + double timeInMs = sw.Elapsed.TotalMilliseconds; + Console.WriteLine("{0}: {1}ms", name, timeInMs); } } - static int[] GetUnsortedData() + // Helper for the sort tests to get some pseudo-random input + static int[] GetUnsortedData(int length) { - int[] unsortedData = new int[Size]; + int[] unsortedData = new int[length]; Random r = new Random(42); for (int i = 0; i < unsortedData.Length; ++i) { @@ -86,273 +107,333 @@ namespace Span } return unsortedData; } + #endregion // helpers - [MethodImpl(MethodImplOptions.NoInlining)] - static void TestBubbleSortSpan(Span span) + // Tests that implement some vary basic algorithms (fill/sort) over spans and arrays + #region Algorithm tests + + #region TestFillAllSpan + [Benchmark(InnerIterationCount = FillAllIterations)] + [InlineData(DefaultLength)] + public static void FillAllSpan(int length) { - bool swap; - int temp; - int n = span.Length - 1; - do { - swap = false; - for (int i = 0; i < n; i++) { - if (span[i] > span[i + 1]) { - temp = span[i]; - span[i] = span[i + 1]; - span[i + 1] = temp; - swap = true; - } + byte[] a = new byte[length]; + + Invoke((int innerIterationCount) => + { + Span s = new Span(a); + for (int i = 0; i < innerIterationCount; ++i) + { + TestFillAllSpan(s); } - --n; - } - while (swap); + }, + "TestFillAllSpan({0})", length); } [MethodImpl(MethodImplOptions.NoInlining)] - static void TestBubbleSortArray(int[] data) + static void TestFillAllSpan(Span span) { - bool swap; - int temp; - int n = data.Length - 1; - do { - swap = false; - for (int i = 0; i < n; i++) { - if (data[i] > data[i + 1]) { - temp = data[i]; - data[i] = data[i + 1]; - data[i + 1] = temp; - swap = true; - } - } - --n; + for (int i = 0; i < span.Length; ++i) + { + span[i] = unchecked((byte)i); } - while (swap); } + #endregion - static void TestQuickSortSpan(Span data) + #region TestFillAllArray + [Benchmark(InnerIterationCount = FillAllIterations)] + [InlineData(DefaultLength)] + public static void FillAllArray(int length) { - QuickSortSpan(data); + byte[] a = new byte[length]; + + Invoke((int innerIterationCount) => + { + for (int i = 0; i < innerIterationCount; ++i) + { + TestFillAllArray(a); + } + }, + "TestFillArray({0})", length); } [MethodImpl(MethodImplOptions.NoInlining)] - static void QuickSortSpan(Span data) + static void TestFillAllArray(byte[] data) { - if (data.Length <= 1) { - return; + for (int i = 0; i < data.Length; ++i) + { + data[i] = unchecked((byte)i); } + } + #endregion - int lo = 0; - int hi = data.Length - 1; - int i, j; - int pivot, temp; - for (i = lo, j = hi, pivot = data[hi]; i < j;) { - while (i < j && data[i] <= pivot) { - ++i; - } - while (j > i && data[j] >= pivot) { - --j; - } - if (i < j) { - temp = data[i]; - data[i] = data[j]; - data[j] = temp; + #region TestFillAllReverseSpan + [Benchmark(InnerIterationCount = FillAllIterations)] + [InlineData(DefaultLength)] + public static void FillAllReverseSpan(int length) + { + byte[] a = new byte[length]; + + Invoke((int innerIterationCount) => + { + Span s = new Span(a); + for (int i = 0; i < innerIterationCount; ++i) + { + TestFillAllReverseSpan(s); } + }, + "TestFillAllReverseSpan({0})", length); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + static void TestFillAllReverseSpan(Span span) + { + for (int i = span.Length; --i >= 0;) + { + span[i] = unchecked((byte)i); } - if (i != hi) { - temp = data[i]; - data[i] = pivot; - data[hi] = temp; - } + } + #endregion - QuickSortSpan(data.Slice(0, i)); - QuickSortSpan(data.Slice(i + 1)); + #region TestFillAllReverseArray + [Benchmark(InnerIterationCount = FillAllIterations)] + [InlineData(DefaultLength)] + public static void FillAllReverseArray(int length) + { + byte[] a = new byte[length]; + + Invoke((int innerIterationCount) => + { + for (int i = 0; i < innerIterationCount; ++i) + { + TestFillAllReverseArray(a); + } + }, + "TestFillAllReverseArray({0})", length); } - static void TestQuickSortArray(int[] data) + [MethodImpl(MethodImplOptions.NoInlining)] + static void TestFillAllReverseArray(byte[] data) { - QuickSortArray(data, 0, data.Length - 1); + for (int i = data.Length; --i >= 0;) + { + data[i] = unchecked((byte)i); + } + } + #endregion + + #region TestQuickSortSpan + [Benchmark(InnerIterationCount = QuickSortIterations)] + [InlineData(DefaultLength)] + public static void QuickSortSpan(int length) + { + int[] data = new int[length]; + int[] unsortedData = GetUnsortedData(length); + + Invoke((int innerIterationCount) => + { + Span span = new Span(data); + + for (int i = 0; i < innerIterationCount; ++i) + { + Array.Copy(unsortedData, data, length); + TestQuickSortSpan(span); + } + }, + "TestQuickSortSpan({0})", length); } [MethodImpl(MethodImplOptions.NoInlining)] - static void QuickSortArray(int[] data, int lo, int hi) + static void TestQuickSortSpan(Span data) { - if (lo >= hi) { + if (data.Length <= 1) + { return; } + int lo = 0; + int hi = data.Length - 1; int i, j; int pivot, temp; - for (i = lo, j = hi, pivot = data[hi]; i < j;) { - while (i < j && data[i] <= pivot) { + for (i = lo, j = hi, pivot = data[hi]; i < j;) + { + while (i < j && data[i] <= pivot) + { ++i; } - while (j > i && data[j] >= pivot) { + while (j > i && data[j] >= pivot) + { --j; } - if (i < j) { + if (i < j) + { temp = data[i]; data[i] = data[j]; data[j] = temp; } } - if (i != hi) { + if (i != hi) + { temp = data[i]; data[i] = pivot; data[hi] = temp; } - QuickSortArray(data, lo, i - 1); - QuickSortArray(data, i + 1, hi); - }*/ + TestQuickSortSpan(data.Slice(0, i)); + TestQuickSortSpan(data.Slice(i + 1)); + } #endregion - // XUNIT-PERF tests - #region XUNIT-PERF tests - /*[Benchmark] - public static void FillAllSpan() + #region TestBubbleSortSpan + [Benchmark(InnerIterationCount = BubbleSortIterations)] + [InlineData(DefaultLength)] + public static void BubbleSortSpan(int length) { - byte[] a = new byte[Size]; - Span s = new Span(a); - foreach (var iteration in Benchmark.Iterations) + int[] data = new int[length]; + int[] unsortedData = GetUnsortedData(length); + + Invoke((int innerIterationCount) => { - using (iteration.StartMeasurement()) + Span span = new Span(data); + + for (int i = 0; i < innerIterationCount; i++) { - for (int i = 0; i < FillAllIterations; i++) - { - TestFillAllSpan(s); - } + Array.Copy(unsortedData, data, length); + TestBubbleSortSpan(span); } - } + }, + "TestBubbleSortSpan({0})", length); } - [Benchmark] - public static void FillAllArray() + [MethodImpl(MethodImplOptions.NoInlining)] + static void TestBubbleSortSpan(Span span) { - byte[] a = new byte[Size]; - foreach (var iteration in Benchmark.Iterations) + bool swap; + int temp; + int n = span.Length - 1; + do { - using (iteration.StartMeasurement()) + swap = false; + for (int i = 0; i < n; i++) { - for (int i = 0; i < FillAllIterations; i++) + if (span[i] > span[i + 1]) { - TestFillAllArray(a); + temp = span[i]; + span[i] = span[i + 1]; + span[i + 1] = temp; + swap = true; } } + --n; } + while (swap); } + #endregion - [Benchmark] - public static void FillAllReverseSpan() + #region TestQuickSortArray + [Benchmark(InnerIterationCount = QuickSortIterations)] + [InlineData(DefaultLength)] + public static void QuickSortArray(int length) { - byte[] a = new byte[Size]; - Span s = new Span(a); - foreach (var iteration in Benchmark.Iterations) + int[] data = new int[length]; + int[] unsortedData = GetUnsortedData(length); + + Invoke((int innerIterationCount) => { - using (iteration.StartMeasurement()) + for (int i = 0; i < innerIterationCount; i++) { - for (int i = 0; i < FillAllIterations; i++) - { - TestFillAllReverseSpan(s); - } + Array.Copy(unsortedData, data, length); + TestQuickSortArray(data, 0, data.Length - 1); } - } + }, + "TestQuickSortArray({0})", length); } - [Benchmark] - public static void FillAllReverseArray() + [MethodImpl(MethodImplOptions.NoInlining)] + static void TestQuickSortArray(int[] data, int lo, int hi) { - byte[] a = new byte[Size]; - foreach (var iteration in Benchmark.Iterations) + if (lo >= hi) { - using (iteration.StartMeasurement()) - { - for (int i = 0; i < FillAllIterations; i++) - { - TestFillAllReverseArray(a); - } - } + return; } - } - [Benchmark] - public static void QuickSortSpan() - { - int[] data = new int[Size]; - int[] unsortedData = GetUnsortedData(); - Span span = new Span(data); - - foreach (var iteration in Benchmark.Iterations) + int i, j; + int pivot, temp; + for (i = lo, j = hi, pivot = data[hi]; i < j;) { - using (iteration.StartMeasurement()) + while (i < j && data[i] <= pivot) { - for (int i = 0; i < QuickSortIterations; i++) - { - Array.Copy(unsortedData, data, Size); - TestQuickSortSpan(span); - } + ++i; } - } - } - - [Benchmark] - public static void BubbleSortSpan() - { - int[] data = new int[Size]; - int[] unsortedData = GetUnsortedData(); - Span span = new Span(data); - - foreach (var iteration in Benchmark.Iterations) - { - using (iteration.StartMeasurement()) + while (j > i && data[j] >= pivot) { - for (int i = 0; i < BubbleSortIterations; i++) - { - Array.Copy(unsortedData, data, Size); - TestBubbleSortSpan(span); - } + --j; + } + if (i < j) + { + temp = data[i]; + data[i] = data[j]; + data[j] = temp; } } + if (i != hi) + { + temp = data[i]; + data[i] = pivot; + data[hi] = temp; + } + + TestQuickSortArray(data, lo, i - 1); + TestQuickSortArray(data, i + 1, hi); } + #endregion - [Benchmark] - public static void QuickSortArray() + #region TestBubbleSortArray + [Benchmark(InnerIterationCount = BubbleSortIterations)] + [InlineData(DefaultLength)] + public static void BubbleSortArray(int length) { - int[] data = new int[Size]; - int[] unsortedData = GetUnsortedData(); + int[] data = new int[length]; + int[] unsortedData = GetUnsortedData(length); - foreach (var iteration in Benchmark.Iterations) + Invoke((int innerIterationCount) => { - using (iteration.StartMeasurement()) + for (int i = 0; i < innerIterationCount; i++) { - for (int i = 0; i < QuickSortIterations; i++) - { - Array.Copy(unsortedData, data, Size); - TestQuickSortArray(data); - } + Array.Copy(unsortedData, data, length); + TestBubbleSortArray(data); } - } + }, + "TestBubbleSortArray({0})", length); } - [Benchmark] - public static void BubbleSortArray() + [MethodImpl(MethodImplOptions.NoInlining)] + static void TestBubbleSortArray(int[] data) { - int[] data = new int[Size]; - int[] unsortedData = GetUnsortedData(); - - foreach (var iteration in Benchmark.Iterations) + bool swap; + int temp; + int n = data.Length - 1; + do { - using (iteration.StartMeasurement()) + swap = false; + for (int i = 0; i < n; i++) { - for (int i = 0; i < BubbleSortIterations; i++) + if (data[i] > data[i + 1]) { - Array.Copy(unsortedData, data, Size); - TestBubbleSortArray(data); + temp = data[i]; + data[i] = data[i + 1]; + data[i + 1] = temp; + swap = true; } } + --n; } - }*/ + while (swap); + } #endregion + #endregion // Algorithm tests + // TestSpanAPIs (For comparison with Array and Slow Span) #region TestSpanAPIs @@ -364,12 +445,7 @@ namespace Span [InlineData(1000)] public static void TestSpanConstructorByte(int length) { - var array = new byte[length]; - Span span; - foreach (var iteration in Benchmark.Iterations) - using (iteration.StartMeasurement()) - for (int i = 0; i < Benchmark.InnerIterationCount; i++) - span = new Span(array); + InvokeTestSpanConstructor(length); } [Benchmark(InnerIterationCount = BaseIterations / 100)] @@ -379,15 +455,33 @@ namespace Span [InlineData(1000)] public static void TestSpanConstructorString(int length) { - var array = new string[length]; - Span span; - foreach (var iteration in Benchmark.Iterations) - using (iteration.StartMeasurement()) - for (int i = 0; i < Benchmark.InnerIterationCount; i++) - span = new Span(array); + InvokeTestSpanConstructor(length); + } + + static void InvokeTestSpanConstructor(int length) + { + var array = new T[length]; + + Invoke((int innerIterationCount) => TestSpanConstructor(array, innerIterationCount, false), + "TestSpanConstructor<{0}>({1})", typeof(T).Name, length); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + static void TestSpanConstructor(T[] array, int iterationCount, bool untrue) + { + var sink = Sink.Instance; + + for (int i = 0; i < iterationCount; i++) + { + var span = new Span(array); + // Under a condition that we know is false but the jit doesn't, + // add a read from 'span' to make sure it's not dead, and an assignment + // to 'array' so the constructor call won't get hoisted. + if (untrue) { sink.Data = span[0]; array = new T[iterationCount]; } + } } #endregion - + #region TestSpanDangerousCreate [Benchmark(InnerIterationCount = BaseIterations)] [InlineData(1)] @@ -396,13 +490,7 @@ namespace Span [InlineData(1000)] public static void TestSpanDangerousCreateByte(int length) { - TestClass testClass = new TestClass(); - testClass.C0 = new byte[length]; - Span span; - foreach (var iteration in Benchmark.Iterations) - using (iteration.StartMeasurement()) - for (int i = 0; i < Benchmark.InnerIterationCount; i++) - span = Span.DangerousCreate(testClass, ref testClass.C0[0], testClass.C0.Length); + InvokeTestSpanDangerousCreate(length); } [Benchmark(InnerIterationCount = BaseIterations)] @@ -412,13 +500,31 @@ namespace Span [InlineData(1000)] public static void TestSpanDangerousCreateString(int length) { - TestClass testClass = new TestClass(); - testClass.C0 = new string[length]; - Span span; - foreach (var iteration in Benchmark.Iterations) - using (iteration.StartMeasurement()) - for (int i = 0; i < Benchmark.InnerIterationCount; i++) - span = Span.DangerousCreate(testClass, ref testClass.C0[0], testClass.C0.Length); + InvokeTestSpanDangerousCreate(length); + } + + static void InvokeTestSpanDangerousCreate(int length) + { + TestClass testClass = new TestClass(); + testClass.C0 = new T[length]; + + Invoke((int innerIterationCount) => TestSpanDangerousCreate(testClass, innerIterationCount, false), + "TestSpanDangerousCreate<{0}>({1})", typeof(T).Name, length); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + static void TestSpanDangerousCreate(TestClass testClass, int iterationCount, bool untrue) + { + var sink = Sink.Instance; + + for (int i = 0; i < iterationCount; i++) + { + var span = Span.DangerousCreate(testClass, ref testClass.C0[0], testClass.C0.Length); + // Under a condition that we know is false but the jit doesn't, + // add a read from 'span' to make sure it's not dead, and an assignment + // to 'testClass' so the DangerousCreate call won't get hoisted. + if (untrue) { sink.Data = span[0]; testClass = new TestClass(); } + } } #endregion @@ -430,14 +536,7 @@ namespace Span [InlineData(1000)] public static void TestSpanDangerousGetPinnableReferenceByte(int length) { - var array = new byte[length]; - var span = new Span(array); - foreach (var iteration in Benchmark.Iterations) - using (iteration.StartMeasurement()) - for (int i = 0; i < Benchmark.InnerIterationCount; i++) - { - ref byte temp = ref span.DangerousGetPinnableReference(); - } + InvokeTestSpanDangerousGetPinnableReference(length); } [Benchmark(InnerIterationCount = BaseIterations)] @@ -447,33 +546,80 @@ namespace Span [InlineData(1000)] public static void TestSpanDangerousGetPinnableReferenceString(int length) { - var array = new string[length]; - var span = new Span(array); - foreach (var iteration in Benchmark.Iterations) - using (iteration.StartMeasurement()) - for (int i = 0; i < Benchmark.InnerIterationCount; i++) - { - ref string temp = ref span.DangerousGetPinnableReference(); - } + InvokeTestSpanDangerousGetPinnableReference(length); + } + + static void InvokeTestSpanDangerousGetPinnableReference(int length) + { + var array = new T[length]; + + Invoke((int innerIterationCount) => TestSpanDangerousGetPinnableReference(array, innerIterationCount), + "TestSpanDangerousGetPinnableReference<{0}>({1})", typeof(T).Name, length); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + static void TestSpanDangerousGetPinnableReference(T[] array, int iterationCount) + { + var sink = Sink.Instance; + var span = new Span(array); + + for (int i = 0; i < iterationCount; i++) + { + ref T temp = ref span.DangerousGetPinnableReference(); + sink.Data = temp; + } } #endregion - #region TestSpanIndex + #region TestSpanIndexHoistable [Benchmark(InnerIterationCount = BaseIterations)] [InlineData(1)] [InlineData(10)] [InlineData(100)] [InlineData(1000)] - public static void TestSpanIndexByte(int length) + public static void TestSpanIndexHoistableByte(int length) { - var array = new byte[length]; - var span = new Span(array); - byte temp; - foreach (var iteration in Benchmark.Iterations) - using (iteration.StartMeasurement()) - for (int i = 0; i < Benchmark.InnerIterationCount; i++) - temp = span[length/2]; + InvokeTestSpanIndexHoistable(length); + } + + [Benchmark(InnerIterationCount = BaseIterations)] + [InlineData(1)] + [InlineData(10)] + [InlineData(100)] + [InlineData(1000)] + public static void TestSpanIndexHoistableString(int length) + { + InvokeTestSpanIndexHoistable(length); + } + + static void InvokeTestSpanIndexHoistable(int length) + { + var array = new T[length]; + + Invoke((int innerIterationCount) => TestSpanIndexHoistable(array, length, innerIterationCount), + "TestSpanIndexHoistable<{0}>({1})", typeof(T).Name, length); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + static void TestSpanIndexHoistable(T[] array, int length, int iterationCount) + { + var sink = Sink.Instance; + var span = new Span(array); + + for (int i = 0; i < iterationCount; i++) + sink.Data = span[length/2]; + } + #endregion + #region TestArrayIndexHoistable + [Benchmark(InnerIterationCount = BaseIterations)] + [InlineData(1)] + [InlineData(10)] + [InlineData(100)] + [InlineData(1000)] + public static void TestArrayIndexHoistableByte(int length) + { + InvokeTestArrayIndexHoistable(length); } [Benchmark(InnerIterationCount = BaseIterations)] @@ -481,48 +627,108 @@ namespace Span [InlineData(10)] [InlineData(100)] [InlineData(1000)] - public static void TestSpanIndexString(int length) + public static void TestArrayIndexHoistableString(int length) { - var array = new string[length]; - var span = new Span(array); - string temp; - foreach (var iteration in Benchmark.Iterations) - using (iteration.StartMeasurement()) - for (int i = 0; i < Benchmark.InnerIterationCount; i++) - temp = span[length / 2]; + InvokeTestArrayIndexHoistable(length); + } + + static void InvokeTestArrayIndexHoistable(int length) + { + var array = new T[length]; + Invoke((int innerIterationCount) => TestArrayIndexHoistable(array, length, innerIterationCount), + "TestArrayIndexHoistable<{0}>({1})", typeof(T).Name, length); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + static void TestArrayIndexHoistable(T[] array, int length, int iterationCount) + { + var sink = Sink.Instance; + + for (int i = 0; i < iterationCount; i++) + sink.Data = array[length / 2]; } #endregion - #region TestArrayIndex + #region TestSpanIndexVariant [Benchmark(InnerIterationCount = BaseIterations)] [InlineData(1)] [InlineData(10)] [InlineData(100)] [InlineData(1000)] - public static void TestArrayIndexByte(int length) + public static void TestSpanIndexVariantByte(int length) { - var array = new byte[length]; - byte temp; - foreach (var iteration in Benchmark.Iterations) - using (iteration.StartMeasurement()) - for (int i = 0; i < Benchmark.InnerIterationCount; i++) - temp = array[length / 2]; + InvokeTestSpanIndexVariant(length); + } + [Benchmark(InnerIterationCount = BaseIterations)] + [InlineData(1)] + [InlineData(10)] + [InlineData(100)] + [InlineData(1000)] + public static void TestSpanIndexVariantString(int length) + { + InvokeTestSpanIndexVariant(length); + } + + static void InvokeTestSpanIndexVariant(int length) + { + var array = new T[length]; + + Invoke((int innerIterationCount) => TestSpanIndexVariant(array, length, innerIterationCount), + "TestSpanIndexVariant<{0}>({1})", typeof(T).Name, length); } + [MethodImpl(MethodImplOptions.NoInlining)] + static void TestSpanIndexVariant(T[] array, int length, int iterationCount) + { + var sink = Sink.Instance; + var span = new Span(array); + int mask = (length < 2 ? 0 : (length < 8 ? 1 : 7)); + + for (int i = 0; i < iterationCount; i++) + sink.Data = span[i & mask]; + } + #endregion + + #region TestArrayIndexVariant [Benchmark(InnerIterationCount = BaseIterations)] [InlineData(1)] [InlineData(10)] [InlineData(100)] [InlineData(1000)] - public static void TestArrayIndexString(int length) + public static void TestArrayIndexVariantByte(int length) { - var array = new string[length]; - string temp; - foreach (var iteration in Benchmark.Iterations) - using (iteration.StartMeasurement()) - for (int i = 0; i < Benchmark.InnerIterationCount; i++) - temp = array[length / 2]; + InvokeTestArrayIndexVariant(length); + } + + [Benchmark(InnerIterationCount = BaseIterations)] + [InlineData(1)] + [InlineData(10)] + [InlineData(100)] + [InlineData(1000)] + public static void TestArrayIndexVariantString(int length) + { + InvokeTestArrayIndexVariant(length); + } + + static void InvokeTestArrayIndexVariant(int length) + { + var array = new T[length]; + + Invoke((int innerIterationCount) => TestArrayIndexVariant(array, length, innerIterationCount), + "TestArrayIndexVariant<{0}>({1})", typeof(T).Name, length); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + static void TestArrayIndexVariant(T[] array, int length, int iterationCount) + { + var sink = Sink.Instance; + int mask = (length < 2 ? 0 : (length < 8 ? 1 : 7)); + + for (int i = 0; i < iterationCount; i++) + { + sink.Data = array[i & mask]; + } } #endregion @@ -534,13 +740,7 @@ namespace Span [InlineData(1000)] public static void TestSpanSliceByte(int length) { - var array = new byte[length]; - var span = new Span(array); - Span temp; - foreach (var iteration in Benchmark.Iterations) - using (iteration.StartMeasurement()) - for (int i = 0; i < Benchmark.InnerIterationCount; i++) - temp = span.Slice(length / 2); + InvokeTestSpanSlice(length); } [Benchmark(InnerIterationCount = BaseIterations)] @@ -550,16 +750,34 @@ namespace Span [InlineData(1000)] public static void TestSpanSliceString(int length) { - var array = new string[length]; - var span = new Span(array); - Span temp; - foreach (var iteration in Benchmark.Iterations) - using (iteration.StartMeasurement()) - for (int i = 0; i < Benchmark.InnerIterationCount; i++) - temp = span.Slice(length / 2); + InvokeTestSpanSlice(length); + } + + static void InvokeTestSpanSlice(int length) + { + var array = new T[length]; + + Invoke((int innerIterationCount) => TestSpanSlice(array, length, innerIterationCount, false), + "TestSpanSlice<{0}>({1})", typeof(T).Name, length); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + static void TestSpanSlice(T[] array, int length, int iterationCount, bool untrue) + { + var span = new Span(array); + var sink = Sink.Instance; + + for (int i = 0; i < iterationCount; i++) + { + var slice = span.Slice(length / 2); + // Under a condition that we know is false but the jit doesn't, + // add a read from 'span' to make sure it's not dead, and an assignment + // to 'array' so the slice call won't get hoisted. + if (untrue) { sink.Data = slice[0]; array = new T[iterationCount]; } + } } #endregion - + #region TestSpanToArray [Benchmark(InnerIterationCount = BaseIterations / 100)] [InlineData(1)] @@ -568,13 +786,7 @@ namespace Span [InlineData(1000)] public static void TestSpanToArrayByte(int length) { - var array = new byte[length]; - var span = new Span(array); - byte[] temp; - foreach (var iteration in Benchmark.Iterations) - using (iteration.StartMeasurement()) - for (int i = 0; i < Benchmark.InnerIterationCount; i++) - temp = span.ToArray(); + InvokeTestSpanToArray(length); } [Benchmark(InnerIterationCount = BaseIterations / 100)] @@ -584,16 +796,28 @@ namespace Span [InlineData(1000)] public static void TestSpanToArrayString(int length) { - var array = new string[length]; - var span = new Span(array); - string[] temp; - foreach (var iteration in Benchmark.Iterations) - using (iteration.StartMeasurement()) - for (int i = 0; i < Benchmark.InnerIterationCount; i++) - temp = span.ToArray(); + InvokeTestSpanToArray(length); + } + + static void InvokeTestSpanToArray(int length) + { + var array = new T[length]; + + Invoke((int innerIterationCount) => TestSpanToArray(array, length, innerIterationCount), + "TestSpanToArray<{0}>({1})", typeof(T).Name, length); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + static void TestSpanToArray(T[] array, int length, int iterationCount) + { + var span = new Span(array); + var sink = Sink.Instance; + + for (int i = 0; i < iterationCount; i++) + sink.Data = span.ToArray(); } #endregion - + #region TestSpanCopyTo [Benchmark(InnerIterationCount = BaseIterations / 10)] [InlineData(1)] @@ -602,14 +826,7 @@ namespace Span [InlineData(1000)] public static void TestSpanCopyToByte(int length) { - var array = new byte[length]; - var span = new Span(array); - var destArray = new byte[array.Length]; - var destination = new Span(destArray); - foreach (var iteration in Benchmark.Iterations) - using (iteration.StartMeasurement()) - for (int i = 0; i < Benchmark.InnerIterationCount; i++) - span.CopyTo(destination); + InvokeTestSpanCopyTo(length); } [Benchmark(InnerIterationCount = BaseIterations / 100)] @@ -619,14 +836,26 @@ namespace Span [InlineData(1000)] public static void TestSpanCopyToString(int length) { - var array = new string[length]; - var span = new Span(array); - var destArray = new string[array.Length]; - var destination = new Span(destArray); - foreach (var iteration in Benchmark.Iterations) - using (iteration.StartMeasurement()) - for (int i = 0; i < Benchmark.InnerIterationCount; i++) - span.CopyTo(destination); + InvokeTestSpanCopyTo(length); + } + + static void InvokeTestSpanCopyTo(int length) + { + var array = new T[length]; + var destArray = new T[array.Length]; + + Invoke((int innerIterationCount) => TestSpanCopyTo(array, destArray, innerIterationCount), + "TestSpanCopyTo<{0}>({1})", typeof(T).Name, length); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + static void TestSpanCopyTo(T[] array, T[] destArray, int iterationCount) + { + var span = new Span(array); + var destination = new Span(destArray); + + for (int i = 0; i < iterationCount; i++) + span.CopyTo(destination); } #endregion @@ -638,12 +867,7 @@ namespace Span [InlineData(1000)] public static void TestArrayCopyToByte(int length) { - var array = new byte[length]; - var destination = new byte[array.Length]; - foreach (var iteration in Benchmark.Iterations) - using (iteration.StartMeasurement()) - for (int i = 0; i < Benchmark.InnerIterationCount; i++) - array.CopyTo(destination, 0); + InvokeTestArrayCopyTo(length); } [Benchmark(InnerIterationCount = BaseIterations / 100)] @@ -653,12 +877,23 @@ namespace Span [InlineData(1000)] public static void TestArrayCopyToString(int length) { - var array = new string[length]; - var destination = new string[array.Length]; - foreach (var iteration in Benchmark.Iterations) - using (iteration.StartMeasurement()) - for (int i = 0; i < Benchmark.InnerIterationCount; i++) - array.CopyTo(destination, 0); + InvokeTestArrayCopyTo(length); + } + + static void InvokeTestArrayCopyTo(int length) + { + var array = new T[length]; + var destination = new T[array.Length]; + + Invoke((int innerIterationCount) => TestArrayCopyTo(array, destination, innerIterationCount), + "TestArrayCopyTo<{0}>({1})", typeof(T).Name, length); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + static void TestArrayCopyTo(T[] array, T[] destination, int iterationCount) + { + for (int i = 0; i < iterationCount; i++) + array.CopyTo(destination, 0); } #endregion @@ -670,12 +905,7 @@ namespace Span [InlineData(1000)] public static void TestSpanFillByte(int length) { - var array = new byte[length]; - var span = new Span(array); - foreach (var iteration in Benchmark.Iterations) - using (iteration.StartMeasurement()) - for (int i = 0; i < Benchmark.InnerIterationCount; i++) - span.Fill(default(byte)); + InvokeTestSpanFill(length); } [Benchmark(InnerIterationCount = BaseIterations / 100)] @@ -685,12 +915,23 @@ namespace Span [InlineData(1000)] public static void TestSpanFillString(int length) { - var array = new string[length]; - var span = new Span(array); - foreach (var iteration in Benchmark.Iterations) - using (iteration.StartMeasurement()) - for (int i = 0; i < Benchmark.InnerIterationCount; i++) - span.Fill(default(string)); + InvokeTestSpanFill(length); + } + + static void InvokeTestSpanFill(int length) + { + var array = new T[length]; + + Invoke((int innerIterationCount) => TestSpanFill(array, innerIterationCount), + "TestSpanFill<{0}>({1})", typeof(T).Name, length); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + static void TestSpanFill(T[] array, int iterationCount) + { + var span = new Span(array); + for (int i = 0; i < iterationCount; i++) + span.Fill(default(T)); } #endregion @@ -702,12 +943,7 @@ namespace Span [InlineData(1000)] public static void TestSpanClearByte(int length) { - var array = new byte[length]; - var span = new Span(array); - foreach (var iteration in Benchmark.Iterations) - using (iteration.StartMeasurement()) - for (int i = 0; i < Benchmark.InnerIterationCount; i++) - span.Clear(); + InvokeTestSpanClear(length); } [Benchmark(InnerIterationCount = BaseIterations / 10)] @@ -717,12 +953,23 @@ namespace Span [InlineData(1000)] public static void TestSpanClearString(int length) { - var array = new string[length]; - var span = new Span(array); - foreach (var iteration in Benchmark.Iterations) - using (iteration.StartMeasurement()) - for (int i = 0; i < Benchmark.InnerIterationCount; i++) - span.Clear(); + InvokeTestSpanClear(length); + } + + static void InvokeTestSpanClear(int length) + { + var array = new T[length]; + + Invoke((int innerIterationCount) => TestSpanClear(array, innerIterationCount), + "TestSpanClear<{0}>({1})", typeof(T).Name, length); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + static void TestSpanClear(T[] array, int iterationCount) + { + var span = new Span(array); + for (int i = 0; i < iterationCount; i++) + span.Clear(); } #endregion @@ -734,11 +981,7 @@ namespace Span [InlineData(1000)] public static void TestArrayClearByte(int length) { - var array = new byte[length]; - foreach (var iteration in Benchmark.Iterations) - using (iteration.StartMeasurement()) - for (int i = 0; i < Benchmark.InnerIterationCount; i++) - Array.Clear(array, 0, length); + InvokeTestArrayClear(length); } [Benchmark(InnerIterationCount = BaseIterations / 10)] @@ -748,11 +991,22 @@ namespace Span [InlineData(1000)] public static void TestArrayClearString(int length) { - var array = new string[length]; - foreach (var iteration in Benchmark.Iterations) - using (iteration.StartMeasurement()) - for (int i = 0; i < Benchmark.InnerIterationCount; i++) - Array.Clear(array, 0, length); + InvokeTestArrayClear(length); + } + + static void InvokeTestArrayClear(int length) + { + var array = new T[length]; + + Invoke((int innerIterationCount) => TestArrayClear(array, length, innerIterationCount), + "TestArrayClear<{0}>({1})", typeof(T).Name, length); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + static void TestArrayClear(T[] array, int length, int iterationCount) + { + for (int i = 0; i < iterationCount; i++) + Array.Clear(array, 0, length); } #endregion @@ -764,14 +1018,7 @@ namespace Span [InlineData(1000)] public static void TestSpanAsBytesByte(int length) { - var array = new byte[length]; - var span = new Span(array); - foreach (var iteration in Benchmark.Iterations) - using (iteration.StartMeasurement()) - for (int i = 0; i < Benchmark.InnerIterationCount; i++) - { - Span temp = span.AsBytes(); - } + InvokeTestSpanAsBytes(length); } [Benchmark(InnerIterationCount = BaseIterations)] @@ -781,14 +1028,33 @@ namespace Span [InlineData(1000)] public static void TestSpanAsBytesInt(int length) { - var array = new int[length]; - var span = new Span(array); - foreach (var iteration in Benchmark.Iterations) - using (iteration.StartMeasurement()) - for (int i = 0; i < Benchmark.InnerIterationCount; i++) - { - Span temp = span.AsBytes(); - } + InvokeTestSpanAsBytes(length); + } + + static void InvokeTestSpanAsBytes(int length) + where T : struct + { + var array = new T[length]; + + Invoke((int innerIterationCount) => TestSpanAsBytes(array, innerIterationCount, false), + "TestSpanAsBytes<{0}>({1})", typeof(T).Name, length); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + static void TestSpanAsBytes(T[] array, int iterationCount, bool untrue) + where T : struct + { + var sink = Sink.Instance; + var span = new Span(array); + + for (int i = 0; i < iterationCount; i++) + { + var byteSpan = span.AsBytes(); + // Under a condition that we know is false but the jit doesn't, + // add a read from 'byteSpan' to make sure it's not dead, and an assignment + // to 'span' so the AsBytes call won't get hoisted. + if (untrue) { sink.Data = byteSpan[0]; span = new Span(); } + } } #endregion @@ -800,14 +1066,7 @@ namespace Span [InlineData(1000)] public static void TestSpanNonPortableCastFromByteToInt(int length) { - var array = new byte[length]; - var span = new Span(array); - foreach (var iteration in Benchmark.Iterations) - using (iteration.StartMeasurement()) - for (int i = 0; i < Benchmark.InnerIterationCount; i++) - { - Span temp = span.NonPortableCast(); - } + InvokeTestSpanNonPortableCast(length); } [Benchmark(InnerIterationCount = BaseIterations)] @@ -817,14 +1076,35 @@ namespace Span [InlineData(1000)] public static void TestSpanNonPortableCastFromIntToByte(int length) { - var array = new int[length]; - var span = new Span(array); - foreach (var iteration in Benchmark.Iterations) - using (iteration.StartMeasurement()) - for (int i = 0; i < Benchmark.InnerIterationCount; i++) - { - Span temp = span.NonPortableCast(); - } + InvokeTestSpanNonPortableCast(length); + } + + static void InvokeTestSpanNonPortableCast(int length) + where From : struct + where To : struct + { + var array = new From[length]; + + Invoke((int innerIterationCount) => TestSpanNonPortableCast(array, innerIterationCount, false), + "TestSpanNonPortableCast<{0}, {1}>({2})", typeof(From).Name, typeof(To).Name, length); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + static void TestSpanNonPortableCast(From[] array, int iterationCount, bool untrue) + where From : struct + where To : struct + { + var sink = Sink.Instance; + var span = new Span(array); + + for (int i = 0; i < iterationCount; i++) + { + var toSpan = span.NonPortableCast(); + // Under a condition that we know is false but the jit doesn't, + // add a read from 'toSpan' to make sure it's not dead, and an assignment + // to 'span' so the AsBytes call won't get hoisted. + if (untrue) { sink.Data = toSpan[0]; span = new Span(); } + } } #endregion @@ -834,7 +1114,7 @@ namespace Span [InlineData(10)] [InlineData(100)] [InlineData(1000)] - public static void TestSpanSliceStringChar(int length) + public static void TestSpanSliceStringCharWrapper(int length) { StringBuilder sb = new StringBuilder(); Random rand = new Random(42); @@ -846,148 +1126,67 @@ namespace Span } string s = sb.ToString(); - foreach (var iteration in Benchmark.Iterations) - using (iteration.StartMeasurement()) - for (int i = 0; i < Benchmark.InnerIterationCount; i++) - { - ReadOnlySpan temp = s.Slice(); - } + Invoke((int innerIterationCount) => TestSpanSliceStringChar(s, innerIterationCount, false), + "TestSpanSliceStringChar({0})", length); } - #endregion - #endregion - - // EXE-based testing - #region EXE-base testing - /*static void FillAllSpanBase() - { - byte[] a = new byte[Size]; - Span s = new Span(a); - for (int i = 0; i < FillAllIterations; i++) - { - TestFillAllSpan(s); - } - } - - static void FillAllArrayBase() - { - byte[] a = new byte[Size]; - for (int i = 0; i < FillAllIterations; i++) - { - TestFillAllArray(a); - } - } - - static void FillAllReverseSpanBase() - { - byte[] a = new byte[Size]; - Span s = new Span(a); - for (int i = 0; i < FillAllIterations; i++) - { - TestFillAllReverseSpan(s); - } - } - - static void FillAllReverseArrayBase() - { - byte[] a = new byte[Size]; - for (int i = 0; i < FillAllIterations; i++) - { - TestFillAllReverseArray(a); - } - } - - static void QuickSortSpanBase() - { - int[] data = new int[Size]; - int[] unsortedData = GetUnsortedData(); - Span span = new Span(data); - - for (int i = 0; i < QuickSortIterations; i++) - { - Array.Copy(unsortedData, data, Size); - TestQuickSortSpan(span); - } - } - - static void BubbleSortSpanBase() + [MethodImpl(MethodImplOptions.NoInlining)] + static void TestSpanSliceStringChar(string s, int iterationCount, bool untrue) { - int[] data = new int[Size]; - int[] unsortedData = GetUnsortedData(); - Span span = new Span(data); + var sink = Sink.Instance; - for (int i = 0; i < BubbleSortIterations; i++) + for (int i = 0; i < iterationCount; i++) { - Array.Copy(unsortedData, data, Size); - TestBubbleSortSpan(span); + var charSpan = s.Slice(); + // Under a condition that we know is false but the jit doesn't, + // add a read from 'charSpan' to make sure it's not dead, and an assignment + // to 's' so the AsBytes call won't get hoisted. + if (untrue) { sink.Data = charSpan[0]; s = "block hoisting the call to Slice()"; } } } + #endregion - static void QuickSortArrayBase() - { - int[] data = new int[Size]; - int[] unsortedData = GetUnsortedData(); + #endregion // TestSpanAPIs - for (int i = 0; i < QuickSortIterations; i++) - { - Array.Copy(unsortedData, data, Size); - TestQuickSortArray(data); - } - } - static void BubbleSortArrayBase() + public static int Main(string[] args) { - int[] data = new int[Size]; - int[] unsortedData = GetUnsortedData(); + // When we call into Invoke, it'll need to know this isn't xunit-perf running + IsXunitInvocation = false; - for (int i = 0; i < BubbleSortIterations; i++) + // Now simulate xunit-perf's benchmark discovery so we know what tests to invoke + TypeInfo t = typeof(SpanBench).GetTypeInfo(); + foreach(MethodInfo m in t.DeclaredMethods) { - Array.Copy(unsortedData, data, Size); - TestBubbleSortArray(data); - } - }*/ - #endregion - - static double Bench(Action f) - { - Stopwatch sw = Stopwatch.StartNew(); - f(); - sw.Stop(); - return sw.Elapsed.TotalMilliseconds; - } - - static IEnumerable MakeArgs(params string[] args) - { - return args.Select(arg => new object[] { arg }); - } - - static IEnumerable TestFuncs = MakeArgs( - /*"FillAllSpanBase", "FillAllArrayBase", - "FillAllReverseSpanBase", "FillAllReverseArrayBase", - "BubbleSortSpanBase", "BubbleSortArrayBase", - "QuickSortSpanBase", "QuickSortArrayBase"*/ - ); + BenchmarkAttribute benchAttr = m.GetCustomAttribute(); + if (benchAttr != null) + { + // All benchmark methods in this test set the InnerIterationCount on their BenchmarkAttribute. + // Take max of specified count and 1 since some tests use expressions for their count that + // evaluate to 0 under DEBUG. + CommandLineInnerIterationCount = Math.Max((int)benchAttr.InnerIterationCount, 1); - static Action LookupFunc(object o) - { - TypeInfo t = typeof(SpanBench).GetTypeInfo(); - MethodInfo m = t.GetDeclaredMethod((string) o); - return m.CreateDelegate(typeof(Action)) as Action; - } + // Request a warm-up iteration before measuring this benchmark method. + DoWarmUp = true; - public static int Main(string[] args) - { - bool result = true; + // Get the benchmark to measure as a delegate taking the number of inner-loop iterations to run + var invokeMethod = m.CreateDelegate(typeof(Action)) as Action; - foreach(object[] o in TestFuncs) - { - string funcName = (string) o[0]; - Action func = LookupFunc(funcName); - double timeInMs = Bench(func); - Console.WriteLine("{0}: {1}ms", funcName, timeInMs); + // All the benchmarks methods in this test use [InlineData] to specify how many times and with + // what arguments they should be run. + foreach (InlineDataAttribute dataAttr in m.GetCustomAttributes()) + { + foreach (object[] data in dataAttr.GetData(m)) + { + // All the benchmark methods in this test take a single int parameter + invokeMethod((int)data[0]); + } + } + } } - return (result ? 100 : -1); + // The only failure modes are crash/exception. + return 100; } } } \ No newline at end of file