Add benchmarks mandelbrot and fannkuch (#12933)

* Add benchmarks mandelbrot and fannkuch

Adding mandelbrot and fankuch-redux from Benchmarks Game to
complete the representation of benchmarks from this list
http://benchmarksgame.alioth.debian.org/u64q/csharp.html

* Update source acknolegments and edit return codes

* Add checksums to verify benchmark runs

This commit adds checksums, which are pre-computed for each input
case, to verify that the fannkuch and mandelbrot benchmarks
actualy compute the desired function

* Revert version bump

* Remove XunitPerfHarness from Main

* Remove an unnecessary line

* Add iterations to fannkuch to inscrease runtime

diff --git a/tests/src/JIT/Performance/CodeQuality/BenchmarksGame/fannkuch-redux/fannkuch-redux.csharp.cs b/tests/src/JIT/Performance/CodeQuality/BenchmarksGame/fannkuch-redux/fannkuch-redux.csharp.cs
new file mode 100644 (file)
index 0000000..ea7a189
--- /dev/null
+++ b/tests/src/JIT/Performance/CodeQuality/BenchmarksGame/fannkuch-redux/fannkuch-redux.csharp.cs
@@ -0,0 +1,120 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+/* The Computer Language Benchmarks Game
+   http://benchmarksgame.alioth.debian.org/
+
+   contributed by Isaac Gouy, transliterated from Mike Pall's Lua program
+
+   posted to Benchmarks Game as fannkuck-redux C# .NET Core #2
+   (http://benchmarksgame.alioth.debian.org/u64q/program.php?test=fannkuchredux&lang=csharpcore&id=2)
+   modified for use with xunit-performance
+*/
+
+using System.Reflection;
+using System.Runtime.CompilerServices;
+using Microsoft.Xunit.Performance;
+using Microsoft.Xunit.Performance.Api;
+using Xunit;
+
+namespace BenchmarksGame
+{
+    public class FannkuchRedux
+    {
+
+        [MethodImpl(MethodImplOptions.NoInlining)]
+        private static int[] Bench(int n, bool verbose = false)
+        {
+            int[] p = new int[n], q = new int[n], s = new int[n];
+            int sign = 1, maxflips = 0, sum = 0, m = n - 1;
+            for (int i = 0; i < n; i++) { p[i] = i; q[i] = i; s[i] = i; }
+            do
+            {
+                // Copy and flip.
+                var q0 = p[0];                                     // Cache 0th element.
+                if (q0 != 0)
+                {
+                    for (int i = 1; i < n; i++) q[i] = p[i];             // Work on a copy.
+                    var flips = 1;
+                    do
+                    {
+                        var qq = q[q0];
+                        if (qq == 0)
+                        {                                // ... until 0th element is 0.
+                            sum += sign * flips;
+                            if (flips > maxflips) maxflips = flips;   // New maximum?
+                            break;
+                        }
+                        q[q0] = q0;
+                        if (q0 >= 3)
+                        {
+                            int i = 1, j = q0 - 1, t;
+                            do { t = q[i]; q[i] = q[j]; q[j] = t; i++; j--; } while (i < j);
+                        }
+                        q0 = qq; flips++;
+                    } while (true);
+                }
+                // Permute.
+                if (sign == 1)
+                {
+                    var t = p[1]; p[1] = p[0]; p[0] = t; sign = -1; // Rotate 0<-1.
+                }
+                else
+                {
+                    var t = p[1]; p[1] = p[2]; p[2] = t; sign = 1;  // Rotate 0<-1 and 0<-1<-2.
+                    for (int i = 2; i < n; i++)
+                    {
+                        var sx = s[i];
+                        if (sx != 0) { s[i] = sx - 1; break; }
+                        if (i == m) return new int[] { sum, maxflips };  // Out of permutations.
+                        s[i] = i;
+                        // Rotate 0<-...<-i+1.
+                        t = p[0]; for (int j = 0; j <= i; j++) { p[j] = p[j + 1]; }
+                        p[i + 1] = t;
+                    }
+                }
+            } while (true);
+        }
+
+        // Commented out data left in source to provide checksums for each case
+        // Checksums calculated from the origonal source referenced at top of this source
+
+        [Benchmark(InnerIterationCount = 7)]
+        //[InlineData(7, 228)]
+        //[InlineData(8, 1616)]
+        //[InlineData(9, 8629)]
+        [InlineData(10, 73196)]
+        //[InlineData(11, 556355)]
+        //[InlineData(12, 3968050)]
+        public static void Test(int n, int checksum)
+        {
+            foreach (var iteration in Benchmark.Iterations)
+            {
+                int[] pfannkuchen = null;
+                using (iteration.StartMeasurement())
+                {
+                    for (int i = 0; i < Benchmark.InnerIterationCount; ++i)
+                    {
+                        pfannkuchen = Bench(n);
+                    }
+                }
+                Assert.Equal(checksum, pfannkuchen[0]);
+            }
+        }
+
+        public static bool VerifyBench(int n, int checksum)
+        {
+            int[] pfannkuchen = Bench(n);
+            return pfannkuchen[0] == checksum;
+        }
+
+        public static int Main(string[] args)
+        {
+            const int n = 7;
+            const int checksum = 228;
+
+            bool verified = VerifyBench(n, checksum);
+            return (verified ? 100 : -1);
+        }
+    }
+}

diff --git a/tests/src/JIT/Performance/CodeQuality/BenchmarksGame/fannkuch-redux/fannkuch-redux.csproj b/tests/src/JIT/Performance/CodeQuality/BenchmarksGame/fannkuch-redux/fannkuch-redux.csproj
new file mode 100644 (file)
index 0000000..aed483b
--- /dev/null
+++ b/tests/src/JIT/Performance/CodeQuality/BenchmarksGame/fannkuch-redux/fannkuch-redux.csproj
@@ -0,0 +1,38 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="12.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory), dir.props))\dir.props" />
+  <PropertyGroup>
+    <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
+    <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
+    <SchemaVersion>2.0</SchemaVersion>
+    <ProjectGuid>{95DFC527-4DC1-495E-97D7-E94EE1F7140D}</ProjectGuid>
+    <OutputType>Exe</OutputType>
+    <ProjectTypeGuids>{786C830F-07A1-408B-BD7F-6EE04809D6DB};{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}</ProjectTypeGuids>
+    <SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\..\</SolutionDir>
+    <NuGetTargetMoniker>.NETStandard,Version=v1.4</NuGetTargetMoniker>
+    <NuGetTargetMonikerShort>netstandard1.4</NuGetTargetMonikerShort>
+  </PropertyGroup>
+  <!-- Default configurations to help VS understand the configurations -->
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' "></PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' " />
+  <PropertyGroup>
+    <DebugType>pdbonly</DebugType>
+    <Optimize>true</Optimize>
+  </PropertyGroup>
+  <ItemGroup>
+    <CodeAnalysisDependentAssemblyPaths Condition=" '$(VS100COMNTOOLS)' != '' " Include="$(VS100COMNTOOLS)..\IDE\PrivateAssemblies">
+      <Visible>False</Visible>
+    </CodeAnalysisDependentAssemblyPaths>
+  </ItemGroup>
+  <ItemGroup>
+    <Service Include="{82A7F48D-3B50-4B1E-B82E-3ADA8210C358}" />
+  </ItemGroup>
+  <ItemGroup>
+    <Compile Include="fannkuch-redux.csharp.cs" />
+  </ItemGroup>
+  <Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory), dir.targets))\dir.targets" />
+  <PropertyGroup Condition=" '$(MsBuildProjectDirOverride)' != '' "></PropertyGroup>
+  <PropertyGroup>
+    <ProjectAssetsFile>$(JitPackagesConfigFileDirectory)benchmark\obj\project.assets.json</ProjectAssetsFile>
+  </PropertyGroup>
+</Project>
\ No newline at end of file

diff --git a/tests/src/JIT/Performance/CodeQuality/BenchmarksGame/mandelbrot/mandelbrot.csharp.cs b/tests/src/JIT/Performance/CodeQuality/BenchmarksGame/mandelbrot/mandelbrot.csharp.cs
new file mode 100644 (file)
index 0000000..aa38976
--- /dev/null
+++ b/tests/src/JIT/Performance/CodeQuality/BenchmarksGame/mandelbrot/mandelbrot.csharp.cs
@@ -0,0 +1,146 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+/* The Computer Language Benchmarks Game
+   http://benchmarksgame.alioth.debian.org/
+
+   started with Java #2 program (Krause/Whipkey/Bennet/AhnTran/Enotus/Stalcup)
+   adapted for C# by Jan de Vaan
+   simplified and optimised to use TPL by Anthony Lloyd
+
+   posted to Benchmarks Game as mandelbrot C# .NET Core #4
+   (http://benchmarksgame.alioth.debian.org/u64q/program.php?test=mandelbrot&lang=csharpcore&id=4)
+   modified to remove concurrency and operate with xunit-performance
+*/
+
+using System;
+using System.Reflection;
+using System.Runtime.CompilerServices;
+using System.Security.Cryptography;
+using Microsoft.Xunit.Performance;
+using Microsoft.Xunit.Performance.Api;
+using Xunit;
+
+namespace BenchmarksGame
+{
+    public class Mandelbrot
+    {
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        static byte getByte(double[] Crb, double Ciby, int x, int y)
+        {
+            int res = 0;
+            for (int i = 0; i < 8; i += 2)
+            {
+                double Crbx = Crb[x + i], Crbx1 = Crb[x + i + 1];
+                double Zr1 = Crbx, Zr2 = Crbx1;
+                double Zi1 = Ciby, Zi2 = Ciby;
+
+                int b = 0;
+                int j = 49;
+                do
+                {
+                    double nZr1 = Zr1 * Zr1 - Zi1 * Zi1 + Crbx;
+                    Zi1 = Zr1 * Zi1 + Zr1 * Zi1 + Ciby;
+                    Zr1 = nZr1;
+
+                    double nZr2 = Zr2 * Zr2 - Zi2 * Zi2 + Crbx1;
+                    Zi2 = Zr2 * Zi2 + Zr2 * Zi2 + Ciby;
+                    Zr2 = nZr2;
+
+                    if (Zr1 * Zr1 + Zi1 * Zi1 > 4)
+                    {
+                        b |= 2;
+                        if (b == 3)
+                            break;
+                    }
+                    if (Zr2 * Zr2 + Zi2 * Zi2 > 4)
+                    {
+                        b |= 1;
+                        if (b == 3)
+                            break;
+                    }
+                } while (--j > 0);
+                res = (res << 2) + b;
+            }
+            return (byte)(res ^ -1);
+        }
+
+        [MethodImpl(MethodImplOptions.NoInlining)]
+        private static byte[] Bench(int n, bool verbose = false)
+        {
+            double invN = 2.0 / n;
+            var Crb = new double[n + 7];
+            for (int i = 0; i < n; i++)
+            {
+                Crb[i] = i * invN - 1.5;
+            }
+            int lineLen = (n - 1) / 8 + 1;
+            var data = new byte[n * lineLen];
+            for (int i = 0; i < n; i++)
+            {
+                var Cibi = i * invN - 1.0;
+                var offset = i * lineLen;
+                for (int x = 0; x < lineLen; x++)
+                    data[offset + x] = getByte(Crb, Cibi, x * 8, i);
+            };
+
+            if (verbose)
+            {
+                Console.Out.WriteLine("P4\n{0} {0}", n);
+                Console.OpenStandardOutput().Write(data, 0, data.Length);
+            }
+
+            return data;
+        }
+
+        // Commented out data left in source to provide checksums for each case
+        // Checksums calculated from the origonal source referenced at top of this source
+
+        [Benchmark]
+        //[InlineData(500,  "54-01-EE-C8-46-9B-AB-FA-54-9F-45-CE-98-89-66-A9")]
+        //[InlineData(1000, "B2-13-51-CE-B0-29-2C-4E-75-5E-91-19-18-E4-0C-D9")]
+        //[InlineData(2000, "5A-21-55-9B-7B-18-2F-34-9B-33-C5-F9-B5-2C-40-56")]
+        //[InlineData(3000, "E5-82-85-0A-3C-89-69-B1-A8-21-63-52-75-B3-C8-33")]
+        [InlineData(4000, "C7-E6-66-43-66-73-F8-A8-D3-B4-D7-97-2F-FC-A1-D3")]
+        //[InlineData(5000, "6D-36-F1-F6-37-8F-34-EB-52-F9-2D-11-89-12-B2-2F")]
+        //[InlineData(6000, "8B-05-78-EB-2E-0E-98-F2-C7-39-76-ED-0F-A9-D2-B8")]
+        //[InlineData(7000, "01-F8-F2-2A-AB-70-C7-BA-E3-64-19-E7-D2-84-DF-57")]
+        //[InlineData(8000, "C8-ED-D7-FB-65-66-3A-D9-C6-04-9E-96-E8-CA-4F-2C")]
+        public static void Test(int n, string checksum)
+        {
+            foreach (var iteration in Benchmark.Iterations)
+            {
+                byte[] bitmap = null;
+                using (iteration.StartMeasurement())
+                {
+                    bitmap = Bench(n);
+                }
+                using (var md5 = MD5.Create())
+                {
+                    byte[] hash = md5.ComputeHash(bitmap);
+                    Assert.Equal(checksum, BitConverter.ToString(hash));
+                }
+            }
+        }
+
+        public static bool VerifyBench(int n, string checksum)
+        {
+            byte[] bitmap = Bench(n);
+            using (var md5 = MD5.Create())
+            {
+                byte[] hash = md5.ComputeHash(bitmap);
+                return checksum == BitConverter.ToString(hash);
+            }
+        }
+
+        public static int Main(string[] args)
+        {
+            const int n = 500;
+            const string checksum = "54-01-EE-C8-46-9B-AB-FA-54-9F-45-CE-98-89-66-A9";
+
+            bool verified = VerifyBench(n, checksum);
+            return (verified ? 100 : -1);
+        }
+    }
+}

diff --git a/tests/src/JIT/Performance/CodeQuality/BenchmarksGame/mandelbrot/mandelbrot.csproj b/tests/src/JIT/Performance/CodeQuality/BenchmarksGame/mandelbrot/mandelbrot.csproj
new file mode 100644 (file)
index 0000000..0ef3cdf
--- /dev/null
+++ b/tests/src/JIT/Performance/CodeQuality/BenchmarksGame/mandelbrot/mandelbrot.csproj
@@ -0,0 +1,38 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="12.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory), dir.props))\dir.props" />
+  <PropertyGroup>
+    <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
+    <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
+    <SchemaVersion>2.0</SchemaVersion>
+    <ProjectGuid>{95DFC527-4DC1-495E-97D7-E94EE1F7140D}</ProjectGuid>
+    <OutputType>Exe</OutputType>
+    <ProjectTypeGuids>{786C830F-07A1-408B-BD7F-6EE04809D6DB};{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}</ProjectTypeGuids>
+    <SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\..\</SolutionDir>
+    <NuGetTargetMoniker>.NETStandard,Version=v1.4</NuGetTargetMoniker>
+    <NuGetTargetMonikerShort>netstandard1.4</NuGetTargetMonikerShort>
+  </PropertyGroup>
+  <!-- Default configurations to help VS understand the configurations -->
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' "></PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' " />
+  <PropertyGroup>
+    <DebugType>pdbonly</DebugType>
+    <Optimize>true</Optimize>
+  </PropertyGroup>
+  <ItemGroup>
+    <CodeAnalysisDependentAssemblyPaths Condition=" '$(VS100COMNTOOLS)' != '' " Include="$(VS100COMNTOOLS)..\IDE\PrivateAssemblies">
+      <Visible>False</Visible>
+    </CodeAnalysisDependentAssemblyPaths>
+  </ItemGroup>
+  <ItemGroup>
+    <Service Include="{82A7F48D-3B50-4B1E-B82E-3ADA8210C358}" />
+  </ItemGroup>
+  <ItemGroup>
+    <Compile Include="mandelbrot.csharp.cs" />
+  </ItemGroup>
+  <Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory), dir.targets))\dir.targets" />
+  <PropertyGroup Condition=" '$(MsBuildProjectDirOverride)' != '' "></PropertyGroup>
+  <PropertyGroup>
+    <ProjectAssetsFile>$(JitPackagesConfigFileDirectory)benchmark\obj\project.assets.json</ProjectAssetsFile>
+  </PropertyGroup>
+</Project>
\ No newline at end of file