From: Swaroop Sridhar Date: Fri, 14 Apr 2017 03:12:00 +0000 (-0700) Subject: Add LinkBench to CoreCLR BenchView X-Git-Tag: accepted/tizen/base/20180629.140029~1347 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=15c6b56ff8edfc5a25b98dfbcac7b10710b8c937;p=platform%2Fupstream%2Fcoreclr.git Add LinkBench to CoreCLR BenchView This change adds Linker tests as scenario tests tracked in BenchView for CoreCLR benchmarks. The Benchmarks added are: HelloWorld, WebAPI, CoreFx, MusicStore (MSIL and ReadyToRun), Roslyn. The metrics are: MSIL size before and after ILLINK Total publish-dir size before and after ILLINK % Reduction in MSIL and Total dir-size after linking. --- diff --git a/Linked b/Linked new file mode 100644 index 0000000..675a79b Binary files /dev/null and b/Linked differ diff --git a/tests/src/performance/linkbench/assets/MusicStore/MusicStoreReflection.xml b/tests/src/performance/linkbench/assets/MusicStore/MusicStoreReflection.xml new file mode 100644 index 0000000..3673c65 --- /dev/null +++ b/tests/src/performance/linkbench/assets/MusicStore/MusicStoreReflection.xml @@ -0,0 +1,27 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/src/performance/linkbench/assets/Roslyn/RoslynRoots.txt b/tests/src/performance/linkbench/assets/Roslyn/RoslynRoots.txt new file mode 100644 index 0000000..10f149e --- /dev/null +++ b/tests/src/performance/linkbench/assets/Roslyn/RoslynRoots.txt @@ -0,0 +1,3 @@ +Microsoft.CodeAnalysis +Microsoft.CodeAnalysis.CSharp +System.Private.CoreLib diff --git a/tests/src/performance/linkbench/assets/Roslyn/RoslynRoots.xml b/tests/src/performance/linkbench/assets/Roslyn/RoslynRoots.xml new file mode 100644 index 0000000..afc1ce6 --- /dev/null +++ b/tests/src/performance/linkbench/assets/Roslyn/RoslynRoots.xml @@ -0,0 +1,5 @@ + + + + + diff --git a/tests/src/performance/linkbench/assets/Roslyn/illinkcsproj b/tests/src/performance/linkbench/assets/Roslyn/illinkcsproj new file mode 100644 index 0000000..1247c5e --- /dev/null +++ b/tests/src/performance/linkbench/assets/Roslyn/illinkcsproj @@ -0,0 +1,9 @@ + + + netcoreapp2.0 + win7-x86;win7-x64 + + + + + diff --git a/tests/src/performance/linkbench/linkbench.cs b/tests/src/performance/linkbench/linkbench.cs new file mode 100644 index 0000000..3d29957 --- /dev/null +++ b/tests/src/performance/linkbench/linkbench.cs @@ -0,0 +1,372 @@ +using System; +using System.Diagnostics; +using System.Collections.Generic; +using System.IO; +using System.Reflection; +using System.Runtime.Loader; +using System.Text; +using System.Globalization; +using System.Linq; +using System.Xml.Linq; +using Microsoft.Xunit.Performance; +using Microsoft.Xunit.Performance.Api; +using Xunit; +using Xunit.Abstractions; + +namespace LinkBench +{ + public class Benchmark + { + public string Name; + + public string UnlinkedDir; + public string LinkedDir; + public double UnlinkedMsilSize; + public double LinkedMsilSize; + public double UnlinkedDirSize; + public double LinkedDirSize; + public double MsilSizeReduction; + public double DirSizeReduction; + + private DirectoryInfo unlinkedDirInfo; + private DirectoryInfo linkedDirInfo; + private double certDiff; + const double MB = 1024 * 1024; + + public Benchmark(string _Name, string _UnlinkedDir, string _LinkedDir) + { + Name = _Name; + UnlinkedDir = _UnlinkedDir; + LinkedDir = _LinkedDir; + unlinkedDirInfo = new DirectoryInfo(UnlinkedDir); + linkedDirInfo = new DirectoryInfo(LinkedDir); + } + + public void Compute() + { + ComputeCertDiff(); + UnlinkedMsilSize = GetMSILSize(UnlinkedDir); + LinkedMsilSize = GetMSILSize(LinkedDir); + UnlinkedDirSize = GetDirSize(unlinkedDirInfo); + LinkedDirSize = GetDirSize(linkedDirInfo); + + MsilSizeReduction = (UnlinkedMsilSize - LinkedMsilSize) / UnlinkedMsilSize * 100; + DirSizeReduction = (UnlinkedDirSize - LinkedDirSize) / UnlinkedDirSize * 100; + } + + // Compute total size of a directory, in MegaBytes + // Includes all files and subdirectories recursively + private double GetDirSize(DirectoryInfo dir) + { + double size = 0; + FileInfo[] files = dir.GetFiles(); + foreach (FileInfo fileInfo in files) + { + size += fileInfo.Length; + } + DirectoryInfo[] subDirs = dir.GetDirectories(); + foreach (DirectoryInfo dirInfo in subDirs) + { + size += GetDirSize(dirInfo); + } + + return size / MB; + } + + // Compute the size of MSIL files in a directory, in MegaBytes + // Top level only, excludes crossgen files. + private double GetMSILSize(string dir) + { + string[] files = Directory.GetFiles(dir); + long msilSize = 0; + + foreach (string file in files) + { + if (file.EndsWith(".ni.dll") || file.EndsWith(".ni.exe")) + { + continue; + } + try + { + AssemblyLoadContext.GetAssemblyName(file); + } + catch (BadImageFormatException) + { + continue; + } + + msilSize += new FileInfo(file).Length; + } + + return msilSize / MB; + } + + // Gets the size of the Certificate header in a MSIL or ReadyToRun binary. + private long GetCertSize(string file) + { + Process p = new Process(); + p.StartInfo.UseShellExecute = false; + p.StartInfo.RedirectStandardOutput = true; + p.StartInfo.FileName = LinkBench.ScriptDir + "GetCert.cmd"; + p.StartInfo.Arguments = file; + p.Start(); + string output = p.StandardOutput.ReadToEnd(); + p.WaitForExit(); + long size = Int32.Parse(output.Substring(18, 8), + NumberStyles.AllowLeadingWhite | NumberStyles.AllowTrailingWhite | NumberStyles.HexNumber); + return size; + } + + // Get the total size difference for all certificates in all managed binaries + // in the unlinked and linked directories. + private double ComputeCertDiff() + { + string[] files = Directory.GetFiles(LinkedDir); + long totalDiff = 0; + + foreach (string file in files) + { + try + { + AssemblyLoadContext.GetAssemblyName(file); + } + catch (BadImageFormatException) + { + continue; + } + + FileInfo fileInfo = new FileInfo(file); + long linkedCert = GetCertSize(file); + long unlinkedCert = GetCertSize(UnlinkedDir + "\\" + fileInfo.Name); + totalDiff += (unlinkedCert - linkedCert); + } + + return totalDiff / MB; + } + } + + public class LinkBench + { + private static ScenarioConfiguration scenarioConfiguration = new ScenarioConfiguration(new TimeSpan(2000000)); + private static MetricModel SizeMetric = new MetricModel { Name = "Size", DisplayName = "File Size", Unit = "MB" }; + private static MetricModel PercMetric = new MetricModel { Name = "Perc", DisplayName = "% Reduction", Unit = "%" }; + public static string Workspace; + public static string ScriptDir; + public static string AssetsDir; + private static Benchmark CurrentBenchmark; + + public static int Main(String [] args) + { + // Workspace is the ROOT of the coreclr tree. + // If CORECLR_REPO is not set, the script assumes that the location of sandbox + // is \coreclr\sandbox. + bool doClone = true; + bool doBuild = true; + + for(int i=0; i < args.Length; i++) + { + if (String.Compare(args[i], "noclone", true) == 0) + { + doClone = false; + } + else if (String.Compare(args[i], "nobuild", true) == 0) + { + doClone = false; + doBuild = false; + } + else + { + Console.WriteLine("Unknown argument"); + return -4; + } + } + + Workspace = Environment.GetEnvironmentVariable("CORECLR_REPO"); + if (Workspace == null) + { + Workspace = Directory.GetParent(Directory.GetCurrentDirectory()).FullName; + } + if (Workspace == null) + { + Console.WriteLine("CORECLR_REPO not found"); + return -1; + } + + string LinkBenchDir = Workspace + "\\tests\\src\\performance\\linkbench\\"; + ScriptDir = LinkBenchDir + "scripts\\"; + AssetsDir = LinkBenchDir + "assets\\"; + + Benchmark[] Benchmarks = + { + new Benchmark("HelloWorld", + "LinkBench\\HelloWorld\\bin\\release\\netcoreapp2.0\\win10-x64\\publish", + "LinkBench\\HelloWorld\\bin\\release\\netcoreapp2.0\\win10-x64\\linked"), + new Benchmark("WebAPI", + "LinkBench\\WebAPI\\bin\\release\\netcoreapp2.0\\win10-x64\\publish", + "LinkBench\\WebAPI\\bin\\release\\netcoreapp2.0\\win10-x64\\linked"), + new Benchmark("MusicStore", + "LinkBench\\JitBench\\src\\MusicStore\\bin\\release\\netcoreapp2.0\\win10-x64\\publish", + "LinkBench\\JitBench\\src\\MusicStore\\bin\\release\\netcoreapp2.0\\win10-x64\\linked"), + new Benchmark("MusicStore_R2R", + "LinkBench\\JitBench\\src\\MusicStore\\bin\\release\\netcoreapp2.0\\win10-x64\\publish_r2r", + "LinkBench\\JitBench\\src\\MusicStore\\bin\\release\\netcoreapp2.0\\win10-x64\\linked_r2r"), + new Benchmark("Corefx", + "LinkBench\\corefx\\bin\\ILLinkTrimAssembly\\netcoreapp-Windows_NT-Release-x64\\pretrimmed", + "LinkBench\\corefx\\bin\\ILLinkTrimAssembly\\netcoreapp-Windows_NT-Release-x64\\trimmed"), + new Benchmark("Roslyn", + "LinkBench\\roslyn\\Binaries\\Release\\Exes\\CscCore", + "LinkBench\\roslyn\\Binaries\\Release\\Exes\\Linked"), + }; + + // Update the build files to facilitate the link step + if(doClone) + { + if(!Setup()) + { + return -2; + } + } + + if (doBuild) + { + // Run the setup Script, which clones, builds and links the benchmarks. + using (var setup = new Process()) + { + setup.StartInfo.FileName = ScriptDir + "build.cmd"; + setup.StartInfo.Arguments = AssetsDir; + setup.Start(); + setup.WaitForExit(); + if (setup.ExitCode != 0) + { + Console.WriteLine("Setup failed"); + return -3; + } + } + } + + // Since this is a size measurement scenario, there are no iterations + // to perform. So, create a process that does nothing, to satisfy XUnit. + // All size measurements are performed PostRun() + var emptyCmd = new ProcessStartInfo() + { + FileName = ScriptDir + "empty.cmd" + }; + + for (int i = 0; i < Benchmarks.Length; i++) + { + CurrentBenchmark = Benchmarks[i]; + string[] scriptArgs = { "--perf:runid", CurrentBenchmark.Name }; + + using (var h = new XunitPerformanceHarness(scriptArgs)) + { + h.RunScenario(emptyCmd, null, null, PostRun, scenarioConfiguration); + } + } + + return 0; + } + + private static ScenarioBenchmark PostRun() + { + // The XUnit output doesn't print the benchmark name, so print it now. + Console.WriteLine("{0}", CurrentBenchmark.Name); + + var scenario = new ScenarioBenchmark(CurrentBenchmark.Name) + { + Namespace = "LinkBench" + }; + + CurrentBenchmark.Compute(); + + addMeasurement(ref scenario, "MSIL Unlinked", SizeMetric, CurrentBenchmark.UnlinkedMsilSize); + addMeasurement(ref scenario, "MSIL Linked", SizeMetric, CurrentBenchmark.LinkedMsilSize); + addMeasurement(ref scenario, "MSIL %Reduction", PercMetric, CurrentBenchmark.MsilSizeReduction); + addMeasurement(ref scenario, "Total Uninked", SizeMetric, CurrentBenchmark.UnlinkedDirSize); + addMeasurement(ref scenario, "Total Linked", SizeMetric, CurrentBenchmark.LinkedDirSize); + addMeasurement(ref scenario, "Total %Reduction", PercMetric, CurrentBenchmark.DirSizeReduction); + + return scenario; + } + + private static bool Setup() + { + // Clone the benchmarks + using (var setup = new Process()) + { + setup.StartInfo.FileName = ScriptDir + "clone.cmd"; + Console.WriteLine("Run {0}", setup.StartInfo.FileName); + setup.Start(); + setup.WaitForExit(); + if (setup.ExitCode != 0) + { + Console.WriteLine("clone failed"); + return false; + } + } + + //Update the project files + AddLinkerReference("LinkBench\\HelloWorld\\HelloWorld.csproj"); + AddLinkerReference("LinkBench\\WebAPI\\WebAPI.csproj"); + AddLinkerReference("LinkBench\\JitBench\\src\\MusicStore\\MusicStore.csproj"); + RemoveCrossgenTarget("LinkBench\\JitBench\\src\\MusicStore\\MusicStore.csproj"); + + return true; + } + + private static void AddLinkerReference(string csproj) + { + var xdoc = XDocument.Load(csproj); + var ns = xdoc.Root.GetDefaultNamespace(); + bool added = false; + foreach (var el in xdoc.Root.Elements(ns + "ItemGroup")) + { + if (el.Elements(ns + "PackageReference").Any()) + { + el.Add(new XElement(ns + "PackageReference", + new XAttribute("Include", "ILLink.Tasks"), + new XAttribute("Version", "0.1.0-preview"))); + added = true; + break; + } + } + if (!added) + { + xdoc.Root.Add(new XElement(ns + "ItemGroup", + new XElement(ns + "PackageReference", + new XAttribute("Include", "ILLink.Tasks"), + new XAttribute("Version", "0.1.0-preview")))); + added = true; + } + using (var fs = new FileStream(csproj, FileMode.Create)) + { + xdoc.Save(fs); + } + } + + private static void RemoveCrossgenTarget(string csproj) + { + var xdoc = XDocument.Load(csproj); + var ns = xdoc.Root.GetDefaultNamespace(); + var target = xdoc.Root.Element(ns + "Target"); + target.Remove(); + using (var fs = new FileStream(csproj, FileMode.Create)) + { + xdoc.Save(fs); + } + } + + private static void addMeasurement(ref ScenarioBenchmark scenario, string name, MetricModel metric, double value) + { + var iteration = new IterationModel + { + Iteration = new Dictionary() + }; + iteration.Iteration.Add(metric.Name, value); + + var size = new ScenarioTestModel(name); + size.Performance.Metrics.Add(metric); + size.Performance.IterationModels.Add(iteration); + scenario.Tests.Add(size); + } + } +} diff --git a/tests/src/performance/linkbench/linkbench.csproj b/tests/src/performance/linkbench/linkbench.csproj new file mode 100644 index 0000000..2a3048a --- /dev/null +++ b/tests/src/performance/linkbench/linkbench.csproj @@ -0,0 +1,41 @@ + + + + + Debug + AnyCPU + LinkBench + 2.0 + {507E3CC2-5D95-414D-9F01-2A106FC177DC} + exe + Properties + 512 + {786C830F-07A1-408B-BD7F-6EE04809D6DB};{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC} + $(ProgramFiles)\Common Files\microsoft shared\VSTT\11.0\UITestExtensionPackages + ..\..\ + 7a9bfb7d + $(DefineConstants);STATIC + .NETStandard,Version=v1.4 + + + + + + + + ..\project.json + ..\project.lock.json + + + + False + + + + + + + + + + diff --git a/tests/src/performance/linkbench/scripts/build.cmd b/tests/src/performance/linkbench/scripts/build.cmd new file mode 100644 index 0000000..88b7eb1 --- /dev/null +++ b/tests/src/performance/linkbench/scripts/build.cmd @@ -0,0 +1,116 @@ +@echo off + +REM Usage: Build.cmd +setlocal +call "C:\Program Files (x86)\Microsoft Visual Studio 14.0\Common7\Tools\VsDevCmd.bat" + +set ROOT=%cd%\LinkBench +set AssetDir=%1 +set ExitCode=0 +mkdir LinkBench 2> nul +pushd %ROOT% + +echo Build ** HelloWorld ** +cd %ROOT%\HelloWorld +dotnet restore -r win10-x64 +dotnet publish -c release -r win10-x64 +dotnet msbuild /t:Link /p:LinkerMode=sdk /p:RuntimeIdentifier=win10-x64 /v:n /p:Configuration=release +if errorlevel 1 set ExitCode=1 +echo -- Done -- + +echo Build ** WebAPI ** +cd %ROOT%\WebAPI +dotnet restore -r win10-x64 +dotnet publish -c release -r win10-x64 +dotnet msbuild /t:Link /p:LinkerMode=sdk /p:RuntimeIdentifier=win10-x64 /v:n /p:Configuration=release +if errorlevel 1 set ExitCode=1 +echo -- Done -- + +echo Build ** MusicStore ** +cd %ROOT%\JitBench\src\MusicStore +copy %AssetDir%\MusicStore\MusicStoreReflection.xml . +dotnet restore -r win10-x64 +dotnet publish -c release -r win10-x64 +dotnet msbuild /t:Link /p:LinkerMode=sdk /p:RuntimeIdentifier=win10-x64 /v:n /p:LinkerRootFiles=MusicStoreReflection.xml /p:Configuration=release +if errorlevel 1 set ExitCode=1 +echo -- Done -- + +echo Build ** MusicStore Ready2Run ** +cd %ROOT%\JitBench\src\MusicStore +powershell -noprofile -executionPolicy RemoteSigned -file Get-Crossgen.ps1 +pushd bin\release\netcoreapp2.0\win10-x64\ +call :SetupR2R publish +if errorlevel 1 set ExitCode=1 +call :SetupR2R linked +if errorlevel 1 set ExitCode=1 +echo -- Done -- + +echo Build ** CoreFX ** +cd %ROOT%\corefx +set BinPlaceILLinkTrimAssembly=true +call build.cmd -release +if errorlevel 1 set ExitCode=1 +echo -- Done -- + +echo Build ** Roslyn ** +cd %ROOT%\roslyn +copy %AssetDir%\Roslyn\RoslynRoots.txt . +copy %AssetDir%\Roslyn\RoslynRoots.xml . +set RoslynRoot=%cd% +REM Build Roslyn +call restore.cmd +msbuild /m Roslyn.sln /p:Configuration=Release +REM Fetch ILLink +mkdir illink +cd illink +copy %AssetDir%\Roslyn\illinkcsproj illink.csproj +dotnet restore illink.csproj -r win10-x64 --packages bin +cd .. +REM Create Linker Directory +cd Binaries\Release\Exes +mkdir Linked +cd CscCore +REM Copy Unmanaged Assets +FOR /F "delims=" %%I IN ('DIR /b *') DO ( + corflags %%I >nul 2> nul + if errorlevel 1 copy %%I ..\Linked >nul +) +copy *.ni.dll ..\Linked +REM Run Linker +dotnet %RoslynRoot%\illink\bin\illink.tasks\0.1.0-preview\tools\illink.dll -t -c link -a @%RoslynRoot%\RoslynRoots.txt -x %RoslynRoot%\RoslynRoots.xml -l none -out ..\Linked +if errorlevel 1 set ExitCode=1 +echo -- Done -- +popd + +:Done +exit /b %ExitCode% + +:SetupR2R +REM Create R2R directory and copy all contents from MSIL to R2R directory +mkdir %1_r2r +xcopy /E /Y /Q %1 %1_r2r +REM Generate Ready2Run images for all MSIL files by running crossgen +cd %1_r2r +copy ..\..\..\..\..\crossgen.exe +FOR /F %%I IN ('dir /b *.dll ^| find /V /I ".ni.dll" ^| find /V /I "System.Private.CoreLib" ^| find /V /I "mscorlib.dll"') DO ( + REM Don't crossgen Corlib, since the native image already exists. + REM For all other MSIL files (corflags returns 0), run crossgen + corflags %%I >nul 2>nul + if not errorlevel 1 ( + crossgen /Platform_Assemblies_Paths . %%I >nul 2>nul + if errorlevel 1 ( + exit /b 1 + ) + ) +) +del crossgen.exe + +REM Remove the original MSIL files, rename the Ready2Run files .ni.dll --> .dll +FOR /F "delims=" %%I IN ('dir /b *.dll') DO ( + if exist %%~nI.ni.dll ( + del %%I + ren %%~nI.ni.dll %%I + ) +) +cd .. +exit /b 0 diff --git a/tests/src/performance/linkbench/scripts/clone.cmd b/tests/src/performance/linkbench/scripts/clone.cmd new file mode 100644 index 0000000..ae28d41 --- /dev/null +++ b/tests/src/performance/linkbench/scripts/clone.cmd @@ -0,0 +1,30 @@ +@echo off + +rmdir /s /q LinkBench + +set ROOT=%cd%\LinkBench +mkdir LinkBench 2> nul +pushd %ROOT% + +mkdir HelloWorld +cd HelloWorld +dotnet new console +if errorlevel 1 exit /b 1 +cd .. + +mkdir WebAPI +cd WebAPI +dotnet new webapi +if errorlevel 1 exit /b 1 +cd .. + +git clone https://github.com/aspnet/JitBench -b dev +if errorlevel 1 exit /b 1 + +git clone http://github.com/dotnet/corefx +if errorlevel 1 exit /b 1 + +git clone https://github.com/dotnet/roslyn.git +if errorlevel 1 exit /b 1 + +popd \ No newline at end of file diff --git a/tests/src/performance/linkbench/scripts/empty.cmd b/tests/src/performance/linkbench/scripts/empty.cmd new file mode 100644 index 0000000..83cb140 --- /dev/null +++ b/tests/src/performance/linkbench/scripts/empty.cmd @@ -0,0 +1 @@ +@echo off diff --git a/tests/src/performance/linkbench/scripts/getcert.cmd b/tests/src/performance/linkbench/scripts/getcert.cmd new file mode 100644 index 0000000..e02f72d --- /dev/null +++ b/tests/src/performance/linkbench/scripts/getcert.cmd @@ -0,0 +1,2 @@ +@echo off +"%VS140COMNTOOLS%\..\..\VC\bin\amd64\dumpbin.exe" /headers %1 | findstr /C:"Certificates Directory