From 4501c887f665db6132c3778394f29fa60b9c05fc Mon Sep 17 00:00:00 2001 From: Michelle McDaniel Date: Tue, 27 Jun 2017 14:55:32 -0700 Subject: [PATCH] Update throughput exclusion list There are many assemblies in our throughput benchmark package that are reference or forwarding assemblies. There is no jit work in these assemblies, so we want to exclude them. This change also adds an iterations parameter so users can specify a specific number of iterations for testing. Commit migrated from https://github.com/dotnet/coreclr/commit/ddaa2cd5de83fa90012991a18bc8c4edd4e97755 --- src/coreclr/tests/scripts/run-throughput-perf.py | 56 +++++++++++++++++++----- 1 file changed, 45 insertions(+), 11 deletions(-) diff --git a/src/coreclr/tests/scripts/run-throughput-perf.py b/src/coreclr/tests/scripts/run-throughput-perf.py index 05352ac..4fa43e9 100644 --- a/src/coreclr/tests/scripts/run-throughput-perf.py +++ b/src/coreclr/tests/scripts/run-throughput-perf.py @@ -34,18 +34,49 @@ import csv # List of dlls we want to exclude dll_exclude_list = { - 'Windows_NT': [ + 'All': [ # Require Newtonsoft.Json "Microsoft.DotNet.ProjectModel.dll", "Microsoft.Extensions.DependencyModel.dll", # Require System.Security.Principal.Windows "System.Net.Requests.dll", "System.Net.Security.dll", - "System.Net.Sockets.dll" + "System.Net.Sockets.dll", + # Moving target. Not a reliable measurement + "System.Private.CoreLib.dll", + # Reference and forwarding assemblies + "System.AppContext.dll", + "System.Diagnostics.Contracts.dll", + "System.Dynamic.Runtime.dll", + "System.Globalization.dll", + "System.Globalization.Calendars.dll", + "System.IO.dll", + "System.IO.FileSystem.Primitives.dll", + "System.Reflection.dll", + "System.Reflection.Emit.dll", + "System.Reflection.Emit.ILGeneration.dll", + "System.Reflection.Emit.Lightweight.dll", + "System.Reflection.Extensions.dll", + "System.Reflection.Primitives.dll", + "System.Resources.ResourceManager.dll", + "System.Runtime.Handles.dll", + "System.Runtime.Loader.dll", + "System.Runtime.Serialization.Json.dll", + "System.Runtime.Serialization.Xml.dll", + "System.Security.Principal.dll", + "System.Text.Encoding.dll", + "System.Text.Encoding.Extensions.dll", + "System.Threading.ThreadPool.dll", + "System.Threading.Timer.dll", + "System.Xml.ReaderWriter.dll", + "System.Xml.XDocument.dll", + "System.Xml.XmlDocument.dll", + "System.Xml.XmlSerializer.dll", + "System.Xml.XPath.dll" + ], + 'Windows_NT': [ ], 'Linux' : [ - # Required System.Runtime.WindowsRuntime - "System.Runtime.WindowsRuntime.UI.Xaml.dll" ] } @@ -84,6 +115,7 @@ parser.add_argument('-os', dest='operating_system', default='Windows_NT') parser.add_argument('-clr_root', dest='clr_root', default=None) parser.add_argument('-assembly_root', dest='assembly_root', default=None) parser.add_argument('-benchview_path', dest='benchview_path', default=None) +parser.add_argument('-iterations', dest='iterations', default=5, type=int) ########################################################################## # Helper Functions @@ -108,6 +140,7 @@ def validate_args(args): clr_root = args.clr_root assembly_root = args.assembly_root benchview_path = args.benchview_path + iterations = args.iterations def validate_arg(arg, check): """ Validate an individual arg @@ -140,6 +173,7 @@ def validate_args(args): validate_arg(arch, lambda item: item in valid_archs[os_group]) validate_arg(build_type, lambda item: item in valid_build_types) validate_arg(run_type, lambda item: item in valid_run_types) + validate_arg(iterations, lambda item: item > 0) if clr_root is None: raise Exception('--clr_root must be set') @@ -157,7 +191,7 @@ def validate_args(args): benchview_path = os.path.normpath(benchview_path) validate_arg(benchview_path, lambda item: os.path.isdir(benchview_path)) - args = (arch, operating_system, os_group, build_type, run_type, clr_root, assembly_root, benchview_path) + args = (arch, operating_system, os_group, build_type, run_type, clr_root, assembly_root, benchview_path, iterations) # Log configuration log('Configuration:') @@ -166,6 +200,7 @@ def validate_args(args): log(' os_group: %s' % os_group) log(' build_type: %s' % build_type) log(' run_type: %s' % run_type) + log(' iterations: %d' % iterations) log(' clr_root: %s' % clr_root) log(' assembly_root: %s' % assembly_root) if not benchview_path is None: @@ -250,7 +285,7 @@ def runIterations(dll_name, dll_path, iterations, crossgen_path, jit_path, assem # Time.clock() returns seconds, with a resolution of 0.4 microseconds, so multiply by the multiplier to get milliseconds multiplier = 1000 - for iteration in range(0,iterations): + for iteration in range(0,iterations + 1): proc = subprocess.Popen(run_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) start_time = timeit.default_timer() @@ -277,7 +312,7 @@ def main(args): global os_group_list global python_exe_list - architecture, operating_system, os_group, build_type, run_type, clr_root, assembly_root, benchview_path = validate_args(args) + architecture, operating_system, os_group, build_type, run_type, clr_root, assembly_root, benchview_path, iterations = validate_args(args) arch = architecture current_dir = os.getcwd() @@ -304,8 +339,6 @@ def main(args): # Replace assembly_root's System.Private.CoreLib with built System.Private.CoreLib. shutil.copyfile(os.path.join(bin_path, 'System.Private.CoreLib.dll'), os.path.join(assembly_root, 'System.Private.CoreLib.dll')) - iterations = 6 - python_exe = python_exe_list[os_group] # Run throughput testing @@ -314,7 +347,8 @@ def main(args): if (dll_file_name.endswith(".dll") and (not ".ni." in dll_file_name) and ("Microsoft" in dll_file_name or "System" in dll_file_name) and - (not dll_file_name in dll_exclude_list[os_group])): + (not dll_file_name in dll_exclude_list[os_group]) and + (not dll_file_name in dll_exclude_list["All"])): dll_name = dll_file_name.replace(".dll", "") dll_path = os.path.join(assembly_root, dll_file_name) dll_elapsed_times = runIterations(dll_file_name, dll_path, iterations, crossgen_path, jit_path, assembly_root) @@ -344,7 +378,7 @@ def main(args): else: # Write output to console if we are not publishing log("%s" % (dll_name)) - log("Duration: [%s]" % (", ".join(str(x) for x in dll_elapsed_times))) + log("Duration: [%s]" % (", ".join(str(x) for x in dll_elapsed_times[1:]))) # Upload the data if not benchview_path is None: -- 2.7.4