Update throughput exclusion list

There are many assemblies in our throughput benchmark package that are
reference or forwarding assemblies. There is no jit work in these
assemblies, so we want to exclude them.

This change also adds an iterations parameter so users can specify a
specific number of iterations for testing.

Commit migrated from https://github.com/dotnet/coreclr/commit/ddaa2cd5de83fa90012991a18bc8c4edd4e97755

diff --git a/src/coreclr/tests/scripts/run-throughput-perf.py b/src/coreclr/tests/scripts/run-throughput-perf.py
index 05352ac..4fa43e9 100644 (file)
--- a/src/coreclr/tests/scripts/run-throughput-perf.py
+++ b/src/coreclr/tests/scripts/run-throughput-perf.py
@@ -34,18 +34,49 @@ import csv
 
 # List of dlls we want to exclude
 dll_exclude_list = {
-    'Windows_NT': [
+    'All': [
         # Require Newtonsoft.Json
         "Microsoft.DotNet.ProjectModel.dll",
         "Microsoft.Extensions.DependencyModel.dll",
         # Require System.Security.Principal.Windows
         "System.Net.Requests.dll",
         "System.Net.Security.dll",
-        "System.Net.Sockets.dll"
+        "System.Net.Sockets.dll",
+        # Moving target. Not a reliable measurement
+        "System.Private.CoreLib.dll",
+        # Reference and forwarding assemblies
+        "System.AppContext.dll",
+        "System.Diagnostics.Contracts.dll",
+        "System.Dynamic.Runtime.dll",
+        "System.Globalization.dll",
+        "System.Globalization.Calendars.dll",
+        "System.IO.dll",
+        "System.IO.FileSystem.Primitives.dll",
+        "System.Reflection.dll",
+        "System.Reflection.Emit.dll",
+        "System.Reflection.Emit.ILGeneration.dll",
+        "System.Reflection.Emit.Lightweight.dll",
+        "System.Reflection.Extensions.dll",
+        "System.Reflection.Primitives.dll",
+        "System.Resources.ResourceManager.dll",
+        "System.Runtime.Handles.dll",
+        "System.Runtime.Loader.dll",
+        "System.Runtime.Serialization.Json.dll",
+        "System.Runtime.Serialization.Xml.dll",
+        "System.Security.Principal.dll",
+        "System.Text.Encoding.dll",
+        "System.Text.Encoding.Extensions.dll",
+        "System.Threading.ThreadPool.dll",
+        "System.Threading.Timer.dll",
+        "System.Xml.ReaderWriter.dll",
+        "System.Xml.XDocument.dll",
+        "System.Xml.XmlDocument.dll",
+        "System.Xml.XmlSerializer.dll",
+        "System.Xml.XPath.dll"
+    ],
+    'Windows_NT': [
     ],
     'Linux' : [
-        # Required System.Runtime.WindowsRuntime
-        "System.Runtime.WindowsRuntime.UI.Xaml.dll"
     ]
 }
 
@@ -84,6 +115,7 @@ parser.add_argument('-os', dest='operating_system', default='Windows_NT')
 parser.add_argument('-clr_root', dest='clr_root', default=None)
 parser.add_argument('-assembly_root', dest='assembly_root', default=None)
 parser.add_argument('-benchview_path', dest='benchview_path', default=None)
+parser.add_argument('-iterations', dest='iterations', default=5, type=int)
 
 ##########################################################################
 # Helper Functions
@@ -108,6 +140,7 @@ def validate_args(args):
     clr_root = args.clr_root
     assembly_root = args.assembly_root
     benchview_path = args.benchview_path
+    iterations = args.iterations
 
     def validate_arg(arg, check):
         """ Validate an individual arg
@@ -140,6 +173,7 @@ def validate_args(args):
     validate_arg(arch, lambda item: item in valid_archs[os_group])
     validate_arg(build_type, lambda item: item in valid_build_types)
     validate_arg(run_type, lambda item: item in valid_run_types)
+    validate_arg(iterations, lambda item: item > 0)
 
     if clr_root is None:
         raise Exception('--clr_root must be set')
@@ -157,7 +191,7 @@ def validate_args(args):
         benchview_path = os.path.normpath(benchview_path)
         validate_arg(benchview_path, lambda item: os.path.isdir(benchview_path))
 
-    args = (arch, operating_system, os_group, build_type, run_type, clr_root, assembly_root, benchview_path)
+    args = (arch, operating_system, os_group, build_type, run_type, clr_root, assembly_root, benchview_path, iterations)
 
     # Log configuration
     log('Configuration:')
@@ -166,6 +200,7 @@ def validate_args(args):
     log(' os_group: %s' % os_group)
     log(' build_type: %s' % build_type)
     log(' run_type: %s' % run_type)
+    log(' iterations: %d' % iterations)
     log(' clr_root: %s' % clr_root)
     log(' assembly_root: %s' % assembly_root)
     if not benchview_path is None:
@@ -250,7 +285,7 @@ def runIterations(dll_name, dll_path, iterations, crossgen_path, jit_path, assem
     # Time.clock() returns seconds, with a resolution of 0.4 microseconds, so multiply by the multiplier to get milliseconds
     multiplier = 1000
 
-    for iteration in range(0,iterations):
+    for iteration in range(0,iterations + 1):
         proc = subprocess.Popen(run_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 
         start_time = timeit.default_timer()
@@ -277,7 +312,7 @@ def main(args):
     global os_group_list
     global python_exe_list
 
-    architecture, operating_system, os_group, build_type, run_type, clr_root, assembly_root, benchview_path = validate_args(args)
+    architecture, operating_system, os_group, build_type, run_type, clr_root, assembly_root, benchview_path, iterations = validate_args(args)
     arch = architecture
 
     current_dir = os.getcwd()
@@ -304,8 +339,6 @@ def main(args):
     # Replace assembly_root's System.Private.CoreLib with built System.Private.CoreLib.
     shutil.copyfile(os.path.join(bin_path, 'System.Private.CoreLib.dll'), os.path.join(assembly_root, 'System.Private.CoreLib.dll'))
 
-    iterations = 6
-
     python_exe = python_exe_list[os_group]
 
     # Run throughput testing
@@ -314,7 +347,8 @@ def main(args):
         if (dll_file_name.endswith(".dll") and
                 (not ".ni." in dll_file_name) and
                 ("Microsoft" in dll_file_name or "System" in dll_file_name) and
-                (not dll_file_name in dll_exclude_list[os_group])):
+                (not dll_file_name in dll_exclude_list[os_group]) and
+                (not dll_file_name in dll_exclude_list["All"])):
             dll_name = dll_file_name.replace(".dll", "")
             dll_path = os.path.join(assembly_root, dll_file_name)
             dll_elapsed_times = runIterations(dll_file_name, dll_path, iterations, crossgen_path, jit_path, assembly_root)
@@ -344,7 +378,7 @@ def main(args):
                 else:
                     # Write output to console if we are not publishing
                     log("%s" % (dll_name))
-                    log("Duration: [%s]" % (", ".join(str(x) for x in dll_elapsed_times)))
+                    log("Duration: [%s]" % (", ".join(str(x) for x in dll_elapsed_times[1:])))
 
     # Upload the data
     if not benchview_path is None: