From 8f679991b8fc9494dc548620f48f841d38a6dc24 Mon Sep 17 00:00:00 2001 From: Bruce Forstall Date: Thu, 5 Dec 2019 10:14:33 -0800 Subject: [PATCH] Update superpmi.py script for new 'runtime' repo layout (#541) Updated a few comments as well. --- src/coreclr/scripts/coreclr_arguments.py | 14 ++--- src/coreclr/scripts/superpmi.py | 97 +++++++++++++++++++------------- 2 files changed, 64 insertions(+), 47 deletions(-) diff --git a/src/coreclr/scripts/coreclr_arguments.py b/src/coreclr/scripts/coreclr_arguments.py index 65641ba..5e62cfb 100644 --- a/src/coreclr/scripts/coreclr_arguments.py +++ b/src/coreclr/scripts/coreclr_arguments.py @@ -63,7 +63,7 @@ class CoreclrArguments: self.arch = None self.build_type = None self.core_root = None - self.coreclr_repo_location = None + self.runtime_repo_location = None self.default_build_type = default_build_type @@ -205,7 +205,7 @@ class CoreclrArguments: return None def check_and_return_test_location(test_location): - default_test_location = os.path.join(self.coreclr_repo_location, "..", "..", "artifacts", "tests", "coreclr", "%s.%s.%s" % (self.host_os, self.arch, self.build_type)) + default_test_location = os.path.join(self.runtime_repo_location, "artifacts", "tests", "coreclr", "%s.%s.%s" % (self.host_os, self.arch, self.build_type)) if os.path.isdir(default_test_location) or not self.require_built_test_dir: return default_test_location @@ -227,7 +227,7 @@ class CoreclrArguments: return core_root def check_and_return_default_product_location(product_location): - default_product_location = os.path.join(self.bin_location, "Product", "%s.%s.%s" % (self.host_os, self.arch, self.build_type)) + default_product_location = os.path.join(self.artifacts_location, "bin", "coreclr", "%s.%s.%s" % (self.host_os, self.arch, self.build_type)) if os.path.isdir(default_product_location) or not self.require_built_product_dir: return default_product_location @@ -236,8 +236,8 @@ class CoreclrArguments: return product_location - self.coreclr_repo_location = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - self.bin_location = os.path.join(self.coreclr_repo_location, "artifacts") + self.runtime_repo_location = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) + self.artifacts_location = os.path.join(self.runtime_repo_location, "artifacts") self.verify(args, "host_os", @@ -248,7 +248,7 @@ class CoreclrArguments: self.verify(args, "arch", check_arch, - "Unsupported architecture: %s.\nSupported architectures: %s" % (args.arch, ", ".join(self.valid_arches))) + "Unsupported architecture.\nSupported architectures: %s" % (", ".join(self.valid_arches))) self.verify(args, "build_type", @@ -269,4 +269,4 @@ class CoreclrArguments: self.verify(args, "product_location", check_and_return_default_product_location, - "Error, incorrect product_location.") \ No newline at end of file + "Error, incorrect product_location.") diff --git a/src/coreclr/scripts/superpmi.py b/src/coreclr/scripts/superpmi.py index 4001b13..17cfc03 100755 --- a/src/coreclr/scripts/superpmi.py +++ b/src/coreclr/scripts/superpmi.py @@ -47,9 +47,12 @@ from coreclr_arguments import * # Argument Parser ################################################################################ -description = ("""Script to handle running SuperPMI Collections, and replays. In addition, this +description = ("""Script to handle running SuperPMI collections and replays. In addition, this script provides support for SuperPMI ASM diffs. Note that some of the options -provided by this script are also provided in our SuperPMI collect test.""") +provided by this script are also provided in our SuperPMI collect test. + +Help for each individual command can be shown by asking for help on the individual command, for example +`superpmi.py collect --help`.""") superpmi_collect_help = """ Command to run SuperPMI collect over. Note that there cannot be any dotnet cli command invoked inside this command, as they will fail due @@ -82,10 +85,10 @@ collect_parser.add_argument("-build_type", dest="build_type", nargs='?', default collect_parser.add_argument("-test_location", dest="test_location", nargs="?", default=None, help="Test location. This is optional") collect_parser.add_argument("-pmi_assemblies", dest="pmi_assemblies", nargs="+", default=[], help="Pass a sequence of managed dlls or directories to recurisvely run pmi over while collecting.") collect_parser.add_argument("-core_root", dest="core_root", nargs='?', default=None, help="Location of the Core_Root location. If not passed it will be deduced if possible.") -collect_parser.add_argument("-product_location", dest="product_location", nargs='?', default=None, help="Location of the built product, this is optional.") -collect_parser.add_argument("-coreclr_repo_location", dest="coreclr_repo_location", default=os.path.dirname(os.path.dirname(os.path.abspath(__file__))), help="Location of the coreclr repo. Optional.") +collect_parser.add_argument("-product_location", dest="product_location", nargs='?', default=None, help="Location of the built Product directory. Optional.") +collect_parser.add_argument("-runtime_repo_root", dest="runtime_repo_root", default=os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))), help="Path of the dotnet/runtime repo root directory. Optional.") collect_parser.add_argument("-test_env", dest="test_env", default=None, help="Test env to pass to the coreclr tests if collecting over the tests.") -collect_parser.add_argument("-output_mch_path", dest="output_mch_path", default=None, help="Location to drop the final mch file. By default it will drop to artifacts/mch/$(buildType).$(arch).$(config)/$(buildType).$(arch).$(config).mch") +collect_parser.add_argument("-output_mch_path", dest="output_mch_path", default=None, help="Location to drop the final mch file. By default it will drop to artifacts/mch/$(os).$(arch).$(build_type)/$(os).$(arch).$(build_type).mch") collect_parser.add_argument("--pmi", dest="pmi", default=False, action="store_true", help="Use pmi on a set of directories or assemblies") collect_parser.add_argument("-mch_files", dest="mch_files", nargs='+', default=None, help="Pass a sequence of mch files which will be merged.") @@ -108,7 +111,7 @@ collect_parser.add_argument("--skip_cleanup", dest="skip_cleanup", default=False replay_parser = subparsers.add_parser("replay") # Add required arguments -replay_parser.add_argument("collection", nargs='?', default="default", help="Which collection type to run. Default is to run everything. Use superpmi list to find potential collections") +replay_parser.add_argument("collection", nargs='?', default="default", help="Which collection type to run. Default is to run everything. Use 'superpmi list-collections' to find potential collections.") replay_parser.add_argument("-jit_path", nargs='?', help="Path to clrjit. defaults to core_root jit.") replay_parser.add_argument("-mch_file", nargs=1, help=superpmi_replay_help) @@ -121,8 +124,8 @@ replay_parser.add_argument("-arch", dest="arch", nargs='?', default="x64") replay_parser.add_argument("-build_type", dest="build_type", nargs='?', default="Checked") replay_parser.add_argument("-test_location", dest="test_location", nargs="?", default=None) replay_parser.add_argument("-core_root", dest="core_root", nargs='?', default=None) -replay_parser.add_argument("-product_location", dest="product_location", nargs='?', default=None) -replay_parser.add_argument("-coreclr_repo_location", dest="coreclr_repo_location", default=os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +replay_parser.add_argument("-product_location", dest="product_location", nargs='?', default=None, help="Location of the built Product directory. Optional.") +replay_parser.add_argument("-runtime_repo_root", dest="runtime_repo_root", default=os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))), help="Path of the dotnet/runtime repo root directory. Optional.") replay_parser.add_argument("-test_env", dest="test_env", default=None) replay_parser.add_argument("--skip_cleanup", dest="skip_cleanup", default=False, action="store_true") @@ -134,7 +137,7 @@ asm_diff_parser = subparsers.add_parser("asmdiffs") # Add required arguments asm_diff_parser.add_argument("base_jit_path", nargs=1, help="Path to baseline clrjit.") asm_diff_parser.add_argument("diff_jit_path", nargs=1, help="Path to diff clrjit.") -asm_diff_parser.add_argument("collection", nargs='?', default="default", help="Which collection type to run. Default is to run everything. Use superpmi list to find potential collections") +asm_diff_parser.add_argument("collection", nargs='?', default="default", help="Which collection type to run. Default is to run everything. Use 'superpmi list-collections' to find potential collections.") asm_diff_parser.add_argument("-mch_file", nargs=1, help=superpmi_replay_help) @@ -147,8 +150,8 @@ asm_diff_parser.add_argument("-arch", dest="arch", nargs='?', default="x64") asm_diff_parser.add_argument("-build_type", dest="build_type", nargs='?', default="Checked") asm_diff_parser.add_argument("-test_location", dest="test_location", nargs="?", default=None) asm_diff_parser.add_argument("-core_root", dest="core_root", nargs='?', default=None) -asm_diff_parser.add_argument("-product_location", dest="product_location", nargs='?', default=None) -asm_diff_parser.add_argument("-coreclr_repo_location", dest="coreclr_repo_location", default=os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +asm_diff_parser.add_argument("-product_location", dest="product_location", nargs='?', default=None, help="Location of the built Product directory. Optional.") +asm_diff_parser.add_argument("-runtime_repo_root", dest="runtime_repo_root", default=os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))), help="Path of the dotnet/runtime repo root directory. Optional.") asm_diff_parser.add_argument("-test_env", dest="test_env", default=None) asm_diff_parser.add_argument("--skip_cleanup", dest="skip_cleanup", default=False, action="store_true") @@ -170,7 +173,7 @@ upload_parser.add_argument("-jit_location", nargs=1, default=None, help="Locatio upload_parser.add_argument("-arch", dest="arch", nargs='?', default="x64") upload_parser.add_argument("-build_type", dest="build_type", nargs='?', default="Checked") -upload_parser.add_argument("-coreclr_repo_location", dest="coreclr_repo_location", default=os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +upload_parser.add_argument("-runtime_repo_root", dest="runtime_repo_root", default=os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))), help="Path of the dotnet/runtime repo root directory. Optional.") upload_parser.add_argument("--skip_cleanup", dest="skip_cleanup", default=False, action="store_true") @@ -179,7 +182,7 @@ list_parser = subparsers.add_parser("list-collections") list_parser.add_argument("-arch", dest="arch", nargs='?', default="x64") list_parser.add_argument("-build_type", dest="build_type", nargs='?', default="Checked") -list_parser.add_argument("-coreclr_repo_location", dest="coreclr_repo_location", default=os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +list_parser.add_argument("-runtime_repo_root", dest="runtime_repo_root", default=os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))), help="Path of the dotnet/runtime repo root directory. Optional.") ################################################################################ # Helper classes @@ -814,7 +817,7 @@ class SuperPMIReplay: mc_files = [os.path.join(temp_location, item) for item in os.listdir(temp_location) if item.endswith(".mc")] if len(mc_files) > 0: - repro_location = os.path.join(self.coreclr_args.coreclr_repo_location, "artifacts", "repro", "{}.{}.{}".format(self.coreclr_args.host_os, self.coreclr_args.arch, self.coreclr_args.build_type)) + repro_location = os.path.join(self.coreclr_args.runtime_repo_root, "artifacts", "repro", "{}.{}.{}".format(self.coreclr_args.host_os, self.coreclr_args.arch, self.coreclr_args.build_type)) # Delete existing repro location if os.path.isdir(repro_location): @@ -1015,7 +1018,7 @@ class SuperPMIReplayAsmDiffs: mc_files = [os.path.join(temp_location, item) for item in os.listdir(temp_location) if item.endswith(".mc")] if len(mc_files) > 0: - repro_location = os.path.join(self.coreclr_args.coreclr_repo_location, "artifacts", "repro", "{}.{}.{}".format(self.coreclr_args.host_os, self.coreclr_args.arch, self.coreclr_args.build_type)) + repro_location = os.path.join(self.coreclr_args.runtime_repo_root, "artifacts", "repro", "{}.{}.{}".format(self.coreclr_args.host_os, self.coreclr_args.arch, self.coreclr_args.build_type)) # Delete existing repro location if os.path.isdir(repro_location): @@ -1074,11 +1077,11 @@ class SuperPMIReplayAsmDiffs: mcl_lines = [item.strip() for item in mcl_lines] self.diff_mcl_contents = mcl_lines - bin_asm_location = os.path.join(self.coreclr_args.bin_location, "asm", "asm") + bin_asm_location = os.path.join(self.coreclr_args.artifacts_location, "asm", "asm") count = 0 while os.path.isdir(bin_asm_location): - new_bin_asm_location = os.path.join(self.coreclr_args.bin_location, "asm", "asm" + str(count)) + new_bin_asm_location = os.path.join(self.coreclr_args.artifacts_location, "asm", "asm" + str(count)) count += 1 @@ -1089,11 +1092,11 @@ class SuperPMIReplayAsmDiffs: base_asm_location = os.path.join(bin_asm_location, "base") diff_asm_location = os.path.join(bin_asm_location, "diff") - bin_dump_location = os.path.join(self.coreclr_args.bin_location, "jit_dump", "jit_dump") + bin_dump_location = os.path.join(self.coreclr_args.artifacts_location, "jit_dump", "jit_dump") count = 0 while os.path.isdir(bin_dump_location): - new_base_dump_location = os.path.join(self.coreclr_args.bin_location, "jit_dump", "jit_dump" + str(count)) + new_base_dump_location = os.path.join(self.coreclr_args.artifacts_location, "jit_dump", "jit_dump" + str(count)) count += 1 @@ -1267,7 +1270,7 @@ class SuperPMIReplayAsmDiffs: diff_txt = None # Change the working directory to the core root we will call SuperPMI from. - # This is done to allow libcorcedistools to be loaded correctly on unix + # This is done to allow libcoredistools to be loaded correctly on unix # as the loadlibrary path will be relative to the current directory. with ChangeDir(self.coreclr_args.core_root) as dir: @@ -1622,7 +1625,7 @@ def download_mch(coreclr_args, specific_mch=None, include_baseline_jit=False): """ urls = list_superpmi_container_via_rest_api(coreclr_args) - default_mch_dir = os.path.join(coreclr_args.bin_location, "mch", "{}.{}.{}".format(coreclr_args.host_os, coreclr_args.arch, coreclr_args.build_type)) + default_mch_dir = os.path.join(coreclr_args.artifacts_location, "mch", "{}.{}.{}".format(coreclr_args.host_os, coreclr_args.arch, coreclr_args.build_type)) if not os.path.isdir(default_mch_dir): os.makedirs(default_mch_dir) @@ -1756,19 +1759,19 @@ def setup_args(args): lambda mode: mode in ["collect", "replay", "asmdiffs", "upload", "list-collections"], 'Incorrect mode passed, please choose from ["collect", "replay", "asmdiffs", "upload", "list-collections"]') - default_coreclr_bin_mch_location = os.path.join(coreclr_args.bin_location, "mch", "{}.{}.{}".format(coreclr_args.host_os, coreclr_args.arch, coreclr_args.build_type)) + default_coreclr_bin_mch_location = os.path.join(coreclr_args.artifacts_location, "mch", "{}.{}.{}".format(coreclr_args.host_os, coreclr_args.arch, coreclr_args.build_type)) def setup_mch_arg(arg): - default_mch_location = os.path.join(coreclr_args.bin_location, "mch", "{}.{}.{}".format(coreclr_args.host_os, coreclr_args.arch, coreclr_args.build_type), "{}.{}.{}.mch".format(coreclr_args.host_os, coreclr_args.arch, coreclr_args.build_type)) + default_mch_location = os.path.join(coreclr_args.artifacts_location, "mch", "{}.{}.{}".format(coreclr_args.host_os, coreclr_args.arch, coreclr_args.build_type), "{}.{}.{}.mch".format(coreclr_args.host_os, coreclr_args.arch, coreclr_args.build_type)) - if os.path.isfile(default_mch_location) and not args.force_download and coreclr_args.collection is "default": + if os.path.isfile(default_mch_location) and not args.force_download and coreclr_args.collection == "default": return default_mch_location # Download the mch else: index = download_index(coreclr_args) - mch_location = os.path.join(coreclr_args.bin_location, "mch", "{}.{}.{}".format(coreclr_args.host_os, coreclr_args.arch, coreclr_args.build_type), index[coreclr_args.collection]) + mch_location = os.path.join(coreclr_args.artifacts_location, "mch", "{}.{}.{}".format(coreclr_args.host_os, coreclr_args.arch, coreclr_args.build_type), index[coreclr_args.collection]) if not os.path.isfile(mch_location): download_mch(coreclr_args, specific_mch=index[coreclr_args.collection], include_baseline_jit=True) @@ -1917,19 +1920,26 @@ def setup_args(args): "Unable to set break_on_error") standard_location = False - if coreclr_args.bin_location.lower() in coreclr_args.jit_path.lower(): + if coreclr_args.product_location.lower() in coreclr_args.jit_path.lower(): standard_location = True determined_arch = None determined_build_type = None if standard_location: - standard_location_split = coreclr_args.jit_path.split(coreclr_args.bin_location) - + # Get os/arch/flavor directory, e.g. split "F:\gh\runtime\artifacts\bin\coreclr\Windows_NT.x64.Checked" with "F:\gh\runtime\artifacts\bin\coreclr" + # yielding + # [0]: "" + # [1]: "\Windows_NT.x64.Checked" + standard_location_split = os.path.dirname(coreclr_args.jit_path).split(os.path.dirname(coreclr_args.product_location)) assert(coreclr_args.host_os in standard_location_split[1]) - specialized_path = standard_location_split[1].split(coreclr_args.host_os) - specialized_path = specialized_path[1].split("/")[0] + # Get arch/flavor. Remove leading slash. + specialized_path = standard_location_split[1].split(os.path.sep)[1] + # Split components: "Windows_NT.x64.Checked" into: + # [0]: "Windows_NT" + # [1]: "x64" + # [2]: "Checked" determined_split = specialized_path.split(".") determined_arch = determined_split[1] @@ -2027,19 +2037,26 @@ def setup_args(args): "Unable to set diff_jit_dump.") standard_location = False - if coreclr_args.bin_location.lower() in coreclr_args.base_jit_path.lower(): + if coreclr_args.product_location.lower() in coreclr_args.base_jit_path.lower(): standard_location = True determined_arch = None determined_build_type = None if standard_location: - standard_location_split = coreclr_args.base_jit_path.split(coreclr_args.bin_location) - + # Get os/arch/flavor directory, e.g. split "F:\gh\runtime\artifacts\bin\coreclr\Windows_NT.x64.Checked" with "F:\gh\runtime\artifacts\bin\coreclr" + # yielding + # [0]: "" + # [1]: "\Windows_NT.x64.Checked" + standard_location_split = os.path.dirname(coreclr_args.jit_path).split(os.path.dirname(coreclr_args.product_location)) assert(coreclr_args.host_os in standard_location_split[1]) - specialized_path = standard_location_split[1].split(coreclr_args.host_os) - specialized_path = specialized_path[1].split("/")[0] + # Get arch/flavor. Remove leading slash. + specialized_path = standard_location_split[1].split(os.path.sep)[1] + # Split components: "Windows_NT.x64.Checked" into: + # [0]: "Windows_NT" + # [1]: "x64" + # [2]: "Checked" determined_split = specialized_path.split(".") determined_arch = determined_split[1] @@ -2100,7 +2117,7 @@ def main(args): # await/async requires python >= 3.5 if sys.version_info.major < 3 and sys.version_info.minor < 5: print("Error, language features require the latest python version.") - print("Please install python 3.7 or greater") + print("Please install python 3.8 or greater") return 1 @@ -2146,7 +2163,7 @@ def main(args): print("") print("MCH Path: {}".format(mch_file)) - print("Jit Path: {}".format(jit_path)) + print("JIT Path: {}".format(jit_path)) replay = SuperPMIReplay(coreclr_args, mch_file, jit_path) success = replay.replay() @@ -2162,7 +2179,7 @@ def main(args): begin_time = datetime.datetime.now() - print("SuperPMI Replay") + print("SuperPMI ASM diffs") print("------------------------------------------------------------") print("Start time: {}".format(begin_time.strftime("%H:%M:%S"))) @@ -2173,8 +2190,8 @@ def main(args): print("") print("MCH Path: {}".format(mch_file)) - print("Base Jit Path: {}".format(base_jit_path)) - print("Diff Jit Path: {}".format(diff_jit_path)) + print("Base JIT Path: {}".format(base_jit_path)) + print("Diff JIT Path: {}".format(diff_jit_path)) asm_diffs = SuperPMIReplayAsmDiffs(coreclr_args, mch_file, base_jit_path, diff_jit_path) success = asm_diffs.replay_with_asm_diffs(coreclr_args.previous_temp_location) -- 2.7.4