scripts/superpmi.py

   1 #!/usr/bin/env python
   2 #
   3 ## Licensed to the .NET Foundation under one or more agreements.
   4 ## The .NET Foundation licenses this file to you under the MIT license.
   5 ## See the LICENSE file in the project root for more information.
   6 #
   7 ##
   8 # Title               : superpmi.py
   9 #
  10 # Notes:
  11 #
  12 # Script to handle running SuperPMI Collections, and replays. In addition, this
  13 # script provides support for SuperPMI ASM diffs. Note that some of the options
  14 # provided by this script are also provided in our SuperPMI collect test. The
  15 # test can be found here: https://github.com/dotnet/coreclr/blob/master/tests/src/JIT/superpmi/superpmicollect.cs.
  16 #
  17 ################################################################################
  18 ################################################################################
  19
  20 import argparse
  21 import datetime
  22 import json
  23 import math
  24 import os
  25 import platform
  26 import shutil
  27 import subprocess
  28 import sys
  29 import tempfile
  30 import time
  31 import re
  32 import string
  33 import zipfile
  34
  35 import xml.etree.ElementTree
  36
  37 from collections import defaultdict
  38 from sys import platform as _platform
  39
  40 # Version specific imports
  41
  42 if sys.version_info.major < 3:
  43     import urllib
  44 else:
  45     import urllib.request
  46
  47 from coreclr_arguments import *
  48
  49 ################################################################################
  50 # Argument Parser
  51 ################################################################################
  52
  53 description = ("""Script to handle running SuperPMI Collections, and replays. In addition, this
  54 script provides support for SuperPMI ASM diffs. Note that some of the options
  55 provided by this script are also provided in our SuperPMI collect test.""")
  56
  57 superpmi_collect_help = """ Command to run SuperPMI collect over. Note that there
  58 cannot be any dotnet cli command invoked inside this command, as they will fail due
  59 to the shim altjit being set.
  60 """
  61
  62 superpmi_replay_help = """ Location of the mch file to run a replay over. Note
  63 that this may either be a location to a path on disk or a uri to download the
  64 mch file and replay it.
  65 """
  66
  67 parser = argparse.ArgumentParser(description=description)
  68
  69 subparsers = parser.add_subparsers(dest='mode')
  70
  71 # subparser for collect
  72 collect_parser = subparsers.add_parser("collect")
  73
  74 # Add required arguments
  75 collect_parser.add_argument("collection_command", nargs=1, help=superpmi_collect_help)
  76 collect_parser.add_argument("collection_args", nargs=1, help="Arguments to pass to the SuperPMI collect command.")
  77
  78 collect_parser.add_argument("--break_on_assert", dest="break_on_assert", default=False, action="store_true")
  79 collect_parser.add_argument("--break_on_error", dest="break_on_error", default=False, action="store_true")
  80
  81 collect_parser.add_argument("-log_file", dest="log_file", default=None)
  82
  83 collect_parser.add_argument("-arch", dest="arch", nargs='?', default="x64")
  84 collect_parser.add_argument("-build_type", dest="build_type", nargs='?', default="Checked")
  85 collect_parser.add_argument("-test_location", dest="test_location", nargs="?", default=None)
  86 collect_parser.add_argument("-core_root", dest="core_root", nargs='?', default=None)
  87 collect_parser.add_argument("-product_location", dest="product_location", nargs='?', default=None)
  88 collect_parser.add_argument("-coreclr_repo_location", dest="coreclr_repo_location", default=os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
  89 collect_parser.add_argument("-test_env", dest="test_env", default=None)
  90 collect_parser.add_argument("-output_mch_path", dest="output_mch_path", default=None)
  91 collect_parser.add_argument("-run_from_coreclr_dir", dest="run_from_coreclr_dir", default=False)
  92
  93 collect_parser.add_argument("--use_zapdisable", dest="use_zapdisable", default=False, action="store_true", help="Allow redundant calls to the systems libraries for more coverage.")
  94
  95 collect_parser.add_argument("--assume_unclean_mch", dest="assume_unclean_mch", default=False, action="store_true", help="Force clean the mch file. This is useful if the dataset is large and there are expected dups.")
  96
  97 # Allow for continuing a collection in progress
  98 collect_parser.add_argument("-existing_temp_dir", dest="existing_temp_dir", default=None, nargs="?")
  99 collect_parser.add_argument("--has_run_collection_command", dest="has_run_collection_command", default=False, action="store_true")
 100 collect_parser.add_argument("--has_merged_mch", dest="has_merged_mch", default=False, action="store_true")
 101 collect_parser.add_argument("--has_verified_clean_mch", dest="has_verified_clean_mch", default=False, action="store_true")
 102
 103 collect_parser.add_argument("--skip_collect_mc_files", dest="skip_collect_mc_files", default=False, action="store_true")
 104 collect_parser.add_argument("--skip_cleanup", dest="skip_cleanup", default=False, action="store_true")
 105
 106 # subparser for replay
 107 replay_parser = subparsers.add_parser("replay")
 108
 109 # Add required arguments
 110 replay_parser.add_argument("jit_path", nargs=1, help="Path to clrjit.")
 111
 112 replay_parser.add_argument("-mch_file", nargs=1, help=superpmi_replay_help)
 113 replay_parser.add_argument("-log_file", dest="log_file", default=None)
 114
 115 replay_parser.add_argument("--break_on_assert", dest="break_on_assert", default=False, action="store_true")
 116 replay_parser.add_argument("--break_on_error", dest="break_on_error", default=False, action="store_true")
 117
 118 replay_parser.add_argument("-arch", dest="arch", nargs='?', default="x64")
 119 replay_parser.add_argument("-build_type", dest="build_type", nargs='?', default="Checked")
 120 replay_parser.add_argument("-test_location", dest="test_location", nargs="?", default=None)
 121 replay_parser.add_argument("-core_root", dest="core_root", nargs='?', default=None)
 122 replay_parser.add_argument("-product_location", dest="product_location", nargs='?', default=None)
 123 replay_parser.add_argument("-coreclr_repo_location", dest="coreclr_repo_location", default=os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 124 replay_parser.add_argument("-test_env", dest="test_env", default=None)
 125 replay_parser.add_argument("-output_mch_path", dest="output_mch_path", default=None)
 126 replay_parser.add_argument("-run_from_coreclr_dir", dest="run_from_coreclr_dir", default=False)
 127
 128 replay_parser.add_argument("--skip_collect_mc_files", dest="skip_collect_mc_files", default=False, action="store_true")
 129 replay_parser.add_argument("--skip_cleanup", dest="skip_cleanup", default=False, action="store_true")
 130 replay_parser.add_argument("--force_download", dest="force_download", default=False, action="store_true")
 131
 132 # subparser for asmDiffs
 133 asm_diff_parser = subparsers.add_parser("asmdiffs")
 134
 135 # Add required arguments
 136 asm_diff_parser.add_argument("base_jit_path", nargs=1, help="Path to baseline clrjit.")
 137 asm_diff_parser.add_argument("diff_jit_path", nargs=1, help="Path to diff clrjit.")
 138
 139 asm_diff_parser.add_argument("-mch_file", nargs=1, help=superpmi_replay_help)
 140
 141 asm_diff_parser.add_argument("-log_file", dest="log_file", default=None)
 142 asm_diff_parser.add_argument("--break_on_assert", dest="break_on_assert", default=False, action="store_true")
 143 asm_diff_parser.add_argument("--break_on_error", dest="break_on_error", default=False, action="store_true")
 144
 145 asm_diff_parser.add_argument("-arch", dest="arch", nargs='?', default="x64")
 146 asm_diff_parser.add_argument("-build_type", dest="build_type", nargs='?', default="Checked")
 147 asm_diff_parser.add_argument("-test_location", dest="test_location", nargs="?", default=None)
 148 asm_diff_parser.add_argument("-core_root", dest="core_root", nargs='?', default=None)
 149 asm_diff_parser.add_argument("-product_location", dest="product_location", nargs='?', default=None)
 150 asm_diff_parser.add_argument("-coreclr_repo_location", dest="coreclr_repo_location", default=os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 151 asm_diff_parser.add_argument("-test_env", dest="test_env", default=None)
 152 asm_diff_parser.add_argument("-output_mch_path", dest="output_mch_path", default=None)
 153 asm_diff_parser.add_argument("-run_from_coreclr_dir", dest="run_from_coreclr_dir", default=False)
 154
 155 asm_diff_parser.add_argument("--skip_collect_mc_files", dest="skip_collect_mc_files", default=False, action="store_true")
 156 asm_diff_parser.add_argument("--skip_cleanup", dest="skip_cleanup", default=False, action="store_true")
 157 asm_diff_parser.add_argument("--force_download", dest="force_download", default=False, action="store_true")
 158
 159 asm_diff_parser.add_argument("--diff_with_code", dest="diff_with_code", default=False, action="store_true")
 160 asm_diff_parser.add_argument("--diff_with_code_only", dest="diff_with_code_only", default=False, action="store_true", help="Only run the diff command, do not run SuperPMI to regenerate diffs.")
 161
 162 asm_diff_parser.add_argument("--diff_jit_dump", dest="diff_jit_dump", default=False, action="store_true")
 163 asm_diff_parser.add_argument("--diff_jit_dump_only", dest="diff_jit_dump_only", default=False, action="store_true", help="Only diff jitdumps, not asm.")
 164
 165 ################################################################################
 166 # Helper classes
 167 ################################################################################
 168
 169 class TempDir:
 170     def __init__(self, path=None):
 171         self.dir = tempfile.mkdtemp() if path is None else path
 172         self.cwd = None
 173
 174     def __enter__(self):
 175         self.cwd = os.getcwd()
 176         os.chdir(self.dir)
 177
 178         return self.dir
 179
 180     def __exit__(self, exc_type, exc_val, exc_tb):
 181         os.chdir(self.cwd)
 182         if not args.skip_cleanup:
 183             shutil.rmtree(self.dir)
 184
 185 class ChangeDir:
 186     def __init__(self, dir):
 187         self.dir = dir
 188         self.cwd = None
 189
 190     def __enter__(self):
 191         self.cwd = os.getcwd()
 192         os.chdir(self.dir)
 193
 194     def __exit__(self, exc_type, exc_val, exc_tb):
 195         os.chdir(self.cwd)
 196
 197 ################################################################################
 198 # SuperPMI Collect
 199 ################################################################################
 200
 201 class SuperPMICollect:
 202     """ SuperPMI Collect class
 203
 204     Notes:
 205         The object is responsible for setting up a super pmi collection given
 206         the arguments passed into the script.
 207     """
 208
 209     def __init__(self, args):
 210         """ Constructor
 211
 212         Args:
 213             args (CoreclrArguments): parsed args
 214
 215         """
 216
 217         if args.host_os == "OSX":
 218             self.standalone_jit_name = "libclrjit.dylib"
 219             self.collection_shim_name = "libsuperpmi-shim-collector.dylib"
 220             self.superpmi_tool_name = "superpmi"
 221             self.mcs_tool_name = "mcs"
 222         elif args.host_os == "Linux":
 223             self.standalone_jit_name = "libclrjit.so"
 224             self.collection_shim_name = "libsuperpmi-shim-collector.so"
 225             self.superpmi_tool_name = "superpmi"
 226             self.mcs_tool_name = "mcs"
 227         elif args.host_os == "Windows_NT":
 228             self.standalone_jit_name = "clrjit.dll"
 229             self.collection_shim_name = "superpmi-shim-collector.dll"
 230             self.superpmi_tool_name = "superpmi.exe"
 231             self.mcs_tool_name = "mcs.exe"
 232         else:
 233             raise RuntimeError("Unsupported OS.")
 234
 235         self.jit_path = os.path.join(args.core_root, self.standalone_jit_name)
 236         self.superpmi_path = os.path.join(args.core_root, self.superpmi_tool_name)
 237         self.mcs_path = os.path.join(args.core_root, self.mcs_tool_name)
 238
 239         self.coreclr_args = args
 240
 241         self.command = self.coreclr_args.collection_command
 242         self.args = self.coreclr_args.collection_args
 243
 244     ############################################################################
 245     # Instance Methods
 246     ############################################################################
 247
 248     def collect(self):
 249         """ Do the SuperPMI Collection.
 250         """
 251
 252         # Pathname for a temporary .MCL file used for noticing SuperPMI replay failures against base MCH.
 253         self.base_fail_mcl_file = None
 254
 255         # Pathname for a temporary .MCL file used for noticing SuperPMI replay failures against final MCH.
 256         self.final_fail_mcl_file = None
 257
 258         # The base .MCH file path
 259         self.base_mch_file = None
 260
 261         # Clean .MCH file path
 262         self.clean_mch_file = None
 263
 264         # Final .MCH file path
 265         self.final_mch_file = None
 266
 267         # The .TOC file path for the clean thin unique .MCH file
 268         self.toc_file = None
 269
 270         self.save_the_final_mch_file = False
 271
 272         # Do a basic SuperPMI collect and validation:
 273         #   1. Collect MC files by running a set of sample apps.
 274         #   2. Merge the MC files into a single MCH using "mcs -merge *.mc -recursive".
 275         #   3. Create a clean MCH by running SuperPMI over the MCH, and using "mcs -strip" to filter
 276         #       out any failures (if any).
 277         #    4. Create a thin unique MCH by using "mcs -removeDup -thin".
 278         #    5. Create a TOC using "mcs -toc".
 279         #    6. Verify the resulting MCH file is error-free when running SuperPMI against it with the
 280         #       same JIT used for collection.
 281         #
 282         #    MCH files are big. If we don't need them anymore, clean them up right away to avoid
 283         #    running out of disk space in disk constrained situations.
 284
 285         passed = False
 286
 287         try:
 288             with TempDir(self.coreclr_args.existing_temp_dir) as temp_location:
 289                 # Setup all of the temp locations
 290                 self.base_fail_mcl_file = os.path.join(temp_location, "basefail.mcl")
 291                 self.final_fail_mcl_file = os.path.join(temp_location, "finalfail.mcl")
 292
 293                 self.base_mch_file = os.path.join(temp_location, "base.mch")
 294                 self.clean_mch_file = os.path.join(temp_location, "clean.mch")
 295
 296                 self.temp_location = temp_location
 297
 298                 if self.coreclr_args.output_mch_path is not None:
 299                     self.save_the_final_mch_file = True
 300                     self.final_mch_file = os.path.abspath(self.coreclr_args.output_mch_path)
 301                     self.toc_file = self.final_mch_file + ".mct"
 302                 else:
 303                     self.save_the_final_mch_file = True
 304
 305                     default_coreclr_bin_mch_location = self.coreclr_args.default_coreclr_bin_mch_location
 306
 307                     self.final_mch_file = os.path.join(default_coreclr_bin_mch_location, "{}.{}.{}.mch".format(self.coreclr_args.host_os, self.coreclr_args.arch, self.coreclr_args.build_type))
 308                     self.toc_file = "{}.mct".format(self.final_mch_file)
 309
 310
 311                 # If we have passed existing_temp_dir, then we have a few flags we need
 312                 # to check to see where we are in the collection process. Note that this
 313                 # functionality exists to help not lose progress during a SuperPMI collection.
 314
 315
 316                 # It is not unreasonable for the SuperPMI collection to take many hours
 317                 # therefore allow re-use of a collection in progress
 318
 319                 if not self.coreclr_args.has_run_collection_command:
 320                     self.__collect_mc_files__(self.command, self.args)
 321
 322                 if not self.coreclr_args.has_merged_mch:
 323                     self.__merge_mc_files__()
 324
 325                 if not self.coreclr_args.has_verified_clean_mch:
 326                     self.__create_clean_mch_file__()
 327                     self.__create_thin_unique_mch__()
 328                     self.__create_toc__()
 329                     self.__verify_final_mch__()
 330
 331                 passed = True
 332
 333         except Exception as exception:
 334             print(exception)
 335
 336         return passed
 337
 338     ############################################################################
 339     # Helper Methods
 340     ############################################################################
 341
 342     def __collect_mc_files__(self, command, args):
 343         """ Do the actual SuperPMI collection for a command
 344
 345         Args:
 346             command (str)   : script/executable to run
 347             args ([str])    : arguments to pass
 348
 349         Returns:
 350             None
 351         """
 352
 353         if not self.coreclr_args.skip_collect_mc_files:
 354             assert os.path.isdir(self.temp_location)
 355
 356             # Set environment variables.
 357             env_copy = os.environ.copy()
 358             env_copy["SuperPMIShimLogPath"] = self.temp_location
 359             env_copy["SuperPMIShimPath"] = self.jit_path
 360             env_copy["COMPlus_AltJit"] = "*"
 361             env_copy["COMPlus_AltJitName"] = self.collection_shim_name
 362
 363             if self.coreclr_args.use_zapdisable:
 364                 env_copy["COMPlus_ZapDisable"] = "1"
 365
 366             print("Starting collection.")
 367             print("")
 368             print_platform_specific_environment_vars(self.coreclr_args, "SuperPMIShimLogPath", self.temp_location)
 369             print_platform_specific_environment_vars(self.coreclr_args, "SuperPMIShimPath", self.jit_path)
 370             print_platform_specific_environment_vars(self.coreclr_args, "COMPlus_AltJit", "*")
 371             print_platform_specific_environment_vars(self.coreclr_args, "COMPlus_AltJitName", self.collection_shim_name)
 372             print("")
 373             print("%s %s" % (command, " ".join(args)))
 374
 375             assert isinstance(command, str)
 376             assert isinstance(args, list)
 377
 378             return_code = 1
 379
 380             command = [command] + args
 381             proc = subprocess.Popen(command, env=env_copy)
 382
 383             proc.communicate()
 384             return_code = proc.returncode
 385
 386         contents = os.listdir(self.temp_location)
 387         mc_contents = [os.path.join(self.temp_location, item) for item in contents if ".mc" in item]
 388
 389         if len(mc_contents) == 0:
 390             raise RuntimeError("No .mc files generated.")
 391
 392         self.mc_contents = mc_contents
 393
 394     def __merge_mc_files__(self):
 395         """ Merge the mc files that were generated
 396
 397         Notes:
 398             mcs -merge <s_baseMchFile> <s_tempDir>\*.mc -recursive
 399
 400         """
 401
 402         pattern = os.path.join(self.temp_location, "*.mc")
 403
 404         command = [self.mcs_path, "-merge", self.base_mch_file, pattern, "-recursive"]
 405         print("Invoking: " + " ".join(command))
 406         proc = subprocess.Popen(command)
 407
 408         proc.communicate()
 409
 410         if not os.path.isfile(self.mcs_path):
 411             raise RuntimeError("mch file failed to be generated at: %s" % self.mcs_path)
 412
 413         contents = os.listdir(self.temp_location)
 414         mc_contents = [os.path.join(self.temp_location, item) for item in contents if ".mc" in item and not ".mch" in item]
 415
 416         # All the individual MC files are no longer necessary, now that we have
 417         # merged them into the base.mch. Delete them.
 418         if not self.coreclr_args.skip_cleanup:
 419             for item in mc_contents:
 420                 os.remove(item)
 421
 422     def __create_clean_mch_file__(self):
 423         """ Create a clean mch file based on the original
 424
 425         Notes:
 426             <SuperPMIPath> -p -f <s_baseFailMclFile> <s_baseMchFile> <jitPath>
 427
 428             if <s_baseFailMclFile> is non-empty:
 429                 <mcl> -strip <s_baseFailMclFile> <s_baseMchFile> <s_cleanMchFile>
 430             else
 431                 # no need to copy, just change the names
 432                 clean_mch_file = base_mch_file
 433             del <s_baseFailMclFile>
 434         """
 435
 436         command = [self.superpmi_path, "-p", "-f", self.base_fail_mcl_file, self.base_mch_file, self.jit_path]
 437         print (" ".join(command))
 438         proc = subprocess.Popen(command)
 439
 440         proc.communicate()
 441
 442         if os.path.isfile(self.base_fail_mcl_file) and os.stat(self.base_fail_mcl_file).st_size != 0:
 443             command = [self.mcs_path, "-strip", self.base_fail_mcl_file, self.base_mch_file, self.clean_mch_file]
 444             print (" ".join(command))
 445             proc = subprocess.Popen(command)
 446
 447             proc.communicate()
 448         else:
 449             self.clean_mch_file = self.base_mch_file
 450             self.base_mch_file = None
 451
 452         if not os.path.isfile(self.clean_mch_file):
 453             raise RuntimeError("Clean mch file failed to be generated.")
 454
 455         if not self.coreclr_args.skip_cleanup:
 456             if os.path.isfile(self.base_fail_mcl_file):
 457                 os.remove(self.base_fail_mcl_file)
 458                 self.base_fail_mcl_file = None
 459
 460             # The base file is no longer used (unless there was no cleaning done, in which case
 461             # self.base_mch_file has been set to None and clean_mch_File is the base file).
 462             if os.path.isfile(self.base_mch_file):
 463                 os.remove(self.base_mch_file)
 464                 self.base_mch_file = None
 465
 466     def __create_thin_unique_mch__(self):
 467         """  Create a thin unique MCH
 468
 469         Notes:
 470             <mcl> -removeDup -thin <s_cleanMchFile> <s_finalMchFile>
 471         """
 472
 473         command = [self.mcs_path, "-removeDup", "-thin", self.clean_mch_file, self.final_mch_file]
 474         proc = subprocess.Popen(command)
 475         proc.communicate()
 476
 477         if not os.path.isfile(self.final_mch_file):
 478             raise RuntimeError("Error, final mch file not created correctly.")
 479
 480         if not self.coreclr_args.skip_cleanup:
 481             os.remove(self.clean_mch_file)
 482             self.clean_mch_file = None
 483
 484     def __create_toc__(self):
 485         """ Create a TOC file
 486
 487         Notes:
 488             <mcl> -toc <s_finalMchFile>
 489         """
 490
 491         command = [self.mcs_path, "-toc", self.final_mch_file]
 492         proc = subprocess.Popen(command)
 493         proc.communicate()
 494
 495         if not os.path.isfile(self.toc_file):
 496             raise RuntimeError("Error, toc file not created correctly.")
 497
 498     def __verify_final_mch__(self):
 499         """ Verify the resulting MCH file is error-free when running SuperPMI against it with the same JIT used for collection.
 500
 501         Notes:
 502             <superPmiPath> -p -f <s_finalFailMclFile> <s_finalMchFile> <jitPath>
 503         """
 504
 505         spmi_replay = SuperPMIReplay(self.coreclr_args, self.final_mch_file, self.jit_path)
 506         passed = spmi_replay.replay()
 507
 508         if not passed:
 509             raise RuntimeError("Error unclean replay.")
 510
 511 ################################################################################
 512 # SuperPMI Replay
 513 ################################################################################
 514
 515 class SuperPMIReplay:
 516     """ SuperPMI Replay class
 517
 518     Notes:
 519         The object is responsible for replaying the mch final given to the
 520         instance of the class
 521     """
 522
 523     def __init__(self, coreclr_args, mch_file, jit_path):
 524         """ Constructor
 525
 526         Args:
 527             args (CoreclrArguments) : parsed args
 528             mch_file (str)          : final mch file from the collection
 529             jit_path (str)          : path to clrjit/libclrjit
 530
 531         """
 532
 533         self.jit_path = jit_path
 534         self.mch_file = mch_file
 535         self.superpmi_path = os.path.join(coreclr_args.core_root, "superpmi")
 536
 537         self.coreclr_args = coreclr_args
 538
 539     ############################################################################
 540     # Instance Methods
 541     ############################################################################
 542
 543     def replay(self):
 544         """ Replay the given SuperPMI collection
 545
 546         Returns:
 547             sucessful_replay (bool)
 548         """
 549
 550         return_code = False
 551
 552         # Possible return codes from SuperPMI
 553         #
 554         # 0  : success
 555         # -1 : general fatal error (e.g., failed to initialize, failed to read files)
 556         # -2 : JIT failed to initialize
 557         # 1  : there were compilation failures
 558         # 2  : there were assembly diffs
 559
 560         with TempDir() as temp_location:
 561             print("Starting SuperPMI replay.")
 562             print("")
 563             print("Temp Location: {}".format(temp_location))
 564             print("")
 565
 566             self.fail_mcl_file = os.path.join(temp_location, "fail.mcl")
 567
 568             # TODO: add aljit support
 569             #
 570             # Set: -jitoption force AltJit=* -jitoption force AltJitNgen=*
 571             force_altjit_options = [
 572                 "-jitoption",
 573                 "force",
 574                 "AltJit=",
 575                 "-jitoption",
 576                 "force",
 577                 "AltJitNgen="
 578             ]
 579
 580             flags = [
 581                 "-p", # Parallel
 582                 "-f", # Failing mc List
 583                 self.fail_mcl_file,
 584                 "-r", # Repro name, create .mc repro files
 585                 os.path.join(temp_location, "repro")
 586             ]
 587
 588             flags += force_altjit_options
 589
 590             if self.coreclr_args.break_on_assert:
 591                 flags += [
 592                     "-boa" # break on assert
 593                 ]
 594
 595             if self.coreclr_args.break_on_error:
 596                 flags += [
 597                     "-boe" # break on error
 598                 ]
 599
 600             if self.coreclr_args.log_file != None:
 601                 flags += [
 602                     "-w",
 603                     self.coreclr_args.log_file
 604                 ]
 605
 606             command = [self.superpmi_path] + flags + [self.jit_path, self.mch_file]
 607
 608             print("Invoking: " + " ".join(command))
 609             proc = subprocess.Popen(command)
 610             proc.communicate()
 611
 612             return_code = proc.returncode
 613
 614             if return_code == 0:
 615                 print("Clean SuperPMI Replay")
 616                 return_code = True
 617
 618             if os.path.isfile(self.fail_mcl_file) and os.stat(self.fail_mcl_file).st_size != 0:
 619                 # Unclean replay.
 620                 #
 621                 # Save the contents of the fail.mcl file to dig into failures.
 622
 623                 assert(return_code != 0)
 624
 625                 if return_code == -1:
 626                     print("General fatal error.")
 627                 elif return_code == -2:
 628                     print("Jit failed to initialize.")
 629                 elif return_code == 1:
 630                     print("Complition failures.")
 631                 else:
 632                     print("Unknown error code.")
 633
 634                 self.fail_mcl_contents = None
 635                 with open(self.fail_mcl_file) as file_handle:
 636                     self.fail_mcl_contents = file_handle.read()
 637
 638                 # If there are any .mc files, drop them into bin/repro/<host_os>.<arch>.<build_type>/*.mc
 639                 mc_files = [os.path.join(temp_location, item) for item in os.listdir(temp_location) if item.endswith(".mc")]
 640
 641                 if len(mc_files) > 0:
 642                     repro_location = os.path.join(self.coreclr_args.coreclr_repo_location, "bin", "repro", "{}.{}.{}".format(self.coreclr_args.host_os, self.coreclr_args.arch, self.coreclr_args.build_type))
 643
 644                     # Delete existing repro location
 645                     if os.path.isdir(repro_location):
 646                         shutil.rmtree(repro_location)
 647
 648                     assert(not os.path.isdir(repro_location))
 649
 650                     os.makedirs(repro_location)
 651
 652                     repro_files = []
 653                     for item in mc_files:
 654                         repro_files.append(os.path.join(repro_location, os.path.basename(item)))
 655                         shutil.copy2(item, repro_location)
 656
 657                     print("")
 658                     print("Repro .mc files:")
 659                     print("")
 660
 661                     for item in repro_files:
 662                         print(item)
 663
 664                     print("")
 665
 666                     print("To run an specific failure:")
 667                     print("")
 668                     print("<SuperPMI_path>/SuperPMI <core_root|product_dir>/clrjit.dll|libclrjit.so|libclrjit.dylib <<coreclr_path>/bin/repro/<host_os>.<arch>.<build_type>/1xxxx.mc")
 669                     print("")
 670
 671                 else:
 672                     print(self.fail_mcl_contents)
 673
 674             if not self.coreclr_args.skip_cleanup:
 675                 if os.path.isfile(self.fail_mcl_file):
 676                     os.remove(self.fail_mcl_file)
 677                     self.fail_mcl_file = None
 678
 679                 return_code == False
 680
 681         return return_code
 682
 683 ################################################################################
 684 # SuperPMI Replay/AsmDiffs
 685 ################################################################################
 686
 687 class SuperPMIReplayAsmDiffs:
 688     """ SuperPMI Replay AsmDiffs class
 689
 690     Notes:
 691         The object is responsible for replaying the mch final given to the
 692         instance of the class and doing diffs on the two passed jits.
 693     """
 694
 695     def __init__(self, coreclr_args, mch_file, base_jit_path, diff_jit_path):
 696         """ Constructor
 697
 698         Args:
 699             args (CoreclrArguments) : parsed args
 700             mch_file (str)          : final mch file from the collection
 701             base_jit_path (str)     : path to clrjit/libclrjit
 702             diff_jit_path (str)     : path to clrjit/libclrjit
 703
 704         """
 705
 706         self.base_jit_path = base_jit_path
 707         self.diff_jit_path = diff_jit_path
 708
 709         self.mch_file = mch_file
 710         self.superpmi_path = os.path.join(coreclr_args.core_root, "superpmi")
 711
 712         self.coreclr_args = coreclr_args
 713
 714     ############################################################################
 715     # Instance Methods
 716     ############################################################################
 717
 718     def replay_with_asm_diffs(self):
 719         """ Replay the given SuperPMI collection
 720
 721         Returns:
 722             sucessful_replay (bool)
 723         """
 724
 725         return_code = False
 726
 727         # Possible return codes from SuperPMI
 728         #
 729         # 0  : success
 730         # -1 : general fatal error (e.g., failed to initialize, failed to read files)
 731         # -2 : JIT failed to initialize
 732         # 1  : there were compilation failures
 733         # 2  : there were assembly diffs
 734
 735         with TempDir() as temp_location:
 736             print("Starting SuperPMI AsmDiffs.")
 737             print("")
 738             print("Temp Location: {}".format(temp_location))
 739             print("")
 740
 741             self.fail_mcl_file = os.path.join(temp_location, "fail.mcl")
 742             self.diff_mcl_file = os.path.join(temp_location, "diff.mcl")
 743
 744             # TODO: add aljit support
 745             #
 746             # Set: -jitoption force AltJit=* -jitoption force AltJitNgen=*
 747             force_altjit_options = [
 748                 "-jitoption",
 749                 "force",
 750                 "AltJit=",
 751                 "-jitoption",
 752                 "force",
 753                 "AltJitNgen=",
 754                 "-jit2option",
 755                 "force",
 756                 "AltJit=",
 757                 "-jit2option",
 758                 "force",
 759                 "AltJitNgen="
 760             ]
 761
 762             flags = [
 763                 "-a", # Asm diffs
 764                 "-p", # Parallel
 765                 "-f", # Failing mc List
 766                 self.fail_mcl_file,
 767                 "-diffMCList", # Create all of the diffs in an mcl file
 768                 self.diff_mcl_file,
 769                 "-r", # Repro name, create .mc repro files
 770                 os.path.join(temp_location, "repro")
 771             ]
 772
 773             flags += force_altjit_options
 774
 775             if self.coreclr_args.break_on_assert:
 776                 flags += [
 777                     "-boa" # break on assert
 778                 ]
 779
 780             if self.coreclr_args.break_on_error:
 781                 flags += [
 782                     "-boe" # break on error
 783                 ]
 784
 785             if self.coreclr_args.log_file != None:
 786                 flags += [
 787                     "-w",
 788                     self.coreclr_args.log_file
 789                 ]
 790
 791             if not self.coreclr_args.diff_with_code_only:
 792                 # Change the working directory to the core root we will call SuperPMI from.
 793                 # This is done to allow libcoredistools to be loaded correctly on unix
 794                 # as the loadlibrary path will be relative to the current directory.
 795                 with ChangeDir(self.coreclr_args.core_root) as dir:
 796                     command = [self.superpmi_path] + flags + [self.base_jit_path, self.diff_jit_path, self.mch_file]
 797
 798                     print("Invoking: " + " ".join(command))
 799                     proc = subprocess.Popen(command)
 800                     proc.communicate()
 801
 802                 return_code = proc.returncode
 803
 804                 if return_code == 0:
 805                     print("Clean SuperPMI Replay")
 806
 807             else:
 808                 return_code = 2
 809
 810             if os.path.isfile(self.fail_mcl_file) and os.stat(self.fail_mcl_file).st_size != 0:
 811                 # Unclean replay.
 812                 #
 813                 # Save the contents of the fail.mcl file to dig into failures.
 814
 815                 assert(return_code != 0)
 816
 817                 if return_code == -1:
 818                     print("General fatal error.")
 819                 elif return_code == -2:
 820                     print("Jit failed to initialize.")
 821                 elif return_code == 1:
 822                     print("Complition failures.")
 823                 elif return_code == 139 and self.coreclr_args != "Windows_NT":
 824                     print("Fatal error, SuperPMI has returned SIG_SEV (segmentation fault).")
 825                 else:
 826                     print("Unknown error code.")
 827
 828                 self.fail_mcl_contents = None
 829                 mcl_lines = []
 830                 with open(self.fail_mcl_file) as file_handle:
 831                     mcl_lines = file_handle.readlines()
 832                     mcl_lines = [item.strip() for item in mcl_lines]
 833                     self.fail_mcl_contents = os.linesep.join(mcl_lines)
 834
 835                 # If there are any .mc files, drop them into bin/repro/<host_os>.<arch>.<build_type>/*.mc
 836                 mc_files = [os.path.join(temp_location, item) for item in os.listdir(temp_location) if item.endswith(".mc")]
 837
 838                 if len(mc_files) > 0:
 839                     repro_location = os.path.join(self.coreclr_args.coreclr_repo_location, "bin", "repro", "{}.{}.{}".format(self.coreclr_args.host_os, self.coreclr_args.arch, self.coreclr_args.build_type))
 840
 841                     # Delete existing repro location
 842                     if os.path.isdir(repro_location):
 843                         shutil.rmtree(repro_location)
 844
 845                     assert(not os.path.isdir(repro_location))
 846
 847                     os.makedirs(repro_location)
 848
 849                     repro_files = []
 850                     for item in mc_files:
 851                         repro_files.append(os.path.join(repro_location, os.path.basename(item)))
 852                         shutil.copy2(item, repro_location)
 853
 854                     print("")
 855                     print("Repro .mc files:")
 856                     print("")
 857
 858                     for item in repro_files:
 859                         print(item)
 860
 861                     print("")
 862
 863                     print("To run an specific failure:")
 864                     print("")
 865                     print("<SuperPMI_path>/SuperPMI <core_root|product_dir>/clrjit.dll|libclrjit.so|libclrjit.dylib <<coreclr_path>/bin/repro/<host_os>.<arch>.<build_type>/1xxxx.mc")
 866                     print("")
 867
 868                 print(self.fail_mcl_contents)
 869
 870             # There were diffs. Go through each method that created diffs and
 871             # create a base/diff asm file with diffable asm. In addition, create
 872             # a standalone .mc for easy iteration.
 873             if os.path.isfile(self.diff_mcl_file) and os.stat(self.diff_mcl_file).st_size != 0 or self.coreclr_args.diff_with_code_only:
 874                 # AsmDiffs.
 875                 #
 876                 # Save the contents of the fail.mcl file to dig into failures.
 877
 878                 assert(return_code != 0)
 879
 880                 if return_code == -1:
 881                     print("General fatal error.")
 882                 elif return_code == -2:
 883                     print("Jit failed to initialize.")
 884                 elif return_code == 1:
 885                     print("Complition failures.")
 886                 elif return_code == 139 and self.coreclr_args != "Windows_NT":
 887                     print("Fatal error, SuperPMI has returned SIG_SEV (segmentation fault).")
 888                 else:
 889                     print("Unknown error code.")
 890
 891                 if not self.coreclr_args.diff_with_code_only:
 892                     self.diff_mcl_contents = None
 893                     with open(self.diff_mcl_file) as file_handle:
 894                         mcl_lines = file_handle.readlines()
 895                         mcl_lines = [item.strip() for item in mcl_lines]
 896                         self.diff_mcl_contents = mcl_lines
 897
 898                 base_asm_location = os.path.join(self.coreclr_args.bin_location, "asm", "base")
 899                 diff_asm_location = os.path.join(self.coreclr_args.bin_location, "asm", "diff")
 900
 901                 base_dump_location = os.path.join(self.coreclr_args.bin_location, "jit_dump", "base")
 902                 diff_dump_location = os.path.join(self.coreclr_args.bin_location, "jit_dump", "diff")
 903
 904                 if not self.coreclr_args.diff_with_code_only:
 905                     # Delete the old asm.
 906
 907                     # Create a diff and baseline directory
 908                     if os.path.isdir(base_asm_location):
 909                         shutil.rmtree(base_asm_location)
 910                     if os.path.isdir(diff_asm_location):
 911                         shutil.rmtree(diff_asm_location)
 912
 913                     os.makedirs(base_asm_location)
 914                     os.makedirs(diff_asm_location)
 915
 916                     assert(os.path.isdir(base_asm_location))
 917                     assert(os.path.isdir(diff_asm_location))
 918
 919                     assert(len(os.listdir(base_asm_location)) == 0)
 920                     assert(len(os.listdir(diff_asm_location)) == 0)
 921
 922                     if self.coreclr_args.diff_jit_dump:
 923                         # Create a diff and baseline directory for jit_dumps
 924                         if os.path.isdir(base_dump_location):
 925                             shutil.rmtree(base_dump_location)
 926                         if os.path.isdir(diff_dump_location):
 927                             shutil.rmtree(diff_dump_location)
 928
 929                         os.makedirs(base_dump_location)
 930                         os.makedirs(diff_dump_location)
 931
 932                         assert(os.path.isdir(base_dump_location))
 933                         assert(os.path.isdir(diff_dump_location))
 934
 935                         assert(len(os.listdir(base_dump_location)) == 0)
 936                         assert(len(os.listdir(diff_dump_location)) == 0)
 937
 938                 text_differences = []
 939                 jit_dump_differences = []
 940
 941                 if not self.coreclr_args.diff_with_code_only:
 942                     for item in self.diff_mcl_contents:
 943                         # Setup to call SuperPMI for both the diff jit and the base
 944                         # jit
 945
 946                         # TODO: add aljit support
 947                         #
 948                         # Set: -jitoption force AltJit=* -jitoption force AltJitNgen=*
 949                         force_altjit_options = [
 950                             "-jitoption",
 951                             "force",
 952                             "AltJit=",
 953                             "-jitoption",
 954                             "force",
 955                             "AltJitNgen="
 956                         ]
 957
 958                         flags = [
 959                             "-c",
 960                             item,
 961                             "-v",
 962                             "q" # only log from the jit.
 963                         ]
 964
 965                         flags += force_altjit_options
 966
 967                         asm_env = os.environ.copy()
 968                         asm_env["COMPlus_JitDisasm"] = "*"
 969                         asm_env["COMPlus_JitUnwindDump"] = "*"
 970                         asm_env["COMPlus_JitEHDump"] = "*"
 971                         asm_env["COMPlus_JitDiffableDasm"] = "1"
 972                         asm_env["COMPlus_NgenDisasm"] = "*"
 973                         asm_env["COMPlus_NgenDump"] = "*"
 974                         asm_env["COMPlus_NgenUnwindDump"] = "*"
 975                         asm_env["COMPlus_NgenEHDump"] = "*"
 976                         asm_env["COMPlus_JitEnableNoWayAssert"] = "1"
 977                         asm_env["COMPlus_JitNoForceFallback"] = "1"
 978                         asm_env["COMPlus_JitRequired"] = "1"
 979
 980                         jit_dump_env = os.environ.copy()
 981                         jit_dump_env["COMPlus_JitEnableNoWayAssert"] = "1"
 982                         jit_dump_env["COMPlus_JitNoForceFallback"] = "1"
 983                         jit_dump_env["COMPlus_JitRequired"] = "1"
 984                         jit_dump_env["COMPlus_JitDump"] = "*"
 985
 986                         # Change the working directory to the core root we will call SuperPMI from.
 987                         # This is done to allow libcoredistools to be loaded correctly on unix
 988                         # as the loadlibrary path will be relative to the current directory.
 989                         with ChangeDir(self.coreclr_args.core_root) as dir:
 990                             command = [self.superpmi_path] + flags + [self.base_jit_path, self.mch_file]
 991
 992                             # Generate diff and base asm
 993                             base_txt = None
 994                             diff_txt = None
 995
 996                             with open(os.path.join(base_asm_location, "{}.asm".format(item)), 'w') as file_handle:
 997                                 print("Invoking: " + " ".join(command))
 998                                 proc = subprocess.Popen(command, env=asm_env, stdout=file_handle)
 999                                 proc.communicate()
1000
1001                             command = [self.superpmi_path] + flags + [self.diff_jit_path, self.mch_file]
1002
1003                             with open(os.path.join(diff_asm_location, "{}.asm".format(item)), 'w') as file_handle:
1004                                 print("Invoking: " + " ".join(command))
1005                                 proc = subprocess.Popen(command, env=asm_env, stdout=file_handle)
1006                                 proc.communicate()
1007
1008                             with open(os.path.join(base_asm_location, "{}.asm".format(item))) as file_handle:
1009                                 base_txt = file_handle.read()
1010
1011                             with open(os.path.join(diff_asm_location, "{}.asm".format(item))) as file_handle:
1012                                 diff_txt = file_handle.read()
1013
1014                             if base_txt != diff_txt:
1015                                 text_differences.append(item)
1016
1017                             if self.coreclr_args.diff_jit_dump:
1018                                 # Generate jit dumps
1019                                 base_txt = None
1020                                 diff_txt = None
1021
1022                                 command = [self.superpmi_path] + flags + [self.base_jit_path, self.mch_file]
1023
1024                                 with open(os.path.join(base_dump_location, "{}.txt".format(item)), 'w') as file_handle:
1025                                     print("Invoking: " + " ".join(command))
1026                                     proc = subprocess.Popen(command, env=jit_dump_env, stdout=file_handle)
1027                                     proc.communicate()
1028
1029                                 command = [self.superpmi_path] + flags + [self.diff_jit_path, self.mch_file]
1030
1031                                 with open(os.path.join(diff_dump_location, "{}.txt".format(item)), 'w') as file_handle:
1032                                     print("Invoking: " + " ".join(command))
1033                                     proc = subprocess.Popen(command, env=jit_dump_env, stdout=file_handle)
1034                                     proc.communicate()
1035
1036                                 with open(os.path.join(base_dump_location, "{}.txt".format(item))) as file_handle:
1037                                     base_txt = file_handle.read()
1038
1039                                 with open(os.path.join(diff_dump_location, "{}.txt".format(item))) as file_handle:
1040                                     diff_txt = file_handle.read()
1041
1042                                 if base_txt != diff_txt:
1043                                     jit_dump_differences.append(item)
1044
1045                 else:
1046                     # We have already generated asm under <coreclr_bin_path>/asm/base and <coreclr_bin_path>/asm/diff
1047                     for item in os.listdir(base_asm_location):
1048                         base_asm_file = os.path.join(base_asm_location, item)
1049                         diff_asm_file = os.path.join(diff_asm_location, item)
1050
1051                         base_txt = None
1052                         diff_txt = None
1053
1054                         # Every file should have a diff asm file.
1055                         assert os.path.isfile(diff_asm_file)
1056
1057                         with open(base_asm_file) as file_handle:
1058                             base_txt = file_handle.read()
1059
1060                         with open(diff_asm_file) as file_handle:
1061                             diff_txt = file_handle.read()
1062
1063                         if base_txt != diff_txt:
1064                             text_differences.append(item[:-4])
1065
1066                     if self.coreclr_args.diff_jit_dump:
1067                         for item in os.listdir(base_dump_location):
1068                             base_dump_file = os.path.join(base_dump_location, item)
1069                             diff_dump_file = os.path.join(diff_dump_location, item)
1070
1071                             base_txt = None
1072                             diff_txt = None
1073
1074                             # Every file should have a diff asm file.
1075                             assert os.path.isfile(diff_dump_file)
1076
1077                             with open(base_dump_file) as file_handle:
1078                                 base_txt = file_handle.read()
1079
1080                             with open(diff_dump_file) as file_handle:
1081                                 diff_txt = file_handle.read()
1082
1083                             if base_txt != diff_txt:
1084                                 jit_dump_differences.append(item[:-4])
1085
1086                 if not self.coreclr_args.diff_with_code_only:
1087                     print("Differences found, to replay SuperPMI use <path_to_SuperPMI> -jitoption force AltJit= -jitoption force AltJitNgen= -c ### <path_to_jit> <path_to_mcl>")
1088                     print("")
1089                     print("Binary differences found with superpmi -a")
1090                     print("")
1091                     print("Method numbers with binary differences:")
1092                     print(self.diff_mcl_contents)
1093                     print("")
1094
1095                 if len(text_differences) > 0:
1096                     print("Textual differences found, the asm is located under %s and %s" % (base_asm_location, diff_asm_location))
1097                     print("")
1098                     print("Method numbers with textual differences:")
1099
1100                     print(text_differences)
1101
1102                     if self.coreclr_args.diff_with_code and not self.coreclr_args.diff_jit_dump_only:
1103                         batch_command = ["cmd", "/c"] if platform.system() == "Windows" else []
1104                         for index, item in enumerate(text_differences):
1105                             command = batch_command + [
1106                                 "code",
1107                                 "-d",
1108                                 os.path.join(base_asm_location, "{}.asm".format(item)),
1109                                 os.path.join(diff_asm_location, "{}.asm".format(item))
1110                             ]
1111                             print("Invoking: " + " ".join(command))
1112                             proc = subprocess.Popen(command)
1113
1114                             if index > 5:
1115                                 break
1116
1117                     print("")
1118                 else:
1119                     print("No textual differences. Is this an issue with libcoredistools?")
1120
1121                 if len(jit_dump_differences) > 0:
1122                     print("Diffs found in the JitDump generated. These files are located under <coreclr_dir>/bin/jit_dump/base and <coreclr_dir>/bin/jit_dump/diff")
1123                     print("")
1124                     print("Method numbers with textual differences:")
1125
1126                     print(jit_dump_differences)
1127
1128                     if self.coreclr_args.diff_with_code:
1129                         batch_command = ["cmd", "/c"] if platform.system() == "Windows" else []
1130                         for index, item in enumerate(text_differences):
1131                             command = batch_command + [
1132                                 "code",
1133                                 "-d",
1134                                 os.path.join(base_dump_location, "{}.txt".format(item)),
1135                                 os.path.join(diff_dump_location, "{}.txt".format(item))
1136                             ]
1137                             print("Invoking: " + " ".join(command))
1138                             proc = subprocess.Popen(command)
1139
1140                             if index > 5:
1141                                 break
1142
1143                     print("")
1144
1145             if not self.coreclr_args.skip_cleanup:
1146                 if os.path.isfile(self.fail_mcl_file):
1147                     os.remove(self.fail_mcl_file)
1148                     self.fail_mcl_file = None
1149
1150         return return_code
1151
1152 ################################################################################
1153 # Helper Methods
1154 ################################################################################
1155
1156 def determine_coredis_tools(coreclr_args):
1157     """ Determine the coredistools location
1158
1159     Args:
1160         coreclr_args (CoreclrArguments) : parsed args
1161
1162     Returns:
1163         coredistools_location (str)     : path of libcoredistools.dylib|so|dll
1164
1165     Notes:
1166         If unable to find libcoredist tools, download it from azure storage.
1167     """
1168
1169     coredistools_dll_name = None
1170     if coreclr_args.host_os.lower() == "osx":
1171         coredistools_dll_name = "libcoredistools.dylib"
1172     elif coreclr_args.host_os.lower() == "linux":
1173         coredistools_dll_name = "libcoredistools.so"
1174     elif coreclr_args.host_os.lower() == "windows_nt":
1175         coredistools_dll_name = "coredistools.dll"
1176     else:
1177         raise RuntimeError("Unknown host os: {}").format(coreclr_args.host_os)
1178
1179     coredistools_uri = "https://clrjit.blob.core.windows.net/superpmi/libcoredistools/{}-{}/{}".format(coreclr_args.host_os.lower(), coreclr_args.arch.lower(), coredistools_dll_name)
1180
1181     coredistools_location = os.path.join(coreclr_args.core_root, coredistools_dll_name)
1182     if not os.path.isfile(coredistools_location):
1183         urlretrieve = urllib.urlretrieve if sys.version_info.major < 3 else urllib.request.urlretrieve
1184         urlretrieve(coredistools_uri, coredistools_location)
1185
1186     assert os.path.isfile(coredistools_location)
1187     return coredistools_location
1188
1189 def determine_jit_name(coreclr_args):
1190     """ Determine the jit based on the os
1191
1192     Args:
1193         coreclr_args (CoreclrArguments): parsed args
1194
1195     Return:
1196         jit_name(str) : name of the jit for this os
1197     """
1198
1199     if coreclr_args.host_os == "OSX":
1200         return "libclrjit.dylib"
1201     elif coreclr_args.host_os == "Linux":
1202         return "libclrjit.so"
1203     elif coreclr_args.host_os == "Windows_NT":
1204         return "clrjit.dll"
1205     else:
1206         raise RuntimeError("Unknown os.")
1207
1208 def print_platform_specific_environment_vars(coreclr_args, var, value):
1209     """ Print environment variables as set {}={} or export {}={}
1210
1211     Args:
1212         coreclr_args (CoreclrArguments): parsed args
1213         var   (str): variable to set
1214         value (str): value being set.
1215     """
1216
1217     if coreclr_args.host_os == "Windows_NT":
1218         print("set {}={}".format(var, value))
1219     else:
1220         print("export {}={}".format(var, value))
1221
1222 def setup_args(args):
1223     """ Setup the args for SuperPMI to use.
1224
1225     Args:
1226         args (ArgParse): args parsed by arg parser
1227
1228     Returns:
1229         args (CoreclrArguments)
1230
1231     """
1232     coreclr_args = CoreclrArguments(args, require_built_core_root=True, require_built_product_dir=False, require_built_test_dir=False, default_build_type="Checked")
1233
1234     coreclr_args.verify(args,
1235                         "skip_cleanup",
1236                         lambda unused: True,
1237                         "Unable to set skip_cleanup")
1238
1239     coreclr_args.verify(args,
1240                         "mode",
1241                         lambda mode: mode in ["collect", "replay", "asmdiffs"],
1242                         'Incorrect mode passed, please choose from ["collect", "replay", "asmdiffs"]')
1243
1244     coreclr_args.verify(args,
1245                         "run_from_coreclr_dir",
1246                         lambda unused: True,
1247                         "Error setting run_from_coreclr_dir")
1248
1249     default_coreclr_bin_mch_location = os.path.join(coreclr_args.bin_location, "mch", "{}.{}.{}".format(coreclr_args.host_os, coreclr_args.arch, coreclr_args.build_type))
1250
1251     def setup_mch_arg(arg):
1252         default_mch_location = os.path.join(coreclr_args.bin_location, "mch", "{}.{}.{}".format(coreclr_args.host_os, coreclr_args.arch, coreclr_args.build_type), "{}.{}.{}.mch".format(coreclr_args.host_os, coreclr_args.arch, coreclr_args.build_type))
1253
1254         if os.path.isfile(default_mch_location) and not args.force_download:
1255             return default_mch_location
1256
1257         # Download the mch
1258         else:
1259             uri_mch_location = "https://clrjit.blob.core.windows.net/superpmi/{}/{}/{}/{}.{}.{}.mch.zip".format(coreclr_args.host_os, coreclr_args.arch, coreclr_args.build_type, coreclr_args.host_os, coreclr_args.arch, coreclr_args.build_type)
1260
1261             with TempDir() as temp_location:
1262                 urlretrieve = urllib.urlretrieve if sys.version_info.major < 3 else urllib.request.urlretrieve
1263                 zipfilename = os.path.join(temp_location, "temp.zip")
1264                 urlretrieve(uri_mch_location, zipfilename)
1265
1266                 default_mch_dir = os.path.join(coreclr_args.bin_location, "mch", "{}.{}.{}".format(coreclr_args.host_os, coreclr_args.arch, coreclr_args.build_type))
1267
1268                 # Clean all the files out of the default location.
1269                 default_mch_dir_items = [os.path.join(default_mch_dir, item) for item in os.listdir(default_mch_dir)]
1270                 for item in default_mch_dir_items:
1271                     if os.path.isdir(item):
1272                         shutil.rmtree(item)
1273                     else:
1274                         os.remove(item)
1275
1276                 if not os.path.isdir(default_mch_dir):
1277                     os.makedirs(default_mch_dir)
1278
1279                 with zipfile.ZipFile(zipfilename, "r") as file_handle:
1280                     file_handle.extractall(temp_location)
1281
1282                 items = [os.path.join(temp_location, item) for item in os.listdir(temp_location) if not item.endswith(".zip")]
1283
1284                 for item in items:
1285                     shutil.copy2(item, default_mch_dir)
1286
1287             return default_mch_location
1288
1289     if not os.path.isdir(default_coreclr_bin_mch_location):
1290         os.makedirs(default_coreclr_bin_mch_location)
1291
1292     coreclr_args.verify(default_coreclr_bin_mch_location,
1293                         "default_coreclr_bin_mch_location",
1294                         lambda unused: True,
1295                         "Error setting default_coreclr_bin_mch_location")
1296
1297     if coreclr_args.mode == "collect":
1298         coreclr_args.verify(args,
1299                             "collection_command",
1300                             lambda command_list: len(command_list) == 1,
1301                             "Unable to find script.",
1302                             modify_arg=lambda arg: arg[0],
1303                             modify_after_validation=True)
1304         coreclr_args.verify(args,
1305                             "collection_args",
1306                             lambda unused: True,
1307                             "Unable to set collection_args",
1308                             modify_arg=lambda collection_args: collection_args[0].split(" ") if collection_args is not None else collection_args)
1309
1310         coreclr_args.verify(args,
1311                             "output_mch_path",
1312                             lambda unused: True,
1313                             "Unable to set output_mch_path")
1314
1315         coreclr_args.verify(args,
1316                             "skip_collect_mc_files",
1317                             lambda unused: True,
1318                             "Unable to set skip_collect_mc_files")
1319
1320         coreclr_args.verify(args,
1321                             "existing_temp_dir",
1322                             lambda unused: True,
1323                             "Unable to set existing_temp_dir.")
1324
1325         coreclr_args.verify(args,
1326                             "assume_unclean_mch",
1327                             lambda unused: True,
1328                             "Unable to set assume_unclean_mch.")
1329
1330         coreclr_args.verify(args,
1331                             "has_run_collection_command",
1332                             lambda unused: True,
1333                             "Unable to set has_run_collection_command.")
1334
1335         coreclr_args.verify(args,
1336                             "has_merged_mch",
1337                             lambda unused: True,
1338                             "Unable to set has_merged_mch.")
1339
1340         coreclr_args.verify(args,
1341                             "has_verified_clean_mch",
1342                             lambda unused: True,
1343                             "Unable to set has_verified_clean_mch.")
1344
1345         coreclr_args.verify(args,
1346                             "break_on_assert",
1347                             lambda unused: True,
1348                             "Unable to set break_on_assert")
1349
1350         coreclr_args.verify(args,
1351                             "break_on_error",
1352                             lambda unused: True,
1353                             "Unable to set break_on_error")
1354
1355         coreclr_args.verify(args,
1356                             "use_zapdisable",
1357                             lambda unused: True,
1358                             "Unable to set use_zapdisable")
1359
1360         jit_location = os.path.join(coreclr_args.core_root, determine_jit_name(coreclr_args))
1361         assert(os.path.isfile(jit_location))
1362
1363     elif coreclr_args.mode == "replay":
1364         coreclr_args.verify(args,
1365                             "mch_file",
1366                             lambda mch_file: os.path.isfile(mch_file),
1367                             lambda mch_file: "Incorrect file path to mch_file: {}".format(mch_file),
1368                             modify_arg=lambda arg: arg[0] if arg is not None else setup_mch_arg(arg))
1369
1370         coreclr_args.verify(args,
1371                             "jit_path",
1372                             lambda jit_path: os.path.isfile(jit_path),
1373                             "Unable to set jit_path",
1374                             modify_arg=lambda arg: arg[0])
1375
1376         coreclr_args.verify(args,
1377                             "log_file",
1378                             lambda unused: True,
1379                             "Unable to set log_file.")
1380
1381         coreclr_args.verify(args,
1382                             "break_on_assert",
1383                             lambda unused: True,
1384                             "Unable to set break_on_assert")
1385
1386         coreclr_args.verify(args,
1387                             "break_on_error",
1388                             lambda unused: True,
1389                             "Unable to set break_on_error")
1390
1391         standard_location = False
1392         if coreclr_args.bin_location.lower() in coreclr_args.jit_path.lower():
1393             standard_location = True
1394
1395         determined_arch = None
1396         determined_build_type = None
1397         if standard_location:
1398             standard_location_split = coreclr_args.jit_path.split(coreclr_args.bin_location)
1399
1400             assert(coreclr_args.host_os in standard_location_split[1])
1401             specialized_path = standard_location_split[1].split(coreclr_args.host_os)
1402
1403             specialized_path = specialized_path[1].split("/")[0]
1404
1405             determined_split = specialized_path.split(".")
1406
1407             determined_arch = determined_split[1]
1408             determined_build_type = determined_split[2]
1409
1410         # Make a more intelligent decision about the arch and build type
1411         # based on the path of the jit passed
1412         if standard_location and not coreclr_args.build_type in coreclr_args.jit_path:
1413             coreclr_args.verify(determined_arch.lower(),
1414                                 "arch",
1415                                 lambda unused: True,
1416                                 "Unable to set arch")
1417
1418             coreclr_args.verify(determined_build_type,
1419                                 "build_type",
1420                                 coreclr_args.check_build_type,
1421                                 "Invalid build_type")
1422
1423         coreclr_args.verify(args,
1424                             "mch_file",
1425                             lambda mch_file: os.path.isfile(mch_file),
1426                             lambda mch_file: "Incorrect file path to mch_file: {}".format(mch_file),
1427                             modify_arg=lambda arg: arg[0] if arg is not None else setup_mch_arg(arg))
1428
1429     elif coreclr_args.mode == "asmdiffs":
1430         coreclr_args.verify(args,
1431                             "base_jit_path",
1432                             lambda unused: True,
1433                             "Unable to set base_jit_path",
1434                             modify_arg=lambda arg: arg[0])
1435
1436         coreclr_args.verify(args,
1437                             "diff_jit_path",
1438                             lambda jit_path: True,
1439                             "Unable to set base_jit_path",
1440                             modify_arg=lambda arg: arg[0])
1441
1442         coreclr_args.verify(args,
1443                             "log_file",
1444                             lambda unused: True,
1445                             "Unable to set log_file.")
1446
1447         coreclr_args.verify(args,
1448                             "break_on_assert",
1449                             lambda unused: True,
1450                             "Unable to set break_on_assert")
1451
1452         coreclr_args.verify(args,
1453                             "break_on_error",
1454                             lambda unused: True,
1455                             "Unable to set break_on_error")
1456
1457         coreclr_args.verify(args,
1458                             "diff_with_code",
1459                             lambda unused: True,
1460                             "Unable to set diff_with_code.")
1461
1462         coreclr_args.verify(args,
1463                             "diff_with_code_only",
1464                             lambda unused: True,
1465                             "Unable to set diff_with_code_only.")
1466
1467         if coreclr_args.diff_with_code_only:
1468             # Set diff with code if we are not running SuperPMI to regenerate diffs.
1469             # This avoids having to re-run generating asm diffs.
1470             coreclr_args.verify(True,
1471                                 "diff_with_code",
1472                                 lambda unused: True,
1473                                 "Unable to set diff_with_code.")
1474
1475         coreclr_args.verify(args,
1476                             "diff_jit_dump",
1477                             lambda unused: True,
1478                             "Unable to set diff_jit_dump.")
1479
1480         coreclr_args.verify(args,
1481                             "diff_jit_dump_only",
1482                             lambda unused: True,
1483                             "Unable to set diff_jit_dump_only.")
1484
1485         if coreclr_args.diff_jit_dump_only:
1486             coreclr_args.verify(True,
1487                                 "diff_jit_dump",
1488                                 lambda unused: True,
1489                                 "Unable to set diff_jit_dump.")
1490
1491         standard_location = False
1492         if coreclr_args.bin_location.lower() in coreclr_args.base_jit_path.lower():
1493             standard_location = True
1494
1495         determined_arch = None
1496         determined_build_type = None
1497         if standard_location:
1498             standard_location_split = coreclr_args.base_jit_path.split(coreclr_args.bin_location)
1499
1500             assert(coreclr_args.host_os in standard_location_split[1])
1501             specialized_path = standard_location_split[1].split(coreclr_args.host_os)
1502
1503             specialized_path = specialized_path[1].split("/")[0]
1504
1505             determined_split = specialized_path.split(".")
1506
1507             determined_arch = determined_split[1]
1508             determined_build_type = determined_split[2]
1509
1510         # Make a more intelligent decision about the arch and build type
1511         # based on the path of the jit passed
1512         if standard_location and not coreclr_args.build_type in coreclr_args.base_jit_path:
1513             coreclr_args.verify(determined_build_type,
1514                                 "build_type",
1515                                 coreclr_args.check_build_type,
1516                                 "Invalid build_type")
1517
1518         if standard_location and not coreclr_args.arch in coreclr_args.base_jit_path:
1519             coreclr_args.verify(determined_arch.lower(),
1520                                 "arch",
1521                                 lambda unused: True,
1522                                 "Unable to set arch")
1523
1524         coreclr_args.verify(determine_coredis_tools(coreclr_args),
1525                             "coredistools_location",
1526                             lambda coredistools_path: os.path.isfile(coredistools_path),
1527                             "Unable to find coredistools.")
1528
1529         coreclr_args.verify(args,
1530                             "mch_file",
1531                             lambda mch_file: os.path.isfile(mch_file),
1532                             lambda mch_file: "Incorrect file path to mch_file: {}".format(mch_file),
1533                             modify_arg=lambda arg: arg[0] if arg is not None else setup_mch_arg(arg))
1534
1535     return coreclr_args
1536
1537 ################################################################################
1538 # main
1539 ################################################################################
1540
1541 def main(args):
1542     """ Main method
1543     """
1544
1545     # Force tieried compilation off. It will effect both collection and replay
1546     os.environ["COMPlus_TieredCompilation"] = "0"
1547
1548     coreclr_args = setup_args(args)
1549     success = True
1550
1551     if coreclr_args.mode == "collect":
1552         # Start a new SuperPMI Collection.
1553
1554         begin_time = datetime.datetime.now()
1555
1556         print("SuperPMI Collect")
1557         print("------------------------------------------------------------")
1558         print("Start time: {}".format(begin_time.strftime("%H:%M:%S")))
1559
1560         collection = SuperPMICollect(coreclr_args)
1561         success = collection.collect()
1562
1563         print("Finished SuperPMI collect")
1564
1565         if coreclr_args.output_mch_path != None:
1566             print("mch path: {}".format(coreclr_args.output_mch_path))
1567
1568         end_time = datetime.datetime.now()
1569
1570         print("Finish time: {}".format(end_time.strftime("%H:%M:%S")))
1571
1572     elif coreclr_args.mode == "replay":
1573         # Start a new SuperPMI Replay
1574
1575         begin_time = datetime.datetime.now()
1576
1577         print("SuperPMI Replay")
1578         print("------------------------------------------------------------")
1579         print("Start time: {}".format(begin_time.strftime("%H:%M:%S")))
1580
1581         mch_file = coreclr_args.mch_file
1582         jit_path = coreclr_args.jit_path
1583
1584         print("")
1585
1586         print("MCH Path: {}".format(mch_file))
1587         print("Jit Path: {}".format(jit_path))
1588
1589         replay = SuperPMIReplay(coreclr_args, mch_file, jit_path)
1590         success = replay.replay()
1591
1592         print("Finished SuperPMI replay")
1593
1594         end_time = datetime.datetime.now()
1595
1596         print("Finish time: {}".format(end_time.strftime("%H:%M:%S")))
1597
1598     elif coreclr_args.mode == "asmdiffs":
1599         # Start a new SuperPMI Replay with AsmDiffs
1600
1601         begin_time = datetime.datetime.now()
1602
1603         print("SuperPMI Replay")
1604         print("------------------------------------------------------------")
1605         print("Start time: {}".format(begin_time.strftime("%H:%M:%S")))
1606
1607         mch_file = coreclr_args.mch_file
1608         base_jit_path = coreclr_args.base_jit_path
1609         diff_jit_path = coreclr_args.diff_jit_path
1610
1611         print("")
1612
1613         print("MCH Path: {}".format(mch_file))
1614         print("Base Jit Path: {}".format(base_jit_path))
1615         print("Diff Jit Path: {}".format(diff_jit_path))
1616
1617         asm_diffs = SuperPMIReplayAsmDiffs(coreclr_args, mch_file, base_jit_path, diff_jit_path)
1618         success = asm_diffs.replay_with_asm_diffs()
1619
1620         print("Finished SuperPMI replay")
1621
1622         end_time = datetime.datetime.now()
1623
1624         print("Finish time: {}".format(end_time.strftime("%H:%M:%S")))
1625
1626     return 0 if success else 1
1627
1628 ################################################################################
1629 # __main__
1630 ################################################################################
1631
1632 if __name__ == "__main__":
1633     args = parser.parse_args()
1634     sys.exit(main(args))