From e825146f966cac35cc8b8958899edca19bb7e781 Mon Sep 17 00:00:00 2001 From: Jarret Shook Date: Wed, 13 Nov 2019 09:42:15 -0800 Subject: [PATCH] Run the format job in a temp dir (dotnet/coreclr#27852) This will allow us to use whatever version of dotnet we want. Commit migrated from https://github.com/dotnet/coreclr/commit/4fe26a4653b5dcc1259b26a7439f96f452cb0d9e --- src/coreclr/tests/scripts/format.py | 223 +++++++++++++++++++++--------------- 1 file changed, 129 insertions(+), 94 deletions(-) diff --git a/src/coreclr/tests/scripts/format.py b/src/coreclr/tests/scripts/format.py index a20368a..bf34116 100644 --- a/src/coreclr/tests/scripts/format.py +++ b/src/coreclr/tests/scripts/format.py @@ -16,10 +16,37 @@ import argparse import os import sys import tarfile +import tempfile import zipfile import subprocess import shutil +class ChangeDir: + def __init__(self, dir): + self.dir = dir + self.cwd = None + + def __enter__(self): + self.cwd = os.getcwd() + os.chdir(self.dir) + + def __exit__(self, exc_type, exc_val, exc_tb): + os.chdir(self.cwd) + +class TempDir: + def __init__(self, path=None): + self.dir = tempfile.mkdtemp() if path is None else path + self.cwd = None + + def __enter__(self): + self.cwd = os.getcwd() + os.chdir(self.dir) + + return self.dir + + def __exit__(self, exc_type, exc_val, exc_tb): + os.chdir(self.cwd) + # Version specific imports if sys.version_info.major < 3: @@ -63,7 +90,8 @@ def main(argv): print('Bad path to coreclr') return -1 - coreclr = args.coreclr + coreclr = args.coreclr.replace('/', os.sep) + platform = args.os arch = args.arch @@ -86,109 +114,116 @@ def main(argv): bootstrapUrl = "https://raw.githubusercontent.com/dotnet/jitutils/master/" + bootstrapFilename - bootstrapPath = os.path.join(coreclr, bootstrapFilename) - urlretrieve(bootstrapUrl, bootstrapPath) - - if not os.path.isfile(bootstrapPath): - print("Did not download bootstrap!") - return -1 + with TempDir() as temp_location: + bootstrapPath = os.path.join(temp_location, bootstrapFilename) - # On *nix platforms, we need to make the bootstrap file executable + assert len(os.listdir(os.path.dirname(bootstrapPath))) == 0 - if platform == 'Linux' or platform == 'OSX': - print("Making bootstrap executable") - os.chmod(bootstrapPath, 0o751) + urlretrieve(bootstrapUrl, bootstrapPath) - print(bootstrapPath) + if not os.path.isfile(bootstrapPath): + print("Did not download bootstrap!") + return -1 - # Run bootstrap - if platform == 'Linux' or platform == 'OSX': - print("Running bootstrap") - proc = subprocess.Popen(['bash', bootstrapPath], env=my_env) - output,error = proc.communicate() - elif platform == 'Windows_NT': - proc = subprocess.Popen([bootstrapPath], env=my_env) - output,error = proc.communicate() + # On *nix platforms, we need to make the bootstrap file executable - # Run jit-format + if platform == 'Linux' or platform == 'OSX': + print("Making bootstrap executable") + os.chmod(bootstrapPath, 0o751) - returncode = 0 - jitutilsBin = os.path.join(coreclr, "jitutils", "bin") - my_env["PATH"] = jitutilsBin + os.pathsep + my_env["PATH"] - current_dir = os.getcwd() + print(bootstrapPath) - if not os.path.isdir(jitutilsBin): - print("Jitutils not built!") - return -1 - - jitformat = jitutilsBin - - if platform == 'Linux' or platform == 'OSX': - jitformat = os.path.join(jitformat, "jit-format") - elif platform == 'Windows_NT': - jitformat = os.path.join(jitformat,"jit-format.bat") - errorMessage = "" - - builds = ["Checked", "Debug", "Release"] - projects = ["dll", "standalone", "crossgen"] - - for build in builds: - for project in projects: - proc = subprocess.Popen([jitformat, "-a", arch, "-b", build, "-o", platform, "-c", coreclr, "--verbose", "--projects", project], env=my_env) + # Run bootstrap + if platform == 'Linux' or platform == 'OSX': + print("Running bootstrap") + proc = subprocess.Popen(['bash', bootstrapPath], env=my_env) + output,error = proc.communicate() + elif platform == 'Windows_NT': + proc = subprocess.Popen([bootstrapPath], env=my_env) output,error = proc.communicate() - errorcode = proc.returncode - - if errorcode != 0: - errorMessage += "\tjit-format -a " + arch + " -b " + build + " -o " + platform - errorMessage += " -c --verbose --fix --projects " + project +"\n" - returncode = errorcode - - # Fix mode doesn't return an error, so we have to run the build, then run with - # --fix to generate the patch. This means that it is likely only the first run - # of jit-format will return a formatting failure. - if errorcode == -2: - # If errorcode was -2, no need to run clang-tidy again - proc = subprocess.Popen([jitformat, "--fix", "--untidy", "-a", arch, "-b", build, "-o", platform, "-c", coreclr, "--verbose", "--projects", project], env=my_env) - output,error = proc.communicate() - else: - # Otherwise, must run both - proc = subprocess.Popen([jitformat, "--fix", "-a", arch, "-b", build, "-o", platform, "-c", coreclr, "--verbose", "--projects", project], env=my_env) - output,error = proc.communicate() - - os.chdir(current_dir) - - if returncode != 0: - # Create a patch file - patchFile = open("format.patch", "w") - proc = subprocess.Popen(["git", "diff", "--patch", "-U20"], env=my_env, stdout=patchFile) - output,error = proc.communicate() - if os.path.isdir(jitUtilsPath): - print("Deleting " + jitUtilsPath) - shutil.rmtree(jitUtilsPath, onerror=del_rw) + # Run jit-format + + returncode = 0 + jitutilsBin = os.path.join(os.path.dirname(bootstrapPath), "jitutils", "bin") + my_env["PATH"] = jitutilsBin + os.pathsep + my_env["PATH"] + current_dir = os.getcwd() + + if not os.path.isdir(jitutilsBin): + print("Jitutils not built!") + return -1 + + jitformat = jitutilsBin + + if platform == 'Linux' or platform == 'OSX': + jitformat = os.path.join(jitformat, "jit-format") + elif platform == 'Windows_NT': + jitformat = os.path.join(jitformat,"jit-format.bat") + errorMessage = "" + + builds = ["Checked", "Debug", "Release"] + projects = ["dll", "standalone", "crossgen"] + + for build in builds: + for project in projects: + proc = subprocess.Popen([jitformat, "-a", arch, "-b", build, "-o", platform, "-c", coreclr, "--verbose", "--projects", project], env=my_env) + output,error = proc.communicate() + errorcode = proc.returncode + + if errorcode != 0: + errorMessage += "\tjit-format -a " + arch + " -b " + build + " -o " + platform + errorMessage += " -c --verbose --fix --projects " + project +"\n" + returncode = errorcode + + # Fix mode doesn't return an error, so we have to run the build, then run with + # --fix to generate the patch. This means that it is likely only the first run + # of jit-format will return a formatting failure. + if errorcode == -2: + # If errorcode was -2, no need to run clang-tidy again + proc = subprocess.Popen([jitformat, "--fix", "--untidy", "-a", arch, "-b", build, "-o", platform, "-c", coreclr, "--verbose", "--projects", project], env=my_env) + output,error = proc.communicate() + else: + # Otherwise, must run both + proc = subprocess.Popen([jitformat, "--fix", "-a", arch, "-b", build, "-o", platform, "-c", coreclr, "--verbose", "--projects", project], env=my_env) + output,error = proc.communicate() + + os.chdir(current_dir) + + patchFilePath = os.path.join(coreclr, "format.patch") + + if returncode != 0: + # Create a patch file + print("Creating patch file " + patchFilePath) + patchFile = open(patchFilePath, "w") + proc = subprocess.Popen(["git", "diff", "--patch", "-U20"], env=my_env, stdout=patchFile) + output,error = proc.communicate() - if os.path.isfile(bootstrapPath): - print("Deleting " + bootstrapPath) - os.remove(bootstrapPath) - - if returncode != 0: - print("There were errors in formatting. Please run jit-format locally with: \n") - print(errorMessage) - print("\nOr download and apply generated patch:") - print("1. From the GitHub 'Checks' page on the Pull Request, with the failing Formatting") - print(" job selected (e.g., 'Formatting Linux x64'), click the 'View more details on") - print(" Azure Pipelines' link.") - print("3. Select the 'Summary' tab.") - print("4. Open the 'Build artifacts published' entry.") - print("5. Find the link to the OS/architecture appropriate format patch file.") - print("6. Click on the link to download it.") - print("7. Unzip the patch file.") - print("8. git apply format.patch") - - if (returncode != 0) and (os.environ.get("TF_BUILD") == "True"): - print("##vso[task.logissue type=error](NETCORE_ENGINEERING_TELEMETRY=Build) Format job found errors, please apply the format patch.") - - return returncode + if os.path.isdir(jitUtilsPath): + print("Deleting " + jitUtilsPath) + shutil.rmtree(jitUtilsPath, onerror=del_rw) + + if os.path.isfile(bootstrapPath): + print("Deleting " + bootstrapPath) + os.remove(bootstrapPath) + + if returncode != 0: + print("There were errors in formatting. Please run jit-format locally with: \n") + print(errorMessage) + print("\nOr download and apply generated patch:") + print("1. From the GitHub 'Checks' page on the Pull Request, with the failing Formatting") + print(" job selected (e.g., 'Formatting Linux x64'), click the 'View more details on") + print(" Azure Pipelines' link.") + print("3. Select the 'Summary' tab.") + print("4. Open the 'Build artifacts published' entry.") + print("5. Find the link to the OS/architecture appropriate format patch file.") + print("6. Click on the link to download it.") + print("7. Unzip the patch file.") + print("8. git apply format.patch") + + if (returncode != 0) and (os.environ.get("TF_BUILD") == "True"): + print("##vso[task.logissue type=error](NETCORE_ENGINEERING_TELEMETRY=Build) Format job found errors, please apply the format patch.") + + return returncode if __name__ == '__main__': return_code = main(sys.argv[1:]) -- 2.7.4