Fix format.py

In format.py, we were downloading bootstrap.sh/cmd to the current working
directory, but assuming that it was in coreclr/bootstrap, which is
incorrect. This change downloads it directly to coreclr/bootstrap, so that
we download and run to the same location.

This change also modifies where we install the newer version of dotnetcli
needed for jitutils to the coreclr/dotnetcli-jitutils directory. This
change is necessary because build.cmd->run.cmd->init-tools.cmd removes the
Tools directory, including our dotnetcli directory, so we don't want our
version of the cli to be trashed.

We also clean up our work at the end of the script by removing the
created jitutils and dotnetcli directories, and the bootstrap script.

Finally, this change adds some comments to tell the user what it's doing.

Commit migrated from https://github.com/dotnet/coreclr/commit/4d51ced0cd78abb9439f31d6599600b33d7e8e0d

diff --git a/src/coreclr/netci.groovy b/src/coreclr/netci.groovy
index 071048d..00e2d5b 100755 (executable)
--- a/src/coreclr/netci.groovy
+++ b/src/coreclr/netci.groovy
@@ -1740,7 +1740,7 @@ combinedScenarios.each { scenario ->
                                         buildCommands += "set __TestIntermediateDir=int&&build-test.cmd ${lowerConfiguration} ${arch}"
                                     }
                                     else if (scenario == 'formatting') {
-                                        buildCommands += "python tests\\scripts\\format.py -c %WORKSPACE% -o Windows_NT -a ${arch}"
+                                        buildCommands += "python -u tests\\scripts\\format.py -c %WORKSPACE% -o Windows_NT -a ${arch}"
                                         break
                                     }
                                     else {

diff --git a/src/coreclr/tests/scripts/format.py b/src/coreclr/tests/scripts/format.py
index e89aad7..50a0d3d 100644 (file)
--- a/src/coreclr/tests/scripts/format.py
+++ b/src/coreclr/tests/scripts/format.py
@@ -25,6 +25,10 @@ import shutil
 def expandPath(path):
     return os.path.abspath(os.path.expanduser(path))
 
+def del_rw(action, name, exc):
+    os.chmod(name, 0651)
+    os.remove(name)
+
 def main(argv):
     parser = argparse.ArgumentParser()
     required = parser.add_argument_group('required arguments')
@@ -58,23 +62,35 @@ def main(argv):
     platform = args.os
     arch = args.arch
 
-    # Download dotnetcli
+    my_env = os.environ
+
+    # Download .Net CLI
+
     dotnetcliUrl = ""
     dotnetcliFilename = ""
-    dotnetcliPath = os.path.join(coreclr, 'Tools', 'dotnetcli-jitutils')
+
+    # build.cmd removes the Tools directory, so we need to put our version of jitutils
+    # outside of the Tools directory
+
+    dotnetcliPath = os.path.join(coreclr, 'dotnetcli-jitutils')
 
     # Try to make the dotnetcli-jitutils directory if it doesn't exist
-    try: 
+
+    try:
         os.makedirs(dotnetcliPath)
     except OSError:
         if not os.path.isdir(dotnetcliPath):
             raise
 
-    if platform == 'Linux' or platform == 'OSX':
-        dotnetcliUrl = "https://go.microsoft.com/fwlink/?LinkID=809118"
+    print("Downloading .Net CLI")
+    if platform == 'Linux':
+        dotnetcliUrl = "https://go.microsoft.com/fwlink/?LinkID=809129"
+        dotnetcliFilename = os.path.join(dotnetcliPath, 'dotnetcli-jitutils.tar.gz')
+    elif platform == 'OSX':
+        dotnetcliUrl = "https://go.microsoft.com/fwlink/?LinkID=809128"
         dotnetcliFilename = os.path.join(dotnetcliPath, 'dotnetcli-jitutils.tar.gz')
     elif platform == 'Windows_NT':
-        dotnetcliUrl = "https://go.microsoft.com/fwlink/?LinkID=809115"
+        dotnetcliUrl = "https://go.microsoft.com/fwlink/?LinkID=809126"
         dotnetcliFilename = os.path.join(dotnetcliPath, 'dotnetcli-jitutils.zip')
     else:
         print('Unknown os ', os)
@@ -82,11 +98,14 @@ def main(argv):
 
     response = urllib2.urlopen(dotnetcliUrl)
     request_url = response.geturl()
-    print(request_url)
     testfile = urllib.URLopener()
     testfile.retrieve(request_url, dotnetcliFilename)
 
-    # Install dotnetcli
+    if not os.path.isfile(dotnetcliFilename):
+        print("Did not download .Net CLI!")
+        return -1
+
+    # Install .Net CLI
 
     if platform == 'Linux' or platform == 'OSX':
         tar = tarfile.open(dotnetcliFilename)
@@ -96,13 +115,26 @@ def main(argv):
         with zipfile.ZipFile(dotnetcliFilename, "r") as z:
             z.extractall(dotnetcliPath)
 
+    dotnet = ""
+    if platform == 'Linux' or platform == 'OSX':
+        dotnet = "dotnet"
+    elif platform == 'Windows_NT':
+        dotnet = "dotnet.exe"
+
+
+    if not os.path.isfile(os.path.join(dotnetcliPath, dotnet)):
+        print("Did not extract .Net CLI from download")
+        return -1
+
     # Download bootstrap
+
     bootstrapFilename = ""
 
     jitUtilsPath = os.path.join(coreclr, "jitutils")
 
     if os.path.isdir(jitUtilsPath):
-        shutil.rmtree(dest, ignore_errors=True)
+        print("Deleting " + jitUtilsPath)
+        shutil.rmtree(jitUtilsPath, onerror=del_rw)
 
     if platform == 'Linux' or platform == 'OSX':
         bootstrapFilename = "bootstrap.sh"
@@ -111,37 +143,75 @@ def main(argv):
 
     bootstrapUrl = "https://raw.githubusercontent.com/dotnet/jitutils/master/" + bootstrapFilename
 
-    testfile.retrieve(bootstrapUrl, bootstrapFilename)
+    bootstrapPath = os.path.join(coreclr, bootstrapFilename)
+    testfile.retrieve(bootstrapUrl, bootstrapPath)
+
+    if not os.path.isfile(bootstrapPath):
+        print("Did not download bootstrap!")
+        return -1
+
+    # On *nix platforms, we need to make the bootstrap file executable
 
-    # On Linux platforms, we need to make the bootstrap file executable
     if platform == 'Linux' or platform == 'OSX':
-        os.chmod(bootstrapFilename, 0751)
+        print("Making bootstrap executable")
+        os.chmod(bootstrapPath, 0751)
+
+    print(bootstrapPath)
 
     # Run bootstrap
-    os.environ["PATH"] += os.pathsep + dotnetcliPath
-    proc = subprocess.Popen([os.path.join(coreclr, bootstrapFilename)], shell=True)
 
-    output,error = proc.communicate()
-    print(output)
-    print(error)
+    my_env["PATH"] += os.pathsep + dotnetcliPath
+    if platform == 'Linux' or platform == 'OSX':
+        print("Running bootstrap")
+        proc = subprocess.Popen(['bash', bootstrapPath], env=my_env)
+        output,error = proc.communicate()
+    elif platform == 'Windows_NT':
+        proc = subprocess.Popen([bootstrapPath], env=my_env)
+        output,error = proc.communicate()
 
     # Run jit-format
+
     returncode = 0
-    os.environ["PATH"] += os.pathsep + os.path.join(coreclr, "jitutils", "bin")
+    jitutilsBin = os.path.join(coreclr, "jitutils", "bin")
+    my_env["PATH"] += os.pathsep + jitutilsBin
+    current_dir = os.getcwd()
+
+    if os.path.isdir(jitutilsBin):
+        os.chdir(jitutilsBin)
+    else:
+        print("Jitutils not built!")
+        return -1
+
+    jitformat = ""
+
+    if platform == 'Linux' or platform == 'OSX':
+        jitformat = "jit-format"
+    elif platform == 'Windows_NT':
+        jitformat = "jit-format.cmd"
 
     for build in ["Checked", "Debug", "Release"]:
         for project in ["dll", "standalone", "crossgen"]:
-            proc = subprocess.Popen(["jit-format", "-a", arch, "-b", build, "-o", platform,
-                "-c", coreclr, "--verbose", "--projects", project], shell=True)
+            proc = subprocess.Popen([jitformat, "-a", arch, "-b", build, "-o", platform, "-c", coreclr, "--verbose", "--projects", project], env=my_env)
             output,error = proc.communicate()
             errorcode = proc.returncode
 
-            print(output)
-            print(error)
-
             if errorcode != 0:
                 returncode = errorcode
 
+    os.chdir(current_dir)
+
+    if os.path.isdir(jitUtilsPath):
+        print("Deleting " + jitUtilsPath)
+        shutil.rmtree(jitUtilsPath, onerror=del_rw)
+
+    if os.path.isdir(dotnetcliPath):
+        print("Deleting " + dotnetcliPath)
+        shutil.rmtree(dotnetcliPath, onerror=del_rw)
+
+    if os.path.isfile(bootstrapPath):
+        print("Deleting " + bootstrapPath)
+        os.remove(bootstrapPath)
+
     return returncode
 
 if __name__ == '__main__':