Merge vk-gl-cts/vulkan-cts-1.0.2 into vk-gl-cts/master

[platform/upstream/VK-GL-CTS.git] / external / fetch_sources.py

diff --git a/external/fetch_sources.py b/external/fetch_sources.py
index c6b05e6..0d7c8e5 100644 (file)
--- a/external/fetch_sources.py
+++ b/external/fetch_sources.py
@@ -24,10 +24,10 @@ import os
 import sys
 import shutil
 import tarfile
-import urllib2
 import hashlib
 import argparse
 import subprocess
+import ssl
 
 sys.path.append(os.path.join(os.path.dirname(__file__), "..", "scripts"))
 
@@ -61,7 +61,7 @@ class SourcePackage (Source):
                Source.clean(self)
                self.removeArchives()
 
-       def update (self):
+       def update (self, cmdProtocol = None):
                if not self.isArchiveUpToDate():
                        self.fetchAndVerifyArchive()
 
@@ -94,12 +94,30 @@ class SourcePackage (Source):
                        return None
 
        def storeExtractedChecksum (self, checksum):
-               writeFile(self.getExtractedChecksumFilePath(), checksum)
+               checksum_bytes = checksum.encode("utf-8")
+               writeFile(self.getExtractedChecksumFilePath(), checksum_bytes)
+
+       def connectToUrl (self, url):
+               result = None
+
+               if sys.version_info < (3, 0):
+                       from urllib2 import urlopen
+               else:
+                       from urllib.request import urlopen
+
+               if args.insecure:
+                       print("Ignoring certificate checks")
+                       ssl_context = ssl._create_unverified_context()
+                       result = urlopen(url, context=ssl_context)
+               else:
+                       result = urlopen(url)
+
+               return result
 
        def fetchAndVerifyArchive (self):
-               print "Fetching %s" % self.url
+               print("Fetching %s" % self.url)
 
-               req                     = urllib2.urlopen(self.url)
+               req                     = self.connectToUrl(self.url)
                data            = req.read()
                checksum        = computeChecksum(data)
                dstPath         = os.path.join(EXTERNAL_DIR, self.baseDir, self.archiveDir, self.filename)
@@ -113,7 +131,7 @@ class SourcePackage (Source):
                writeFile(dstPath, data)
 
        def extract (self):
-               print "Extracting %s to %s/%s" % (self.filename, self.baseDir, self.extractDir)
+               print("Extracting %s to %s/%s" % (self.filename, self.baseDir, self.extractDir))
 
                srcPath = os.path.join(EXTERNAL_DIR, self.baseDir, self.archiveDir, self.filename)
                tmpPath = os.path.join(EXTERNAL_DIR, ".extract-tmp-%s" % self.baseDir)
@@ -149,20 +167,59 @@ class SourcePackage (Source):
                        self.postExtract(dstPath)
 
 class GitRepo (Source):
-       def __init__(self, url, revision, baseDir, extractDir = "src"):
+       def __init__(self, httpsUrl, sshUrl, revision, baseDir, extractDir = "src"):
                Source.__init__(self, baseDir, extractDir)
-               self.url                = url
+               self.httpsUrl   = httpsUrl
+               self.sshUrl             = sshUrl
                self.revision   = revision
 
-       def update (self):
+       def detectProtocol(self, cmdProtocol = None):
+               # reuse parent repo protocol
+               proc = subprocess.Popen(['git', 'ls-remote', '--get-url', 'origin'], stdout=subprocess.PIPE)
+               (stdout, stderr) = proc.communicate()
+
+               if proc.returncode != 0:
+                       raise Exception("Failed to execute 'git ls-remote origin', got %d" % proc.returncode)
+               if (stdout[:3] == 'ssh') or (stdout[:3] == 'git'):
+                       protocol = 'ssh'
+               else:
+                       # remote 'origin' doesn't exist, assume 'https' as checkout protocol
+                       protocol = 'https'
+               return protocol
+
+       def selectUrl(self, cmdProtocol = None):
+               try:
+                       if cmdProtocol == None:
+                               protocol = self.detectProtocol(cmdProtocol)
+                       else:
+                               protocol = cmdProtocol
+               except:
+                       # fallback to https on any issues
+                       protocol = 'https'
+
+               if protocol == 'ssh':
+                       if self.sshUrl != None:
+                               url = self.sshUrl
+                       else:
+                               assert self.httpsUrl != None
+                               url = self.httpsUrl
+               else:
+                       assert protocol == 'https'
+                       url = self.httpsUrl
+
+               assert url != None
+               return url
+
+       def update (self, cmdProtocol = None):
                fullDstPath = os.path.join(EXTERNAL_DIR, self.baseDir, self.extractDir)
 
+               url = self.selectUrl(cmdProtocol)
                if not os.path.exists(fullDstPath):
-                       execute(["git", "clone", "--no-checkout", self.url, fullDstPath])
+                       execute(["git", "clone", "--no-checkout", url, fullDstPath])
 
                pushWorkingDir(fullDstPath)
                try:
-                       execute(["git", "fetch", self.url, "+refs/heads/*:refs/remotes/origin/*"])
+                       execute(["git", "fetch", url, "+refs/heads/*:refs/remotes/origin/*"])
                        execute(["git", "checkout", self.revision])
                finally:
                        popWorkingDir()
@@ -185,23 +242,45 @@ PACKAGES = [
                postExtract = postExtractLibpng),
        GitRepo(
                "https://github.com/KhronosGroup/SPIRV-Tools.git",
+               None,
                "ab03b879cab5d09147c257035145f0ad36c45064",
                "spirv-tools"),
        GitRepo(
                "https://github.com/KhronosGroup/glslang.git",
-               "e3aa654c4b0c761b28d7864192ca8ceea6faf70a",
+               None,
+               "36852b838d05b2a1d5a0dd311b86c4971656fe36",
                "glslang"),
        GitRepo(
                "https://github.com/KhronosGroup/SPIRV-Headers.git",
+               None,
                "bd47a9abaefac00be692eae677daed1b977e625c",
                "spirv-headers"),
 ]
 
 def parseArgs ():
+       versionsForInsecure = ((2,7,9), (3,4,3))
+       versionsForInsecureStr = ' or '.join(('.'.join(str(x) for x in v)) for v in versionsForInsecure)
+
        parser = argparse.ArgumentParser(description = "Fetch external sources")
        parser.add_argument('--clean', dest='clean', action='store_true', default=False,
                                                help='Remove sources instead of fetching')
-       return parser.parse_args()
+       parser.add_argument('--insecure', dest='insecure', action='store_true', default=False,
+                                               help="Disable certificate check for external sources."
+                                               " Minimum python version required " + versionsForInsecureStr)
+       parser.add_argument('--protocol', dest='protocol', default=None, choices=['ssh', 'https'],
+                                               help="Select protocol to checkout git repositories.")
+
+       args = parser.parse_args()
+
+       if args.insecure:
+               for versionItem in versionsForInsecure:
+                       if (sys.version_info.major == versionItem[0]):
+                               if sys.version_info < versionItem:
+                                       parser.error("For --insecure minimum required python version is " +
+                                                               versionsForInsecureStr)
+                               break;
+
+       return args
 
 if __name__ == "__main__":
        args = parseArgs()
@@ -210,4 +289,4 @@ if __name__ == "__main__":
                if args.clean:
                        pkg.clean()
                else:
-                       pkg.update()
+                       pkg.update(args.protocol)