import sys
import shutil
import tarfile
-import urllib2
import hashlib
import argparse
import subprocess
+import ssl
sys.path.append(os.path.join(os.path.dirname(__file__), "..", "scripts"))
Source.clean(self)
self.removeArchives()
- def update (self):
+ def update (self, cmdProtocol = None):
if not self.isArchiveUpToDate():
self.fetchAndVerifyArchive()
- # \note No way to verify that extracted contents match archive, re-extract
- Source.clean(self)
- self.extract()
+ if self.getExtractedChecksum() != self.checksum:
+ Source.clean(self)
+ self.extract()
+ self.storeExtractedChecksum(self.checksum)
def removeArchives (self):
archiveDir = os.path.join(EXTERNAL_DIR, pkg.baseDir, pkg.archiveDir)
else:
return False
+ def getExtractedChecksumFilePath (self):
+ return os.path.join(EXTERNAL_DIR, pkg.baseDir, pkg.archiveDir, "extracted")
+
+ def getExtractedChecksum (self):
+ extractedChecksumFile = self.getExtractedChecksumFilePath()
+
+ if os.path.exists(extractedChecksumFile):
+ return readFile(extractedChecksumFile)
+ else:
+ return None
+
+ def storeExtractedChecksum (self, checksum):
+ checksum_bytes = checksum.encode("utf-8")
+ writeFile(self.getExtractedChecksumFilePath(), checksum_bytes)
+
+ def connectToUrl (self, url):
+ result = None
+
+ if sys.version_info < (3, 0):
+ from urllib2 import urlopen
+ else:
+ from urllib.request import urlopen
+
+ if args.insecure:
+ print("Ignoring certificate checks")
+ ssl_context = ssl._create_unverified_context()
+ result = urlopen(url, context=ssl_context)
+ else:
+ result = urlopen(url)
+
+ return result
+
def fetchAndVerifyArchive (self):
- print "Fetching %s" % self.url
+ print("Fetching %s" % self.url)
- req = urllib2.urlopen(self.url)
+ req = self.connectToUrl(self.url)
data = req.read()
checksum = computeChecksum(data)
dstPath = os.path.join(EXTERNAL_DIR, self.baseDir, self.archiveDir, self.filename)
if checksum != self.checksum:
- raise Exception("Checksum mismatch for %s, exepected %s, got %s" % (self.filename, self.checksum, checksum))
+ raise Exception("Checksum mismatch for %s, expected %s, got %s" % (self.filename, self.checksum, checksum))
if not os.path.exists(os.path.dirname(dstPath)):
os.mkdir(os.path.dirname(dstPath))
writeFile(dstPath, data)
def extract (self):
- print "Extracting %s to %s/%s" % (self.filename, self.baseDir, self.extractDir)
+ print("Extracting %s to %s/%s" % (self.filename, self.baseDir, self.extractDir))
srcPath = os.path.join(EXTERNAL_DIR, self.baseDir, self.archiveDir, self.filename)
tmpPath = os.path.join(EXTERNAL_DIR, ".extract-tmp-%s" % self.baseDir)
self.postExtract(dstPath)
class GitRepo (Source):
- def __init__(self, url, revision, baseDir, extractDir = "src"):
+ def __init__(self, httpsUrl, sshUrl, revision, baseDir, extractDir = "src"):
Source.__init__(self, baseDir, extractDir)
- self.url = url
+ self.httpsUrl = httpsUrl
+ self.sshUrl = sshUrl
self.revision = revision
- def update (self):
+ def detectProtocol(self, cmdProtocol = None):
+ # reuse parent repo protocol
+ proc = subprocess.Popen(['git', 'ls-remote', '--get-url', 'origin'], stdout=subprocess.PIPE)
+ (stdout, stderr) = proc.communicate()
+
+ if proc.returncode != 0:
+ raise Exception("Failed to execute 'git ls-remote origin', got %d" % proc.returncode)
+ if (stdout[:3] == 'ssh') or (stdout[:3] == 'git'):
+ protocol = 'ssh'
+ else:
+ # remote 'origin' doesn't exist, assume 'https' as checkout protocol
+ protocol = 'https'
+ return protocol
+
+ def selectUrl(self, cmdProtocol = None):
+ try:
+ if cmdProtocol == None:
+ protocol = self.detectProtocol(cmdProtocol)
+ else:
+ protocol = cmdProtocol
+ except:
+ # fallback to https on any issues
+ protocol = 'https'
+
+ if protocol == 'ssh':
+ if self.sshUrl != None:
+ url = self.sshUrl
+ else:
+ assert self.httpsUrl != None
+ url = self.httpsUrl
+ else:
+ assert protocol == 'https'
+ url = self.httpsUrl
+
+ assert url != None
+ return url
+
+ def update (self, cmdProtocol = None):
fullDstPath = os.path.join(EXTERNAL_DIR, self.baseDir, self.extractDir)
+ url = self.selectUrl(cmdProtocol)
if not os.path.exists(fullDstPath):
- execute(["git", "clone", "--no-checkout", self.url, fullDstPath])
+ execute(["git", "clone", "--no-checkout", url, fullDstPath])
pushWorkingDir(fullDstPath)
try:
- execute(["git", "fetch", self.url, "+refs/heads/*:refs/remotes/origin/*"])
+ execute(["git", "fetch", url, "+refs/heads/*:refs/remotes/origin/*"])
execute(["git", "checkout", self.revision])
finally:
popWorkingDir()
PACKAGES = [
SourcePackage(
- "http://zlib.net/zlib-1.2.8.tar.gz",
- "zlib-1.2.8.tar.gz",
- "36658cb768a54c1d4dec43c3116c27ed893e88b02ecfcb44f2166f9c0b7f2a0d",
+ "http://zlib.net/zlib-1.2.11.tar.gz",
+ "zlib-1.2.11.tar.gz",
+ "c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1",
"zlib"),
SourcePackage(
- "http://prdownloads.sourceforge.net/libpng/libpng-1.6.17.tar.gz",
- "libpng-1.6.17.tar.gz",
- "a18233c99e1dc59a256180e6871d9305a42e91b3f98799b3ceb98e87e9ec5e31",
+ "http://prdownloads.sourceforge.net/libpng/libpng-1.6.27.tar.gz",
+ "libpng-1.6.27.tar.gz",
+ "c9d164ec247f426a525a7b89936694aefbc91fb7a50182b198898b8fc91174b4",
"libpng",
postExtract = postExtractLibpng),
GitRepo(
"https://github.com/KhronosGroup/SPIRV-Tools.git",
- "f7e63786a919040cb2e0e572d960a0650f2c2881",
+ None,
+ "0b0454c42c6b6f6746434bd5c78c5c70f65d9c51",
"spirv-tools"),
GitRepo(
"https://github.com/KhronosGroup/glslang.git",
- "d02dc5d05ad1f63db8d37fda9928a4d59e3c132d",
+ None,
+ "a5c5fb61180e8703ca85f36d618f98e16dc317e2",
"glslang"),
+ GitRepo(
+ "https://github.com/KhronosGroup/SPIRV-Headers.git",
+ None,
+ "2bf02308656f97898c5f7e433712f21737c61e4e",
+ "spirv-headers"),
]
def parseArgs ():
+ versionsForInsecure = ((2,7,9), (3,4,3))
+ versionsForInsecureStr = ' or '.join(('.'.join(str(x) for x in v)) for v in versionsForInsecure)
+
parser = argparse.ArgumentParser(description = "Fetch external sources")
parser.add_argument('--clean', dest='clean', action='store_true', default=False,
help='Remove sources instead of fetching')
- return parser.parse_args()
+ parser.add_argument('--insecure', dest='insecure', action='store_true', default=False,
+ help="Disable certificate check for external sources."
+ " Minimum python version required " + versionsForInsecureStr)
+ parser.add_argument('--protocol', dest='protocol', default=None, choices=['ssh', 'https'],
+ help="Select protocol to checkout git repositories.")
+
+ args = parser.parse_args()
+
+ if args.insecure:
+ for versionItem in versionsForInsecure:
+ if (sys.version_info.major == versionItem[0]):
+ if sys.version_info < versionItem:
+ parser.error("For --insecure minimum required python version is " +
+ versionsForInsecureStr)
+ break;
+
+ return args
if __name__ == "__main__":
args = parseArgs()
if args.clean:
pkg.clean()
else:
- pkg.update()
+ pkg.update(args.protocol)