1 # -*- coding: utf-8 -*-
3 #-------------------------------------------------------------------------
4 # drawElements Quality Program utilities
5 # --------------------------------------
7 # Copyright 2015 The Android Open Source Project
9 # Licensed under the Apache License, Version 2.0 (the "License");
10 # you may not use this file except in compliance with the License.
11 # You may obtain a copy of the License at
13 # http://www.apache.org/licenses/LICENSE-2.0
15 # Unless required by applicable law or agreed to in writing, software
16 # distributed under the License is distributed on an "AS IS" BASIS,
17 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18 # See the License for the specific language governing permissions and
19 # limitations under the License.
21 #-------------------------------------------------------------------------
33 sys.path.append(os.path.join(os.path.dirname(__file__), "..", "scripts"))
35 from build.common import *
37 EXTERNAL_DIR = os.path.realpath(os.path.normpath(os.path.dirname(__file__)))
39 def computeChecksum (data):
40 return hashlib.sha256(data).hexdigest()
42 def onReadonlyRemoveError (func, path, exc_info):
43 os.chmod(path, stat.S_IWRITE)
47 def __init__(self, baseDir, extractDir):
48 self.baseDir = baseDir
49 self.extractDir = extractDir
52 fullDstPath = os.path.join(EXTERNAL_DIR, self.baseDir, self.extractDir)
53 # Remove read-only first
54 readonlydir = os.path.join(fullDstPath, ".git", "objects", "pack")
55 if os.path.exists(readonlydir):
56 shutil.rmtree(readonlydir, onerror = onReadonlyRemoveError )
57 if os.path.exists(fullDstPath):
58 shutil.rmtree(fullDstPath, ignore_errors=False)
60 class SourcePackage (Source):
61 def __init__(self, url, filename, checksum, baseDir, extractDir = "src", postExtract=None):
62 Source.__init__(self, baseDir, extractDir)
64 self.filename = filename
65 self.checksum = checksum
66 self.archiveDir = "packages"
67 self.postExtract = postExtract
73 def update (self, cmdProtocol = None, force = False):
74 if not self.isArchiveUpToDate():
75 self.fetchAndVerifyArchive()
77 if self.getExtractedChecksum() != self.checksum:
80 self.storeExtractedChecksum(self.checksum)
82 def removeArchives (self):
83 archiveDir = os.path.join(EXTERNAL_DIR, pkg.baseDir, pkg.archiveDir)
84 if os.path.exists(archiveDir):
85 shutil.rmtree(archiveDir, ignore_errors=False)
87 def isArchiveUpToDate (self):
88 archiveFile = os.path.join(EXTERNAL_DIR, pkg.baseDir, pkg.archiveDir, pkg.filename)
89 if os.path.exists(archiveFile):
90 return computeChecksum(readBinaryFile(archiveFile)) == self.checksum
94 def getExtractedChecksumFilePath (self):
95 return os.path.join(EXTERNAL_DIR, pkg.baseDir, pkg.archiveDir, "extracted")
97 def getExtractedChecksum (self):
98 extractedChecksumFile = self.getExtractedChecksumFilePath()
100 if os.path.exists(extractedChecksumFile):
101 return readFile(extractedChecksumFile)
105 def storeExtractedChecksum (self, checksum):
106 checksum_bytes = checksum.encode("utf-8")
107 writeBinaryFile(self.getExtractedChecksumFilePath(), checksum_bytes)
109 def connectToUrl (self, url):
112 if sys.version_info < (3, 0):
113 from urllib2 import urlopen
115 from urllib.request import urlopen
118 print("Ignoring certificate checks")
119 ssl_context = ssl._create_unverified_context()
120 result = urlopen(url, context=ssl_context)
122 result = urlopen(url)
126 def fetchAndVerifyArchive (self):
127 print("Fetching %s" % self.url)
129 req = self.connectToUrl(self.url)
131 checksum = computeChecksum(data)
132 dstPath = os.path.join(EXTERNAL_DIR, self.baseDir, self.archiveDir, self.filename)
134 if checksum != self.checksum:
135 raise Exception("Checksum mismatch for %s, expected %s, got %s" % (self.filename, self.checksum, checksum))
137 if not os.path.exists(os.path.dirname(dstPath)):
138 os.mkdir(os.path.dirname(dstPath))
140 writeBinaryFile(dstPath, data)
143 print("Extracting %s to %s/%s" % (self.filename, self.baseDir, self.extractDir))
145 srcPath = os.path.join(EXTERNAL_DIR, self.baseDir, self.archiveDir, self.filename)
146 tmpPath = os.path.join(EXTERNAL_DIR, ".extract-tmp-%s" % self.baseDir)
147 dstPath = os.path.join(EXTERNAL_DIR, self.baseDir, self.extractDir)
148 archive = tarfile.open(srcPath)
150 if os.path.exists(tmpPath):
151 shutil.rmtree(tmpPath, ignore_errors=False)
155 archive.extractall(tmpPath)
158 extractedEntries = os.listdir(tmpPath)
159 if len(extractedEntries) != 1 or not os.path.isdir(os.path.join(tmpPath, extractedEntries[0])):
160 raise Exception("%s doesn't contain single top-level directory" % self.filename)
162 topLevelPath = os.path.join(tmpPath, extractedEntries[0])
164 if not os.path.exists(dstPath):
167 for entry in os.listdir(topLevelPath):
168 if os.path.exists(os.path.join(dstPath, entry)):
169 raise Exception("%s exists already" % entry)
171 shutil.move(os.path.join(topLevelPath, entry), dstPath)
173 shutil.rmtree(tmpPath, ignore_errors=True)
175 if self.postExtract != None:
176 self.postExtract(dstPath)
178 class SourceFile (Source):
179 def __init__(self, url, filename, checksum, baseDir, extractDir = "src"):
180 Source.__init__(self, baseDir, extractDir)
182 self.filename = filename
183 self.checksum = checksum
185 def update (self, cmdProtocol = None, force = False):
186 if not self.isFileUpToDate():
188 self.fetchAndVerifyFile()
190 def isFileUpToDate (self):
191 file = os.path.join(EXTERNAL_DIR, pkg.baseDir, pkg.extractDir, pkg.filename)
192 if os.path.exists(file):
193 data = readFile(file)
194 return computeChecksum(data.encode('utf-8')) == self.checksum
198 def connectToUrl (self, url):
201 if sys.version_info < (3, 0):
202 from urllib2 import urlopen
204 from urllib.request import urlopen
207 print("Ignoring certificate checks")
208 ssl_context = ssl._create_unverified_context()
209 result = urlopen(url, context=ssl_context)
211 result = urlopen(url)
215 def fetchAndVerifyFile (self):
216 print("Fetching %s" % self.url)
218 req = self.connectToUrl(self.url)
220 checksum = computeChecksum(data)
221 dstPath = os.path.join(EXTERNAL_DIR, self.baseDir, self.extractDir, self.filename)
223 if checksum != self.checksum:
224 raise Exception("Checksum mismatch for %s, expected %s, got %s" % (self.filename, self.checksum, checksum))
226 if not os.path.exists(os.path.dirname(dstPath)):
227 os.mkdir(os.path.dirname(dstPath))
229 writeBinaryFile(dstPath, data)
231 class GitRepo (Source):
232 def __init__(self, httpsUrl, sshUrl, revision, baseDir, extractDir = "src", removeTags = []):
233 Source.__init__(self, baseDir, extractDir)
234 self.httpsUrl = httpsUrl
236 self.revision = revision
237 self.removeTags = removeTags
239 def detectProtocol(self, cmdProtocol = None):
240 # reuse parent repo protocol
241 proc = subprocess.Popen(['git', 'ls-remote', '--get-url', 'origin'], stdout=subprocess.PIPE, universal_newlines=True)
242 (stdout, stderr) = proc.communicate()
244 if proc.returncode != 0:
245 raise Exception("Failed to execute 'git ls-remote origin', got %d" % proc.returncode)
246 if (stdout[:3] == 'ssh') or (stdout[:3] == 'git'):
249 # remote 'origin' doesn't exist, assume 'https' as checkout protocol
253 def selectUrl(self, cmdProtocol = None):
255 if cmdProtocol == None:
256 protocol = self.detectProtocol(cmdProtocol)
258 protocol = cmdProtocol
260 # fallback to https on any issues
263 if protocol == 'ssh':
264 if self.sshUrl != None:
267 assert self.httpsUrl != None
270 assert protocol == 'https'
276 def update (self, cmdProtocol = None, force = False):
277 fullDstPath = os.path.join(EXTERNAL_DIR, self.baseDir, self.extractDir)
279 url = self.selectUrl(cmdProtocol)
280 if not os.path.exists(os.path.join(fullDstPath, '.git')):
281 execute(["git", "clone", "--no-checkout", url, fullDstPath])
283 pushWorkingDir(fullDstPath)
285 for tag in self.removeTags:
286 proc = subprocess.Popen(['git', 'tag', '-l', tag], stdout=subprocess.PIPE)
287 (stdout, stderr) = proc.communicate()
288 if proc.returncode == 0:
289 execute(["git", "tag", "-d",tag])
290 force_arg = ['--force'] if force else []
291 execute(["git", "fetch"] + force_arg + ["--tags", url, "+refs/heads/*:refs/remotes/origin/*"])
292 execute(["git", "checkout"] + force_arg + [self.revision])
296 def postExtractLibpng (path):
297 shutil.copy(os.path.join(path, "scripts", "pnglibconf.h.prebuilt"),
298 os.path.join(path, "pnglibconf.h"))
302 "http://zlib.net/zlib-1.2.11.tar.gz",
303 "zlib-1.2.11.tar.gz",
304 "c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1",
307 "http://prdownloads.sourceforge.net/libpng/libpng-1.6.27.tar.gz",
308 "libpng-1.6.27.tar.gz",
309 "c9d164ec247f426a525a7b89936694aefbc91fb7a50182b198898b8fc91174b4",
311 postExtract = postExtractLibpng),
313 "https://raw.githubusercontent.com/baldurk/renderdoc/v1.1/renderdoc/api/app/renderdoc_app.h",
315 "e7b5f0aa5b1b0eadc63a1c624c0ca7f5af133aa857d6a4271b0ef3d0bdb6868e",
318 "https://gitlab.khronos.org/spirv/spirv-tools.git",
319 "git@gitlab.khronos.org:spirv/spirv-tools.git",
320 "49ced6a8d7b61a487a36fba91ae3294c47352aeb",
323 "https://gitlab.khronos.org/GLSL/glslang.git",
324 "git@gitlab.khronos.org:GLSL/glslang.git",
325 "463e8ef3f555c7b648a826b37519b093ab3daca5",
327 removeTags = ["master-tot"]),
329 "https://github.com/KhronosGroup/SPIRV-Headers.git",
331 "b42ba6d92faf6b4938e6f22ddd186dbdacc98d78",
334 "https://gitlab.khronos.org/vulkan/vulkan.git",
335 "git@gitlab.khronos.org:vulkan/vulkan.git",
336 "a19a491443b0f244de88a869b5b0559243b5c214",
339 "https://github.com/google/amber.git",
341 "615ab4863f7d2e31d3037d0c6a0f641fd6fc0d07",
346 versionsForInsecure = ((2,7,9), (3,4,3))
347 versionsForInsecureStr = ' or '.join(('.'.join(str(x) for x in v)) for v in versionsForInsecure)
349 parser = argparse.ArgumentParser(description = "Fetch external sources")
350 parser.add_argument('--clean', dest='clean', action='store_true', default=False,
351 help='Remove sources instead of fetching')
352 parser.add_argument('--insecure', dest='insecure', action='store_true', default=False,
353 help="Disable certificate check for external sources."
354 " Minimum python version required " + versionsForInsecureStr)
355 parser.add_argument('--protocol', dest='protocol', default=None, choices=['ssh', 'https'],
356 help="Select protocol to checkout git repositories.")
357 parser.add_argument('--force', dest='force', action='store_true', default=False,
358 help="Pass --force to git fetch and checkout commands")
360 args = parser.parse_args()
363 for versionItem in versionsForInsecure:
364 if (sys.version_info.major == versionItem[0]):
365 if sys.version_info < versionItem:
366 parser.error("For --insecure minimum required python version is " +
367 versionsForInsecureStr)
372 if __name__ == "__main__":
379 pkg.update(args.protocol, args.force)