1 # -*- coding: utf-8 -*-
3 #-------------------------------------------------------------------------
4 # drawElements Quality Program utilities
5 # --------------------------------------
7 # Copyright 2015 The Android Open Source Project
9 # Licensed under the Apache License, Version 2.0 (the "License");
10 # you may not use this file except in compliance with the License.
11 # You may obtain a copy of the License at
13 # http://www.apache.org/licenses/LICENSE-2.0
15 # Unless required by applicable law or agreed to in writing, software
16 # distributed under the License is distributed on an "AS IS" BASIS,
17 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18 # See the License for the specific language governing permissions and
19 # limitations under the License.
21 #-------------------------------------------------------------------------
32 sys.path.append(os.path.join(os.path.dirname(__file__), "..", "scripts"))
34 from build.common import *
36 EXTERNAL_DIR = os.path.realpath(os.path.normpath(os.path.dirname(__file__)))
38 def computeChecksum (data):
39 return hashlib.sha256(data).hexdigest()
42 def __init__(self, baseDir, extractDir):
43 self.baseDir = baseDir
44 self.extractDir = extractDir
47 fullDstPath = os.path.join(EXTERNAL_DIR, self.baseDir, self.extractDir)
48 if os.path.exists(fullDstPath):
49 shutil.rmtree(fullDstPath, ignore_errors=False)
51 class SourcePackage (Source):
52 def __init__(self, url, filename, checksum, baseDir, extractDir = "src", postExtract=None):
53 Source.__init__(self, baseDir, extractDir)
55 self.filename = filename
56 self.checksum = checksum
57 self.archiveDir = "packages"
58 self.postExtract = postExtract
64 def update (self, cmdProtocol = None):
65 if not self.isArchiveUpToDate():
66 self.fetchAndVerifyArchive()
68 if self.getExtractedChecksum() != self.checksum:
71 self.storeExtractedChecksum(self.checksum)
73 def removeArchives (self):
74 archiveDir = os.path.join(EXTERNAL_DIR, pkg.baseDir, pkg.archiveDir)
75 if os.path.exists(archiveDir):
76 shutil.rmtree(archiveDir, ignore_errors=False)
78 def isArchiveUpToDate (self):
79 archiveFile = os.path.join(EXTERNAL_DIR, pkg.baseDir, pkg.archiveDir, pkg.filename)
80 if os.path.exists(archiveFile):
81 return computeChecksum(readFile(archiveFile)) == self.checksum
85 def getExtractedChecksumFilePath (self):
86 return os.path.join(EXTERNAL_DIR, pkg.baseDir, pkg.archiveDir, "extracted")
88 def getExtractedChecksum (self):
89 extractedChecksumFile = self.getExtractedChecksumFilePath()
91 if os.path.exists(extractedChecksumFile):
92 return readFile(extractedChecksumFile)
96 def storeExtractedChecksum (self, checksum):
97 writeFile(self.getExtractedChecksumFilePath(), checksum)
99 def fetchAndVerifyArchive (self):
100 print "Fetching %s" % self.url
102 req = urllib2.urlopen(self.url)
104 checksum = computeChecksum(data)
105 dstPath = os.path.join(EXTERNAL_DIR, self.baseDir, self.archiveDir, self.filename)
107 if checksum != self.checksum:
108 raise Exception("Checksum mismatch for %s, exepected %s, got %s" % (self.filename, self.checksum, checksum))
110 if not os.path.exists(os.path.dirname(dstPath)):
111 os.mkdir(os.path.dirname(dstPath))
113 writeFile(dstPath, data)
116 print "Extracting %s to %s/%s" % (self.filename, self.baseDir, self.extractDir)
118 srcPath = os.path.join(EXTERNAL_DIR, self.baseDir, self.archiveDir, self.filename)
119 tmpPath = os.path.join(EXTERNAL_DIR, ".extract-tmp-%s" % self.baseDir)
120 dstPath = os.path.join(EXTERNAL_DIR, self.baseDir, self.extractDir)
121 archive = tarfile.open(srcPath)
123 if os.path.exists(tmpPath):
124 shutil.rmtree(tmpPath, ignore_errors=False)
128 archive.extractall(tmpPath)
131 extractedEntries = os.listdir(tmpPath)
132 if len(extractedEntries) != 1 or not os.path.isdir(os.path.join(tmpPath, extractedEntries[0])):
133 raise Exception("%s doesn't contain single top-level directory" % self.filename)
135 topLevelPath = os.path.join(tmpPath, extractedEntries[0])
137 if not os.path.exists(dstPath):
140 for entry in os.listdir(topLevelPath):
141 if os.path.exists(os.path.join(dstPath, entry)):
142 raise Exception("%s exists already" % entry)
144 shutil.move(os.path.join(topLevelPath, entry), dstPath)
146 shutil.rmtree(tmpPath, ignore_errors=True)
148 if self.postExtract != None:
149 self.postExtract(dstPath)
151 class GitRepo (Source):
152 def __init__(self, httpsUrl, sshUrl, revision, baseDir, extractDir = "src"):
153 Source.__init__(self, baseDir, extractDir)
154 self.httpsUrl = httpsUrl
156 self.revision = revision
158 def detectProtocol(self, cmdProtocol = None):
159 # reuse parent repo protocol
160 proc = subprocess.Popen(['git', 'ls-remote', '--get-url', 'origin'], stdout=subprocess.PIPE)
161 (stdout, stderr) = proc.communicate()
163 if proc.returncode != 0:
164 raise Exception("Failed to execute 'git ls-remote origin', got %d" % proc.returncode)
165 if (stdout[:3] == 'ssh') or (stdout[:3] == 'git'):
168 # remote 'origin' doesn't exist, assume 'https' as checkout protocol
171 def selectUrl(self, cmdProtocol = None):
173 if cmdProtocol == None:
174 protocol = detectProtocol(self, cmdProtocol)
176 protocol = cmdProtocol
178 # fallback to https on any issues
181 if protocol == 'ssh':
182 if self.sshUrl != None:
185 assert self.httpsUrl != None
188 assert protocol == 'https'
194 def update (self, cmdProtocol = None):
195 fullDstPath = os.path.join(EXTERNAL_DIR, self.baseDir, self.extractDir)
197 url = self.selectUrl(cmdProtocol)
198 if not os.path.exists(fullDstPath):
199 execute(["git", "clone", "--no-checkout", url, fullDstPath])
201 pushWorkingDir(fullDstPath)
203 execute(["git", "fetch", url, "+refs/heads/*:refs/remotes/origin/*"])
204 execute(["git", "checkout", self.revision])
208 def postExtractLibpng (path):
209 shutil.copy(os.path.join(path, "scripts", "pnglibconf.h.prebuilt"),
210 os.path.join(path, "pnglibconf.h"))
214 "http://zlib.net/zlib-1.2.11.tar.gz",
215 "zlib-1.2.11.tar.gz",
216 "c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1",
219 "http://prdownloads.sourceforge.net/libpng/libpng-1.6.27.tar.gz",
220 "libpng-1.6.27.tar.gz",
221 "c9d164ec247f426a525a7b89936694aefbc91fb7a50182b198898b8fc91174b4",
223 postExtract = postExtractLibpng),
225 "https://github.com/KhronosGroup/SPIRV-Tools.git",
227 "ab03b879cab5d09147c257035145f0ad36c45064",
230 "https://github.com/KhronosGroup/glslang.git",
232 "e3aa654c4b0c761b28d7864192ca8ceea6faf70a",
235 "https://github.com/KhronosGroup/SPIRV-Headers.git",
237 "bd47a9abaefac00be692eae677daed1b977e625c",
242 parser = argparse.ArgumentParser(description = "Fetch external sources")
243 parser.add_argument('--clean', dest='clean', action='store_true', default=False,
244 help='Remove sources instead of fetching')
245 parser.add_argument('--protocol', dest='protocol', default=None, choices=['ssh', 'https'],
246 help="Select protocol to checkout git repositories.")
247 return parser.parse_args()
249 if __name__ == "__main__":
256 pkg.update(args.protocol)