Fix texture mip level size in framebuffer fetch tests am: be0d165a5d
[platform/upstream/VK-GL-CTS.git] / external / fetch_sources.py
1 # -*- coding: utf-8 -*-
2
3 #-------------------------------------------------------------------------
4 # drawElements Quality Program utilities
5 # --------------------------------------
6 #
7 # Copyright 2015 The Android Open Source Project
8 #
9 # Licensed under the Apache License, Version 2.0 (the "License");
10 # you may not use this file except in compliance with the License.
11 # You may obtain a copy of the License at
12 #
13 #      http://www.apache.org/licenses/LICENSE-2.0
14 #
15 # Unless required by applicable law or agreed to in writing, software
16 # distributed under the License is distributed on an "AS IS" BASIS,
17 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18 # See the License for the specific language governing permissions and
19 # limitations under the License.
20 #
21 #-------------------------------------------------------------------------
22
23 import os
24 import sys
25 import shutil
26 import tarfile
27 import hashlib
28 import argparse
29 import subprocess
30 import ssl
31
32 sys.path.append(os.path.join(os.path.dirname(__file__), "..", "scripts"))
33
34 from build.common import *
35
36 EXTERNAL_DIR    = os.path.realpath(os.path.normpath(os.path.dirname(__file__)))
37
38 def computeChecksum (data):
39         return hashlib.sha256(data).hexdigest()
40
41 class Source:
42         def __init__(self, baseDir, extractDir):
43                 self.baseDir            = baseDir
44                 self.extractDir         = extractDir
45
46         def clean (self):
47                 fullDstPath = os.path.join(EXTERNAL_DIR, self.baseDir, self.extractDir)
48                 if os.path.exists(fullDstPath):
49                         shutil.rmtree(fullDstPath, ignore_errors=False)
50
51 class SourcePackage (Source):
52         def __init__(self, url, filename, checksum, baseDir, extractDir = "src", postExtract=None):
53                 Source.__init__(self, baseDir, extractDir)
54                 self.url                        = url
55                 self.filename           = filename
56                 self.checksum           = checksum
57                 self.archiveDir         = "packages"
58                 self.postExtract        = postExtract
59
60         def clean (self):
61                 Source.clean(self)
62                 self.removeArchives()
63
64         def update (self, cmdProtocol = None):
65                 if not self.isArchiveUpToDate():
66                         self.fetchAndVerifyArchive()
67
68                 if self.getExtractedChecksum() != self.checksum:
69                         Source.clean(self)
70                         self.extract()
71                         self.storeExtractedChecksum(self.checksum)
72
73         def removeArchives (self):
74                 archiveDir = os.path.join(EXTERNAL_DIR, pkg.baseDir, pkg.archiveDir)
75                 if os.path.exists(archiveDir):
76                         shutil.rmtree(archiveDir, ignore_errors=False)
77
78         def isArchiveUpToDate (self):
79                 archiveFile = os.path.join(EXTERNAL_DIR, pkg.baseDir, pkg.archiveDir, pkg.filename)
80                 if os.path.exists(archiveFile):
81                         return computeChecksum(readFile(archiveFile)) == self.checksum
82                 else:
83                         return False
84
85         def getExtractedChecksumFilePath (self):
86                 return os.path.join(EXTERNAL_DIR, pkg.baseDir, pkg.archiveDir, "extracted")
87
88         def getExtractedChecksum (self):
89                 extractedChecksumFile = self.getExtractedChecksumFilePath()
90
91                 if os.path.exists(extractedChecksumFile):
92                         return readFile(extractedChecksumFile)
93                 else:
94                         return None
95
96         def storeExtractedChecksum (self, checksum):
97                 checksum_bytes = checksum.encode("utf-8")
98                 writeFile(self.getExtractedChecksumFilePath(), checksum_bytes)
99
100         def connectToUrl (self, url):
101                 result = None
102
103                 if sys.version_info < (3, 0):
104                         from urllib2 import urlopen
105                 else:
106                         from urllib.request import urlopen
107
108                 if args.insecure:
109                         print("Ignoring certificate checks")
110                         ssl_context = ssl._create_unverified_context()
111                         result = urlopen(url, context=ssl_context)
112                 else:
113                         result = urlopen(url)
114
115                 return result
116
117         def fetchAndVerifyArchive (self):
118                 print("Fetching %s" % self.url)
119
120                 req                     = self.connectToUrl(self.url)
121                 data            = req.read()
122                 checksum        = computeChecksum(data)
123                 dstPath         = os.path.join(EXTERNAL_DIR, self.baseDir, self.archiveDir, self.filename)
124
125                 if checksum != self.checksum:
126                         raise Exception("Checksum mismatch for %s, expected %s, got %s" % (self.filename, self.checksum, checksum))
127
128                 if not os.path.exists(os.path.dirname(dstPath)):
129                         os.mkdir(os.path.dirname(dstPath))
130
131                 writeFile(dstPath, data)
132
133         def extract (self):
134                 print("Extracting %s to %s/%s" % (self.filename, self.baseDir, self.extractDir))
135
136                 srcPath = os.path.join(EXTERNAL_DIR, self.baseDir, self.archiveDir, self.filename)
137                 tmpPath = os.path.join(EXTERNAL_DIR, ".extract-tmp-%s" % self.baseDir)
138                 dstPath = os.path.join(EXTERNAL_DIR, self.baseDir, self.extractDir)
139                 archive = tarfile.open(srcPath)
140
141                 if os.path.exists(tmpPath):
142                         shutil.rmtree(tmpPath, ignore_errors=False)
143
144                 os.mkdir(tmpPath)
145
146                 archive.extractall(tmpPath)
147                 archive.close()
148
149                 extractedEntries = os.listdir(tmpPath)
150                 if len(extractedEntries) != 1 or not os.path.isdir(os.path.join(tmpPath, extractedEntries[0])):
151                         raise Exception("%s doesn't contain single top-level directory" % self.filename)
152
153                 topLevelPath = os.path.join(tmpPath, extractedEntries[0])
154
155                 if not os.path.exists(dstPath):
156                         os.mkdir(dstPath)
157
158                 for entry in os.listdir(topLevelPath):
159                         if os.path.exists(os.path.join(dstPath, entry)):
160                                 raise Exception("%s exists already" % entry)
161
162                         shutil.move(os.path.join(topLevelPath, entry), dstPath)
163
164                 shutil.rmtree(tmpPath, ignore_errors=True)
165
166                 if self.postExtract != None:
167                         self.postExtract(dstPath)
168
169 class GitRepo (Source):
170         def __init__(self, httpsUrl, sshUrl, revision, baseDir, extractDir = "src"):
171                 Source.__init__(self, baseDir, extractDir)
172                 self.httpsUrl   = httpsUrl
173                 self.sshUrl             = sshUrl
174                 self.revision   = revision
175
176         def detectProtocol(self, cmdProtocol = None):
177                 # reuse parent repo protocol
178                 proc = subprocess.Popen(['git', 'ls-remote', '--get-url', 'origin'], stdout=subprocess.PIPE)
179                 (stdout, stderr) = proc.communicate()
180
181                 if proc.returncode != 0:
182                         raise Exception("Failed to execute 'git ls-remote origin', got %d" % proc.returncode)
183                 if (stdout[:3] == 'ssh') or (stdout[:3] == 'git'):
184                         protocol = 'ssh'
185                 else:
186                         # remote 'origin' doesn't exist, assume 'https' as checkout protocol
187                         protocol = 'https'
188                 return protocol
189
190         def selectUrl(self, cmdProtocol = None):
191                 try:
192                         if cmdProtocol == None:
193                                 protocol = self.detectProtocol(cmdProtocol)
194                         else:
195                                 protocol = cmdProtocol
196                 except:
197                         # fallback to https on any issues
198                         protocol = 'https'
199
200                 if protocol == 'ssh':
201                         if self.sshUrl != None:
202                                 url = self.sshUrl
203                         else:
204                                 assert self.httpsUrl != None
205                                 url = self.httpsUrl
206                 else:
207                         assert protocol == 'https'
208                         url = self.httpsUrl
209
210                 assert url != None
211                 return url
212
213         def update (self, cmdProtocol = None):
214                 fullDstPath = os.path.join(EXTERNAL_DIR, self.baseDir, self.extractDir)
215
216                 url = self.selectUrl(cmdProtocol)
217                 if not os.path.exists(fullDstPath):
218                         execute(["git", "clone", "--no-checkout", url, fullDstPath])
219
220                 pushWorkingDir(fullDstPath)
221                 try:
222                         execute(["git", "fetch", url, "+refs/heads/*:refs/remotes/origin/*"])
223                         execute(["git", "checkout", self.revision])
224                 finally:
225                         popWorkingDir()
226
227 def postExtractLibpng (path):
228         shutil.copy(os.path.join(path, "scripts", "pnglibconf.h.prebuilt"),
229                                 os.path.join(path, "pnglibconf.h"))
230
231 PACKAGES = [
232         SourcePackage(
233                 "http://zlib.net/zlib-1.2.11.tar.gz",
234                 "zlib-1.2.11.tar.gz",
235                 "c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1",
236                 "zlib"),
237         SourcePackage(
238                 "http://prdownloads.sourceforge.net/libpng/libpng-1.6.27.tar.gz",
239                 "libpng-1.6.27.tar.gz",
240                 "c9d164ec247f426a525a7b89936694aefbc91fb7a50182b198898b8fc91174b4",
241                 "libpng",
242                 postExtract = postExtractLibpng),
243         GitRepo(
244                 "https://github.com/KhronosGroup/SPIRV-Tools.git",
245                 None,
246                 "0b0454c42c6b6f6746434bd5c78c5c70f65d9c51",
247                 "spirv-tools"),
248         GitRepo(
249                 "https://github.com/KhronosGroup/glslang.git",
250                 None,
251                 "a5c5fb61180e8703ca85f36d618f98e16dc317e2",
252                 "glslang"),
253         GitRepo(
254                 "https://github.com/KhronosGroup/SPIRV-Headers.git",
255                 None,
256                 "2bf02308656f97898c5f7e433712f21737c61e4e",
257                 "spirv-headers"),
258 ]
259
260 def parseArgs ():
261         versionsForInsecure = ((2,7,9), (3,4,3))
262         versionsForInsecureStr = ' or '.join(('.'.join(str(x) for x in v)) for v in versionsForInsecure)
263
264         parser = argparse.ArgumentParser(description = "Fetch external sources")
265         parser.add_argument('--clean', dest='clean', action='store_true', default=False,
266                                                 help='Remove sources instead of fetching')
267         parser.add_argument('--insecure', dest='insecure', action='store_true', default=False,
268                                                 help="Disable certificate check for external sources."
269                                                 " Minimum python version required " + versionsForInsecureStr)
270         parser.add_argument('--protocol', dest='protocol', default=None, choices=['ssh', 'https'],
271                                                 help="Select protocol to checkout git repositories.")
272
273         args = parser.parse_args()
274
275         if args.insecure:
276                 for versionItem in versionsForInsecure:
277                         if (sys.version_info.major == versionItem[0]):
278                                 if sys.version_info < versionItem:
279                                         parser.error("For --insecure minimum required python version is " +
280                                                                 versionsForInsecureStr)
281                                 break;
282
283         return args
284
285 if __name__ == "__main__":
286         args = parseArgs()
287
288         for pkg in PACKAGES:
289                 if args.clean:
290                         pkg.clean()
291                 else:
292                         pkg.update(args.protocol)