Merge gerrit/vulkan-cts-1.0.0 into gerrit/vulkan-cts-1.0.1
[platform/upstream/VK-GL-CTS.git] / external / fetch_sources.py
1 # -*- coding: utf-8 -*-
2
3 #-------------------------------------------------------------------------
4 # drawElements Quality Program utilities
5 # --------------------------------------
6 #
7 # Copyright 2015 The Android Open Source Project
8 #
9 # Licensed under the Apache License, Version 2.0 (the "License");
10 # you may not use this file except in compliance with the License.
11 # You may obtain a copy of the License at
12 #
13 #      http://www.apache.org/licenses/LICENSE-2.0
14 #
15 # Unless required by applicable law or agreed to in writing, software
16 # distributed under the License is distributed on an "AS IS" BASIS,
17 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18 # See the License for the specific language governing permissions and
19 # limitations under the License.
20 #
21 #-------------------------------------------------------------------------
22
23 import os
24 import sys
25 import shutil
26 import tarfile
27 import urllib2
28 import hashlib
29 import argparse
30 import subprocess
31
32 sys.path.append(os.path.join(os.path.dirname(__file__), "..", "scripts"))
33
34 from build.common import *
35
36 EXTERNAL_DIR    = os.path.realpath(os.path.normpath(os.path.dirname(__file__)))
37
38 def computeChecksum (data):
39         return hashlib.sha256(data).hexdigest()
40
41 class Source:
42         def __init__(self, baseDir, extractDir):
43                 self.baseDir            = baseDir
44                 self.extractDir         = extractDir
45
46         def clean (self):
47                 fullDstPath = os.path.join(EXTERNAL_DIR, self.baseDir, self.extractDir)
48                 if os.path.exists(fullDstPath):
49                         shutil.rmtree(fullDstPath, ignore_errors=False)
50
51 class SourcePackage (Source):
52         def __init__(self, url, filename, checksum, baseDir, extractDir = "src", postExtract=None):
53                 Source.__init__(self, baseDir, extractDir)
54                 self.url                        = url
55                 self.filename           = filename
56                 self.checksum           = checksum
57                 self.archiveDir         = "packages"
58                 self.postExtract        = postExtract
59
60         def clean (self):
61                 Source.clean(self)
62                 self.removeArchives()
63
64         def update (self):
65                 if not self.isArchiveUpToDate():
66                         self.fetchAndVerifyArchive()
67
68                 # \note No way to verify that extracted contents match archive, re-extract
69                 Source.clean(self)
70                 self.extract()
71
72         def removeArchives (self):
73                 archiveDir = os.path.join(EXTERNAL_DIR, pkg.baseDir, pkg.archiveDir)
74                 if os.path.exists(archiveDir):
75                         shutil.rmtree(archiveDir, ignore_errors=False)
76
77         def isArchiveUpToDate (self):
78                 archiveFile = os.path.join(EXTERNAL_DIR, pkg.baseDir, pkg.archiveDir, pkg.filename)
79                 if os.path.exists(archiveFile):
80                         return computeChecksum(readFile(archiveFile)) == self.checksum
81                 else:
82                         return False
83
84         def fetchAndVerifyArchive (self):
85                 print "Fetching %s" % self.url
86
87                 req                     = urllib2.urlopen(self.url)
88                 data            = req.read()
89                 checksum        = computeChecksum(data)
90                 dstPath         = os.path.join(EXTERNAL_DIR, self.baseDir, self.archiveDir, self.filename)
91
92                 if checksum != self.checksum:
93                         raise Exception("Checksum mismatch for %s, exepected %s, got %s" % (self.filename, self.checksum, checksum))
94
95                 if not os.path.exists(os.path.dirname(dstPath)):
96                         os.mkdir(os.path.dirname(dstPath))
97
98                 writeFile(dstPath, data)
99
100         def extract (self):
101                 print "Extracting %s to %s/%s" % (self.filename, self.baseDir, self.extractDir)
102
103                 srcPath = os.path.join(EXTERNAL_DIR, self.baseDir, self.archiveDir, self.filename)
104                 tmpPath = os.path.join(EXTERNAL_DIR, ".extract-tmp-%s" % self.baseDir)
105                 dstPath = os.path.join(EXTERNAL_DIR, self.baseDir, self.extractDir)
106                 archive = tarfile.open(srcPath)
107
108                 if os.path.exists(tmpPath):
109                         shutil.rmtree(tmpPath, ignore_errors=False)
110
111                 os.mkdir(tmpPath)
112
113                 archive.extractall(tmpPath)
114                 archive.close()
115
116                 extractedEntries = os.listdir(tmpPath)
117                 if len(extractedEntries) != 1 or not os.path.isdir(os.path.join(tmpPath, extractedEntries[0])):
118                         raise Exception("%s doesn't contain single top-level directory" % self.filename)
119
120                 topLevelPath = os.path.join(tmpPath, extractedEntries[0])
121
122                 if not os.path.exists(dstPath):
123                         os.mkdir(dstPath)
124
125                 for entry in os.listdir(topLevelPath):
126                         if os.path.exists(os.path.join(dstPath, entry)):
127                                 raise Exception("%s exists already" % entry)
128
129                         shutil.move(os.path.join(topLevelPath, entry), dstPath)
130
131                 shutil.rmtree(tmpPath, ignore_errors=True)
132
133                 if self.postExtract != None:
134                         self.postExtract(dstPath)
135
136 class GitRepo (Source):
137         def __init__(self, url, revision, baseDir, extractDir = "src"):
138                 Source.__init__(self, baseDir, extractDir)
139                 self.url                = url
140                 self.revision   = revision
141
142         def update (self):
143                 fullDstPath = os.path.join(EXTERNAL_DIR, self.baseDir, self.extractDir)
144
145                 if not os.path.exists(fullDstPath):
146                         execute(["git", "clone", "--no-checkout", self.url, fullDstPath])
147
148                 pushWorkingDir(fullDstPath)
149                 try:
150                         execute(["git", "fetch", self.url, "+refs/heads/*:refs/remotes/origin/*"])
151                         execute(["git", "checkout", self.revision])
152                 finally:
153                         popWorkingDir()
154
155 def postExtractLibpng (path):
156         shutil.copy(os.path.join(path, "scripts", "pnglibconf.h.prebuilt"),
157                                 os.path.join(path, "pnglibconf.h"))
158
159 PACKAGES = [
160         SourcePackage(
161                 "http://zlib.net/zlib-1.2.10.tar.gz",
162                 "zlib-1.2.10.tar.gz",
163                 "8d7e9f698ce48787b6e1c67e6bff79e487303e66077e25cb9784ac8835978017",
164                 "zlib"),
165         SourcePackage(
166                 "http://prdownloads.sourceforge.net/libpng/libpng-1.6.27.tar.gz",
167                 "libpng-1.6.27.tar.gz",
168                 "c9d164ec247f426a525a7b89936694aefbc91fb7a50182b198898b8fc91174b4",
169                 "libpng",
170                 postExtract = postExtractLibpng),
171         GitRepo(
172                 "https://github.com/KhronosGroup/SPIRV-Tools.git",
173                 "d12a10d2dd0cc4236ef227707c11f991b9c0d544",
174                 "spirv-tools"),
175         GitRepo(
176                 "https://github.com/KhronosGroup/glslang.git",
177                 "e3aa654c4b0c761b28d7864192ca8ceea6faf70a",
178                 "glslang"),
179 ]
180
181 def parseArgs ():
182         parser = argparse.ArgumentParser(description = "Fetch external sources")
183         parser.add_argument('--clean', dest='clean', action='store_true', default=False,
184                                                 help='Remove sources instead of fetching')
185         return parser.parse_args()
186
187 if __name__ == "__main__":
188         args = parseArgs()
189
190         for pkg in PACKAGES:
191                 if args.clean:
192                         pkg.clean()
193                 else:
194                         pkg.update()