Fix missing dependency on sparse binds
[platform/upstream/VK-GL-CTS.git] / scripts / make_release.py
1 # -*- coding: utf-8 -*-
2
3 #-------------------------------------------------------------------------
4 # drawElements Quality Program utilities
5 # --------------------------------------
6 #
7 # Copyright 2015 The Android Open Source Project
8 #
9 # Licensed under the Apache License, Version 2.0 (the "License");
10 # you may not use this file except in compliance with the License.
11 # You may obtain a copy of the License at
12 #
13 #      http://www.apache.org/licenses/LICENSE-2.0
14 #
15 # Unless required by applicable law or agreed to in writing, software
16 # distributed under the License is distributed on an "AS IS" BASIS,
17 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18 # See the License for the specific language governing permissions and
19 # limitations under the License.
20 #
21 #-------------------------------------------------------------------------
22
23 import os
24 import re
25 import sys
26 import copy
27 import zlib
28 import time
29 import shlex
30 import shutil
31 import fnmatch
32 import tarfile
33 import argparse
34 import platform
35 import datetime
36 import tempfile
37 import posixpath
38 import subprocess
39
40 from ctsbuild.common import *
41 from ctsbuild.config import *
42 from ctsbuild.build import *
43
44 pythonExecutable = sys.executable or "python"
45
46 def die (msg):
47         print(msg)
48         sys.exit(-1)
49
50 def removeLeadingPath (path, basePath):
51         # Both inputs must be normalized already
52         assert os.path.normpath(path) == path
53         assert os.path.normpath(basePath) == basePath
54         return path[len(basePath) + 1:]
55
56 def findFile (candidates):
57         for file in candidates:
58                 if os.path.exists(file):
59                         return file
60         return None
61
62 def getFileList (basePath):
63         allFiles        = []
64         basePath        = os.path.normpath(basePath)
65         for root, dirs, files in os.walk(basePath):
66                 for file in files:
67                         relPath = removeLeadingPath(os.path.normpath(os.path.join(root, file)), basePath)
68                         allFiles.append(relPath)
69         return allFiles
70
71 def toDatetime (dateTuple):
72         Y, M, D = dateTuple
73         return datetime.datetime(Y, M, D)
74
75 class PackageBuildInfo:
76         def __init__ (self, releaseConfig, srcBasePath, dstBasePath, tmpBasePath):
77                 self.releaseConfig      = releaseConfig
78                 self.srcBasePath        = srcBasePath
79                 self.dstBasePath        = dstBasePath
80                 self.tmpBasePath        = tmpBasePath
81
82         def getReleaseConfig (self):
83                 return self.releaseConfig
84
85         def getReleaseVersion (self):
86                 return self.releaseConfig.getVersion()
87
88         def getReleaseId (self):
89                 # Release id is crc32(releaseConfig + release)
90                 return zlib.crc32(self.releaseConfig.getName() + self.releaseConfig.getVersion()) & 0xffffffff
91
92         def getSrcBasePath (self):
93                 return self.srcBasePath
94
95         def getTmpBasePath (self):
96                 return self.tmpBasePath
97
98 class DstFile (object):
99         def __init__ (self, dstFile):
100                 self.dstFile = dstFile
101
102         def makeDir (self):
103                 dirName = os.path.dirname(self.dstFile)
104                 if not os.path.exists(dirName):
105                         os.makedirs(dirName)
106
107         def make (self, packageBuildInfo):
108                 assert False # Should not be called
109
110 class CopyFile (DstFile):
111         def __init__ (self, srcFile, dstFile):
112                 super(CopyFile, self).__init__(dstFile)
113                 self.srcFile = srcFile
114
115         def make (self, packageBuildInfo):
116                 self.makeDir()
117                 if os.path.exists(self.dstFile):
118                         die("%s already exists" % self.dstFile)
119                 shutil.copyfile(self.srcFile, self.dstFile)
120
121 class GenReleaseInfoFileTarget (DstFile):
122         def __init__ (self, dstFile):
123                 super(GenReleaseInfoFileTarget, self).__init__(dstFile)
124
125         def make (self, packageBuildInfo):
126                 self.makeDir()
127
128                 scriptPath = os.path.normpath(os.path.join(packageBuildInfo.srcBasePath, "framework", "qphelper", "gen_release_info.py"))
129                 execute([
130                                 pythonExecutable,
131                                 "-B", # no .py[co]
132                                 scriptPath,
133                                 "--name=%s" % packageBuildInfo.getReleaseVersion(),
134                                 "--id=0x%08x" % packageBuildInfo.getReleaseId(),
135                                 "--out=%s" % self.dstFile
136                         ])
137
138 class GenCMake (DstFile):
139         def __init__ (self, srcFile, dstFile, replaceVars):
140                 super(GenCMake, self).__init__(dstFile)
141                 self.srcFile            = srcFile
142                 self.replaceVars        = replaceVars
143
144         def make (self, packageBuildInfo):
145                 self.makeDir()
146                 print("    GenCMake: %s" % removeLeadingPath(self.dstFile, packageBuildInfo.dstBasePath))
147                 src = readFile(self.srcFile)
148                 for var, value in self.replaceVars:
149                         src = re.sub('set\(%s\s+"[^"]*"' % re.escape(var),
150                                                  'set(%s "%s"' % (var, value), src)
151                 writeFile(self.dstFile, src)
152
153 def createFileTargets (srcBasePath, dstBasePath, files, filters):
154         usedFiles       = set() # Files that are already included by other filters
155         targets         = []
156
157         for isMatch, createFileObj in filters:
158                 # Build list of files that match filter
159                 matchingFiles = []
160                 for file in files:
161                         if not file in usedFiles and isMatch(file):
162                                 matchingFiles.append(file)
163
164                 # Build file objects, add to used set
165                 for file in matchingFiles:
166                         usedFiles.add(file)
167                         targets.append(createFileObj(os.path.join(srcBasePath, file), os.path.join(dstBasePath, file)))
168
169         return targets
170
171 # Generates multiple file targets based on filters
172 class FileTargetGroup:
173         def __init__ (self, srcBasePath, dstBasePath, filters, srcBasePathFunc=PackageBuildInfo.getSrcBasePath):
174                 self.srcBasePath        = srcBasePath
175                 self.dstBasePath        = dstBasePath
176                 self.filters            = filters
177                 self.getSrcBasePath     = srcBasePathFunc
178
179         def make (self, packageBuildInfo):
180                 fullSrcPath             = os.path.normpath(os.path.join(self.getSrcBasePath(packageBuildInfo), self.srcBasePath))
181                 fullDstPath             = os.path.normpath(os.path.join(packageBuildInfo.dstBasePath, self.dstBasePath))
182
183                 allFiles                = getFileList(fullSrcPath)
184                 targets                 = createFileTargets(fullSrcPath, fullDstPath, allFiles, self.filters)
185
186                 # Make all file targets
187                 for file in targets:
188                         file.make(packageBuildInfo)
189
190 # Single file target
191 class SingleFileTarget:
192         def __init__ (self, srcFile, dstFile, makeTarget):
193                 self.srcFile    = srcFile
194                 self.dstFile    = dstFile
195                 self.makeTarget = makeTarget
196
197         def make (self, packageBuildInfo):
198                 fullSrcPath             = os.path.normpath(os.path.join(packageBuildInfo.srcBasePath, self.srcFile))
199                 fullDstPath             = os.path.normpath(os.path.join(packageBuildInfo.dstBasePath, self.dstFile))
200
201                 target = self.makeTarget(fullSrcPath, fullDstPath)
202                 target.make(packageBuildInfo)
203
204 class BuildTarget:
205         def __init__ (self, baseConfig, generator, targets = None):
206                 self.baseConfig = baseConfig
207                 self.generator  = generator
208                 self.targets    = targets
209
210         def make (self, packageBuildInfo):
211                 print("    Building %s" % self.baseConfig.getBuildDir())
212
213                 # Create config with full build dir path
214                 config = BuildConfig(os.path.join(packageBuildInfo.getTmpBasePath(), self.baseConfig.getBuildDir()),
215                                                          self.baseConfig.getBuildType(),
216                                                          self.baseConfig.getArgs(),
217                                                          srcPath = os.path.join(packageBuildInfo.dstBasePath, "src"))
218
219                 assert not os.path.exists(config.getBuildDir())
220                 build(config, self.generator, self.targets)
221
222 class BuildAndroidTarget:
223         def __init__ (self, dstFile):
224                 self.dstFile = dstFile
225
226         def make (self, packageBuildInfo):
227                 print("    Building Android binary")
228
229                 buildRoot = os.path.join(packageBuildInfo.tmpBasePath, "android-build")
230
231                 assert not os.path.exists(buildRoot)
232                 os.makedirs(buildRoot)
233
234                 # Execute build script
235                 scriptPath = os.path.normpath(os.path.join(packageBuildInfo.dstBasePath, "src", "android", "scripts", "build.py"))
236                 execute([
237                                 pythonExecutable,
238                                 "-B", # no .py[co]
239                                 scriptPath,
240                                 "--build-root=%s" % buildRoot,
241                         ])
242
243                 srcFile         = os.path.normpath(os.path.join(buildRoot, "package", "bin", "dEQP-debug.apk"))
244                 dstFile         = os.path.normpath(os.path.join(packageBuildInfo.dstBasePath, self.dstFile))
245
246                 CopyFile(srcFile, dstFile).make(packageBuildInfo)
247
248 class FetchExternalSourcesTarget:
249         def __init__ (self):
250                 pass
251
252         def make (self, packageBuildInfo):
253                 scriptPath = os.path.normpath(os.path.join(packageBuildInfo.dstBasePath, "src", "external", "fetch_sources.py"))
254                 execute([
255                                 pythonExecutable,
256                                 "-B", # no .py[co]
257                                 scriptPath,
258                         ])
259
260 class RemoveSourcesTarget:
261         def __init__ (self):
262                 pass
263
264         def make (self, packageBuildInfo):
265                 shutil.rmtree(os.path.join(packageBuildInfo.dstBasePath, "src"), ignore_errors=False)
266
267 class Module:
268         def __init__ (self, name, targets):
269                 self.name               = name
270                 self.targets    = targets
271
272         def make (self, packageBuildInfo):
273                 for target in self.targets:
274                         target.make(packageBuildInfo)
275
276 class ReleaseConfig:
277         def __init__ (self, name, version, modules, sources = True):
278                 self.name                       = name
279                 self.version            = version
280                 self.modules            = modules
281                 self.sources            = sources
282
283         def getName (self):
284                 return self.name
285
286         def getVersion (self):
287                 return self.version
288
289         def getModules (self):
290                 return self.modules
291
292         def packageWithSources (self):
293                 return self.sources
294
295 def matchIncludeExclude (includePatterns, excludePatterns, filename):
296         components = os.path.normpath(filename).split(os.sep)
297         for pattern in excludePatterns:
298                 for component in components:
299                         if fnmatch.fnmatch(component, pattern):
300                                 return False
301
302         for pattern in includePatterns:
303                 for component in components:
304                         if fnmatch.fnmatch(component, pattern):
305                                 return True
306
307         return False
308
309 def copyFileFilter (includePatterns, excludePatterns=[]):
310         return (lambda f: matchIncludeExclude(includePatterns, excludePatterns, f),
311                         lambda s, d: CopyFile(s, d))
312
313 def makeFileCopyGroup (srcDir, dstDir, includePatterns, excludePatterns=[]):
314         return FileTargetGroup(srcDir, dstDir, [copyFileFilter(includePatterns, excludePatterns)])
315
316 def makeTmpFileCopyGroup (srcDir, dstDir, includePatterns, excludePatterns=[]):
317         return FileTargetGroup(srcDir, dstDir, [copyFileFilter(includePatterns, excludePatterns)], PackageBuildInfo.getTmpBasePath)
318
319 def makeFileCopy (srcFile, dstFile):
320         return SingleFileTarget(srcFile, dstFile, lambda s, d: CopyFile(s, d))
321
322 def getReleaseFileName (configName, releaseName):
323         today = datetime.date.today()
324         return "dEQP-%s-%04d-%02d-%02d-%s" % (releaseName, today.year, today.month, today.day, configName)
325
326 def getTempDir ():
327         dirName = os.path.join(tempfile.gettempdir(), "dEQP-Releases")
328         if not os.path.exists(dirName):
329                 os.makedirs(dirName)
330         return dirName
331
332 def makeRelease (releaseConfig):
333         releaseName                     = getReleaseFileName(releaseConfig.getName(), releaseConfig.getVersion())
334         tmpPath                         = getTempDir()
335         srcBasePath                     = DEQP_DIR
336         dstBasePath                     = os.path.join(tmpPath, releaseName)
337         tmpBasePath                     = os.path.join(tmpPath, releaseName + "-tmp")
338         packageBuildInfo        = PackageBuildInfo(releaseConfig, srcBasePath, dstBasePath, tmpBasePath)
339         dstArchiveName          = releaseName + ".tar.bz2"
340
341         print("Creating release %s to %s" % (releaseName, tmpPath))
342
343         # Remove old temporary dirs
344         for path in [dstBasePath, tmpBasePath]:
345                 if os.path.exists(path):
346                         shutil.rmtree(path, ignore_errors=False)
347
348         # Make all modules
349         for module in releaseConfig.getModules():
350                 print("  Processing module %s" % module.name)
351                 module.make(packageBuildInfo)
352
353         # Remove sources?
354         if not releaseConfig.packageWithSources():
355                 shutil.rmtree(os.path.join(dstBasePath, "src"), ignore_errors=False)
356
357         # Create archive
358         print("Creating %s" % dstArchiveName)
359         archive = tarfile.open(dstArchiveName, 'w:bz2')
360         archive.add(dstBasePath, arcname=releaseName)
361         archive.close()
362
363         # Remove tmp dirs
364         for path in [dstBasePath, tmpBasePath]:
365                 if os.path.exists(path):
366                         shutil.rmtree(path, ignore_errors=False)
367
368         print("Done!")
369
370 # Module declarations
371
372 SRC_FILE_PATTERNS       = ["*.h", "*.hpp", "*.c", "*.cpp", "*.m", "*.mm", "*.inl", "*.java", "*.aidl", "CMakeLists.txt", "LICENSE.txt", "*.cmake"]
373 TARGET_PATTERNS         = ["*.cmake", "*.h", "*.lib", "*.dll", "*.so", "*.txt"]
374
375 BASE = Module("Base", [
376         makeFileCopy            ("LICENSE",                                                                     "src/LICENSE"),
377         makeFileCopy            ("CMakeLists.txt",                                                      "src/CMakeLists.txt"),
378         makeFileCopyGroup       ("targets",                                                                     "src/targets",                                                  TARGET_PATTERNS),
379         makeFileCopyGroup       ("execserver",                                                          "src/execserver",                                               SRC_FILE_PATTERNS),
380         makeFileCopyGroup       ("executor",                                                            "src/executor",                                                 SRC_FILE_PATTERNS),
381         makeFileCopy            ("modules/CMakeLists.txt",                                      "src/modules/CMakeLists.txt"),
382         makeFileCopyGroup       ("external",                                                            "src/external",                                                 ["CMakeLists.txt", "*.py"]),
383
384         # Stylesheet for displaying test logs on browser
385         makeFileCopyGroup       ("doc/testlog-stylesheet",                                      "doc/testlog-stylesheet",                               ["*"]),
386
387         # Non-optional parts of framework
388         makeFileCopy            ("framework/CMakeLists.txt",                            "src/framework/CMakeLists.txt"),
389         makeFileCopyGroup       ("framework/delibs",                                            "src/framework/delibs",                                 SRC_FILE_PATTERNS),
390         makeFileCopyGroup       ("framework/common",                                            "src/framework/common",                                 SRC_FILE_PATTERNS),
391         makeFileCopyGroup       ("framework/qphelper",                                          "src/framework/qphelper",                               SRC_FILE_PATTERNS),
392         makeFileCopyGroup       ("framework/platform",                                          "src/framework/platform",                               SRC_FILE_PATTERNS),
393         makeFileCopyGroup       ("framework/opengl",                                            "src/framework/opengl",                                 SRC_FILE_PATTERNS, ["simplereference"]),
394         makeFileCopyGroup       ("framework/egl",                                                       "src/framework/egl",                                    SRC_FILE_PATTERNS),
395
396         # android sources
397         makeFileCopyGroup       ("android/package/src",                                         "src/android/package/src",                              SRC_FILE_PATTERNS),
398         makeFileCopy            ("android/package/AndroidManifest.xml",         "src/android/package/AndroidManifest.xml"),
399         makeFileCopyGroup       ("android/package/res",                                         "src/android/package/res",                              ["*.png", "*.xml"]),
400         makeFileCopyGroup       ("android/scripts",                                                     "src/android/scripts", [
401                 "common.py",
402                 "build.py",
403                 "resources.py",
404                 "install.py",
405                 "launch.py",
406                 "debug.py"
407                 ]),
408
409         # Release info
410         GenReleaseInfoFileTarget("src/framework/qphelper/qpReleaseInfo.inl")
411 ])
412
413 DOCUMENTATION = Module("Documentation", [
414         makeFileCopyGroup       ("doc/pdf",                                                                     "doc",                                                                  ["*.pdf"]),
415         makeFileCopyGroup       ("doc",                                                                         "doc",                                                                  ["porting_layer_changes_*.txt"]),
416 ])
417
418 GLSHARED = Module("Shared GL Tests", [
419         # Optional framework components
420         makeFileCopyGroup       ("framework/randomshaders",                                     "src/framework/randomshaders",                  SRC_FILE_PATTERNS),
421         makeFileCopyGroup       ("framework/opengl/simplereference",            "src/framework/opengl/simplereference", SRC_FILE_PATTERNS),
422         makeFileCopyGroup       ("framework/referencerenderer",                         "src/framework/referencerenderer",              SRC_FILE_PATTERNS),
423
424         makeFileCopyGroup       ("modules/glshared",                                            "src/modules/glshared",                                 SRC_FILE_PATTERNS),
425 ])
426
427 GLES2 = Module("GLES2", [
428         makeFileCopyGroup       ("modules/gles2",                                                       "src/modules/gles2",                                    SRC_FILE_PATTERNS),
429         makeFileCopyGroup       ("data/gles2",                                                          "src/data/gles2",                                               ["*.*"]),
430         makeFileCopyGroup       ("doc/testspecs/GLES2",                                         "doc/testspecs/GLES2",                                  ["*.txt"])
431 ])
432
433 GLES3 = Module("GLES3", [
434         makeFileCopyGroup       ("modules/gles3",                                                       "src/modules/gles3",                                    SRC_FILE_PATTERNS),
435         makeFileCopyGroup       ("data/gles3",                                                          "src/data/gles3",                                               ["*.*"]),
436         makeFileCopyGroup       ("doc/testspecs/GLES3",                                         "doc/testspecs/GLES3",                                  ["*.txt"])
437 ])
438
439 GLES31 = Module("GLES31", [
440         makeFileCopyGroup       ("modules/gles31",                                                      "src/modules/gles31",                                   SRC_FILE_PATTERNS),
441         makeFileCopyGroup       ("data/gles31",                                                         "src/data/gles31",                                              ["*.*"]),
442         makeFileCopyGroup       ("doc/testspecs/GLES31",                                        "doc/testspecs/GLES31",                                 ["*.txt"])
443 ])
444
445 EGL = Module("EGL", [
446         makeFileCopyGroup       ("modules/egl",                                                         "src/modules/egl",                                              SRC_FILE_PATTERNS)
447 ])
448
449 INTERNAL = Module("Internal", [
450         makeFileCopyGroup       ("modules/internal",                                            "src/modules/internal",                                 SRC_FILE_PATTERNS),
451         makeFileCopyGroup       ("data/internal",                                                       "src/data/internal",                                    ["*.*"]),
452 ])
453
454 EXTERNAL_SRCS = Module("External sources", [
455         FetchExternalSourcesTarget()
456 ])
457
458 ANDROID_BINARIES = Module("Android Binaries", [
459         BuildAndroidTarget      ("bin/android/dEQP.apk"),
460         makeFileCopyGroup       ("targets/android",                                                     "bin/android",                                                  ["*.bat", "*.sh"]),
461 ])
462
463 COMMON_BUILD_ARGS       = ['-DPNG_SRC_PATH=%s' % os.path.realpath(os.path.join(DEQP_DIR, '..', 'libpng'))]
464 NULL_X32_CONFIG         = BuildConfig('null-x32',       'Release', ['-DDEQP_TARGET=null', '-DCMAKE_C_FLAGS=-m32', '-DCMAKE_CXX_FLAGS=-m32'] + COMMON_BUILD_ARGS)
465 NULL_X64_CONFIG         = BuildConfig('null-x64',       'Release', ['-DDEQP_TARGET=null', '-DCMAKE_C_FLAGS=-m64', '-DCMAKE_CXX_FLAGS=-m64'] + COMMON_BUILD_ARGS)
466 GLX_X32_CONFIG          = BuildConfig('glx-x32',        'Release', ['-DDEQP_TARGET=x11_glx', '-DCMAKE_C_FLAGS=-m32', '-DCMAKE_CXX_FLAGS=-m32'] + COMMON_BUILD_ARGS)
467 GLX_X64_CONFIG          = BuildConfig('glx-x64',        'Release', ['-DDEQP_TARGET=x11_glx', '-DCMAKE_C_FLAGS=-m64', '-DCMAKE_CXX_FLAGS=-m64'] + COMMON_BUILD_ARGS)
468
469 EXCLUDE_BUILD_FILES = ["CMakeFiles", "*.a", "*.cmake"]
470
471 LINUX_X32_COMMON_BINARIES = Module("Linux x32 Common Binaries", [
472         BuildTarget                     (NULL_X32_CONFIG, ANY_UNIX_GENERATOR),
473         makeTmpFileCopyGroup(NULL_X32_CONFIG.getBuildDir() + "/execserver",             "bin/linux32",                                  ["*"],  EXCLUDE_BUILD_FILES),
474         makeTmpFileCopyGroup(NULL_X32_CONFIG.getBuildDir() + "/executor",               "bin/linux32",                                  ["*"],  EXCLUDE_BUILD_FILES),
475 ])
476
477 LINUX_X64_COMMON_BINARIES = Module("Linux x64 Common Binaries", [
478         BuildTarget                     (NULL_X64_CONFIG, ANY_UNIX_GENERATOR),
479         makeTmpFileCopyGroup(NULL_X64_CONFIG.getBuildDir() + "/execserver",             "bin/linux64",                                  ["*"],  EXCLUDE_BUILD_FILES),
480         makeTmpFileCopyGroup(NULL_X64_CONFIG.getBuildDir() + "/executor",               "bin/linux64",                                  ["*"],  EXCLUDE_BUILD_FILES),
481 ])
482
483 # Special module to remove src dir, for example after binary build
484 REMOVE_SOURCES = Module("Remove sources from package", [
485         RemoveSourcesTarget()
486 ])
487
488 # Release configuration
489
490 ALL_MODULES             = [
491         BASE,
492         DOCUMENTATION,
493         GLSHARED,
494         GLES2,
495         GLES3,
496         GLES31,
497         EGL,
498         INTERNAL,
499         EXTERNAL_SRCS,
500 ]
501
502 ALL_BINARIES    = [
503         LINUX_X64_COMMON_BINARIES,
504         ANDROID_BINARIES,
505 ]
506
507 RELEASE_CONFIGS = {
508         "src":          ALL_MODULES,
509         "src-bin":      ALL_MODULES + ALL_BINARIES,
510         "bin":          ALL_MODULES + ALL_BINARIES + [REMOVE_SOURCES],
511 }
512
513 def parseArgs ():
514         parser = argparse.ArgumentParser(description = "Build release package")
515         parser.add_argument("-c",
516                                                 "--config",
517                                                 dest="config",
518                                                 choices=RELEASE_CONFIGS.keys(),
519                                                 required=True,
520                                                 help="Release configuration")
521         parser.add_argument("-n",
522                                                 "--name",
523                                                 dest="name",
524                                                 required=True,
525                                                 help="Package-specific name")
526         parser.add_argument("-v",
527                                                 "--version",
528                                                 dest="version",
529                                                 required=True,
530                                                 help="Version code")
531         return parser.parse_args()
532
533 if __name__ == "__main__":
534         args    = parseArgs()
535         config  = ReleaseConfig(args.name, args.version, RELEASE_CONFIGS[args.config])
536         makeRelease(config)