Merge upstream-vulkan-cts-1.0-dev into master am: fb20fc5f5d am: c0d5f3c3bd am: 93a21...
[platform/upstream/VK-GL-CTS.git] / scripts / make_release.py
1 # -*- coding: utf-8 -*-
2
3 #-------------------------------------------------------------------------
4 # drawElements Quality Program utilities
5 # --------------------------------------
6 #
7 # Copyright 2015 The Android Open Source Project
8 #
9 # Licensed under the Apache License, Version 2.0 (the "License");
10 # you may not use this file except in compliance with the License.
11 # You may obtain a copy of the License at
12 #
13 #      http://www.apache.org/licenses/LICENSE-2.0
14 #
15 # Unless required by applicable law or agreed to in writing, software
16 # distributed under the License is distributed on an "AS IS" BASIS,
17 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18 # See the License for the specific language governing permissions and
19 # limitations under the License.
20 #
21 #-------------------------------------------------------------------------
22
23 import os
24 import re
25 import sys
26 import copy
27 import zlib
28 import time
29 import shlex
30 import shutil
31 import fnmatch
32 import tarfile
33 import argparse
34 import platform
35 import datetime
36 import tempfile
37 import posixpath
38 import subprocess
39
40 from build.common import *
41 from build.config import *
42 from build.build import *
43
44 def die (msg):
45         print msg
46         sys.exit(-1)
47
48 def removeLeadingPath (path, basePath):
49         # Both inputs must be normalized already
50         assert os.path.normpath(path) == path
51         assert os.path.normpath(basePath) == basePath
52         return path[len(basePath) + 1:]
53
54 def findFile (candidates):
55         for file in candidates:
56                 if os.path.exists(file):
57                         return file
58         return None
59
60 def getFileList (basePath):
61         allFiles        = []
62         basePath        = os.path.normpath(basePath)
63         for root, dirs, files in os.walk(basePath):
64                 for file in files:
65                         relPath = removeLeadingPath(os.path.normpath(os.path.join(root, file)), basePath)
66                         allFiles.append(relPath)
67         return allFiles
68
69 def toDatetime (dateTuple):
70         Y, M, D = dateTuple
71         return datetime.datetime(Y, M, D)
72
73 class PackageBuildInfo:
74         def __init__ (self, releaseConfig, srcBasePath, dstBasePath, tmpBasePath):
75                 self.releaseConfig      = releaseConfig
76                 self.srcBasePath        = srcBasePath
77                 self.dstBasePath        = dstBasePath
78                 self.tmpBasePath        = tmpBasePath
79
80         def getReleaseConfig (self):
81                 return self.releaseConfig
82
83         def getReleaseVersion (self):
84                 return self.releaseConfig.getVersion()
85
86         def getReleaseId (self):
87                 # Release id is crc32(releaseConfig + release)
88                 return zlib.crc32(self.releaseConfig.getName() + self.releaseConfig.getVersion()) & 0xffffffff
89
90         def getSrcBasePath (self):
91                 return self.srcBasePath
92
93         def getTmpBasePath (self):
94                 return self.tmpBasePath
95
96 class DstFile (object):
97         def __init__ (self, dstFile):
98                 self.dstFile = dstFile
99
100         def makeDir (self):
101                 dirName = os.path.dirname(self.dstFile)
102                 if not os.path.exists(dirName):
103                         os.makedirs(dirName)
104
105         def make (self, packageBuildInfo):
106                 assert False # Should not be called
107
108 class CopyFile (DstFile):
109         def __init__ (self, srcFile, dstFile):
110                 super(CopyFile, self).__init__(dstFile)
111                 self.srcFile = srcFile
112
113         def make (self, packageBuildInfo):
114                 self.makeDir()
115                 if os.path.exists(self.dstFile):
116                         die("%s already exists" % self.dstFile)
117                 shutil.copyfile(self.srcFile, self.dstFile)
118
119 class GenReleaseInfoFileTarget (DstFile):
120         def __init__ (self, dstFile):
121                 super(GenReleaseInfoFileTarget, self).__init__(dstFile)
122
123         def make (self, packageBuildInfo):
124                 self.makeDir()
125
126                 scriptPath = os.path.normpath(os.path.join(packageBuildInfo.srcBasePath, "framework", "qphelper", "gen_release_info.py"))
127                 execute([
128                                 "python",
129                                 "-B", # no .py[co]
130                                 scriptPath,
131                                 "--name=%s" % packageBuildInfo.getReleaseVersion(),
132                                 "--id=0x%08x" % packageBuildInfo.getReleaseId(),
133                                 "--out=%s" % self.dstFile
134                         ])
135
136 class GenCMake (DstFile):
137         def __init__ (self, srcFile, dstFile, replaceVars):
138                 super(GenCMake, self).__init__(dstFile)
139                 self.srcFile            = srcFile
140                 self.replaceVars        = replaceVars
141
142         def make (self, packageBuildInfo):
143                 self.makeDir()
144                 print "    GenCMake: %s" % removeLeadingPath(self.dstFile, packageBuildInfo.dstBasePath)
145                 src = readFile(self.srcFile)
146                 for var, value in self.replaceVars:
147                         src = re.sub('set\(%s\s+"[^"]*"' % re.escape(var),
148                                                  'set(%s "%s"' % (var, value), src)
149                 writeFile(self.dstFile, src)
150
151 def createFileTargets (srcBasePath, dstBasePath, files, filters):
152         usedFiles       = set() # Files that are already included by other filters
153         targets         = []
154
155         for isMatch, createFileObj in filters:
156                 # Build list of files that match filter
157                 matchingFiles = []
158                 for file in files:
159                         if not file in usedFiles and isMatch(file):
160                                 matchingFiles.append(file)
161
162                 # Build file objects, add to used set
163                 for file in matchingFiles:
164                         usedFiles.add(file)
165                         targets.append(createFileObj(os.path.join(srcBasePath, file), os.path.join(dstBasePath, file)))
166
167         return targets
168
169 # Generates multiple file targets based on filters
170 class FileTargetGroup:
171         def __init__ (self, srcBasePath, dstBasePath, filters, srcBasePathFunc=PackageBuildInfo.getSrcBasePath):
172                 self.srcBasePath        = srcBasePath
173                 self.dstBasePath        = dstBasePath
174                 self.filters            = filters
175                 self.getSrcBasePath     = srcBasePathFunc
176
177         def make (self, packageBuildInfo):
178                 fullSrcPath             = os.path.normpath(os.path.join(self.getSrcBasePath(packageBuildInfo), self.srcBasePath))
179                 fullDstPath             = os.path.normpath(os.path.join(packageBuildInfo.dstBasePath, self.dstBasePath))
180
181                 allFiles                = getFileList(fullSrcPath)
182                 targets                 = createFileTargets(fullSrcPath, fullDstPath, allFiles, self.filters)
183
184                 # Make all file targets
185                 for file in targets:
186                         file.make(packageBuildInfo)
187
188 # Single file target
189 class SingleFileTarget:
190         def __init__ (self, srcFile, dstFile, makeTarget):
191                 self.srcFile    = srcFile
192                 self.dstFile    = dstFile
193                 self.makeTarget = makeTarget
194
195         def make (self, packageBuildInfo):
196                 fullSrcPath             = os.path.normpath(os.path.join(packageBuildInfo.srcBasePath, self.srcFile))
197                 fullDstPath             = os.path.normpath(os.path.join(packageBuildInfo.dstBasePath, self.dstFile))
198
199                 target = self.makeTarget(fullSrcPath, fullDstPath)
200                 target.make(packageBuildInfo)
201
202 class BuildTarget:
203         def __init__ (self, baseConfig, generator, targets = None):
204                 self.baseConfig = baseConfig
205                 self.generator  = generator
206                 self.targets    = targets
207
208         def make (self, packageBuildInfo):
209                 print "    Building %s" % self.baseConfig.getBuildDir()
210
211                 # Create config with full build dir path
212                 config = BuildConfig(os.path.join(packageBuildInfo.getTmpBasePath(), self.baseConfig.getBuildDir()),
213                                                          self.baseConfig.getBuildType(),
214                                                          self.baseConfig.getArgs(),
215                                                          srcPath = os.path.join(packageBuildInfo.dstBasePath, "src"))
216
217                 assert not os.path.exists(config.getBuildDir())
218                 build(config, self.generator, self.targets)
219
220 class BuildAndroidTarget:
221         def __init__ (self, dstFile):
222                 self.dstFile = dstFile
223
224         def make (self, packageBuildInfo):
225                 print "    Building Android binary"
226
227                 buildRoot = os.path.join(packageBuildInfo.tmpBasePath, "android-build")
228
229                 assert not os.path.exists(buildRoot)
230                 os.makedirs(buildRoot)
231
232                 # Execute build script
233                 scriptPath = os.path.normpath(os.path.join(packageBuildInfo.dstBasePath, "src", "android", "scripts", "build.py"))
234                 execute([
235                                 "python",
236                                 "-B", # no .py[co]
237                                 scriptPath,
238                                 "--build-root=%s" % buildRoot,
239                         ])
240
241                 srcFile         = os.path.normpath(os.path.join(buildRoot, "package", "bin", "dEQP-debug.apk"))
242                 dstFile         = os.path.normpath(os.path.join(packageBuildInfo.dstBasePath, self.dstFile))
243
244                 CopyFile(srcFile, dstFile).make(packageBuildInfo)
245
246 class FetchExternalSourcesTarget:
247         def __init__ (self):
248                 pass
249
250         def make (self, packageBuildInfo):
251                 scriptPath = os.path.normpath(os.path.join(packageBuildInfo.dstBasePath, "src", "external", "fetch_sources.py"))
252                 execute([
253                                 "python",
254                                 "-B", # no .py[co]
255                                 scriptPath,
256                         ])
257
258 class RemoveSourcesTarget:
259         def __init__ (self):
260                 pass
261
262         def make (self, packageBuildInfo):
263                 shutil.rmtree(os.path.join(packageBuildInfo.dstBasePath, "src"), ignore_errors=False)
264
265 class Module:
266         def __init__ (self, name, targets):
267                 self.name               = name
268                 self.targets    = targets
269
270         def make (self, packageBuildInfo):
271                 for target in self.targets:
272                         target.make(packageBuildInfo)
273
274 class ReleaseConfig:
275         def __init__ (self, name, version, modules, sources = True):
276                 self.name                       = name
277                 self.version            = version
278                 self.modules            = modules
279                 self.sources            = sources
280
281         def getName (self):
282                 return self.name
283
284         def getVersion (self):
285                 return self.version
286
287         def getModules (self):
288                 return self.modules
289
290         def packageWithSources (self):
291                 return self.sources
292
293 def matchIncludeExclude (includePatterns, excludePatterns, filename):
294         components = os.path.normpath(filename).split(os.sep)
295         for pattern in excludePatterns:
296                 for component in components:
297                         if fnmatch.fnmatch(component, pattern):
298                                 return False
299
300         for pattern in includePatterns:
301                 for component in components:
302                         if fnmatch.fnmatch(component, pattern):
303                                 return True
304
305         return False
306
307 def copyFileFilter (includePatterns, excludePatterns=[]):
308         return (lambda f: matchIncludeExclude(includePatterns, excludePatterns, f),
309                         lambda s, d: CopyFile(s, d))
310
311 def makeFileCopyGroup (srcDir, dstDir, includePatterns, excludePatterns=[]):
312         return FileTargetGroup(srcDir, dstDir, [copyFileFilter(includePatterns, excludePatterns)])
313
314 def makeTmpFileCopyGroup (srcDir, dstDir, includePatterns, excludePatterns=[]):
315         return FileTargetGroup(srcDir, dstDir, [copyFileFilter(includePatterns, excludePatterns)], PackageBuildInfo.getTmpBasePath)
316
317 def makeFileCopy (srcFile, dstFile):
318         return SingleFileTarget(srcFile, dstFile, lambda s, d: CopyFile(s, d))
319
320 def getReleaseFileName (configName, releaseName):
321         today = datetime.date.today()
322         return "dEQP-%s-%04d-%02d-%02d-%s" % (releaseName, today.year, today.month, today.day, configName)
323
324 def getTempDir ():
325         dirName = os.path.join(tempfile.gettempdir(), "dEQP-Releases")
326         if not os.path.exists(dirName):
327                 os.makedirs(dirName)
328         return dirName
329
330 def makeRelease (releaseConfig):
331         releaseName                     = getReleaseFileName(releaseConfig.getName(), releaseConfig.getVersion())
332         tmpPath                         = getTempDir()
333         srcBasePath                     = DEQP_DIR
334         dstBasePath                     = os.path.join(tmpPath, releaseName)
335         tmpBasePath                     = os.path.join(tmpPath, releaseName + "-tmp")
336         packageBuildInfo        = PackageBuildInfo(releaseConfig, srcBasePath, dstBasePath, tmpBasePath)
337         dstArchiveName          = releaseName + ".tar.bz2"
338
339         print "Creating release %s to %s" % (releaseName, tmpPath)
340
341         # Remove old temporary dirs
342         for path in [dstBasePath, tmpBasePath]:
343                 if os.path.exists(path):
344                         shutil.rmtree(path, ignore_errors=False)
345
346         # Make all modules
347         for module in releaseConfig.getModules():
348                 print "  Processing module %s" % module.name
349                 module.make(packageBuildInfo)
350
351         # Remove sources?
352         if not releaseConfig.packageWithSources():
353                 shutil.rmtree(os.path.join(dstBasePath, "src"), ignore_errors=False)
354
355         # Create archive
356         print "Creating %s" % dstArchiveName
357         archive = tarfile.open(dstArchiveName, 'w:bz2')
358         archive.add(dstBasePath, arcname=releaseName)
359         archive.close()
360
361         # Remove tmp dirs
362         for path in [dstBasePath, tmpBasePath]:
363                 if os.path.exists(path):
364                         shutil.rmtree(path, ignore_errors=False)
365
366         print "Done!"
367
368 # Module declarations
369
370 SRC_FILE_PATTERNS       = ["*.h", "*.hpp", "*.c", "*.cpp", "*.m", "*.mm", "*.inl", "*.java", "*.aidl", "CMakeLists.txt", "LICENSE.txt", "*.cmake"]
371 TARGET_PATTERNS         = ["*.cmake", "*.h", "*.lib", "*.dll", "*.so", "*.txt"]
372
373 BASE = Module("Base", [
374         makeFileCopy            ("LICENSE",                                                                     "src/LICENSE"),
375         makeFileCopy            ("CMakeLists.txt",                                                      "src/CMakeLists.txt"),
376         makeFileCopyGroup       ("targets",                                                                     "src/targets",                                                  TARGET_PATTERNS),
377         makeFileCopyGroup       ("execserver",                                                          "src/execserver",                                               SRC_FILE_PATTERNS),
378         makeFileCopyGroup       ("executor",                                                            "src/executor",                                                 SRC_FILE_PATTERNS),
379         makeFileCopy            ("modules/CMakeLists.txt",                                      "src/modules/CMakeLists.txt"),
380         makeFileCopyGroup       ("external",                                                            "src/external",                                                 ["CMakeLists.txt", "*.py"]),
381
382         # Stylesheet for displaying test logs on browser
383         makeFileCopyGroup       ("doc/testlog-stylesheet",                                      "doc/testlog-stylesheet",                               ["*"]),
384
385         # Non-optional parts of framework
386         makeFileCopy            ("framework/CMakeLists.txt",                            "src/framework/CMakeLists.txt"),
387         makeFileCopyGroup       ("framework/delibs",                                            "src/framework/delibs",                                 SRC_FILE_PATTERNS),
388         makeFileCopyGroup       ("framework/common",                                            "src/framework/common",                                 SRC_FILE_PATTERNS),
389         makeFileCopyGroup       ("framework/qphelper",                                          "src/framework/qphelper",                               SRC_FILE_PATTERNS),
390         makeFileCopyGroup       ("framework/platform",                                          "src/framework/platform",                               SRC_FILE_PATTERNS),
391         makeFileCopyGroup       ("framework/opengl",                                            "src/framework/opengl",                                 SRC_FILE_PATTERNS, ["simplereference"]),
392         makeFileCopyGroup       ("framework/egl",                                                       "src/framework/egl",                                    SRC_FILE_PATTERNS),
393
394         # android sources
395         makeFileCopyGroup       ("android/package/src",                                         "src/android/package/src",                              SRC_FILE_PATTERNS),
396         makeFileCopy            ("android/package/AndroidManifest.xml",         "src/android/package/AndroidManifest.xml"),
397         makeFileCopyGroup       ("android/package/res",                                         "src/android/package/res",                              ["*.png", "*.xml"]),
398         makeFileCopyGroup       ("android/scripts",                                                     "src/android/scripts", [
399                 "common.py",
400                 "build.py",
401                 "resources.py",
402                 "install.py",
403                 "launch.py",
404                 "debug.py"
405                 ]),
406
407         # Release info
408         GenReleaseInfoFileTarget("src/framework/qphelper/qpReleaseInfo.inl")
409 ])
410
411 DOCUMENTATION = Module("Documentation", [
412         makeFileCopyGroup       ("doc/pdf",                                                                     "doc",                                                                  ["*.pdf"]),
413         makeFileCopyGroup       ("doc",                                                                         "doc",                                                                  ["porting_layer_changes_*.txt"]),
414 ])
415
416 GLSHARED = Module("Shared GL Tests", [
417         # Optional framework components
418         makeFileCopyGroup       ("framework/randomshaders",                                     "src/framework/randomshaders",                  SRC_FILE_PATTERNS),
419         makeFileCopyGroup       ("framework/opengl/simplereference",            "src/framework/opengl/simplereference", SRC_FILE_PATTERNS),
420         makeFileCopyGroup       ("framework/referencerenderer",                         "src/framework/referencerenderer",              SRC_FILE_PATTERNS),
421
422         makeFileCopyGroup       ("modules/glshared",                                            "src/modules/glshared",                                 SRC_FILE_PATTERNS),
423 ])
424
425 GLES2 = Module("GLES2", [
426         makeFileCopyGroup       ("modules/gles2",                                                       "src/modules/gles2",                                    SRC_FILE_PATTERNS),
427         makeFileCopyGroup       ("data/gles2",                                                          "src/data/gles2",                                               ["*.*"]),
428         makeFileCopyGroup       ("doc/testspecs/GLES2",                                         "doc/testspecs/GLES2",                                  ["*.txt"])
429 ])
430
431 GLES3 = Module("GLES3", [
432         makeFileCopyGroup       ("modules/gles3",                                                       "src/modules/gles3",                                    SRC_FILE_PATTERNS),
433         makeFileCopyGroup       ("data/gles3",                                                          "src/data/gles3",                                               ["*.*"]),
434         makeFileCopyGroup       ("doc/testspecs/GLES3",                                         "doc/testspecs/GLES3",                                  ["*.txt"])
435 ])
436
437 GLES31 = Module("GLES31", [
438         makeFileCopyGroup       ("modules/gles31",                                                      "src/modules/gles31",                                   SRC_FILE_PATTERNS),
439         makeFileCopyGroup       ("data/gles31",                                                         "src/data/gles31",                                              ["*.*"]),
440         makeFileCopyGroup       ("doc/testspecs/GLES31",                                        "doc/testspecs/GLES31",                                 ["*.txt"])
441 ])
442
443 EGL = Module("EGL", [
444         makeFileCopyGroup       ("modules/egl",                                                         "src/modules/egl",                                              SRC_FILE_PATTERNS)
445 ])
446
447 INTERNAL = Module("Internal", [
448         makeFileCopyGroup       ("modules/internal",                                            "src/modules/internal",                                 SRC_FILE_PATTERNS),
449         makeFileCopyGroup       ("data/internal",                                                       "src/data/internal",                                    ["*.*"]),
450 ])
451
452 EXTERNAL_SRCS = Module("External sources", [
453         FetchExternalSourcesTarget()
454 ])
455
456 ANDROID_BINARIES = Module("Android Binaries", [
457         BuildAndroidTarget      ("bin/android/dEQP.apk"),
458         makeFileCopyGroup       ("targets/android",                                                     "bin/android",                                                  ["*.bat", "*.sh"]),
459 ])
460
461 COMMON_BUILD_ARGS       = ['-DPNG_SRC_PATH=%s' % os.path.realpath(os.path.join(DEQP_DIR, '..', 'libpng'))]
462 NULL_X32_CONFIG         = BuildConfig('null-x32',       'Release', ['-DDEQP_TARGET=null', '-DCMAKE_C_FLAGS=-m32', '-DCMAKE_CXX_FLAGS=-m32'] + COMMON_BUILD_ARGS)
463 NULL_X64_CONFIG         = BuildConfig('null-x64',       'Release', ['-DDEQP_TARGET=null', '-DCMAKE_C_FLAGS=-m64', '-DCMAKE_CXX_FLAGS=-m64'] + COMMON_BUILD_ARGS)
464 GLX_X32_CONFIG          = BuildConfig('glx-x32',        'Release', ['-DDEQP_TARGET=x11_glx', '-DCMAKE_C_FLAGS=-m32', '-DCMAKE_CXX_FLAGS=-m32'] + COMMON_BUILD_ARGS)
465 GLX_X64_CONFIG          = BuildConfig('glx-x64',        'Release', ['-DDEQP_TARGET=x11_glx', '-DCMAKE_C_FLAGS=-m64', '-DCMAKE_CXX_FLAGS=-m64'] + COMMON_BUILD_ARGS)
466
467 EXCLUDE_BUILD_FILES = ["CMakeFiles", "*.a", "*.cmake"]
468
469 LINUX_X32_COMMON_BINARIES = Module("Linux x32 Common Binaries", [
470         BuildTarget                     (NULL_X32_CONFIG, ANY_UNIX_GENERATOR),
471         makeTmpFileCopyGroup(NULL_X32_CONFIG.getBuildDir() + "/execserver",             "bin/linux32",                                  ["*"],  EXCLUDE_BUILD_FILES),
472         makeTmpFileCopyGroup(NULL_X32_CONFIG.getBuildDir() + "/executor",               "bin/linux32",                                  ["*"],  EXCLUDE_BUILD_FILES),
473 ])
474
475 LINUX_X64_COMMON_BINARIES = Module("Linux x64 Common Binaries", [
476         BuildTarget                     (NULL_X64_CONFIG, ANY_UNIX_GENERATOR),
477         makeTmpFileCopyGroup(NULL_X64_CONFIG.getBuildDir() + "/execserver",             "bin/linux64",                                  ["*"],  EXCLUDE_BUILD_FILES),
478         makeTmpFileCopyGroup(NULL_X64_CONFIG.getBuildDir() + "/executor",               "bin/linux64",                                  ["*"],  EXCLUDE_BUILD_FILES),
479 ])
480
481 # Special module to remove src dir, for example after binary build
482 REMOVE_SOURCES = Module("Remove sources from package", [
483         RemoveSourcesTarget()
484 ])
485
486 # Release configuration
487
488 ALL_MODULES             = [
489         BASE,
490         DOCUMENTATION,
491         GLSHARED,
492         GLES2,
493         GLES3,
494         GLES31,
495         EGL,
496         INTERNAL,
497         EXTERNAL_SRCS,
498 ]
499
500 ALL_BINARIES    = [
501         LINUX_X64_COMMON_BINARIES,
502         ANDROID_BINARIES,
503 ]
504
505 RELEASE_CONFIGS = {
506         "src":          ALL_MODULES,
507         "src-bin":      ALL_MODULES + ALL_BINARIES,
508         "bin":          ALL_MODULES + ALL_BINARIES + [REMOVE_SOURCES],
509 }
510
511 def parseArgs ():
512         parser = argparse.ArgumentParser(description = "Build release package")
513         parser.add_argument("-c",
514                                                 "--config",
515                                                 dest="config",
516                                                 choices=RELEASE_CONFIGS.keys(),
517                                                 required=True,
518                                                 help="Release configuration")
519         parser.add_argument("-n",
520                                                 "--name",
521                                                 dest="name",
522                                                 required=True,
523                                                 help="Package-specific name")
524         parser.add_argument("-v",
525                                                 "--version",
526                                                 dest="version",
527                                                 required=True,
528                                                 help="Version code")
529         return parser.parse_args()
530
531 if __name__ == "__main__":
532         args    = parseArgs()
533         config  = ReleaseConfig(args.name, args.version, RELEASE_CONFIGS[args.config])
534         makeRelease(config)