1 # -*- coding: utf-8 -*-
3 #-------------------------------------------------------------------------
4 # drawElements Quality Program utilities
5 # --------------------------------------
7 # Copyright 2015 The Android Open Source Project
9 # Licensed under the Apache License, Version 2.0 (the "License");
10 # you may not use this file except in compliance with the License.
11 # You may obtain a copy of the License at
13 # http://www.apache.org/licenses/LICENSE-2.0
15 # Unless required by applicable law or agreed to in writing, software
16 # distributed under the License is distributed on an "AS IS" BASIS,
17 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18 # See the License for the specific language governing permissions and
19 # limitations under the License.
21 #-------------------------------------------------------------------------
40 from build.common import *
41 from build.config import *
42 from build.build import *
48 def removeLeadingPath (path, basePath):
49 # Both inputs must be normalized already
50 assert os.path.normpath(path) == path
51 assert os.path.normpath(basePath) == basePath
52 return path[len(basePath) + 1:]
54 def findFile (candidates):
55 for file in candidates:
56 if os.path.exists(file):
60 def getFileList (basePath):
62 basePath = os.path.normpath(basePath)
63 for root, dirs, files in os.walk(basePath):
65 relPath = removeLeadingPath(os.path.normpath(os.path.join(root, file)), basePath)
66 allFiles.append(relPath)
69 def toDatetime (dateTuple):
71 return datetime.datetime(Y, M, D)
73 class PackageBuildInfo:
74 def __init__ (self, releaseConfig, srcBasePath, dstBasePath, tmpBasePath):
75 self.releaseConfig = releaseConfig
76 self.srcBasePath = srcBasePath
77 self.dstBasePath = dstBasePath
78 self.tmpBasePath = tmpBasePath
80 def getReleaseConfig (self):
81 return self.releaseConfig
83 def getReleaseVersion (self):
84 return self.releaseConfig.getVersion()
86 def getReleaseId (self):
87 # Release id is crc32(releaseConfig + release)
88 return zlib.crc32(self.releaseConfig.getName() + self.releaseConfig.getVersion()) & 0xffffffff
90 def getSrcBasePath (self):
91 return self.srcBasePath
93 def getTmpBasePath (self):
94 return self.tmpBasePath
96 class DstFile (object):
97 def __init__ (self, dstFile):
98 self.dstFile = dstFile
101 dirName = os.path.dirname(self.dstFile)
102 if not os.path.exists(dirName):
105 def make (self, packageBuildInfo):
106 assert False # Should not be called
108 class CopyFile (DstFile):
109 def __init__ (self, srcFile, dstFile):
110 super(CopyFile, self).__init__(dstFile)
111 self.srcFile = srcFile
113 def make (self, packageBuildInfo):
115 if os.path.exists(self.dstFile):
116 die("%s already exists" % self.dstFile)
117 shutil.copyfile(self.srcFile, self.dstFile)
119 class GenReleaseInfoFileTarget (DstFile):
120 def __init__ (self, dstFile):
121 super(GenReleaseInfoFileTarget, self).__init__(dstFile)
123 def make (self, packageBuildInfo):
126 scriptPath = os.path.normpath(os.path.join(packageBuildInfo.srcBasePath, "framework", "qphelper", "gen_release_info.py"))
131 "--name=%s" % packageBuildInfo.getReleaseVersion(),
132 "--id=0x%08x" % packageBuildInfo.getReleaseId(),
133 "--out=%s" % self.dstFile
136 class GenCMake (DstFile):
137 def __init__ (self, srcFile, dstFile, replaceVars):
138 super(GenCMake, self).__init__(dstFile)
139 self.srcFile = srcFile
140 self.replaceVars = replaceVars
142 def make (self, packageBuildInfo):
144 print " GenCMake: %s" % removeLeadingPath(self.dstFile, packageBuildInfo.dstBasePath)
145 src = readFile(self.srcFile)
146 for var, value in self.replaceVars:
147 src = re.sub('set\(%s\s+"[^"]*"' % re.escape(var),
148 'set(%s "%s"' % (var, value), src)
149 writeFile(self.dstFile, src)
151 def createFileTargets (srcBasePath, dstBasePath, files, filters):
152 usedFiles = set() # Files that are already included by other filters
155 for isMatch, createFileObj in filters:
156 # Build list of files that match filter
159 if not file in usedFiles and isMatch(file):
160 matchingFiles.append(file)
162 # Build file objects, add to used set
163 for file in matchingFiles:
165 targets.append(createFileObj(os.path.join(srcBasePath, file), os.path.join(dstBasePath, file)))
169 # Generates multiple file targets based on filters
170 class FileTargetGroup:
171 def __init__ (self, srcBasePath, dstBasePath, filters, srcBasePathFunc=PackageBuildInfo.getSrcBasePath):
172 self.srcBasePath = srcBasePath
173 self.dstBasePath = dstBasePath
174 self.filters = filters
175 self.getSrcBasePath = srcBasePathFunc
177 def make (self, packageBuildInfo):
178 fullSrcPath = os.path.normpath(os.path.join(self.getSrcBasePath(packageBuildInfo), self.srcBasePath))
179 fullDstPath = os.path.normpath(os.path.join(packageBuildInfo.dstBasePath, self.dstBasePath))
181 allFiles = getFileList(fullSrcPath)
182 targets = createFileTargets(fullSrcPath, fullDstPath, allFiles, self.filters)
184 # Make all file targets
186 file.make(packageBuildInfo)
189 class SingleFileTarget:
190 def __init__ (self, srcFile, dstFile, makeTarget):
191 self.srcFile = srcFile
192 self.dstFile = dstFile
193 self.makeTarget = makeTarget
195 def make (self, packageBuildInfo):
196 fullSrcPath = os.path.normpath(os.path.join(packageBuildInfo.srcBasePath, self.srcFile))
197 fullDstPath = os.path.normpath(os.path.join(packageBuildInfo.dstBasePath, self.dstFile))
199 target = self.makeTarget(fullSrcPath, fullDstPath)
200 target.make(packageBuildInfo)
203 def __init__ (self, baseConfig, generator, targets = None):
204 self.baseConfig = baseConfig
205 self.generator = generator
206 self.targets = targets
208 def make (self, packageBuildInfo):
209 print " Building %s" % self.baseConfig.getBuildDir()
211 # Create config with full build dir path
212 config = BuildConfig(os.path.join(packageBuildInfo.getTmpBasePath(), self.baseConfig.getBuildDir()),
213 self.baseConfig.getBuildType(),
214 self.baseConfig.getArgs(),
215 srcPath = os.path.join(packageBuildInfo.dstBasePath, "src"))
217 assert not os.path.exists(config.getBuildDir())
218 build(config, self.generator, self.targets)
220 class BuildAndroidTarget:
221 def __init__ (self, dstFile):
222 self.dstFile = dstFile
224 def make (self, packageBuildInfo):
225 print " Building Android binary"
227 buildRoot = os.path.join(packageBuildInfo.tmpBasePath, "android-build")
229 assert not os.path.exists(buildRoot)
230 os.makedirs(buildRoot)
232 # Execute build script
233 scriptPath = os.path.normpath(os.path.join(packageBuildInfo.dstBasePath, "src", "android", "scripts", "build.py"))
238 "--build-root=%s" % buildRoot,
241 srcFile = os.path.normpath(os.path.join(buildRoot, "package", "bin", "dEQP-debug.apk"))
242 dstFile = os.path.normpath(os.path.join(packageBuildInfo.dstBasePath, self.dstFile))
244 CopyFile(srcFile, dstFile).make(packageBuildInfo)
246 class FetchExternalSourcesTarget:
250 def make (self, packageBuildInfo):
251 scriptPath = os.path.normpath(os.path.join(packageBuildInfo.dstBasePath, "src", "external", "fetch_sources.py"))
258 class RemoveSourcesTarget:
262 def make (self, packageBuildInfo):
263 shutil.rmtree(os.path.join(packageBuildInfo.dstBasePath, "src"), ignore_errors=False)
266 def __init__ (self, name, targets):
268 self.targets = targets
270 def make (self, packageBuildInfo):
271 for target in self.targets:
272 target.make(packageBuildInfo)
275 def __init__ (self, name, version, modules, sources = True):
277 self.version = version
278 self.modules = modules
279 self.sources = sources
284 def getVersion (self):
287 def getModules (self):
290 def packageWithSources (self):
293 def matchIncludeExclude (includePatterns, excludePatterns, filename):
294 components = os.path.normpath(filename).split(os.sep)
295 for pattern in excludePatterns:
296 for component in components:
297 if fnmatch.fnmatch(component, pattern):
300 for pattern in includePatterns:
301 for component in components:
302 if fnmatch.fnmatch(component, pattern):
307 def copyFileFilter (includePatterns, excludePatterns=[]):
308 return (lambda f: matchIncludeExclude(includePatterns, excludePatterns, f),
309 lambda s, d: CopyFile(s, d))
311 def makeFileCopyGroup (srcDir, dstDir, includePatterns, excludePatterns=[]):
312 return FileTargetGroup(srcDir, dstDir, [copyFileFilter(includePatterns, excludePatterns)])
314 def makeTmpFileCopyGroup (srcDir, dstDir, includePatterns, excludePatterns=[]):
315 return FileTargetGroup(srcDir, dstDir, [copyFileFilter(includePatterns, excludePatterns)], PackageBuildInfo.getTmpBasePath)
317 def makeFileCopy (srcFile, dstFile):
318 return SingleFileTarget(srcFile, dstFile, lambda s, d: CopyFile(s, d))
320 def getReleaseFileName (configName, releaseName):
321 today = datetime.date.today()
322 return "dEQP-%s-%04d-%02d-%02d-%s" % (releaseName, today.year, today.month, today.day, configName)
325 dirName = os.path.join(tempfile.gettempdir(), "dEQP-Releases")
326 if not os.path.exists(dirName):
330 def makeRelease (releaseConfig):
331 releaseName = getReleaseFileName(releaseConfig.getName(), releaseConfig.getVersion())
332 tmpPath = getTempDir()
333 srcBasePath = DEQP_DIR
334 dstBasePath = os.path.join(tmpPath, releaseName)
335 tmpBasePath = os.path.join(tmpPath, releaseName + "-tmp")
336 packageBuildInfo = PackageBuildInfo(releaseConfig, srcBasePath, dstBasePath, tmpBasePath)
337 dstArchiveName = releaseName + ".tar.bz2"
339 print "Creating release %s to %s" % (releaseName, tmpPath)
341 # Remove old temporary dirs
342 for path in [dstBasePath, tmpBasePath]:
343 if os.path.exists(path):
344 shutil.rmtree(path, ignore_errors=False)
347 for module in releaseConfig.getModules():
348 print " Processing module %s" % module.name
349 module.make(packageBuildInfo)
352 if not releaseConfig.packageWithSources():
353 shutil.rmtree(os.path.join(dstBasePath, "src"), ignore_errors=False)
356 print "Creating %s" % dstArchiveName
357 archive = tarfile.open(dstArchiveName, 'w:bz2')
358 archive.add(dstBasePath, arcname=releaseName)
362 for path in [dstBasePath, tmpBasePath]:
363 if os.path.exists(path):
364 shutil.rmtree(path, ignore_errors=False)
368 # Module declarations
370 SRC_FILE_PATTERNS = ["*.h", "*.hpp", "*.c", "*.cpp", "*.m", "*.mm", "*.inl", "*.java", "*.aidl", "CMakeLists.txt", "LICENSE.txt", "*.cmake"]
371 TARGET_PATTERNS = ["*.cmake", "*.h", "*.lib", "*.dll", "*.so", "*.txt"]
373 BASE = Module("Base", [
374 makeFileCopy ("LICENSE", "src/LICENSE"),
375 makeFileCopy ("CMakeLists.txt", "src/CMakeLists.txt"),
376 makeFileCopyGroup ("targets", "src/targets", TARGET_PATTERNS),
377 makeFileCopyGroup ("execserver", "src/execserver", SRC_FILE_PATTERNS),
378 makeFileCopyGroup ("executor", "src/executor", SRC_FILE_PATTERNS),
379 makeFileCopy ("modules/CMakeLists.txt", "src/modules/CMakeLists.txt"),
380 makeFileCopyGroup ("external", "src/external", ["CMakeLists.txt", "*.py"]),
382 # Stylesheet for displaying test logs on browser
383 makeFileCopyGroup ("doc/testlog-stylesheet", "doc/testlog-stylesheet", ["*"]),
385 # Non-optional parts of framework
386 makeFileCopy ("framework/CMakeLists.txt", "src/framework/CMakeLists.txt"),
387 makeFileCopyGroup ("framework/delibs", "src/framework/delibs", SRC_FILE_PATTERNS),
388 makeFileCopyGroup ("framework/common", "src/framework/common", SRC_FILE_PATTERNS),
389 makeFileCopyGroup ("framework/qphelper", "src/framework/qphelper", SRC_FILE_PATTERNS),
390 makeFileCopyGroup ("framework/platform", "src/framework/platform", SRC_FILE_PATTERNS),
391 makeFileCopyGroup ("framework/opengl", "src/framework/opengl", SRC_FILE_PATTERNS, ["simplereference"]),
392 makeFileCopyGroup ("framework/egl", "src/framework/egl", SRC_FILE_PATTERNS),
395 makeFileCopyGroup ("android/package/src", "src/android/package/src", SRC_FILE_PATTERNS),
396 makeFileCopy ("android/package/AndroidManifest.xml", "src/android/package/AndroidManifest.xml"),
397 makeFileCopyGroup ("android/package/res", "src/android/package/res", ["*.png", "*.xml"]),
398 makeFileCopyGroup ("android/scripts", "src/android/scripts", [
408 GenReleaseInfoFileTarget("src/framework/qphelper/qpReleaseInfo.inl")
411 DOCUMENTATION = Module("Documentation", [
412 makeFileCopyGroup ("doc/pdf", "doc", ["*.pdf"]),
413 makeFileCopyGroup ("doc", "doc", ["porting_layer_changes_*.txt"]),
416 GLSHARED = Module("Shared GL Tests", [
417 # Optional framework components
418 makeFileCopyGroup ("framework/randomshaders", "src/framework/randomshaders", SRC_FILE_PATTERNS),
419 makeFileCopyGroup ("framework/opengl/simplereference", "src/framework/opengl/simplereference", SRC_FILE_PATTERNS),
420 makeFileCopyGroup ("framework/referencerenderer", "src/framework/referencerenderer", SRC_FILE_PATTERNS),
422 makeFileCopyGroup ("modules/glshared", "src/modules/glshared", SRC_FILE_PATTERNS),
425 GLES2 = Module("GLES2", [
426 makeFileCopyGroup ("modules/gles2", "src/modules/gles2", SRC_FILE_PATTERNS),
427 makeFileCopyGroup ("data/gles2", "src/data/gles2", ["*.*"]),
428 makeFileCopyGroup ("doc/testspecs/GLES2", "doc/testspecs/GLES2", ["*.txt"])
431 GLES3 = Module("GLES3", [
432 makeFileCopyGroup ("modules/gles3", "src/modules/gles3", SRC_FILE_PATTERNS),
433 makeFileCopyGroup ("data/gles3", "src/data/gles3", ["*.*"]),
434 makeFileCopyGroup ("doc/testspecs/GLES3", "doc/testspecs/GLES3", ["*.txt"])
437 GLES31 = Module("GLES31", [
438 makeFileCopyGroup ("modules/gles31", "src/modules/gles31", SRC_FILE_PATTERNS),
439 makeFileCopyGroup ("data/gles31", "src/data/gles31", ["*.*"]),
440 makeFileCopyGroup ("doc/testspecs/GLES31", "doc/testspecs/GLES31", ["*.txt"])
443 EGL = Module("EGL", [
444 makeFileCopyGroup ("modules/egl", "src/modules/egl", SRC_FILE_PATTERNS)
447 INTERNAL = Module("Internal", [
448 makeFileCopyGroup ("modules/internal", "src/modules/internal", SRC_FILE_PATTERNS),
449 makeFileCopyGroup ("data/internal", "src/data/internal", ["*.*"]),
452 EXTERNAL_SRCS = Module("External sources", [
453 FetchExternalSourcesTarget()
456 ANDROID_BINARIES = Module("Android Binaries", [
457 BuildAndroidTarget ("bin/android/dEQP.apk"),
458 makeFileCopyGroup ("targets/android", "bin/android", ["*.bat", "*.sh"]),
461 COMMON_BUILD_ARGS = ['-DPNG_SRC_PATH=%s' % os.path.realpath(os.path.join(DEQP_DIR, '..', 'libpng'))]
462 NULL_X32_CONFIG = BuildConfig('null-x32', 'Release', ['-DDEQP_TARGET=null', '-DCMAKE_C_FLAGS=-m32', '-DCMAKE_CXX_FLAGS=-m32'] + COMMON_BUILD_ARGS)
463 NULL_X64_CONFIG = BuildConfig('null-x64', 'Release', ['-DDEQP_TARGET=null', '-DCMAKE_C_FLAGS=-m64', '-DCMAKE_CXX_FLAGS=-m64'] + COMMON_BUILD_ARGS)
464 GLX_X32_CONFIG = BuildConfig('glx-x32', 'Release', ['-DDEQP_TARGET=x11_glx', '-DCMAKE_C_FLAGS=-m32', '-DCMAKE_CXX_FLAGS=-m32'] + COMMON_BUILD_ARGS)
465 GLX_X64_CONFIG = BuildConfig('glx-x64', 'Release', ['-DDEQP_TARGET=x11_glx', '-DCMAKE_C_FLAGS=-m64', '-DCMAKE_CXX_FLAGS=-m64'] + COMMON_BUILD_ARGS)
467 EXCLUDE_BUILD_FILES = ["CMakeFiles", "*.a", "*.cmake"]
469 LINUX_X32_COMMON_BINARIES = Module("Linux x32 Common Binaries", [
470 BuildTarget (NULL_X32_CONFIG, ANY_UNIX_GENERATOR),
471 makeTmpFileCopyGroup(NULL_X32_CONFIG.getBuildDir() + "/execserver", "bin/linux32", ["*"], EXCLUDE_BUILD_FILES),
472 makeTmpFileCopyGroup(NULL_X32_CONFIG.getBuildDir() + "/executor", "bin/linux32", ["*"], EXCLUDE_BUILD_FILES),
475 LINUX_X64_COMMON_BINARIES = Module("Linux x64 Common Binaries", [
476 BuildTarget (NULL_X64_CONFIG, ANY_UNIX_GENERATOR),
477 makeTmpFileCopyGroup(NULL_X64_CONFIG.getBuildDir() + "/execserver", "bin/linux64", ["*"], EXCLUDE_BUILD_FILES),
478 makeTmpFileCopyGroup(NULL_X64_CONFIG.getBuildDir() + "/executor", "bin/linux64", ["*"], EXCLUDE_BUILD_FILES),
481 # Special module to remove src dir, for example after binary build
482 REMOVE_SOURCES = Module("Remove sources from package", [
483 RemoveSourcesTarget()
486 # Release configuration
501 LINUX_X64_COMMON_BINARIES,
507 "src-bin": ALL_MODULES + ALL_BINARIES,
508 "bin": ALL_MODULES + ALL_BINARIES + [REMOVE_SOURCES],
512 parser = argparse.ArgumentParser(description = "Build release package")
513 parser.add_argument("-c",
516 choices=RELEASE_CONFIGS.keys(),
518 help="Release configuration")
519 parser.add_argument("-n",
523 help="Package-specific name")
524 parser.add_argument("-v",
529 return parser.parse_args()
531 if __name__ == "__main__":
533 config = ReleaseConfig(args.name, args.version, RELEASE_CONFIGS[args.config])