1 # -*- coding: utf-8 -*-
3 #-------------------------------------------------------------------------
4 # drawElements Quality Program utilities
5 # --------------------------------------
7 # Copyright 2015 The Android Open Source Project
9 # Licensed under the Apache License, Version 2.0 (the "License");
10 # you may not use this file except in compliance with the License.
11 # You may obtain a copy of the License at
13 # http://www.apache.org/licenses/LICENSE-2.0
15 # Unless required by applicable law or agreed to in writing, software
16 # distributed under the License is distributed on an "AS IS" BASIS,
17 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18 # See the License for the specific language governing permissions and
19 # limitations under the License.
21 #-------------------------------------------------------------------------
40 from build.common import *
41 from build.config import *
42 from build.build import *
44 pythonExecutable = sys.executable or "python"
50 def removeLeadingPath (path, basePath):
51 # Both inputs must be normalized already
52 assert os.path.normpath(path) == path
53 assert os.path.normpath(basePath) == basePath
54 return path[len(basePath) + 1:]
56 def findFile (candidates):
57 for file in candidates:
58 if os.path.exists(file):
62 def getFileList (basePath):
64 basePath = os.path.normpath(basePath)
65 for root, dirs, files in os.walk(basePath):
67 relPath = removeLeadingPath(os.path.normpath(os.path.join(root, file)), basePath)
68 allFiles.append(relPath)
71 def toDatetime (dateTuple):
73 return datetime.datetime(Y, M, D)
75 class PackageBuildInfo:
76 def __init__ (self, releaseConfig, srcBasePath, dstBasePath, tmpBasePath):
77 self.releaseConfig = releaseConfig
78 self.srcBasePath = srcBasePath
79 self.dstBasePath = dstBasePath
80 self.tmpBasePath = tmpBasePath
82 def getReleaseConfig (self):
83 return self.releaseConfig
85 def getReleaseVersion (self):
86 return self.releaseConfig.getVersion()
88 def getReleaseId (self):
89 # Release id is crc32(releaseConfig + release)
90 return zlib.crc32(self.releaseConfig.getName() + self.releaseConfig.getVersion()) & 0xffffffff
92 def getSrcBasePath (self):
93 return self.srcBasePath
95 def getTmpBasePath (self):
96 return self.tmpBasePath
98 class DstFile (object):
99 def __init__ (self, dstFile):
100 self.dstFile = dstFile
103 dirName = os.path.dirname(self.dstFile)
104 if not os.path.exists(dirName):
107 def make (self, packageBuildInfo):
108 assert False # Should not be called
110 class CopyFile (DstFile):
111 def __init__ (self, srcFile, dstFile):
112 super(CopyFile, self).__init__(dstFile)
113 self.srcFile = srcFile
115 def make (self, packageBuildInfo):
117 if os.path.exists(self.dstFile):
118 die("%s already exists" % self.dstFile)
119 shutil.copyfile(self.srcFile, self.dstFile)
121 class GenReleaseInfoFileTarget (DstFile):
122 def __init__ (self, dstFile):
123 super(GenReleaseInfoFileTarget, self).__init__(dstFile)
125 def make (self, packageBuildInfo):
128 scriptPath = os.path.normpath(os.path.join(packageBuildInfo.srcBasePath, "framework", "qphelper", "gen_release_info.py"))
133 "--name=%s" % packageBuildInfo.getReleaseVersion(),
134 "--id=0x%08x" % packageBuildInfo.getReleaseId(),
135 "--out=%s" % self.dstFile
138 class GenCMake (DstFile):
139 def __init__ (self, srcFile, dstFile, replaceVars):
140 super(GenCMake, self).__init__(dstFile)
141 self.srcFile = srcFile
142 self.replaceVars = replaceVars
144 def make (self, packageBuildInfo):
146 print(" GenCMake: %s" % removeLeadingPath(self.dstFile, packageBuildInfo.dstBasePath))
147 src = readFile(self.srcFile)
148 for var, value in self.replaceVars:
149 src = re.sub('set\(%s\s+"[^"]*"' % re.escape(var),
150 'set(%s "%s"' % (var, value), src)
151 writeFile(self.dstFile, src)
153 def createFileTargets (srcBasePath, dstBasePath, files, filters):
154 usedFiles = set() # Files that are already included by other filters
157 for isMatch, createFileObj in filters:
158 # Build list of files that match filter
161 if not file in usedFiles and isMatch(file):
162 matchingFiles.append(file)
164 # Build file objects, add to used set
165 for file in matchingFiles:
167 targets.append(createFileObj(os.path.join(srcBasePath, file), os.path.join(dstBasePath, file)))
171 # Generates multiple file targets based on filters
172 class FileTargetGroup:
173 def __init__ (self, srcBasePath, dstBasePath, filters, srcBasePathFunc=PackageBuildInfo.getSrcBasePath):
174 self.srcBasePath = srcBasePath
175 self.dstBasePath = dstBasePath
176 self.filters = filters
177 self.getSrcBasePath = srcBasePathFunc
179 def make (self, packageBuildInfo):
180 fullSrcPath = os.path.normpath(os.path.join(self.getSrcBasePath(packageBuildInfo), self.srcBasePath))
181 fullDstPath = os.path.normpath(os.path.join(packageBuildInfo.dstBasePath, self.dstBasePath))
183 allFiles = getFileList(fullSrcPath)
184 targets = createFileTargets(fullSrcPath, fullDstPath, allFiles, self.filters)
186 # Make all file targets
188 file.make(packageBuildInfo)
191 class SingleFileTarget:
192 def __init__ (self, srcFile, dstFile, makeTarget):
193 self.srcFile = srcFile
194 self.dstFile = dstFile
195 self.makeTarget = makeTarget
197 def make (self, packageBuildInfo):
198 fullSrcPath = os.path.normpath(os.path.join(packageBuildInfo.srcBasePath, self.srcFile))
199 fullDstPath = os.path.normpath(os.path.join(packageBuildInfo.dstBasePath, self.dstFile))
201 target = self.makeTarget(fullSrcPath, fullDstPath)
202 target.make(packageBuildInfo)
205 def __init__ (self, baseConfig, generator, targets = None):
206 self.baseConfig = baseConfig
207 self.generator = generator
208 self.targets = targets
210 def make (self, packageBuildInfo):
211 print(" Building %s" % self.baseConfig.getBuildDir())
213 # Create config with full build dir path
214 config = BuildConfig(os.path.join(packageBuildInfo.getTmpBasePath(), self.baseConfig.getBuildDir()),
215 self.baseConfig.getBuildType(),
216 self.baseConfig.getArgs(),
217 srcPath = os.path.join(packageBuildInfo.dstBasePath, "src"))
219 assert not os.path.exists(config.getBuildDir())
220 build(config, self.generator, self.targets)
222 class BuildAndroidTarget:
223 def __init__ (self, dstFile):
224 self.dstFile = dstFile
226 def make (self, packageBuildInfo):
227 print(" Building Android binary")
229 buildRoot = os.path.join(packageBuildInfo.tmpBasePath, "android-build")
231 assert not os.path.exists(buildRoot)
232 os.makedirs(buildRoot)
234 # Execute build script
235 scriptPath = os.path.normpath(os.path.join(packageBuildInfo.dstBasePath, "src", "android", "scripts", "build.py"))
240 "--build-root=%s" % buildRoot,
243 srcFile = os.path.normpath(os.path.join(buildRoot, "package", "bin", "dEQP-debug.apk"))
244 dstFile = os.path.normpath(os.path.join(packageBuildInfo.dstBasePath, self.dstFile))
246 CopyFile(srcFile, dstFile).make(packageBuildInfo)
248 class FetchExternalSourcesTarget:
252 def make (self, packageBuildInfo):
253 scriptPath = os.path.normpath(os.path.join(packageBuildInfo.dstBasePath, "src", "external", "fetch_sources.py"))
260 class RemoveSourcesTarget:
264 def make (self, packageBuildInfo):
265 shutil.rmtree(os.path.join(packageBuildInfo.dstBasePath, "src"), ignore_errors=False)
268 def __init__ (self, name, targets):
270 self.targets = targets
272 def make (self, packageBuildInfo):
273 for target in self.targets:
274 target.make(packageBuildInfo)
277 def __init__ (self, name, version, modules, sources = True):
279 self.version = version
280 self.modules = modules
281 self.sources = sources
286 def getVersion (self):
289 def getModules (self):
292 def packageWithSources (self):
295 def matchIncludeExclude (includePatterns, excludePatterns, filename):
296 components = os.path.normpath(filename).split(os.sep)
297 for pattern in excludePatterns:
298 for component in components:
299 if fnmatch.fnmatch(component, pattern):
302 for pattern in includePatterns:
303 for component in components:
304 if fnmatch.fnmatch(component, pattern):
309 def copyFileFilter (includePatterns, excludePatterns=[]):
310 return (lambda f: matchIncludeExclude(includePatterns, excludePatterns, f),
311 lambda s, d: CopyFile(s, d))
313 def makeFileCopyGroup (srcDir, dstDir, includePatterns, excludePatterns=[]):
314 return FileTargetGroup(srcDir, dstDir, [copyFileFilter(includePatterns, excludePatterns)])
316 def makeTmpFileCopyGroup (srcDir, dstDir, includePatterns, excludePatterns=[]):
317 return FileTargetGroup(srcDir, dstDir, [copyFileFilter(includePatterns, excludePatterns)], PackageBuildInfo.getTmpBasePath)
319 def makeFileCopy (srcFile, dstFile):
320 return SingleFileTarget(srcFile, dstFile, lambda s, d: CopyFile(s, d))
322 def getReleaseFileName (configName, releaseName):
323 today = datetime.date.today()
324 return "dEQP-%s-%04d-%02d-%02d-%s" % (releaseName, today.year, today.month, today.day, configName)
327 dirName = os.path.join(tempfile.gettempdir(), "dEQP-Releases")
328 if not os.path.exists(dirName):
332 def makeRelease (releaseConfig):
333 releaseName = getReleaseFileName(releaseConfig.getName(), releaseConfig.getVersion())
334 tmpPath = getTempDir()
335 srcBasePath = DEQP_DIR
336 dstBasePath = os.path.join(tmpPath, releaseName)
337 tmpBasePath = os.path.join(tmpPath, releaseName + "-tmp")
338 packageBuildInfo = PackageBuildInfo(releaseConfig, srcBasePath, dstBasePath, tmpBasePath)
339 dstArchiveName = releaseName + ".tar.bz2"
341 print("Creating release %s to %s" % (releaseName, tmpPath))
343 # Remove old temporary dirs
344 for path in [dstBasePath, tmpBasePath]:
345 if os.path.exists(path):
346 shutil.rmtree(path, ignore_errors=False)
349 for module in releaseConfig.getModules():
350 print(" Processing module %s" % module.name)
351 module.make(packageBuildInfo)
354 if not releaseConfig.packageWithSources():
355 shutil.rmtree(os.path.join(dstBasePath, "src"), ignore_errors=False)
358 print("Creating %s" % dstArchiveName)
359 archive = tarfile.open(dstArchiveName, 'w:bz2')
360 archive.add(dstBasePath, arcname=releaseName)
364 for path in [dstBasePath, tmpBasePath]:
365 if os.path.exists(path):
366 shutil.rmtree(path, ignore_errors=False)
370 # Module declarations
372 SRC_FILE_PATTERNS = ["*.h", "*.hpp", "*.c", "*.cpp", "*.m", "*.mm", "*.inl", "*.java", "*.aidl", "CMakeLists.txt", "LICENSE.txt", "*.cmake"]
373 TARGET_PATTERNS = ["*.cmake", "*.h", "*.lib", "*.dll", "*.so", "*.txt"]
375 BASE = Module("Base", [
376 makeFileCopy ("LICENSE", "src/LICENSE"),
377 makeFileCopy ("CMakeLists.txt", "src/CMakeLists.txt"),
378 makeFileCopyGroup ("targets", "src/targets", TARGET_PATTERNS),
379 makeFileCopyGroup ("execserver", "src/execserver", SRC_FILE_PATTERNS),
380 makeFileCopyGroup ("executor", "src/executor", SRC_FILE_PATTERNS),
381 makeFileCopy ("modules/CMakeLists.txt", "src/modules/CMakeLists.txt"),
382 makeFileCopyGroup ("external", "src/external", ["CMakeLists.txt", "*.py"]),
384 # Stylesheet for displaying test logs on browser
385 makeFileCopyGroup ("doc/testlog-stylesheet", "doc/testlog-stylesheet", ["*"]),
387 # Non-optional parts of framework
388 makeFileCopy ("framework/CMakeLists.txt", "src/framework/CMakeLists.txt"),
389 makeFileCopyGroup ("framework/delibs", "src/framework/delibs", SRC_FILE_PATTERNS),
390 makeFileCopyGroup ("framework/common", "src/framework/common", SRC_FILE_PATTERNS),
391 makeFileCopyGroup ("framework/qphelper", "src/framework/qphelper", SRC_FILE_PATTERNS),
392 makeFileCopyGroup ("framework/platform", "src/framework/platform", SRC_FILE_PATTERNS),
393 makeFileCopyGroup ("framework/opengl", "src/framework/opengl", SRC_FILE_PATTERNS, ["simplereference"]),
394 makeFileCopyGroup ("framework/egl", "src/framework/egl", SRC_FILE_PATTERNS),
397 makeFileCopyGroup ("android/package/src", "src/android/package/src", SRC_FILE_PATTERNS),
398 makeFileCopy ("android/package/AndroidManifest.xml", "src/android/package/AndroidManifest.xml"),
399 makeFileCopyGroup ("android/package/res", "src/android/package/res", ["*.png", "*.xml"]),
400 makeFileCopyGroup ("android/scripts", "src/android/scripts", [
410 GenReleaseInfoFileTarget("src/framework/qphelper/qpReleaseInfo.inl")
413 DOCUMENTATION = Module("Documentation", [
414 makeFileCopyGroup ("doc/pdf", "doc", ["*.pdf"]),
415 makeFileCopyGroup ("doc", "doc", ["porting_layer_changes_*.txt"]),
418 GLSHARED = Module("Shared GL Tests", [
419 # Optional framework components
420 makeFileCopyGroup ("framework/randomshaders", "src/framework/randomshaders", SRC_FILE_PATTERNS),
421 makeFileCopyGroup ("framework/opengl/simplereference", "src/framework/opengl/simplereference", SRC_FILE_PATTERNS),
422 makeFileCopyGroup ("framework/referencerenderer", "src/framework/referencerenderer", SRC_FILE_PATTERNS),
424 makeFileCopyGroup ("modules/glshared", "src/modules/glshared", SRC_FILE_PATTERNS),
427 GLES2 = Module("GLES2", [
428 makeFileCopyGroup ("modules/gles2", "src/modules/gles2", SRC_FILE_PATTERNS),
429 makeFileCopyGroup ("data/gles2", "src/data/gles2", ["*.*"]),
430 makeFileCopyGroup ("doc/testspecs/GLES2", "doc/testspecs/GLES2", ["*.txt"])
433 GLES3 = Module("GLES3", [
434 makeFileCopyGroup ("modules/gles3", "src/modules/gles3", SRC_FILE_PATTERNS),
435 makeFileCopyGroup ("data/gles3", "src/data/gles3", ["*.*"]),
436 makeFileCopyGroup ("doc/testspecs/GLES3", "doc/testspecs/GLES3", ["*.txt"])
439 GLES31 = Module("GLES31", [
440 makeFileCopyGroup ("modules/gles31", "src/modules/gles31", SRC_FILE_PATTERNS),
441 makeFileCopyGroup ("data/gles31", "src/data/gles31", ["*.*"]),
442 makeFileCopyGroup ("doc/testspecs/GLES31", "doc/testspecs/GLES31", ["*.txt"])
445 EGL = Module("EGL", [
446 makeFileCopyGroup ("modules/egl", "src/modules/egl", SRC_FILE_PATTERNS)
449 INTERNAL = Module("Internal", [
450 makeFileCopyGroup ("modules/internal", "src/modules/internal", SRC_FILE_PATTERNS),
451 makeFileCopyGroup ("data/internal", "src/data/internal", ["*.*"]),
454 EXTERNAL_SRCS = Module("External sources", [
455 FetchExternalSourcesTarget()
458 ANDROID_BINARIES = Module("Android Binaries", [
459 BuildAndroidTarget ("bin/android/dEQP.apk"),
460 makeFileCopyGroup ("targets/android", "bin/android", ["*.bat", "*.sh"]),
463 COMMON_BUILD_ARGS = ['-DPNG_SRC_PATH=%s' % os.path.realpath(os.path.join(DEQP_DIR, '..', 'libpng'))]
464 NULL_X32_CONFIG = BuildConfig('null-x32', 'Release', ['-DDEQP_TARGET=null', '-DCMAKE_C_FLAGS=-m32', '-DCMAKE_CXX_FLAGS=-m32'] + COMMON_BUILD_ARGS)
465 NULL_X64_CONFIG = BuildConfig('null-x64', 'Release', ['-DDEQP_TARGET=null', '-DCMAKE_C_FLAGS=-m64', '-DCMAKE_CXX_FLAGS=-m64'] + COMMON_BUILD_ARGS)
466 GLX_X32_CONFIG = BuildConfig('glx-x32', 'Release', ['-DDEQP_TARGET=x11_glx', '-DCMAKE_C_FLAGS=-m32', '-DCMAKE_CXX_FLAGS=-m32'] + COMMON_BUILD_ARGS)
467 GLX_X64_CONFIG = BuildConfig('glx-x64', 'Release', ['-DDEQP_TARGET=x11_glx', '-DCMAKE_C_FLAGS=-m64', '-DCMAKE_CXX_FLAGS=-m64'] + COMMON_BUILD_ARGS)
469 EXCLUDE_BUILD_FILES = ["CMakeFiles", "*.a", "*.cmake"]
471 LINUX_X32_COMMON_BINARIES = Module("Linux x32 Common Binaries", [
472 BuildTarget (NULL_X32_CONFIG, ANY_UNIX_GENERATOR),
473 makeTmpFileCopyGroup(NULL_X32_CONFIG.getBuildDir() + "/execserver", "bin/linux32", ["*"], EXCLUDE_BUILD_FILES),
474 makeTmpFileCopyGroup(NULL_X32_CONFIG.getBuildDir() + "/executor", "bin/linux32", ["*"], EXCLUDE_BUILD_FILES),
477 LINUX_X64_COMMON_BINARIES = Module("Linux x64 Common Binaries", [
478 BuildTarget (NULL_X64_CONFIG, ANY_UNIX_GENERATOR),
479 makeTmpFileCopyGroup(NULL_X64_CONFIG.getBuildDir() + "/execserver", "bin/linux64", ["*"], EXCLUDE_BUILD_FILES),
480 makeTmpFileCopyGroup(NULL_X64_CONFIG.getBuildDir() + "/executor", "bin/linux64", ["*"], EXCLUDE_BUILD_FILES),
483 # Special module to remove src dir, for example after binary build
484 REMOVE_SOURCES = Module("Remove sources from package", [
485 RemoveSourcesTarget()
488 # Release configuration
503 LINUX_X64_COMMON_BINARIES,
509 "src-bin": ALL_MODULES + ALL_BINARIES,
510 "bin": ALL_MODULES + ALL_BINARIES + [REMOVE_SOURCES],
514 parser = argparse.ArgumentParser(description = "Build release package")
515 parser.add_argument("-c",
518 choices=RELEASE_CONFIGS.keys(),
520 help="Release configuration")
521 parser.add_argument("-n",
525 help="Package-specific name")
526 parser.add_argument("-v",
531 return parser.parse_args()
533 if __name__ == "__main__":
535 config = ReleaseConfig(args.name, args.version, RELEASE_CONFIGS[args.config])