scripts/make_release.py

   1 # -*- coding: utf-8 -*-
   2
   3 import os
   4 import re
   5 import sys
   6 import copy
   7 import zlib
   8 import time
   9 import shlex
  10 import shutil
  11 import fnmatch
  12 import tarfile
  13 import argparse
  14 import platform
  15 import datetime
  16 import tempfile
  17 import posixpath
  18 import subprocess
  19
  20 from build.common import *
  21 from build.config import *
  22 from build.build import *
  23
  24 def die (msg):
  25         print msg
  26         sys.exit(-1)
  27
  28 def removeLeadingPath (path, basePath):
  29         # Both inputs must be normalized already
  30         assert os.path.normpath(path) == path
  31         assert os.path.normpath(basePath) == basePath
  32         return path[len(basePath) + 1:]
  33
  34 def findFile (candidates):
  35         for file in candidates:
  36                 if os.path.exists(file):
  37                         return file
  38         return None
  39
  40 def getFileList (basePath):
  41         allFiles        = []
  42         basePath        = os.path.normpath(basePath)
  43         for root, dirs, files in os.walk(basePath):
  44                 for file in files:
  45                         relPath = removeLeadingPath(os.path.normpath(os.path.join(root, file)), basePath)
  46                         allFiles.append(relPath)
  47         return allFiles
  48
  49 def toDatetime (dateTuple):
  50         Y, M, D = dateTuple
  51         return datetime.datetime(Y, M, D)
  52
  53 class PackageBuildInfo:
  54         def __init__ (self, releaseConfig, srcBasePath, dstBasePath, tmpBasePath):
  55                 self.releaseConfig      = releaseConfig
  56                 self.srcBasePath        = srcBasePath
  57                 self.dstBasePath        = dstBasePath
  58                 self.tmpBasePath        = tmpBasePath
  59
  60         def getReleaseConfig (self):
  61                 return self.releaseConfig
  62
  63         def getReleaseVersion (self):
  64                 return self.releaseConfig.getVersion()
  65
  66         def getReleaseId (self):
  67                 # Release id is crc32(releaseConfig + release)
  68                 return zlib.crc32(self.releaseConfig.getName() + self.releaseConfig.getVersion()) & 0xffffffff
  69
  70         def getSrcBasePath (self):
  71                 return self.srcBasePath
  72
  73         def getTmpBasePath (self):
  74                 return self.tmpBasePath
  75
  76 class DstFile (object):
  77         def __init__ (self, dstFile):
  78                 self.dstFile = dstFile
  79
  80         def makeDir (self):
  81                 dirName = os.path.dirname(self.dstFile)
  82                 if not os.path.exists(dirName):
  83                         os.makedirs(dirName)
  84
  85         def make (self, packageBuildInfo):
  86                 assert False # Should not be called
  87
  88 class CopyFile (DstFile):
  89         def __init__ (self, srcFile, dstFile):
  90                 super(CopyFile, self).__init__(dstFile)
  91                 self.srcFile = srcFile
  92
  93         def make (self, packageBuildInfo):
  94                 self.makeDir()
  95                 if os.path.exists(self.dstFile):
  96                         die("%s already exists" % self.dstFile)
  97                 shutil.copyfile(self.srcFile, self.dstFile)
  98
  99 class GenReleaseInfoFileTarget (DstFile):
 100         def __init__ (self, dstFile):
 101                 super(GenReleaseInfoFileTarget, self).__init__(dstFile)
 102
 103         def make (self, packageBuildInfo):
 104                 self.makeDir()
 105
 106                 scriptPath = os.path.normpath(os.path.join(packageBuildInfo.srcBasePath, "framework", "qphelper", "gen_release_info.py"))
 107                 execute([
 108                                 "python",
 109                                 "-B", # no .py[co]
 110                                 scriptPath,
 111                                 "--name=%s" % packageBuildInfo.getReleaseVersion(),
 112                                 "--id=0x%08x" % packageBuildInfo.getReleaseId(),
 113                                 "--out=%s" % self.dstFile
 114                         ])
 115
 116 class GenCMake (DstFile):
 117         def __init__ (self, srcFile, dstFile, replaceVars):
 118                 super(GenCMake, self).__init__(dstFile)
 119                 self.srcFile            = srcFile
 120                 self.replaceVars        = replaceVars
 121
 122         def make (self, packageBuildInfo):
 123                 self.makeDir()
 124                 print "    GenCMake: %s" % removeLeadingPath(self.dstFile, packageBuildInfo.dstBasePath)
 125                 src = readFile(self.srcFile)
 126                 for var, value in self.replaceVars:
 127                         src = re.sub('set\(%s\s+"[^"]*"' % re.escape(var),
 128                                                  'set(%s "%s"' % (var, value), src)
 129                 writeFile(self.dstFile, src)
 130
 131 def createFileTargets (srcBasePath, dstBasePath, files, filters):
 132         usedFiles       = set() # Files that are already included by other filters
 133         targets         = []
 134
 135         for isMatch, createFileObj in filters:
 136                 # Build list of files that match filter
 137                 matchingFiles = []
 138                 for file in files:
 139                         if not file in usedFiles and isMatch(file):
 140                                 matchingFiles.append(file)
 141
 142                 # Build file objects, add to used set
 143                 for file in matchingFiles:
 144                         usedFiles.add(file)
 145                         targets.append(createFileObj(os.path.join(srcBasePath, file), os.path.join(dstBasePath, file)))
 146
 147         return targets
 148
 149 # Generates multiple file targets based on filters
 150 class FileTargetGroup:
 151         def __init__ (self, srcBasePath, dstBasePath, filters, srcBasePathFunc=PackageBuildInfo.getSrcBasePath):
 152                 self.srcBasePath        = srcBasePath
 153                 self.dstBasePath        = dstBasePath
 154                 self.filters            = filters
 155                 self.getSrcBasePath     = srcBasePathFunc
 156
 157         def make (self, packageBuildInfo):
 158                 fullSrcPath             = os.path.normpath(os.path.join(self.getSrcBasePath(packageBuildInfo), self.srcBasePath))
 159                 fullDstPath             = os.path.normpath(os.path.join(packageBuildInfo.dstBasePath, self.dstBasePath))
 160
 161                 allFiles                = getFileList(fullSrcPath)
 162                 targets                 = createFileTargets(fullSrcPath, fullDstPath, allFiles, self.filters)
 163
 164                 # Make all file targets
 165                 for file in targets:
 166                         file.make(packageBuildInfo)
 167
 168 # Single file target
 169 class SingleFileTarget:
 170         def __init__ (self, srcFile, dstFile, makeTarget):
 171                 self.srcFile    = srcFile
 172                 self.dstFile    = dstFile
 173                 self.makeTarget = makeTarget
 174
 175         def make (self, packageBuildInfo):
 176                 fullSrcPath             = os.path.normpath(os.path.join(packageBuildInfo.srcBasePath, self.srcFile))
 177                 fullDstPath             = os.path.normpath(os.path.join(packageBuildInfo.dstBasePath, self.dstFile))
 178
 179                 target = self.makeTarget(fullSrcPath, fullDstPath)
 180                 target.make(packageBuildInfo)
 181
 182 class BuildTarget:
 183         def __init__ (self, baseConfig, generator, targets = None):
 184                 self.baseConfig = baseConfig
 185                 self.generator  = generator
 186                 self.targets    = targets
 187
 188         def make (self, packageBuildInfo):
 189                 print "    Building %s" % self.baseConfig.getBuildDir()
 190
 191                 # Create config with full build dir path
 192                 config = BuildConfig(os.path.join(packageBuildInfo.getTmpBasePath(), self.baseConfig.getBuildDir()),
 193                                                          self.baseConfig.getBuildType(),
 194                                                          self.baseConfig.getArgs(),
 195                                                          srcPath = os.path.join(packageBuildInfo.dstBasePath, "src"))
 196
 197                 assert not os.path.exists(config.getBuildDir())
 198                 build(config, self.generator, self.targets)
 199
 200 class BuildAndroidTarget:
 201         def __init__ (self, dstFile):
 202                 self.dstFile = dstFile
 203
 204         def make (self, packageBuildInfo):
 205                 print "    Building Android binary"
 206
 207                 buildRoot = os.path.join(packageBuildInfo.tmpBasePath, "android-build")
 208
 209                 assert not os.path.exists(buildRoot)
 210                 os.makedirs(buildRoot)
 211
 212                 # Execute build script
 213                 scriptPath = os.path.normpath(os.path.join(packageBuildInfo.dstBasePath, "src", "android", "scripts", "build.py"))
 214                 execute([
 215                                 "python",
 216                                 "-B", # no .py[co]
 217                                 scriptPath,
 218                                 "--build-root=%s" % buildRoot,
 219                         ])
 220
 221                 srcFile         = os.path.normpath(os.path.join(buildRoot, "package", "bin", "dEQP-debug.apk"))
 222                 dstFile         = os.path.normpath(os.path.join(packageBuildInfo.dstBasePath, self.dstFile))
 223
 224                 CopyFile(srcFile, dstFile).make(packageBuildInfo)
 225
 226 class FetchExternalSourcesTarget:
 227         def __init__ (self):
 228                 pass
 229
 230         def make (self, packageBuildInfo):
 231                 scriptPath = os.path.normpath(os.path.join(packageBuildInfo.dstBasePath, "src", "external", "fetch_sources.py"))
 232                 execute([
 233                                 "python",
 234                                 "-B", # no .py[co]
 235                                 scriptPath,
 236                         ])
 237
 238 class RemoveSourcesTarget:
 239         def __init__ (self):
 240                 pass
 241
 242         def make (self, packageBuildInfo):
 243                 shutil.rmtree(os.path.join(packageBuildInfo.dstBasePath, "src"), ignore_errors=False)
 244
 245 class Module:
 246         def __init__ (self, name, targets):
 247                 self.name               = name
 248                 self.targets    = targets
 249
 250         def make (self, packageBuildInfo):
 251                 for target in self.targets:
 252                         target.make(packageBuildInfo)
 253
 254 class ReleaseConfig:
 255         def __init__ (self, name, version, modules, sources = True):
 256                 self.name                       = name
 257                 self.version            = version
 258                 self.modules            = modules
 259                 self.sources            = sources
 260
 261         def getName (self):
 262                 return self.name
 263
 264         def getVersion (self):
 265                 return self.version
 266
 267         def getModules (self):
 268                 return self.modules
 269
 270         def packageWithSources (self):
 271                 return self.sources
 272
 273 def matchIncludeExclude (includePatterns, excludePatterns, filename):
 274         components = os.path.normpath(filename).split(os.sep)
 275         for pattern in excludePatterns:
 276                 for component in components:
 277                         if fnmatch.fnmatch(component, pattern):
 278                                 return False
 279
 280         for pattern in includePatterns:
 281                 for component in components:
 282                         if fnmatch.fnmatch(component, pattern):
 283                                 return True
 284
 285         return False
 286
 287 def copyFileFilter (includePatterns, excludePatterns=[]):
 288         return (lambda f: matchIncludeExclude(includePatterns, excludePatterns, f),
 289                         lambda s, d: CopyFile(s, d))
 290
 291 def makeFileCopyGroup (srcDir, dstDir, includePatterns, excludePatterns=[]):
 292         return FileTargetGroup(srcDir, dstDir, [copyFileFilter(includePatterns, excludePatterns)])
 293
 294 def makeTmpFileCopyGroup (srcDir, dstDir, includePatterns, excludePatterns=[]):
 295         return FileTargetGroup(srcDir, dstDir, [copyFileFilter(includePatterns, excludePatterns)], PackageBuildInfo.getTmpBasePath)
 296
 297 def makeFileCopy (srcFile, dstFile):
 298         return SingleFileTarget(srcFile, dstFile, lambda s, d: CopyFile(s, d))
 299
 300 def getReleaseFileName (configName, releaseName):
 301         today = datetime.date.today()
 302         return "dEQP-%s-%04d-%02d-%02d-%s" % (releaseName, today.year, today.month, today.day, configName)
 303
 304 def getTempDir ():
 305         dirName = os.path.join(tempfile.gettempdir(), "dEQP-Releases")
 306         if not os.path.exists(dirName):
 307                 os.makedirs(dirName)
 308         return dirName
 309
 310 def makeRelease (releaseConfig):
 311         releaseName                     = getReleaseFileName(releaseConfig.getName(), releaseConfig.getVersion())
 312         tmpPath                         = getTempDir()
 313         srcBasePath                     = DEQP_DIR
 314         dstBasePath                     = os.path.join(tmpPath, releaseName)
 315         tmpBasePath                     = os.path.join(tmpPath, releaseName + "-tmp")
 316         packageBuildInfo        = PackageBuildInfo(releaseConfig, srcBasePath, dstBasePath, tmpBasePath)
 317         dstArchiveName          = releaseName + ".tar.bz2"
 318
 319         print "Creating release %s to %s" % (releaseName, tmpPath)
 320
 321         # Remove old temporary dirs
 322         for path in [dstBasePath, tmpBasePath]:
 323                 if os.path.exists(path):
 324                         shutil.rmtree(path, ignore_errors=False)
 325
 326         # Make all modules
 327         for module in releaseConfig.getModules():
 328                 print "  Processing module %s" % module.name
 329                 module.make(packageBuildInfo)
 330
 331         # Remove sources?
 332         if not releaseConfig.packageWithSources():
 333                 shutil.rmtree(os.path.join(dstBasePath, "src"), ignore_errors=False)
 334
 335         # Create archive
 336         print "Creating %s" % dstArchiveName
 337         archive = tarfile.open(dstArchiveName, 'w:bz2')
 338         archive.add(dstBasePath, arcname=releaseName)
 339         archive.close()
 340
 341         # Remove tmp dirs
 342         for path in [dstBasePath, tmpBasePath]:
 343                 if os.path.exists(path):
 344                         shutil.rmtree(path, ignore_errors=False)
 345
 346         print "Done!"
 347
 348 # Module declarations
 349
 350 SRC_FILE_PATTERNS       = ["*.h", "*.hpp", "*.c", "*.cpp", "*.m", "*.mm", "*.inl", "*.java", "*.aidl", "CMakeLists.txt", "LICENSE.txt", "*.cmake"]
 351 TARGET_PATTERNS         = ["*.cmake", "*.h", "*.lib", "*.dll", "*.so", "*.txt"]
 352
 353 BASE = Module("Base", [
 354         makeFileCopy            ("LICENSE",                                                                     "src/LICENSE"),
 355         makeFileCopy            ("CMakeLists.txt",                                                      "src/CMakeLists.txt"),
 356         makeFileCopyGroup       ("targets",                                                                     "src/targets",                                                  TARGET_PATTERNS),
 357         makeFileCopyGroup       ("execserver",                                                          "src/execserver",                                               SRC_FILE_PATTERNS),
 358         makeFileCopyGroup       ("executor",                                                            "src/executor",                                                 SRC_FILE_PATTERNS),
 359         makeFileCopy            ("modules/CMakeLists.txt",                                      "src/modules/CMakeLists.txt"),
 360         makeFileCopyGroup       ("external",                                                            "src/external",                                                 ["CMakeLists.txt", "*.py"]),
 361
 362         # Stylesheet for displaying test logs on browser
 363         makeFileCopyGroup       ("doc/testlog-stylesheet",                                      "doc/testlog-stylesheet",                               ["*"]),
 364
 365         # Non-optional parts of framework
 366         makeFileCopy            ("framework/CMakeLists.txt",                            "src/framework/CMakeLists.txt"),
 367         makeFileCopyGroup       ("framework/delibs",                                            "src/framework/delibs",                                 SRC_FILE_PATTERNS),
 368         makeFileCopyGroup       ("framework/common",                                            "src/framework/common",                                 SRC_FILE_PATTERNS),
 369         makeFileCopyGroup       ("framework/qphelper",                                          "src/framework/qphelper",                               SRC_FILE_PATTERNS),
 370         makeFileCopyGroup       ("framework/platform",                                          "src/framework/platform",                               SRC_FILE_PATTERNS),
 371         makeFileCopyGroup       ("framework/opengl",                                            "src/framework/opengl",                                 SRC_FILE_PATTERNS, ["simplereference"]),
 372         makeFileCopyGroup       ("framework/egl",                                                       "src/framework/egl",                                    SRC_FILE_PATTERNS),
 373
 374         # android sources
 375         makeFileCopyGroup       ("android/package/src",                                         "src/android/package/src",                              SRC_FILE_PATTERNS),
 376         makeFileCopy            ("android/package/AndroidManifest.xml",         "src/android/package/AndroidManifest.xml"),
 377         makeFileCopyGroup       ("android/package/res",                                         "src/android/package/res",                              ["*.png", "*.xml"]),
 378         makeFileCopyGroup       ("android/scripts",                                                     "src/android/scripts", [
 379                 "common.py",
 380                 "build.py",
 381                 "resources.py",
 382                 "install.py",
 383                 "launch.py",
 384                 "debug.py"
 385                 ]),
 386
 387         # Release info
 388         GenReleaseInfoFileTarget("src/framework/qphelper/qpReleaseInfo.inl")
 389 ])
 390
 391 DOCUMENTATION = Module("Documentation", [
 392         makeFileCopyGroup       ("doc/pdf",                                                                     "doc",                                                                  ["*.pdf"]),
 393         makeFileCopyGroup       ("doc",                                                                         "doc",                                                                  ["porting_layer_changes_*.txt"]),
 394 ])
 395
 396 GLSHARED = Module("Shared GL Tests", [
 397         # Optional framework components
 398         makeFileCopyGroup       ("framework/randomshaders",                                     "src/framework/randomshaders",                  SRC_FILE_PATTERNS),
 399         makeFileCopyGroup       ("framework/opengl/simplereference",            "src/framework/opengl/simplereference", SRC_FILE_PATTERNS),
 400         makeFileCopyGroup       ("framework/referencerenderer",                         "src/framework/referencerenderer",              SRC_FILE_PATTERNS),
 401
 402         makeFileCopyGroup       ("modules/glshared",                                            "src/modules/glshared",                                 SRC_FILE_PATTERNS),
 403 ])
 404
 405 GLES2 = Module("GLES2", [
 406         makeFileCopyGroup       ("modules/gles2",                                                       "src/modules/gles2",                                    SRC_FILE_PATTERNS),
 407         makeFileCopyGroup       ("data/gles2",                                                          "src/data/gles2",                                               ["*.*"]),
 408         makeFileCopyGroup       ("doc/testspecs/GLES2",                                         "doc/testspecs/GLES2",                                  ["*.txt"])
 409 ])
 410
 411 GLES3 = Module("GLES3", [
 412         makeFileCopyGroup       ("modules/gles3",                                                       "src/modules/gles3",                                    SRC_FILE_PATTERNS),
 413         makeFileCopyGroup       ("data/gles3",                                                          "src/data/gles3",                                               ["*.*"]),
 414         makeFileCopyGroup       ("doc/testspecs/GLES3",                                         "doc/testspecs/GLES3",                                  ["*.txt"])
 415 ])
 416
 417 GLES31 = Module("GLES31", [
 418         makeFileCopyGroup       ("modules/gles31",                                                      "src/modules/gles31",                                   SRC_FILE_PATTERNS),
 419         makeFileCopyGroup       ("data/gles31",                                                         "src/data/gles31",                                              ["*.*"]),
 420         makeFileCopyGroup       ("doc/testspecs/GLES31",                                        "doc/testspecs/GLES31",                                 ["*.txt"])
 421 ])
 422
 423 EGL = Module("EGL", [
 424         makeFileCopyGroup       ("modules/egl",                                                         "src/modules/egl",                                              SRC_FILE_PATTERNS)
 425 ])
 426
 427 INTERNAL = Module("Internal", [
 428         makeFileCopyGroup       ("modules/internal",                                            "src/modules/internal",                                 SRC_FILE_PATTERNS),
 429         makeFileCopyGroup       ("data/internal",                                                       "src/data/internal",                                    ["*.*"]),
 430 ])
 431
 432 EXTERNAL_SRCS = Module("External sources", [
 433         FetchExternalSourcesTarget()
 434 ])
 435
 436 ANDROID_BINARIES = Module("Android Binaries", [
 437         BuildAndroidTarget      ("bin/android/dEQP.apk"),
 438         makeFileCopyGroup       ("targets/android",                                                     "bin/android",                                                  ["*.bat", "*.sh"]),
 439 ])
 440
 441 COMMON_BUILD_ARGS       = ['-DPNG_SRC_PATH=%s' % os.path.realpath(os.path.join(DEQP_DIR, '..', 'libpng'))]
 442 NULL_X32_CONFIG         = BuildConfig('null-x32',       'Release', ['-DDEQP_TARGET=null', '-DCMAKE_C_FLAGS=-m32', '-DCMAKE_CXX_FLAGS=-m32'] + COMMON_BUILD_ARGS)
 443 NULL_X64_CONFIG         = BuildConfig('null-x64',       'Release', ['-DDEQP_TARGET=null', '-DCMAKE_C_FLAGS=-m64', '-DCMAKE_CXX_FLAGS=-m64'] + COMMON_BUILD_ARGS)
 444 GLX_X32_CONFIG          = BuildConfig('glx-x32',        'Release', ['-DDEQP_TARGET=x11_glx', '-DCMAKE_C_FLAGS=-m32', '-DCMAKE_CXX_FLAGS=-m32'] + COMMON_BUILD_ARGS)
 445 GLX_X64_CONFIG          = BuildConfig('glx-x64',        'Release', ['-DDEQP_TARGET=x11_glx', '-DCMAKE_C_FLAGS=-m64', '-DCMAKE_CXX_FLAGS=-m64'] + COMMON_BUILD_ARGS)
 446
 447 EXCLUDE_BUILD_FILES = ["CMakeFiles", "*.a", "*.cmake"]
 448
 449 LINUX_X32_COMMON_BINARIES = Module("Linux x32 Common Binaries", [
 450         BuildTarget                     (NULL_X32_CONFIG, ANY_UNIX_GENERATOR),
 451         makeTmpFileCopyGroup(NULL_X32_CONFIG.getBuildDir() + "/execserver",             "bin/linux32",                                  ["*"],  EXCLUDE_BUILD_FILES),
 452         makeTmpFileCopyGroup(NULL_X32_CONFIG.getBuildDir() + "/executor",               "bin/linux32",                                  ["*"],  EXCLUDE_BUILD_FILES),
 453 ])
 454
 455 LINUX_X64_COMMON_BINARIES = Module("Linux x64 Common Binaries", [
 456         BuildTarget                     (NULL_X64_CONFIG, ANY_UNIX_GENERATOR),
 457         makeTmpFileCopyGroup(NULL_X64_CONFIG.getBuildDir() + "/execserver",             "bin/linux64",                                  ["*"],  EXCLUDE_BUILD_FILES),
 458         makeTmpFileCopyGroup(NULL_X64_CONFIG.getBuildDir() + "/executor",               "bin/linux64",                                  ["*"],  EXCLUDE_BUILD_FILES),
 459 ])
 460
 461 # Special module to remove src dir, for example after binary build
 462 REMOVE_SOURCES = Module("Remove sources from package", [
 463         RemoveSourcesTarget()
 464 ])
 465
 466 # Release configuration
 467
 468 ALL_MODULES             = [
 469         BASE,
 470         DOCUMENTATION,
 471         GLSHARED,
 472         GLES2,
 473         GLES3,
 474         GLES31,
 475         EGL,
 476         INTERNAL,
 477         EXTERNAL_SRCS,
 478 ]
 479
 480 ALL_BINARIES    = [
 481         LINUX_X64_COMMON_BINARIES,
 482         ANDROID_BINARIES,
 483 ]
 484
 485 RELEASE_CONFIGS = {
 486         "src":          ALL_MODULES,
 487         "src-bin":      ALL_MODULES + ALL_BINARIES,
 488         "bin":          ALL_MODULES + ALL_BINARIES + [REMOVE_SOURCES],
 489 }
 490
 491 def parseArgs ():
 492         parser = argparse.ArgumentParser(description = "Build release package")
 493         parser.add_argument("-c",
 494                                                 "--config",
 495                                                 dest="config",
 496                                                 choices=RELEASE_CONFIGS.keys(),
 497                                                 required=True,
 498                                                 help="Release configuration")
 499         parser.add_argument("-n",
 500                                                 "--name",
 501                                                 dest="name",
 502                                                 required=True,
 503                                                 help="Package-specific name")
 504         parser.add_argument("-v",
 505                                                 "--version",
 506                                                 dest="version",
 507                                                 required=True,
 508                                                 help="Version code")
 509         return parser.parse_args()
 510
 511 if __name__ == "__main__":
 512         args    = parseArgs()
 513         config  = ReleaseConfig(args.name, args.version, RELEASE_CONFIGS[args.config])
 514         makeRelease(config)