scripts/make_release.py

   1 # -*- coding: utf-8 -*-
   2
   3 import os
   4 import re
   5 import sys
   6 import copy
   7 import zlib
   8 import time
   9 import shlex
  10 import shutil
  11 import fnmatch
  12 import tarfile
  13 import argparse
  14 import platform
  15 import datetime
  16 import tempfile
  17 import posixpath
  18 import subprocess
  19
  20 from build.common import *
  21 from build.config import *
  22 from build.build import *
  23
  24 def die (msg):
  25         print msg
  26         sys.exit(-1)
  27
  28 def removeLeadingPath (path, basePath):
  29         # Both inputs must be normalized already
  30         assert os.path.normpath(path) == path
  31         assert os.path.normpath(basePath) == basePath
  32         return path[len(basePath) + 1:]
  33
  34 def findFile (candidates):
  35         for file in candidates:
  36                 if os.path.exists(file):
  37                         return file
  38         return None
  39
  40 def getFileList (basePath):
  41         allFiles        = []
  42         basePath        = os.path.normpath(basePath)
  43         for root, dirs, files in os.walk(basePath):
  44                 for file in files:
  45                         relPath = removeLeadingPath(os.path.normpath(os.path.join(root, file)), basePath)
  46                         allFiles.append(relPath)
  47         return allFiles
  48
  49 def toDatetime (dateTuple):
  50         Y, M, D = dateTuple
  51         return datetime.datetime(Y, M, D)
  52
  53 class PackageBuildInfo:
  54         def __init__ (self, releaseConfig, srcBasePath, dstBasePath, tmpBasePath):
  55                 self.releaseConfig      = releaseConfig
  56                 self.srcBasePath        = srcBasePath
  57                 self.dstBasePath        = dstBasePath
  58                 self.tmpBasePath        = tmpBasePath
  59
  60         def getReleaseConfig (self):
  61                 return self.releaseConfig
  62
  63         def getReleaseVersion (self):
  64                 return self.releaseConfig.getVersion()
  65
  66         def getReleaseId (self):
  67                 # Release id is crc32(releaseConfig + release)
  68                 return zlib.crc32(self.releaseConfig.getName() + self.releaseConfig.getVersion()) & 0xffffffff
  69
  70         def getSrcBasePath (self):
  71                 return self.srcBasePath
  72
  73         def getTmpBasePath (self):
  74                 return self.tmpBasePath
  75
  76 class DstFile (object):
  77         def __init__ (self, dstFile):
  78                 self.dstFile = dstFile
  79
  80         def makeDir (self):
  81                 dirName = os.path.dirname(self.dstFile)
  82                 if not os.path.exists(dirName):
  83                         os.makedirs(dirName)
  84
  85         def make (self, packageBuildInfo):
  86                 assert False # Should not be called
  87
  88 class CopyFile (DstFile):
  89         def __init__ (self, srcFile, dstFile):
  90                 super(CopyFile, self).__init__(dstFile)
  91                 self.srcFile = srcFile
  92
  93         def make (self, packageBuildInfo):
  94                 self.makeDir()
  95                 if os.path.exists(self.dstFile):
  96                         die("%s already exists" % self.dstFile)
  97                 shutil.copyfile(self.srcFile, self.dstFile)
  98
  99 class GenInfoFile (DstFile):
 100         def __init__ (self, srcFile, dstFile):
 101                 super(GenInfoFile, self).__init__(dstFile)
 102                 self.srcFile = srcFile
 103
 104         def make (self, packageBuildInfo):
 105                 self.makeDir()
 106                 print "    GenInfoFile: %s" % removeLeadingPath(self.dstFile, packageBuildInfo.dstBasePath)
 107                 src                     = readFile(self.srcFile)
 108
 109                 for define, value in [("DEQP_RELEASE_NAME",     "\"%s\""        % packageBuildInfo.getReleaseVersion()),
 110                                                           ("DEQP_RELEASE_ID",           "0x%08x"        % packageBuildInfo.getReleaseId())]:
 111                         src = re.sub('(#\s*define\s+%s\s+)[^\n\r]+' % re.escape(define), r'\1 %s' % value, src)
 112
 113                 writeFile(self.dstFile, src)
 114
 115 class GenCMake (DstFile):
 116         def __init__ (self, srcFile, dstFile, replaceVars):
 117                 super(GenCMake, self).__init__(dstFile)
 118                 self.srcFile            = srcFile
 119                 self.replaceVars        = replaceVars
 120
 121         def make (self, packageBuildInfo):
 122                 self.makeDir()
 123                 print "    GenCMake: %s" % removeLeadingPath(self.dstFile, packageBuildInfo.dstBasePath)
 124                 src = readFile(self.srcFile)
 125                 for var, value in self.replaceVars:
 126                         src = re.sub('set\(%s\s+"[^"]*"' % re.escape(var),
 127                                                  'set(%s "%s"' % (var, value), src)
 128                 writeFile(self.dstFile, src)
 129
 130 def createFileTargets (srcBasePath, dstBasePath, files, filters):
 131         usedFiles       = set() # Files that are already included by other filters
 132         targets         = []
 133
 134         for isMatch, createFileObj in filters:
 135                 # Build list of files that match filter
 136                 matchingFiles = []
 137                 for file in files:
 138                         if not file in usedFiles and isMatch(file):
 139                                 matchingFiles.append(file)
 140
 141                 # Build file objects, add to used set
 142                 for file in matchingFiles:
 143                         usedFiles.add(file)
 144                         targets.append(createFileObj(os.path.join(srcBasePath, file), os.path.join(dstBasePath, file)))
 145
 146         return targets
 147
 148 # Generates multiple file targets based on filters
 149 class FileTargetGroup:
 150         def __init__ (self, srcBasePath, dstBasePath, filters, srcBasePathFunc=PackageBuildInfo.getSrcBasePath):
 151                 self.srcBasePath        = srcBasePath
 152                 self.dstBasePath        = dstBasePath
 153                 self.filters            = filters
 154                 self.getSrcBasePath     = srcBasePathFunc
 155
 156         def make (self, packageBuildInfo):
 157                 fullSrcPath             = os.path.normpath(os.path.join(self.getSrcBasePath(packageBuildInfo), self.srcBasePath))
 158                 fullDstPath             = os.path.normpath(os.path.join(packageBuildInfo.dstBasePath, self.dstBasePath))
 159
 160                 allFiles                = getFileList(fullSrcPath)
 161                 targets                 = createFileTargets(fullSrcPath, fullDstPath, allFiles, self.filters)
 162
 163                 # Make all file targets
 164                 for file in targets:
 165                         file.make(packageBuildInfo)
 166
 167 # Single file target
 168 class SingleFileTarget:
 169         def __init__ (self, srcFile, dstFile, makeTarget):
 170                 self.srcFile    = srcFile
 171                 self.dstFile    = dstFile
 172                 self.makeTarget = makeTarget
 173
 174         def make (self, packageBuildInfo):
 175                 fullSrcPath             = os.path.normpath(os.path.join(packageBuildInfo.srcBasePath, self.srcFile))
 176                 fullDstPath             = os.path.normpath(os.path.join(packageBuildInfo.dstBasePath, self.dstFile))
 177
 178                 target = self.makeTarget(fullSrcPath, fullDstPath)
 179                 target.make(packageBuildInfo)
 180
 181 class BuildTarget:
 182         def __init__ (self, baseConfig, generator, targets = None):
 183                 self.baseConfig = baseConfig
 184                 self.generator  = generator
 185                 self.targets    = targets
 186
 187         def make (self, packageBuildInfo):
 188                 print "    Building %s" % self.baseConfig.getBuildDir()
 189
 190                 # Create config with full build dir path
 191                 config = BuildConfig(os.path.join(packageBuildInfo.getTmpBasePath(), self.baseConfig.getBuildDir()),
 192                                                          self.baseConfig.getBuildType(),
 193                                                          self.baseConfig.getArgs(),
 194                                                          srcPath = os.path.join(packageBuildInfo.dstBasePath, "src"))
 195
 196                 assert not os.path.exists(config.getBuildDir())
 197                 build(config, self.generator, self.targets)
 198
 199 class BuildAndroidTarget:
 200         def __init__ (self, dstFile):
 201                 self.dstFile = dstFile
 202
 203         def make (self, packageBuildInfo):
 204                 print "    Building Android binary"
 205
 206                 buildRoot = os.path.join(packageBuildInfo.tmpBasePath, "android-build")
 207
 208                 assert not os.path.exists(buildRoot)
 209                 os.makedirs(buildRoot)
 210
 211                 # Execute build script
 212                 scriptPath = os.path.normpath(os.path.join(packageBuildInfo.dstBasePath, "src", "android", "scripts", "build.py"))
 213                 execute([
 214                                 "python",
 215                                 "-B", # no .py[co]
 216                                 scriptPath,
 217                                 "--build-root=%s" % buildRoot,
 218                         ])
 219
 220                 srcFile         = os.path.normpath(os.path.join(buildRoot, "package", "bin", "dEQP-debug.apk"))
 221                 dstFile         = os.path.normpath(os.path.join(packageBuildInfo.dstBasePath, self.dstFile))
 222
 223                 CopyFile(srcFile, dstFile).make(packageBuildInfo)
 224
 225 class FetchExternalSourcesTarget:
 226         def __init__ (self):
 227                 pass
 228
 229         def make (self, packageBuildInfo):
 230                 scriptPath = os.path.normpath(os.path.join(packageBuildInfo.dstBasePath, "src", "external", "fetch_sources.py"))
 231                 execute([
 232                                 "python",
 233                                 "-B", # no .py[co]
 234                                 scriptPath,
 235                         ])
 236
 237 class RemoveSourcesTarget:
 238         def __init__ (self):
 239                 pass
 240
 241         def make (self, packageBuildInfo):
 242                 shutil.rmtree(os.path.join(packageBuildInfo.dstBasePath, "src"), ignore_errors=False)
 243
 244 class Module:
 245         def __init__ (self, name, targets):
 246                 self.name               = name
 247                 self.targets    = targets
 248
 249         def make (self, packageBuildInfo):
 250                 for target in self.targets:
 251                         target.make(packageBuildInfo)
 252
 253 class ReleaseConfig:
 254         def __init__ (self, name, version, modules, sources = True):
 255                 self.name                       = name
 256                 self.version            = version
 257                 self.modules            = modules
 258                 self.sources            = sources
 259
 260         def getName (self):
 261                 return self.name
 262
 263         def getVersion (self):
 264                 return self.version
 265
 266         def getModules (self):
 267                 return self.modules
 268
 269         def packageWithSources (self):
 270                 return self.sources
 271
 272 def matchIncludeExclude (includePatterns, excludePatterns, filename):
 273         components = os.path.normpath(filename).split(os.sep)
 274         for pattern in excludePatterns:
 275                 for component in components:
 276                         if fnmatch.fnmatch(component, pattern):
 277                                 return False
 278
 279         for pattern in includePatterns:
 280                 for component in components:
 281                         if fnmatch.fnmatch(component, pattern):
 282                                 return True
 283
 284         return False
 285
 286 def copyFileFilter (includePatterns, excludePatterns=[]):
 287         return (lambda f: matchIncludeExclude(includePatterns, excludePatterns, f),
 288                         lambda s, d: CopyFile(s, d))
 289
 290 def makeFileCopyGroup (srcDir, dstDir, includePatterns, excludePatterns=[]):
 291         return FileTargetGroup(srcDir, dstDir, [copyFileFilter(includePatterns, excludePatterns)])
 292
 293 def makeTmpFileCopyGroup (srcDir, dstDir, includePatterns, excludePatterns=[]):
 294         return FileTargetGroup(srcDir, dstDir, [copyFileFilter(includePatterns, excludePatterns)], PackageBuildInfo.getTmpBasePath)
 295
 296 def makeFileCopy (srcFile, dstFile):
 297         return SingleFileTarget(srcFile, dstFile, lambda s, d: CopyFile(s, d))
 298
 299 def getReleaseFileName (configName, releaseName):
 300         today = datetime.date.today()
 301         return "dEQP-%s-%04d-%02d-%02d-%s" % (releaseName, today.year, today.month, today.day, configName)
 302
 303 def getTempDir ():
 304         dirName = os.path.join(tempfile.gettempdir(), "dEQP-Releases")
 305         if not os.path.exists(dirName):
 306                 os.makedirs(dirName)
 307         return dirName
 308
 309 def makeRelease (releaseConfig):
 310         releaseName                     = getReleaseFileName(releaseConfig.getName(), releaseConfig.getVersion())
 311         tmpPath                         = getTempDir()
 312         srcBasePath                     = DEQP_DIR
 313         dstBasePath                     = os.path.join(tmpPath, releaseName)
 314         tmpBasePath                     = os.path.join(tmpPath, releaseName + "-tmp")
 315         packageBuildInfo        = PackageBuildInfo(releaseConfig, srcBasePath, dstBasePath, tmpBasePath)
 316         dstArchiveName          = releaseName + ".tar.bz2"
 317
 318         print "Creating release %s to %s" % (releaseName, tmpPath)
 319
 320         # Remove old temporary dirs
 321         for path in [dstBasePath, tmpBasePath]:
 322                 if os.path.exists(path):
 323                         shutil.rmtree(path, ignore_errors=False)
 324
 325         # Make all modules
 326         for module in releaseConfig.getModules():
 327                 print "  Processing module %s" % module.name
 328                 module.make(packageBuildInfo)
 329
 330         # Remove sources?
 331         if not releaseConfig.packageWithSources():
 332                 shutil.rmtree(os.path.join(dstBasePath, "src"), ignore_errors=False)
 333
 334         # Create archive
 335         print "Creating %s" % dstArchiveName
 336         archive = tarfile.open(dstArchiveName, 'w:bz2')
 337         archive.add(dstBasePath, arcname=releaseName)
 338         archive.close()
 339
 340         # Remove tmp dirs
 341         for path in [dstBasePath, tmpBasePath]:
 342                 if os.path.exists(path):
 343                         shutil.rmtree(path, ignore_errors=False)
 344
 345         print "Done!"
 346
 347 # Module declarations
 348
 349 SRC_FILE_PATTERNS       = ["*.h", "*.hpp", "*.c", "*.cpp", "*.m", "*.mm", "*.inl", "*.java", "*.aidl", "CMakeLists.txt", "LICENSE.txt", "*.cmake"]
 350 TARGET_PATTERNS         = ["*.cmake", "*.h", "*.lib", "*.dll", "*.so", "*.txt"]
 351
 352 BASE = Module("Base", [
 353         makeFileCopy            ("LICENSE",                                                                     "src/LICENSE"),
 354         makeFileCopy            ("CMakeLists.txt",                                                      "src/CMakeLists.txt"),
 355         makeFileCopyGroup       ("targets",                                                                     "src/targets",                                                  TARGET_PATTERNS),
 356         makeFileCopyGroup       ("execserver",                                                          "src/execserver",                                               SRC_FILE_PATTERNS),
 357         makeFileCopyGroup       ("executor",                                                            "src/executor",                                                 SRC_FILE_PATTERNS),
 358         makeFileCopy            ("modules/CMakeLists.txt",                                      "src/modules/CMakeLists.txt"),
 359         makeFileCopyGroup       ("external",                                                            "src/external",                                                 ["CMakeLists.txt", "*.py"]),
 360
 361         # Stylesheet for displaying test logs on browser
 362         makeFileCopyGroup       ("doc/testlog-stylesheet",                                      "doc/testlog-stylesheet",                               ["*"]),
 363
 364         # Non-optional parts of framework
 365         # \note qpInfo.c must be processed!
 366         FileTargetGroup         ("framework", "src/framework", [
 367                         # If file is qpInfo.c use GenInfoFile
 368                         (lambda f: f[-8:] == "qpInfo.c", lambda s, d: GenInfoFile(s, d)),
 369                         # Otherwise just CopyFile targets
 370                         copyFileFilter(SRC_FILE_PATTERNS, ["imagedifftester", "randomshaders", "simplereference", "referencerenderer"])
 371                 ]),
 372
 373         # Main modules CMakeLists.txt
 374
 375         # android sources
 376         makeFileCopyGroup       ("android/package/src",                                         "src/android/package/src",                              SRC_FILE_PATTERNS),
 377         makeFileCopy            ("android/package/AndroidManifest.xml",         "src/android/package/AndroidManifest.xml"),
 378         makeFileCopyGroup       ("android/package/res",                                         "src/android/package/res",                              ["*.png", "*.xml"]),
 379         makeFileCopyGroup       ("android/scripts",                                                     "src/android/scripts", [
 380                 "common.py",
 381                 "build.py",
 382                 "resources.py",
 383                 "install.py",
 384                 "launch.py",
 385                 "debug.py"
 386                 ]),
 387 ])
 388
 389 DOCUMENTATION = Module("Documentation", [
 390         makeFileCopyGroup       ("doc/pdf",                                                                     "doc",                                                                  ["*.pdf"]),
 391         makeFileCopyGroup       ("doc",                                                                         "doc",                                                                  ["porting_layer_changes_*.txt"]),
 392 ])
 393
 394 GLSHARED = Module("Shared GL Tests", [
 395         # Optional framework components
 396         makeFileCopyGroup       ("framework/randomshaders",                                     "src/framework/randomshaders",                  SRC_FILE_PATTERNS),
 397         makeFileCopyGroup       ("framework/opengl/simplereference",            "src/framework/opengl/simplereference", SRC_FILE_PATTERNS),
 398         makeFileCopyGroup       ("framework/referencerenderer",                         "src/framework/referencerenderer",              SRC_FILE_PATTERNS),
 399
 400         makeFileCopyGroup       ("modules/glshared",                                            "src/modules/glshared",                                 SRC_FILE_PATTERNS),
 401 ])
 402
 403 GLES2 = Module("GLES2", [
 404         makeFileCopyGroup       ("modules/gles2",                                                       "src/modules/gles2",                                    SRC_FILE_PATTERNS),
 405         makeFileCopyGroup       ("data/gles2",                                                          "src/data/gles2",                                               ["*.*"]),
 406         makeFileCopyGroup       ("doc/testspecs/GLES2",                                         "doc/testspecs/GLES2",                                  ["*.txt"])
 407 ])
 408
 409 GLES3 = Module("GLES3", [
 410         makeFileCopyGroup       ("modules/gles3",                                                       "src/modules/gles3",                                    SRC_FILE_PATTERNS),
 411         makeFileCopyGroup       ("data/gles3",                                                          "src/data/gles3",                                               ["*.*"]),
 412         makeFileCopyGroup       ("doc/testspecs/GLES3",                                         "doc/testspecs/GLES3",                                  ["*.txt"])
 413 ])
 414
 415 GLES31 = Module("GLES31", [
 416         makeFileCopyGroup       ("modules/gles31",                                                      "src/modules/gles31",                                   SRC_FILE_PATTERNS),
 417         makeFileCopyGroup       ("data/gles31",                                                         "src/data/gles31",                                              ["*.*"]),
 418         makeFileCopyGroup       ("doc/testspecs/GLES31",                                        "doc/testspecs/GLES31",                                 ["*.txt"])
 419 ])
 420
 421 EGL = Module("EGL", [
 422         makeFileCopyGroup       ("modules/egl",                                                         "src/modules/egl",                                              SRC_FILE_PATTERNS)
 423 ])
 424
 425 INTERNAL = Module("Internal", [
 426         makeFileCopyGroup       ("modules/internal",                                            "src/modules/internal",                                 SRC_FILE_PATTERNS),
 427         makeFileCopyGroup       ("data/internal",                                                       "src/data/internal",                                    ["*.*"]),
 428 ])
 429
 430 EXTERNAL_SRCS = Module("External sources", [
 431         FetchExternalSourcesTarget()
 432 ])
 433
 434 ANDROID_BINARIES = Module("Android Binaries", [
 435         BuildAndroidTarget      ("bin/android/dEQP.apk"),
 436         makeFileCopyGroup       ("targets/android",                                                     "bin/android",                                                  ["*.bat", "*.sh"]),
 437 ])
 438
 439 COMMON_BUILD_ARGS       = ['-DPNG_SRC_PATH=%s' % os.path.realpath(os.path.join(DEQP_DIR, '..', 'libpng'))]
 440 NULL_X32_CONFIG         = BuildConfig('null-x32',       'Release', ['-DDEQP_TARGET=null', '-DCMAKE_C_FLAGS=-m32', '-DCMAKE_CXX_FLAGS=-m32'] + COMMON_BUILD_ARGS)
 441 NULL_X64_CONFIG         = BuildConfig('null-x64',       'Release', ['-DDEQP_TARGET=null', '-DCMAKE_C_FLAGS=-m64', '-DCMAKE_CXX_FLAGS=-m64'] + COMMON_BUILD_ARGS)
 442 GLX_X32_CONFIG          = BuildConfig('glx-x32',        'Release', ['-DDEQP_TARGET=x11_glx', '-DCMAKE_C_FLAGS=-m32', '-DCMAKE_CXX_FLAGS=-m32'] + COMMON_BUILD_ARGS)
 443 GLX_X64_CONFIG          = BuildConfig('glx-x64',        'Release', ['-DDEQP_TARGET=x11_glx', '-DCMAKE_C_FLAGS=-m64', '-DCMAKE_CXX_FLAGS=-m64'] + COMMON_BUILD_ARGS)
 444
 445 EXCLUDE_BUILD_FILES = ["CMakeFiles", "*.a", "*.cmake"]
 446
 447 LINUX_X32_COMMON_BINARIES = Module("Linux x32 Common Binaries", [
 448         BuildTarget                     (NULL_X32_CONFIG, ANY_UNIX_GENERATOR),
 449         makeTmpFileCopyGroup(NULL_X32_CONFIG.getBuildDir() + "/execserver",             "bin/linux32",                                  ["*"],  EXCLUDE_BUILD_FILES),
 450         makeTmpFileCopyGroup(NULL_X32_CONFIG.getBuildDir() + "/executor",               "bin/linux32",                                  ["*"],  EXCLUDE_BUILD_FILES),
 451 ])
 452
 453 LINUX_X64_COMMON_BINARIES = Module("Linux x64 Common Binaries", [
 454         BuildTarget                     (NULL_X64_CONFIG, ANY_UNIX_GENERATOR),
 455         makeTmpFileCopyGroup(NULL_X64_CONFIG.getBuildDir() + "/execserver",             "bin/linux64",                                  ["*"],  EXCLUDE_BUILD_FILES),
 456         makeTmpFileCopyGroup(NULL_X64_CONFIG.getBuildDir() + "/executor",               "bin/linux64",                                  ["*"],  EXCLUDE_BUILD_FILES),
 457 ])
 458
 459 # Special module to remove src dir, for example after binary build
 460 REMOVE_SOURCES = Module("Remove sources from package", [
 461         RemoveSourcesTarget()
 462 ])
 463
 464 # Release configuration
 465
 466 ALL_MODULES             = [
 467         BASE,
 468         DOCUMENTATION,
 469         GLSHARED,
 470         GLES2,
 471         GLES3,
 472         GLES31,
 473         EGL,
 474         INTERNAL,
 475         EXTERNAL_SRCS,
 476 ]
 477
 478 ALL_BINARIES    = [
 479         LINUX_X64_COMMON_BINARIES,
 480         ANDROID_BINARIES,
 481 ]
 482
 483 RELEASE_CONFIGS = {
 484         "src":          ALL_MODULES,
 485         "src-bin":      ALL_MODULES + ALL_BINARIES,
 486         "bin":          ALL_MODULES + ALL_BINARIES + [REMOVE_SOURCES],
 487 }
 488
 489 def parseArgs ():
 490         parser = argparse.ArgumentParser(description = "Build release package")
 491         parser.add_argument("-c",
 492                                                 "--config",
 493                                                 dest="config",
 494                                                 choices=RELEASE_CONFIGS.keys(),
 495                                                 required=True,
 496                                                 help="Release configuration")
 497         parser.add_argument("-n",
 498                                                 "--name",
 499                                                 dest="name",
 500                                                 required=True,
 501                                                 help="Package-specific name")
 502         parser.add_argument("-v",
 503                                                 "--version",
 504                                                 dest="version",
 505                                                 required=True,
 506                                                 help="Version code")
 507         return parser.parse_args()
 508
 509 if __name__ == "__main__":
 510         args    = parseArgs()
 511         config  = ReleaseConfig(args.name, args.version, RELEASE_CONFIGS[args.config])
 512         makeRelease(config)