scripts/run_internal_tests.py

   1 # -*- coding: utf-8 -*-
   2
   3 #-------------------------------------------------------------------------
   4 # drawElements Quality Program utilities
   5 # --------------------------------------
   6 #
   7 # Copyright 2015 The Android Open Source Project
   8 #
   9 # Licensed under the Apache License, Version 2.0 (the "License");
  10 # you may not use this file except in compliance with the License.
  11 # You may obtain a copy of the License at
  12 #
  13 #      http://www.apache.org/licenses/LICENSE-2.0
  14 #
  15 # Unless required by applicable law or agreed to in writing, software
  16 # distributed under the License is distributed on an "AS IS" BASIS,
  17 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  18 # See the License for the specific language governing permissions and
  19 # limitations under the License.
  20 #
  21 #-------------------------------------------------------------------------
  22
  23 import os
  24 import sys
  25 import shutil
  26 import random
  27 import subprocess
  28
  29 def die (msg):
  30         print msg
  31         exit(-1)
  32
  33 def shellquote(s):
  34         return '"%s"' % s.replace('\\', '\\\\').replace('"', '\"').replace('$', '\$').replace('`', '\`')
  35
  36 def execute (args, workDir = None):
  37         curPath = os.getcwd()
  38         if workDir != None:
  39                 os.chdir(workDir)
  40         retcode = subprocess.call(args)
  41         os.chdir(curPath)
  42         if retcode != 0:
  43                 raise Exception("Failed to execute %s, got %d" % (str(args), retcode))
  44
  45 class Config:
  46         def __init__ (self, name, srcPath, buildPath, genParams, buildParams, testBinaryName, executor = 'executor', execserver = 'execserver', junitTool = 'testlog-to-junit'):
  47                 self.name                               = name
  48                 self.srcPath                    = srcPath
  49                 self.buildPath                  = buildPath
  50                 self.genParams                  = genParams
  51                 self.buildParams                = buildParams
  52                 self.testBinaryName             = testBinaryName
  53                 self.executor                   = executor
  54                 self.execserver                 = execserver
  55                 self.junitTool                  = junitTool
  56
  57 def initBuildDir (config):
  58         if os.path.exists(config.buildPath):
  59                 shutil.rmtree(config.buildPath)
  60
  61         os.makedirs(config.buildPath)
  62         execute(["cmake", os.path.realpath(config.srcPath)] + config.genParams, workDir = config.buildPath)
  63
  64 def prepareBuildDir (config):
  65         # If build dir exists, try to refresh
  66         if os.path.exists(config.buildPath):
  67                 try:
  68                         execute(["cmake", "."], workDir = config.buildPath)
  69                 except:
  70                         print "WARNING: Failed to refresh build dir, recreating"
  71                         initBuildDir(config)
  72         else:
  73                 initBuildDir(config)
  74
  75 def build (config):
  76         prepareBuildDir(config)
  77         execute(["cmake", "--build", "."] + config.buildParams, workDir = config.buildPath)
  78
  79 def runInternalTests (config):
  80         batchResultFile = config.name + ".qpa"
  81         infoLogFile             = config.name + ".txt"
  82         junitFile               = config.name + ".xml"
  83
  84         testWorkDir             = os.path.join(config.buildPath, "modules", "internal")
  85         junitToolPath   = os.path.join(config.buildPath, 'executor', config.junitTool)
  86
  87         # Remove old files
  88         for file in [batchResultFile, junitFile]:
  89                 if os.path.exists(file):
  90                         os.remove(file)
  91
  92         build(config)
  93
  94         # Dump case list
  95         execute([config.testBinaryName, "--deqp-runmode=xml-caselist"], workDir = testWorkDir)
  96
  97         # Run test binary using executor
  98         args = [
  99                 os.path.join(config.buildPath, 'executor', config.executor),
 100                 '--port=%d' % random.randint(50000, 60000),
 101                 '--start-server=%s' % os.path.join(config.buildPath, 'execserver', config.execserver),
 102                 '--binaryname=%s' % config.testBinaryName,
 103                 '--cmdline=--deqp-crashhandler=enable --deqp-watchdog=enable',
 104                 '--workdir=%s' % testWorkDir,
 105                 '--caselistdir=%s' % os.path.join(testWorkDir),
 106                 '--testset=dE-IT.*',
 107                 '--out=%s' % batchResultFile,
 108                 '--info=%s' % infoLogFile
 109         ]
 110         execute(args)
 111
 112         # Convert log to junit format
 113         execute([junitToolPath, batchResultFile, junitFile])
 114
 115 SRC_PATH                = os.path.normpath(os.path.join(os.path.dirname(__file__), ".."))
 116 BASE_BUILD_PATH = os.path.normpath(os.path.join(SRC_PATH, "..", "de-internal-tests"))
 117
 118 CONFIGS = [
 119         Config(
 120                 "win32-vs10-debug",
 121                 SRC_PATH,
 122                 os.path.join(BASE_BUILD_PATH, "win32-vs10-debug"),
 123                 ['-GVisual Studio 10', '-DDEQP_TARGET=no_modules'],
 124                 ['--config', 'Debug', '--', '/m'],
 125                 'Debug\\de-internal-tests.exe',
 126                 'Debug\\executor.exe',
 127                 'Debug\\execserver.exe',
 128                 'Debug\\testlog-to-junit.exe'
 129         ),
 130         Config(
 131                 "win32-vs10-release",
 132                 SRC_PATH,
 133                 os.path.join(BASE_BUILD_PATH, "win32-vs10-release"),
 134                 ['-GVisual Studio 10', '-DDEQP_TARGET=no_modules'],
 135                 ['--config', 'Release', '--', '/m'],
 136                 'Release\\de-internal-tests.exe',
 137                 'Release\\executor.exe',
 138                 'Release\\execserver.exe',
 139                 'Release\\testlog-to-junit.exe'
 140         ),
 141         Config(
 142                 "win64-vs10-debug",
 143                 SRC_PATH,
 144                 os.path.join(BASE_BUILD_PATH, "win64-vs10-debug"),
 145                 ['-GVisual Studio 10 Win64', '-DDEQP_TARGET=no_modules'],
 146                 ['--config', 'Debug', '--', '/m'],
 147                 'Debug\\de-internal-tests.exe',
 148                 'Debug\\executor.exe',
 149                 'Debug\\execserver.exe',
 150                 'Debug\\testlog-to-junit.exe'
 151         ),
 152         Config(
 153                 "win64-vs10-release",
 154                 SRC_PATH,
 155                 os.path.join(BASE_BUILD_PATH, "win64-vs10-release"),
 156                 ['-GVisual Studio 10 Win64', '-DDEQP_TARGET=no_modules'],
 157                 ['--config', 'Release', '--', '/m'],
 158                 'Release\\de-internal-tests.exe',
 159                 'Release\\executor.exe',
 160                 'Release\\execserver.exe',
 161                 'Release\\testlog-to-junit.exe'
 162         ),
 163
 164         # GCC configs
 165         Config(
 166                 "linux32-gcc-debug",
 167                 SRC_PATH,
 168                 os.path.join(BASE_BUILD_PATH, "linux32-gcc-debug"),
 169                 ['-DDEQP_TARGET=no_modules', '-DCMAKE_BUILD_TYPE=Debug', '-DCMAKE_C_FLAGS=-m32', '-DCMAKE_CXX_FLAGS=-m32', '-DCMAKE_LIBRARY_PATH=/usr/lib32;usr/lib/i386-linux-gnu'],
 170                 ['--', '-j', '2'],
 171                 './de-internal-tests'
 172         ),
 173         Config(
 174                 "linux32-gcc-release",
 175                 SRC_PATH,
 176                 os.path.join(BASE_BUILD_PATH, "linux32-gcc-release"),
 177                 ['-DDEQP_TARGET=no_modules', '-DCMAKE_BUILD_TYPE=Release', '-DCMAKE_C_FLAGS=-m32', '-DCMAKE_CXX_FLAGS=-m32', '-DCMAKE_LIBRARY_PATH=/usr/lib32;usr/lib/i386-linux-gnu'],
 178                 ['--', '-j', '2'],
 179                 './de-internal-tests'
 180         ),
 181         Config(
 182                 "linux64-gcc-debug",
 183                 SRC_PATH,
 184                 os.path.join(BASE_BUILD_PATH, "linux64-gcc-debug"),
 185                 ['-DDEQP_TARGET=no_modules', '-DCMAKE_BUILD_TYPE=Debug', '-DCMAKE_C_FLAGS=-m64', '-DCMAKE_CXX_FLAGS=-m64'],
 186                 ['--', '-j', '2'],
 187                 './de-internal-tests'
 188         ),
 189         Config(
 190                 "linux64-gcc-release",
 191                 SRC_PATH,
 192                 os.path.join(BASE_BUILD_PATH, "linux64-gcc-release"),
 193                 ['-DDEQP_TARGET=no_modules', '-DCMAKE_BUILD_TYPE=Release', '-DCMAKE_C_FLAGS=-m64', '-DCMAKE_CXX_FLAGS=-m64'],
 194                 ['--', '-j', '2'],
 195                 './de-internal-tests'
 196         ),
 197
 198         # Clang configs
 199         Config(
 200                 "linux32-clang-debug",
 201                 SRC_PATH,
 202                 os.path.join(BASE_BUILD_PATH, "linux32-clang-debug"),
 203                 ['-DDEQP_TARGET=no_modules', '-DCMAKE_BUILD_TYPE=Debug', '-DCMAKE_C_FLAGS=-m32', '-DCMAKE_CXX_FLAGS=-m32', '-DCMAKE_LIBRARY_PATH=/usr/lib32;usr/lib/i386-linux-gnu', '-DCMAKE_C_COMPILER=clang', '-DCMAKE_CXX_COMPILER=clang++', '-DDE_COMPILER=DE_COMPILER_CLANG'],
 204                 ['--', '-j', '2'],
 205                 './de-internal-tests'
 206         ),
 207         Config(
 208                 "linux32-clang-release",
 209                 SRC_PATH,
 210                 os.path.join(BASE_BUILD_PATH, "linux32-clang-release"),
 211                 ['-DDEQP_TARGET=no_modules', '-DCMAKE_BUILD_TYPE=Release', '-DCMAKE_C_FLAGS=-m32', '-DCMAKE_CXX_FLAGS=-m32', '-DCMAKE_LIBRARY_PATH=/usr/lib32;usr/lib/i386-linux-gnu', '-DCMAKE_C_COMPILER=clang', '-DCMAKE_CXX_COMPILER=clang++', '-DDE_COMPILER=DE_COMPILER_CLANG'],
 212                 ['--', '-j', '2'],
 213                 './de-internal-tests'
 214         ),
 215         Config(
 216                 "linux64-clang-debug",
 217                 SRC_PATH,
 218                 os.path.join(BASE_BUILD_PATH, "linux64-clang-debug"),
 219                 ['-DDEQP_TARGET=no_modules', '-DCMAKE_BUILD_TYPE=Debug', '-DCMAKE_C_FLAGS=-m64', '-DCMAKE_CXX_FLAGS=-m64', '-DCMAKE_C_COMPILER=clang', '-DCMAKE_CXX_COMPILER=clang++', '-DDE_COMPILER=DE_COMPILER_CLANG'],
 220                 ['--', '-j', '2'],
 221                 './de-internal-tests'
 222         ),
 223         Config(
 224                 "linux64-clang-release",
 225                 SRC_PATH,
 226                 os.path.join(BASE_BUILD_PATH, "linux64-clang-release"),
 227                 ['-DDEQP_TARGET=no_modules', '-DCMAKE_BUILD_TYPE=Release', '-DCMAKE_C_FLAGS=-m64', '-DCMAKE_CXX_FLAGS=-m64', '-DCMAKE_C_COMPILER=clang', '-DCMAKE_CXX_COMPILER=clang++', '-DDE_COMPILER=DE_COMPILER_CLANG'],
 228                 ['--', '-j', '2'],
 229                 './de-internal-tests'
 230         )
 231 ]
 232
 233 def findConfig (name):
 234         for config in CONFIGS:
 235                 if config.name == name:
 236                         return config
 237         return None
 238
 239 if __name__ == "__main__":
 240         if len(sys.argv) != 2:
 241                 die("%s: [config]" % sys.argv[0])
 242
 243         config = findConfig(sys.argv[1])
 244         if config == None:
 245                 die("Config '%s' not found" % sys.argv[1])
 246
 247         random.seed()
 248         runInternalTests(config)