am 0fccdf0d: Merge "Remove broken line interpolation tests from MR1 must pass." into...
[platform/upstream/VK-GL-CTS.git] / scripts / run_internal_tests.py
1 # -*- coding: utf-8 -*-
2
3 #-------------------------------------------------------------------------
4 # drawElements Quality Program utilities
5 # --------------------------------------
6 #
7 # Copyright 2015 The Android Open Source Project
8 #
9 # Licensed under the Apache License, Version 2.0 (the "License");
10 # you may not use this file except in compliance with the License.
11 # You may obtain a copy of the License at
12 #
13 #      http://www.apache.org/licenses/LICENSE-2.0
14 #
15 # Unless required by applicable law or agreed to in writing, software
16 # distributed under the License is distributed on an "AS IS" BASIS,
17 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18 # See the License for the specific language governing permissions and
19 # limitations under the License.
20 #
21 #-------------------------------------------------------------------------
22
23 import os
24 import sys
25 import shutil
26 import random
27 import subprocess
28
29 def die (msg):
30         print msg
31         exit(-1)
32
33 def shellquote(s):
34         return '"%s"' % s.replace('\\', '\\\\').replace('"', '\"').replace('$', '\$').replace('`', '\`')
35
36 def execute (args, workDir = None):
37         curPath = os.getcwd()
38         if workDir != None:
39                 os.chdir(workDir)
40         retcode = subprocess.call(args)
41         os.chdir(curPath)
42         if retcode != 0:
43                 raise Exception("Failed to execute %s, got %d" % (str(args), retcode))
44
45 class Config:
46         def __init__ (self, name, srcPath, buildPath, genParams, buildParams, testBinaryName, executor = 'executor', execserver = 'execserver', junitTool = 'testlog-to-junit'):
47                 self.name                               = name
48                 self.srcPath                    = srcPath
49                 self.buildPath                  = buildPath
50                 self.genParams                  = genParams
51                 self.buildParams                = buildParams
52                 self.testBinaryName             = testBinaryName
53                 self.executor                   = executor
54                 self.execserver                 = execserver
55                 self.junitTool                  = junitTool
56
57 def initBuildDir (config):
58         if os.path.exists(config.buildPath):
59                 shutil.rmtree(config.buildPath)
60
61         os.makedirs(config.buildPath)
62         execute(["cmake", os.path.realpath(config.srcPath)] + config.genParams, workDir = config.buildPath)
63
64 def prepareBuildDir (config):
65         # If build dir exists, try to refresh
66         if os.path.exists(config.buildPath):
67                 try:
68                         execute(["cmake", "."], workDir = config.buildPath)
69                 except:
70                         print "WARNING: Failed to refresh build dir, recreating"
71                         initBuildDir(config)
72         else:
73                 initBuildDir(config)
74
75 def build (config):
76         prepareBuildDir(config)
77         execute(["cmake", "--build", "."] + config.buildParams, workDir = config.buildPath)
78
79 def runInternalTests (config):
80         batchResultFile = config.name + ".qpa"
81         infoLogFile             = config.name + ".txt"
82         junitFile               = config.name + ".xml"
83
84         testWorkDir             = os.path.join(config.buildPath, "modules", "internal")
85         junitToolPath   = os.path.join(config.buildPath, 'executor', config.junitTool)
86
87         # Remove old files
88         for file in [batchResultFile, junitFile]:
89                 if os.path.exists(file):
90                         os.remove(file)
91
92         build(config)
93
94         # Dump case list
95         execute([config.testBinaryName, "--deqp-runmode=xml-caselist"], workDir = testWorkDir)
96
97         # Run test binary using executor
98         args = [
99                 os.path.join(config.buildPath, 'executor', config.executor),
100                 '--port=%d' % random.randint(50000, 60000),
101                 '--start-server=%s' % os.path.join(config.buildPath, 'execserver', config.execserver),
102                 '--binaryname=%s' % config.testBinaryName,
103                 '--cmdline=--deqp-crashhandler=enable --deqp-watchdog=enable',
104                 '--workdir=%s' % testWorkDir,
105                 '--caselistdir=%s' % os.path.join(testWorkDir),
106                 '--testset=dE-IT.*',
107                 '--out=%s' % batchResultFile,
108                 '--info=%s' % infoLogFile
109         ]
110         execute(args)
111
112         # Convert log to junit format
113         execute([junitToolPath, batchResultFile, junitFile])
114
115 SRC_PATH                = os.path.normpath(os.path.join(os.path.dirname(__file__), ".."))
116 BASE_BUILD_PATH = os.path.normpath(os.path.join(SRC_PATH, "..", "de-internal-tests"))
117
118 CONFIGS = [
119         Config(
120                 "win32-vs10-debug",
121                 SRC_PATH,
122                 os.path.join(BASE_BUILD_PATH, "win32-vs10-debug"),
123                 ['-GVisual Studio 10', '-DDEQP_TARGET=no_modules'],
124                 ['--config', 'Debug', '--', '/m'],
125                 'Debug\\de-internal-tests.exe',
126                 'Debug\\executor.exe',
127                 'Debug\\execserver.exe',
128                 'Debug\\testlog-to-junit.exe'
129         ),
130         Config(
131                 "win32-vs10-release",
132                 SRC_PATH,
133                 os.path.join(BASE_BUILD_PATH, "win32-vs10-release"),
134                 ['-GVisual Studio 10', '-DDEQP_TARGET=no_modules'],
135                 ['--config', 'Release', '--', '/m'],
136                 'Release\\de-internal-tests.exe',
137                 'Release\\executor.exe',
138                 'Release\\execserver.exe',
139                 'Release\\testlog-to-junit.exe'
140         ),
141         Config(
142                 "win64-vs10-debug",
143                 SRC_PATH,
144                 os.path.join(BASE_BUILD_PATH, "win64-vs10-debug"),
145                 ['-GVisual Studio 10 Win64', '-DDEQP_TARGET=no_modules'],
146                 ['--config', 'Debug', '--', '/m'],
147                 'Debug\\de-internal-tests.exe',
148                 'Debug\\executor.exe',
149                 'Debug\\execserver.exe',
150                 'Debug\\testlog-to-junit.exe'
151         ),
152         Config(
153                 "win64-vs10-release",
154                 SRC_PATH,
155                 os.path.join(BASE_BUILD_PATH, "win64-vs10-release"),
156                 ['-GVisual Studio 10 Win64', '-DDEQP_TARGET=no_modules'],
157                 ['--config', 'Release', '--', '/m'],
158                 'Release\\de-internal-tests.exe',
159                 'Release\\executor.exe',
160                 'Release\\execserver.exe',
161                 'Release\\testlog-to-junit.exe'
162         ),
163
164         # GCC configs
165         Config(
166                 "linux32-gcc-debug",
167                 SRC_PATH,
168                 os.path.join(BASE_BUILD_PATH, "linux32-gcc-debug"),
169                 ['-DDEQP_TARGET=no_modules', '-DCMAKE_BUILD_TYPE=Debug', '-DCMAKE_C_FLAGS=-m32', '-DCMAKE_CXX_FLAGS=-m32', '-DCMAKE_LIBRARY_PATH=/usr/lib32;usr/lib/i386-linux-gnu'],
170                 ['--', '-j', '2'],
171                 './de-internal-tests'
172         ),
173         Config(
174                 "linux32-gcc-release",
175                 SRC_PATH,
176                 os.path.join(BASE_BUILD_PATH, "linux32-gcc-release"),
177                 ['-DDEQP_TARGET=no_modules', '-DCMAKE_BUILD_TYPE=Release', '-DCMAKE_C_FLAGS=-m32', '-DCMAKE_CXX_FLAGS=-m32', '-DCMAKE_LIBRARY_PATH=/usr/lib32;usr/lib/i386-linux-gnu'],
178                 ['--', '-j', '2'],
179                 './de-internal-tests'
180         ),
181         Config(
182                 "linux64-gcc-debug",
183                 SRC_PATH,
184                 os.path.join(BASE_BUILD_PATH, "linux64-gcc-debug"),
185                 ['-DDEQP_TARGET=no_modules', '-DCMAKE_BUILD_TYPE=Debug', '-DCMAKE_C_FLAGS=-m64', '-DCMAKE_CXX_FLAGS=-m64'],
186                 ['--', '-j', '2'],
187                 './de-internal-tests'
188         ),
189         Config(
190                 "linux64-gcc-release",
191                 SRC_PATH,
192                 os.path.join(BASE_BUILD_PATH, "linux64-gcc-release"),
193                 ['-DDEQP_TARGET=no_modules', '-DCMAKE_BUILD_TYPE=Release', '-DCMAKE_C_FLAGS=-m64', '-DCMAKE_CXX_FLAGS=-m64'],
194                 ['--', '-j', '2'],
195                 './de-internal-tests'
196         ),
197
198         # Clang configs
199         Config(
200                 "linux32-clang-debug",
201                 SRC_PATH,
202                 os.path.join(BASE_BUILD_PATH, "linux32-clang-debug"),
203                 ['-DDEQP_TARGET=no_modules', '-DCMAKE_BUILD_TYPE=Debug', '-DCMAKE_C_FLAGS=-m32', '-DCMAKE_CXX_FLAGS=-m32', '-DCMAKE_LIBRARY_PATH=/usr/lib32;usr/lib/i386-linux-gnu', '-DCMAKE_C_COMPILER=clang', '-DCMAKE_CXX_COMPILER=clang++', '-DDE_COMPILER=DE_COMPILER_CLANG'],
204                 ['--', '-j', '2'],
205                 './de-internal-tests'
206         ),
207         Config(
208                 "linux32-clang-release",
209                 SRC_PATH,
210                 os.path.join(BASE_BUILD_PATH, "linux32-clang-release"),
211                 ['-DDEQP_TARGET=no_modules', '-DCMAKE_BUILD_TYPE=Release', '-DCMAKE_C_FLAGS=-m32', '-DCMAKE_CXX_FLAGS=-m32', '-DCMAKE_LIBRARY_PATH=/usr/lib32;usr/lib/i386-linux-gnu', '-DCMAKE_C_COMPILER=clang', '-DCMAKE_CXX_COMPILER=clang++', '-DDE_COMPILER=DE_COMPILER_CLANG'],
212                 ['--', '-j', '2'],
213                 './de-internal-tests'
214         ),
215         Config(
216                 "linux64-clang-debug",
217                 SRC_PATH,
218                 os.path.join(BASE_BUILD_PATH, "linux64-clang-debug"),
219                 ['-DDEQP_TARGET=no_modules', '-DCMAKE_BUILD_TYPE=Debug', '-DCMAKE_C_FLAGS=-m64', '-DCMAKE_CXX_FLAGS=-m64', '-DCMAKE_C_COMPILER=clang', '-DCMAKE_CXX_COMPILER=clang++', '-DDE_COMPILER=DE_COMPILER_CLANG'],
220                 ['--', '-j', '2'],
221                 './de-internal-tests'
222         ),
223         Config(
224                 "linux64-clang-release",
225                 SRC_PATH,
226                 os.path.join(BASE_BUILD_PATH, "linux64-clang-release"),
227                 ['-DDEQP_TARGET=no_modules', '-DCMAKE_BUILD_TYPE=Release', '-DCMAKE_C_FLAGS=-m64', '-DCMAKE_CXX_FLAGS=-m64', '-DCMAKE_C_COMPILER=clang', '-DCMAKE_CXX_COMPILER=clang++', '-DDE_COMPILER=DE_COMPILER_CLANG'],
228                 ['--', '-j', '2'],
229                 './de-internal-tests'
230         )
231 ]
232
233 def findConfig (name):
234         for config in CONFIGS:
235                 if config.name == name:
236                         return config
237         return None
238
239 if __name__ == "__main__":
240         if len(sys.argv) != 2:
241                 die("%s: [config]" % sys.argv[0])
242
243         config = findConfig(sys.argv[1])
244         if config == None:
245                 die("Config '%s' not found" % sys.argv[1])
246
247         random.seed()
248         runInternalTests(config)