am 90c7f555: (-s ours) am e5333493: (-s ours) am 84e26791: Merge "DO NOT MERGE: Remov...
[platform/upstream/VK-GL-CTS.git] / scripts / run_nightly.py
1 # -*- coding: utf-8 -*-
2
3 #-------------------------------------------------------------------------
4 # drawElements Quality Program utilities
5 # --------------------------------------
6 #
7 # Copyright 2015 The Android Open Source Project
8 #
9 # Licensed under the Apache License, Version 2.0 (the "License");
10 # you may not use this file except in compliance with the License.
11 # You may obtain a copy of the License at
12 #
13 #      http://www.apache.org/licenses/LICENSE-2.0
14 #
15 # Unless required by applicable law or agreed to in writing, software
16 # distributed under the License is distributed on an "AS IS" BASIS,
17 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18 # See the License for the specific language governing permissions and
19 # limitations under the License.
20 #
21 #-------------------------------------------------------------------------
22
23 from build.common import *
24 from build.config import *
25 from build.build import *
26
27 import os
28 import sys
29 import string
30 import socket
31 import fnmatch
32 from datetime import datetime
33
34 BASE_NIGHTLY_DIR        = os.path.normpath(os.path.join(DEQP_DIR, "..", "deqp-nightly"))
35 BASE_BUILD_DIR          = os.path.join(BASE_NIGHTLY_DIR, "build")
36 BASE_LOGS_DIR           = os.path.join(BASE_NIGHTLY_DIR, "logs")
37 BASE_REFS_DIR           = os.path.join(BASE_NIGHTLY_DIR, "refs")
38
39 EXECUTOR_PATH           = "executor/executor"
40 LOG_TO_CSV_PATH         = "executor/testlog-to-csv"
41 EXECSERVER_PATH         = "execserver/execserver"
42
43 CASELIST_PATH           = os.path.join(DEQP_DIR, "Candy", "Data")
44
45 COMPARE_NUM_RESULTS     = 4
46 COMPARE_REPORT_NAME     = "nightly-report.html"
47
48 COMPARE_REPORT_TMPL = '''
49 <html>
50 <head>
51 <title>${TITLE}</title>
52 <style type="text/css">
53 <!--
54 body                            { font: serif; font-size: 1em; }
55 table                           { border-spacing: 0; border-collapse: collapse; }
56 td                                      { border-width: 1px; border-style: solid; border-color: #808080; }
57 .Header                         { font-weight: bold; font-size: 1em; border-style: none; }
58 .CasePath                       { }
59 .Pass                           { background: #80ff80; }
60 .Fail                           { background: #ff4040; }
61 .QualityWarning         { background: #ffff00; }
62 .CompabilityWarning     { background: #ffff00; }
63 .Pending                        { background: #808080; }
64 .Running                        { background: #d3d3d3; }
65 .NotSupported           { background: #ff69b4; }
66 .ResourceError          { background: #ff4040; }
67 .InternalError          { background: #ff1493; }
68 .Canceled                       { background: #808080; }
69 .Crash                          { background: #ffa500; }
70 .Timeout                        { background: #ffa500; }
71 .Disabled                       { background: #808080; }
72 .Missing                        { background: #808080; }
73 .Ignored                        { opacity: 0.5; }
74 -->
75 </style>
76 </head>
77 <body>
78 <h1>${TITLE}</h1>
79 <table>
80 ${RESULTS}
81 </table>
82 </body>
83 </html>
84 '''
85
86 class NightlyRunConfig:
87         def __init__(self, name, buildConfig, generator, binaryName, testset, args = [], exclude = [], ignore = []):
88                 self.name                       = name
89                 self.buildConfig        = buildConfig
90                 self.generator          = generator
91                 self.binaryName         = binaryName
92                 self.testset            = testset
93                 self.args                       = args
94                 self.exclude            = exclude
95                 self.ignore                     = ignore
96
97         def getBinaryPath(self, basePath):
98                 return os.path.join(self.buildConfig.getBuildDir(), self.generator.getBinaryPath(self.buildConfig.getBuildType(), basePath))
99
100 class NightlyBuildConfig(BuildConfig):
101         def __init__(self, name, buildType, args):
102                 BuildConfig.__init__(self, os.path.join(BASE_BUILD_DIR, name), buildType, args)
103
104 class TestCaseResult:
105         def __init__ (self, name, statusCode):
106                 self.name               = name
107                 self.statusCode = statusCode
108
109 class MultiResult:
110         def __init__ (self, name, statusCodes):
111                 self.name                       = name
112                 self.statusCodes        = statusCodes
113
114 class BatchResult:
115         def __init__ (self, name):
116                 self.name               = name
117                 self.results    = []
118
119 def parseResultCsv (data):
120         lines   = data.splitlines()[1:]
121         results = []
122
123         for line in lines:
124                 items = line.split(",")
125                 results.append(TestCaseResult(items[0], items[1]))
126
127         return results
128
129 def readTestCaseResultsFromCSV (filename):
130         return parseResultCsv(readFile(filename))
131
132 def readBatchResultFromCSV (filename, batchResultName = None):
133         batchResult = BatchResult(batchResultName if batchResultName != None else os.path.basename(filename))
134         batchResult.results = readTestCaseResultsFromCSV(filename)
135         return batchResult
136
137 def getResultTimestamp ():
138         return datetime.now().strftime("%Y-%m-%d-%H-%M")
139
140 def getCompareFilenames (logsDir):
141         files = []
142         for file in os.listdir(logsDir):
143                 fullPath = os.path.join(logsDir, file)
144                 if os.path.isfile(fullPath) and fnmatch.fnmatch(file, "*.csv"):
145                         files.append(fullPath)
146         files.sort()
147
148         return files[-COMPARE_NUM_RESULTS:]
149
150 def parseAsCSV (logPath, config):
151         args = [config.getBinaryPath(LOG_TO_CSV_PATH), "--mode=all", "--format=csv", logPath]
152         proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
153         out, err = proc.communicate()
154         return out
155
156 def computeUnifiedTestCaseList (batchResults):
157         caseList        = []
158         caseSet         = set()
159
160         for batchResult in batchResults:
161                 for result in batchResult.results:
162                         if not result.name in caseSet:
163                                 caseList.append(result.name)
164                                 caseSet.add(result.name)
165
166         return caseList
167
168 def computeUnifiedResults (batchResults):
169
170         def genResultMap (batchResult):
171                 resMap = {}
172                 for result in batchResult.results:
173                         resMap[result.name] = result
174                 return resMap
175
176         resultMap       = [genResultMap(r) for r in batchResults]
177         caseList        = computeUnifiedTestCaseList(batchResults)
178         results         = []
179
180         for caseName in caseList:
181                 statusCodes = []
182
183                 for i in range(0, len(batchResults)):
184                         result          = resultMap[i][caseName] if caseName in resultMap[i] else None
185                         statusCode      = result.statusCode if result != None else 'Missing'
186                         statusCodes.append(statusCode)
187
188                 results.append(MultiResult(caseName, statusCodes))
189
190         return results
191
192 def allStatusCodesEqual (result):
193         firstCode = result.statusCodes[0]
194         for i in range(1, len(result.statusCodes)):
195                 if result.statusCodes[i] != firstCode:
196                         return False
197         return True
198
199 def computeDiffResults (unifiedResults):
200         diff = []
201         for result in unifiedResults:
202                 if not allStatusCodesEqual(result):
203                         diff.append(result)
204         return diff
205
206 def genCompareReport (batchResults, title, ignoreCases):
207         class TableRow:
208                 def __init__ (self, testCaseName, innerHTML):
209                         self.testCaseName = testCaseName
210                         self.innerHTML = innerHTML
211
212         unifiedResults  = computeUnifiedResults(batchResults)
213         diffResults             = computeDiffResults(unifiedResults)
214         rows                    = []
215
216         # header
217         headerCol = '<td class="Header">Test case</td>\n'
218         for batchResult in batchResults:
219                 headerCol += '<td class="Header">%s</td>\n' % batchResult.name
220         rows.append(TableRow(None, headerCol))
221
222         # results
223         for result in diffResults:
224                 col = '<td class="CasePath">%s</td>\n' % result.name
225                 for statusCode in result.statusCodes:
226                         col += '<td class="%s">%s</td>\n' % (statusCode, statusCode)
227
228                 rows.append(TableRow(result.name, col))
229
230         tableStr = ""
231         for row in rows:
232                 if row.testCaseName is not None and matchesAnyPattern(row.testCaseName, ignoreCases):
233                         tableStr += '<tr class="Ignored">\n%s</tr>\n' % row.innerHTML
234                 else:
235                         tableStr += '<tr>\n%s</tr>\n' % row.innerHTML
236
237         html = COMPARE_REPORT_TMPL
238         html = html.replace("${TITLE}", title)
239         html = html.replace("${RESULTS}", tableStr)
240
241         return html
242
243 def matchesAnyPattern (name, patterns):
244         for pattern in patterns:
245                 if fnmatch.fnmatch(name, pattern):
246                         return True
247         return False
248
249 def statusCodesMatch (refResult, resResult):
250         return refResult == 'Missing' or resResult == 'Missing' or refResult == resResult
251
252 def compareBatchResults (referenceBatch, resultBatch, ignoreCases):
253         unifiedResults  = computeUnifiedResults([referenceBatch, resultBatch])
254         failedCases             = []
255
256         for result in unifiedResults:
257                 if not matchesAnyPattern(result.name, ignoreCases):
258                         refResult               = result.statusCodes[0]
259                         resResult               = result.statusCodes[1]
260
261                         if not statusCodesMatch(refResult, resResult):
262                                 failedCases.append(result)
263
264         return failedCases
265
266 def getUnusedPort ():
267         # \note Not 100%-proof method as other apps may grab this port before we launch execserver
268         s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
269         s.bind(('localhost', 0))
270         addr, port = s.getsockname()
271         s.close()
272         return port
273
274 def runNightly (config):
275         build(config.buildConfig, config.generator)
276
277         # Run parameters
278         timestamp               = getResultTimestamp()
279         logDir                  = os.path.join(BASE_LOGS_DIR, config.name)
280         testLogPath             = os.path.join(logDir, timestamp + ".qpa")
281         infoLogPath             = os.path.join(logDir, timestamp + ".txt")
282         csvLogPath              = os.path.join(logDir, timestamp + ".csv")
283         compareLogPath  = os.path.join(BASE_REFS_DIR, config.name + ".csv")
284         port                    = getUnusedPort()
285
286         if not os.path.exists(logDir):
287                 os.makedirs(logDir)
288
289         if os.path.exists(testLogPath) or os.path.exists(infoLogPath):
290                 raise Exception("Result '%s' already exists", timestamp)
291
292         # Paths, etc.
293         binaryName              = config.generator.getBinaryPath(config.buildConfig.getBuildType(), os.path.basename(config.binaryName))
294         workingDir              = os.path.join(config.buildConfig.getBuildDir(), os.path.dirname(config.binaryName))
295
296         execArgs = [
297                 config.getBinaryPath(EXECUTOR_PATH),
298                 '--start-server=%s' % config.getBinaryPath(EXECSERVER_PATH),
299                 '--port=%d' % port,
300                 '--binaryname=%s' % binaryName,
301                 '--cmdline=%s' % string.join([shellquote(arg) for arg in config.args], " "),
302                 '--workdir=%s' % workingDir,
303                 '--caselistdir=%s' % CASELIST_PATH,
304                 '--testset=%s' % string.join(config.testset, ","),
305                 '--out=%s' % testLogPath,
306                 '--info=%s' % infoLogPath,
307                 '--summary=no'
308         ]
309
310         if len(config.exclude) > 0:
311                 execArgs += ['--exclude=%s' % string.join(config.exclude, ",")]
312
313         execute(execArgs)
314
315         # Translate to CSV for comparison purposes
316         lastResultCsv           = parseAsCSV(testLogPath, config)
317         writeFile(csvLogPath, lastResultCsv)
318
319         if os.path.exists(compareLogPath):
320                 refBatchResult = readBatchResultFromCSV(compareLogPath, "reference")
321         else:
322                 refBatchResult = None
323
324         # Generate comparison report
325         compareFilenames        = getCompareFilenames(logDir)
326         batchResults            = [readBatchResultFromCSV(filename) for filename in compareFilenames]
327
328         if refBatchResult != None:
329                 batchResults = [refBatchResult] + batchResults
330
331         writeFile(COMPARE_REPORT_NAME, genCompareReport(batchResults, config.name, config.ignore))
332         print "Comparison report written to %s" % COMPARE_REPORT_NAME
333
334         # Compare to reference
335         if refBatchResult != None:
336                 curBatchResult          = BatchResult("current")
337                 curBatchResult.results = parseResultCsv(lastResultCsv)
338                 failedCases                     = compareBatchResults(refBatchResult, curBatchResult, config.ignore)
339
340                 print ""
341                 for result in failedCases:
342                         print "MISMATCH: %s: expected %s, got %s" % (result.name, result.statusCodes[0], result.statusCodes[1])
343
344                 print ""
345                 print "%d / %d cases passed, run %s" % (len(curBatchResult.results)-len(failedCases), len(curBatchResult.results), "FAILED" if len(failedCases) > 0 else "passed")
346
347                 if len(failedCases) > 0:
348                         return False
349
350         return True
351
352 # Configurations
353
354 DEFAULT_WIN32_GENERATOR                         = ANY_VS_X32_GENERATOR
355 DEFAULT_WIN64_GENERATOR                         = ANY_VS_X64_GENERATOR
356
357 WGL_X64_RELEASE_BUILD_CFG                       = NightlyBuildConfig("wgl_x64_release", "Release", ['-DDEQP_TARGET=win32_wgl'])
358 ARM_GLES3_EMU_X32_RELEASE_BUILD_CFG     = NightlyBuildConfig("arm_gles3_emu_release", "Release", ['-DDEQP_TARGET=arm_gles3_emu'])
359
360 BASE_ARGS                                                       = ['--deqp-visibility=hidden', '--deqp-watchdog=enable', '--deqp-crashhandler=enable']
361
362 CONFIGS = [
363         NightlyRunConfig(
364                 name                    = "wgl_x64_release_gles2",
365                 buildConfig             = WGL_X64_RELEASE_BUILD_CFG,
366                 generator               = DEFAULT_WIN64_GENERATOR,
367                 binaryName              = "modules/gles2/deqp-gles2",
368                 args                    = ['--deqp-gl-config-name=rgba8888d24s8ms0'] + BASE_ARGS,
369                 testset                 = ["dEQP-GLES2.info.*", "dEQP-GLES2.functional.*", "dEQP-GLES2.usecases.*"],
370                 exclude                 = [
371                                 "dEQP-GLES2.functional.shaders.loops.*while*unconditional_continue*",
372                                 "dEQP-GLES2.functional.shaders.loops.*while*only_continue*",
373                                 "dEQP-GLES2.functional.shaders.loops.*while*double_continue*",
374                         ],
375                 ignore                  = []
376                 ),
377         NightlyRunConfig(
378                 name                    = "wgl_x64_release_gles3",
379                 buildConfig             = WGL_X64_RELEASE_BUILD_CFG,
380                 generator               = DEFAULT_WIN64_GENERATOR,
381                 binaryName              = "modules/gles3/deqp-gles3",
382                 args                    = ['--deqp-gl-config-name=rgba8888d24s8ms0'] + BASE_ARGS,
383                 testset                 = ["dEQP-GLES3.info.*", "dEQP-GLES3.functional.*", "dEQP-GLES3.usecases.*"],
384                 exclude                 = [
385                                 "dEQP-GLES3.functional.shaders.loops.*while*unconditional_continue*",
386                                 "dEQP-GLES3.functional.shaders.loops.*while*only_continue*",
387                                 "dEQP-GLES3.functional.shaders.loops.*while*double_continue*",
388                         ],
389                 ignore                  = [
390                                 "dEQP-GLES3.functional.transform_feedback.*",
391                                 "dEQP-GLES3.functional.occlusion_query.*",
392                                 "dEQP-GLES3.functional.lifetime.*",
393                                 "dEQP-GLES3.functional.fragment_ops.depth_stencil.stencil_ops",
394                         ]
395                 ),
396         NightlyRunConfig(
397                 name                    = "wgl_x64_release_gles31",
398                 buildConfig             = WGL_X64_RELEASE_BUILD_CFG,
399                 generator               = DEFAULT_WIN64_GENERATOR,
400                 binaryName              = "modules/gles31/deqp-gles31",
401                 args                    = ['--deqp-gl-config-name=rgba8888d24s8ms0'] + BASE_ARGS,
402                 testset                 = ["dEQP-GLES31.*"],
403                 exclude                 = [],
404                 ignore                  = [
405                                 "dEQP-GLES31.functional.draw_indirect.negative.command_bad_alignment_3",
406                                 "dEQP-GLES31.functional.draw_indirect.negative.command_offset_not_in_buffer",
407                                 "dEQP-GLES31.functional.vertex_attribute_binding.negative.bind_vertex_buffer_negative_offset",
408                                 "dEQP-GLES31.functional.ssbo.layout.single_basic_type.packed.mediump_uint",
409                                 "dEQP-GLES31.functional.blend_equation_advanced.basic.*",
410                                 "dEQP-GLES31.functional.blend_equation_advanced.srgb.*",
411                                 "dEQP-GLES31.functional.blend_equation_advanced.barrier.*",
412                                 "dEQP-GLES31.functional.uniform_location.*",
413                                 "dEQP-GLES31.functional.debug.negative_coverage.log.state.get_framebuffer_attachment_parameteriv",
414                                 "dEQP-GLES31.functional.debug.negative_coverage.log.state.get_renderbuffer_parameteriv",
415                                 "dEQP-GLES31.functional.debug.error_filters.case_0",
416                                 "dEQP-GLES31.functional.debug.error_filters.case_2",
417                         ]
418                 ),
419         NightlyRunConfig(
420                 name                    = "wgl_x64_release_gl3",
421                 buildConfig             = WGL_X64_RELEASE_BUILD_CFG,
422                 generator               = DEFAULT_WIN64_GENERATOR,
423                 binaryName              = "modules/gl3/deqp-gl3",
424                 args                    = ['--deqp-gl-config-name=rgba8888d24s8ms0'] + BASE_ARGS,
425                 testset                 = ["dEQP-GL3.info.*", "dEQP-GL3.functional.*"],
426                 exclude                 = [
427                                 "dEQP-GL3.functional.shaders.loops.*while*unconditional_continue*",
428                                 "dEQP-GL3.functional.shaders.loops.*while*only_continue*",
429                                 "dEQP-GL3.functional.shaders.loops.*while*double_continue*",
430                         ],
431                 ignore                  = [
432                                 "dEQP-GL3.functional.transform_feedback.*"
433                         ]
434                 ),
435         NightlyRunConfig(
436                 name                    = "arm_gles3_emu_x32_egl",
437                 buildConfig             = ARM_GLES3_EMU_X32_RELEASE_BUILD_CFG,
438                 generator               = DEFAULT_WIN32_GENERATOR,
439                 binaryName              = "modules/egl/deqp-egl",
440                 args                    = BASE_ARGS,
441                 testset                 = ["dEQP-EGL.info.*", "dEQP-EGL.functional.*"],
442                 exclude                 = [
443                                 "dEQP-EGL.functional.sharing.gles2.multithread.*",
444                                 "dEQP-EGL.functional.multithread.*",
445                         ],
446                 ignore                  = []
447                 ),
448         NightlyRunConfig(
449                 name                    = "opencl_x64_release",
450                 buildConfig             = NightlyBuildConfig("opencl_x64_release", "Release", ['-DDEQP_TARGET=opencl_icd']),
451                 generator               = DEFAULT_WIN64_GENERATOR,
452                 binaryName              = "modules/opencl/deqp-opencl",
453                 args                    = ['--deqp-cl-platform-id=2 --deqp-cl-device-ids=1'] + BASE_ARGS,
454                 testset                 = ["dEQP-CL.*"],
455                 exclude                 = ["dEQP-CL.performance.*", "dEQP-CL.robustness.*", "dEQP-CL.stress.memory.*"],
456                 ignore                  = [
457                                 "dEQP-CL.scheduler.random.*",
458                                 "dEQP-CL.language.set_kernel_arg.random_structs.*",
459                                 "dEQP-CL.language.builtin_function.work_item.invalid_get_global_offset",
460                                 "dEQP-CL.language.call_function.arguments.random_structs.*",
461                                 "dEQP-CL.language.call_kernel.random_structs.*",
462                                 "dEQP-CL.language.inf_nan.nan.frexp.float",
463                                 "dEQP-CL.language.inf_nan.nan.lgamma_r.float",
464                                 "dEQP-CL.language.inf_nan.nan.modf.float",
465                                 "dEQP-CL.language.inf_nan.nan.sqrt.float",
466                                 "dEQP-CL.api.multithread.*",
467                                 "dEQP-CL.api.callback.random.nested.*",
468                                 "dEQP-CL.api.memory_migration.out_of_order_host.image2d.single_device_kernel_migrate_validate_abb",
469                                 "dEQP-CL.api.memory_migration.out_of_order.image2d.single_device_kernel_migrate_kernel_validate_abbb",
470                                 "dEQP-CL.image.addressing_filtering12.1d_array.*",
471                                 "dEQP-CL.image.addressing_filtering12.2d_array.*"
472                         ]
473                 )
474 ]
475
476 if __name__ == "__main__":
477         config = None
478
479         if len(sys.argv) == 2:
480                 cfgName = sys.argv[1]
481                 for curCfg in CONFIGS:
482                         if curCfg.name == cfgName:
483                                 config = curCfg
484                                 break
485
486         if config != None:
487                 isOk = runNightly(config)
488                 if not isOk:
489                         sys.exit(-1)
490         else:
491                 print "%s: [config]" % sys.argv[0]
492                 print ""
493                 print "  Available configs:"
494                 for config in CONFIGS:
495                         print "    %s" % config.name
496                 sys.exit(-1)