1 # -*- coding: utf-8 -*-
3 from build.common import *
4 from build.config import *
5 from build.build import *
12 from datetime import datetime
14 BASE_NIGHTLY_DIR = os.path.normpath(os.path.join(DEQP_DIR, "..", "deqp-nightly"))
15 BASE_BUILD_DIR = os.path.join(BASE_NIGHTLY_DIR, "build")
16 BASE_LOGS_DIR = os.path.join(BASE_NIGHTLY_DIR, "logs")
17 BASE_REFS_DIR = os.path.join(BASE_NIGHTLY_DIR, "refs")
19 EXECUTOR_PATH = "executor/executor"
20 LOG_TO_CSV_PATH = "executor/testlog-to-csv"
21 EXECSERVER_PATH = "execserver/execserver"
23 CASELIST_PATH = os.path.join(DEQP_DIR, "Candy", "Data")
25 COMPARE_NUM_RESULTS = 4
26 COMPARE_REPORT_NAME = "nightly-report.html"
28 COMPARE_REPORT_TMPL = '''
31 <title>${TITLE}</title>
32 <style type="text/css">
34 body { font: serif; font-size: 1em; }
35 table { border-spacing: 0; border-collapse: collapse; }
36 td { border-width: 1px; border-style: solid; border-color: #808080; }
37 .Header { font-weight: bold; font-size: 1em; border-style: none; }
39 .Pass { background: #80ff80; }
40 .Fail { background: #ff4040; }
41 .QualityWarning { background: #ffff00; }
42 .CompabilityWarning { background: #ffff00; }
43 .Pending { background: #808080; }
44 .Running { background: #d3d3d3; }
45 .NotSupported { background: #ff69b4; }
46 .ResourceError { background: #ff4040; }
47 .InternalError { background: #ff1493; }
48 .Canceled { background: #808080; }
49 .Crash { background: #ffa500; }
50 .Timeout { background: #ffa500; }
51 .Disabled { background: #808080; }
52 .Missing { background: #808080; }
53 .Ignored { opacity: 0.5; }
66 class NightlyRunConfig:
67 def __init__(self, name, buildConfig, generator, binaryName, testset, args = [], exclude = [], ignore = []):
69 self.buildConfig = buildConfig
70 self.generator = generator
71 self.binaryName = binaryName
72 self.testset = testset
74 self.exclude = exclude
77 def getBinaryPath(self, basePath):
78 return os.path.join(self.buildConfig.getBuildDir(), self.generator.getBinaryPath(self.buildConfig.getBuildType(), basePath))
80 class NightlyBuildConfig(BuildConfig):
81 def __init__(self, name, buildType, args):
82 BuildConfig.__init__(self, os.path.join(BASE_BUILD_DIR, name), buildType, args)
85 def __init__ (self, name, statusCode):
87 self.statusCode = statusCode
90 def __init__ (self, name, statusCodes):
92 self.statusCodes = statusCodes
95 def __init__ (self, name):
99 def parseResultCsv (data):
100 lines = data.splitlines()[1:]
104 items = line.split(",")
105 results.append(TestCaseResult(items[0], items[1]))
109 def readTestCaseResultsFromCSV (filename):
110 return parseResultCsv(readFile(filename))
112 def readBatchResultFromCSV (filename, batchResultName = None):
113 batchResult = BatchResult(batchResultName if batchResultName != None else os.path.basename(filename))
114 batchResult.results = readTestCaseResultsFromCSV(filename)
117 def getResultTimestamp ():
118 return datetime.now().strftime("%Y-%m-%d-%H-%M")
120 def getCompareFilenames (logsDir):
122 for file in os.listdir(logsDir):
123 fullPath = os.path.join(logsDir, file)
124 if os.path.isfile(fullPath) and fnmatch.fnmatch(file, "*.csv"):
125 files.append(fullPath)
128 return files[-COMPARE_NUM_RESULTS:]
130 def parseAsCSV (logPath, config):
131 args = [config.getBinaryPath(LOG_TO_CSV_PATH), "--mode=all", "--format=csv", logPath]
132 proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
133 out, err = proc.communicate()
136 def computeUnifiedTestCaseList (batchResults):
140 for batchResult in batchResults:
141 for result in batchResult.results:
142 if not result.name in caseSet:
143 caseList.append(result.name)
144 caseSet.add(result.name)
148 def computeUnifiedResults (batchResults):
150 def genResultMap (batchResult):
152 for result in batchResult.results:
153 resMap[result.name] = result
156 resultMap = [genResultMap(r) for r in batchResults]
157 caseList = computeUnifiedTestCaseList(batchResults)
160 for caseName in caseList:
163 for i in range(0, len(batchResults)):
164 result = resultMap[i][caseName] if caseName in resultMap[i] else None
165 statusCode = result.statusCode if result != None else 'Missing'
166 statusCodes.append(statusCode)
168 results.append(MultiResult(caseName, statusCodes))
172 def allStatusCodesEqual (result):
173 firstCode = result.statusCodes[0]
174 for i in range(1, len(result.statusCodes)):
175 if result.statusCodes[i] != firstCode:
179 def computeDiffResults (unifiedResults):
181 for result in unifiedResults:
182 if not allStatusCodesEqual(result):
186 def genCompareReport (batchResults, title, ignoreCases):
188 def __init__ (self, testCaseName, innerHTML):
189 self.testCaseName = testCaseName
190 self.innerHTML = innerHTML
192 unifiedResults = computeUnifiedResults(batchResults)
193 diffResults = computeDiffResults(unifiedResults)
197 headerCol = '<td class="Header">Test case</td>\n'
198 for batchResult in batchResults:
199 headerCol += '<td class="Header">%s</td>\n' % batchResult.name
200 rows.append(TableRow(None, headerCol))
203 for result in diffResults:
204 col = '<td class="CasePath">%s</td>\n' % result.name
205 for statusCode in result.statusCodes:
206 col += '<td class="%s">%s</td>\n' % (statusCode, statusCode)
208 rows.append(TableRow(result.name, col))
212 if row.testCaseName is not None and matchesAnyPattern(row.testCaseName, ignoreCases):
213 tableStr += '<tr class="Ignored">\n%s</tr>\n' % row.innerHTML
215 tableStr += '<tr>\n%s</tr>\n' % row.innerHTML
217 html = COMPARE_REPORT_TMPL
218 html = html.replace("${TITLE}", title)
219 html = html.replace("${RESULTS}", tableStr)
223 def matchesAnyPattern (name, patterns):
224 for pattern in patterns:
225 if fnmatch.fnmatch(name, pattern):
229 def statusCodesMatch (refResult, resResult):
230 return refResult == 'Missing' or resResult == 'Missing' or refResult == resResult
232 def compareBatchResults (referenceBatch, resultBatch, ignoreCases):
233 unifiedResults = computeUnifiedResults([referenceBatch, resultBatch])
236 for result in unifiedResults:
237 if not matchesAnyPattern(result.name, ignoreCases):
238 refResult = result.statusCodes[0]
239 resResult = result.statusCodes[1]
241 if not statusCodesMatch(refResult, resResult):
242 failedCases.append(result)
246 def getUnusedPort ():
247 # \note Not 100%-proof method as other apps may grab this port before we launch execserver
248 s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
249 s.bind(('localhost', 0))
250 addr, port = s.getsockname()
254 def runNightly (config):
255 build(config.buildConfig, config.generator)
258 timestamp = getResultTimestamp()
259 logDir = os.path.join(BASE_LOGS_DIR, config.name)
260 testLogPath = os.path.join(logDir, timestamp + ".qpa")
261 infoLogPath = os.path.join(logDir, timestamp + ".txt")
262 csvLogPath = os.path.join(logDir, timestamp + ".csv")
263 compareLogPath = os.path.join(BASE_REFS_DIR, config.name + ".csv")
264 port = getUnusedPort()
266 if not os.path.exists(logDir):
269 if os.path.exists(testLogPath) or os.path.exists(infoLogPath):
270 raise Exception("Result '%s' already exists", timestamp)
273 binaryName = config.generator.getBinaryPath(config.buildConfig.getBuildType(), os.path.basename(config.binaryName))
274 workingDir = os.path.join(config.buildConfig.getBuildDir(), os.path.dirname(config.binaryName))
277 config.getBinaryPath(EXECUTOR_PATH),
278 '--start-server=%s' % config.getBinaryPath(EXECSERVER_PATH),
280 '--binaryname=%s' % binaryName,
281 '--cmdline=%s' % string.join([shellquote(arg) for arg in config.args], " "),
282 '--workdir=%s' % workingDir,
283 '--caselistdir=%s' % CASELIST_PATH,
284 '--testset=%s' % string.join(config.testset, ","),
285 '--out=%s' % testLogPath,
286 '--info=%s' % infoLogPath,
290 if len(config.exclude) > 0:
291 execArgs += ['--exclude=%s' % string.join(config.exclude, ",")]
295 # Translate to CSV for comparison purposes
296 lastResultCsv = parseAsCSV(testLogPath, config)
297 writeFile(csvLogPath, lastResultCsv)
299 if os.path.exists(compareLogPath):
300 refBatchResult = readBatchResultFromCSV(compareLogPath, "reference")
302 refBatchResult = None
304 # Generate comparison report
305 compareFilenames = getCompareFilenames(logDir)
306 batchResults = [readBatchResultFromCSV(filename) for filename in compareFilenames]
308 if refBatchResult != None:
309 batchResults = [refBatchResult] + batchResults
311 writeFile(COMPARE_REPORT_NAME, genCompareReport(batchResults, config.name, config.ignore))
312 print "Comparison report written to %s" % COMPARE_REPORT_NAME
314 # Compare to reference
315 if refBatchResult != None:
316 curBatchResult = BatchResult("current")
317 curBatchResult.results = parseResultCsv(lastResultCsv)
318 failedCases = compareBatchResults(refBatchResult, curBatchResult, config.ignore)
321 for result in failedCases:
322 print "MISMATCH: %s: expected %s, got %s" % (result.name, result.statusCodes[0], result.statusCodes[1])
325 print "%d / %d cases passed, run %s" % (len(curBatchResult.results)-len(failedCases), len(curBatchResult.results), "FAILED" if len(failedCases) > 0 else "passed")
327 if len(failedCases) > 0:
334 DEFAULT_WIN32_GENERATOR = ANY_VS_X32_GENERATOR
335 DEFAULT_WIN64_GENERATOR = ANY_VS_X64_GENERATOR
337 WGL_X64_RELEASE_BUILD_CFG = NightlyBuildConfig("wgl_x64_release", "Release", ['-DDEQP_TARGET=win32_wgl'])
338 ARM_GLES3_EMU_X32_RELEASE_BUILD_CFG = NightlyBuildConfig("arm_gles3_emu_release", "Release", ['-DDEQP_TARGET=arm_gles3_emu'])
340 BASE_ARGS = ['--deqp-visibility=hidden', '--deqp-watchdog=enable', '--deqp-crashhandler=enable']
344 name = "wgl_x64_release_gles2",
345 buildConfig = WGL_X64_RELEASE_BUILD_CFG,
346 generator = DEFAULT_WIN64_GENERATOR,
347 binaryName = "modules/gles2/deqp-gles2",
348 args = ['--deqp-gl-config-name=rgba8888d24s8ms0'] + BASE_ARGS,
349 testset = ["dEQP-GLES2.info.*", "dEQP-GLES2.functional.*", "dEQP-GLES2.usecases.*"],
351 "dEQP-GLES2.functional.shaders.loops.*while*unconditional_continue*",
352 "dEQP-GLES2.functional.shaders.loops.*while*only_continue*",
353 "dEQP-GLES2.functional.shaders.loops.*while*double_continue*",
358 name = "wgl_x64_release_gles3",
359 buildConfig = WGL_X64_RELEASE_BUILD_CFG,
360 generator = DEFAULT_WIN64_GENERATOR,
361 binaryName = "modules/gles3/deqp-gles3",
362 args = ['--deqp-gl-config-name=rgba8888d24s8ms0'] + BASE_ARGS,
363 testset = ["dEQP-GLES3.info.*", "dEQP-GLES3.functional.*", "dEQP-GLES3.usecases.*"],
365 "dEQP-GLES3.functional.shaders.loops.*while*unconditional_continue*",
366 "dEQP-GLES3.functional.shaders.loops.*while*only_continue*",
367 "dEQP-GLES3.functional.shaders.loops.*while*double_continue*",
370 "dEQP-GLES3.functional.transform_feedback.*",
371 "dEQP-GLES3.functional.occlusion_query.*",
372 "dEQP-GLES3.functional.lifetime.*",
373 "dEQP-GLES3.functional.fragment_ops.depth_stencil.stencil_ops",
377 name = "wgl_x64_release_gles31",
378 buildConfig = WGL_X64_RELEASE_BUILD_CFG,
379 generator = DEFAULT_WIN64_GENERATOR,
380 binaryName = "modules/gles31/deqp-gles31",
381 args = ['--deqp-gl-config-name=rgba8888d24s8ms0'] + BASE_ARGS,
382 testset = ["dEQP-GLES31.*"],
385 "dEQP-GLES31.functional.draw_indirect.negative.command_bad_alignment_3",
386 "dEQP-GLES31.functional.draw_indirect.negative.command_offset_not_in_buffer",
387 "dEQP-GLES31.functional.vertex_attribute_binding.negative.bind_vertex_buffer_negative_offset",
388 "dEQP-GLES31.functional.ssbo.layout.single_basic_type.packed.mediump_uint",
389 "dEQP-GLES31.functional.blend_equation_advanced.basic.*",
390 "dEQP-GLES31.functional.blend_equation_advanced.srgb.*",
391 "dEQP-GLES31.functional.blend_equation_advanced.barrier.*",
392 "dEQP-GLES31.functional.uniform_location.*",
393 "dEQP-GLES31.functional.debug.negative_coverage.log.state.get_framebuffer_attachment_parameteriv",
394 "dEQP-GLES31.functional.debug.negative_coverage.log.state.get_renderbuffer_parameteriv",
395 "dEQP-GLES31.functional.debug.error_filters.case_0",
396 "dEQP-GLES31.functional.debug.error_filters.case_2",
400 name = "wgl_x64_release_gl3",
401 buildConfig = WGL_X64_RELEASE_BUILD_CFG,
402 generator = DEFAULT_WIN64_GENERATOR,
403 binaryName = "modules/gl3/deqp-gl3",
404 args = ['--deqp-gl-config-name=rgba8888d24s8ms0'] + BASE_ARGS,
405 testset = ["dEQP-GL3.info.*", "dEQP-GL3.functional.*"],
407 "dEQP-GL3.functional.shaders.loops.*while*unconditional_continue*",
408 "dEQP-GL3.functional.shaders.loops.*while*only_continue*",
409 "dEQP-GL3.functional.shaders.loops.*while*double_continue*",
412 "dEQP-GL3.functional.transform_feedback.*"
416 name = "arm_gles3_emu_x32_egl",
417 buildConfig = ARM_GLES3_EMU_X32_RELEASE_BUILD_CFG,
418 generator = DEFAULT_WIN32_GENERATOR,
419 binaryName = "modules/egl/deqp-egl",
421 testset = ["dEQP-EGL.info.*", "dEQP-EGL.functional.*"],
423 "dEQP-EGL.functional.sharing.gles2.multithread.*",
424 "dEQP-EGL.functional.multithread.*",
429 name = "opencl_x64_release",
430 buildConfig = NightlyBuildConfig("opencl_x64_release", "Release", ['-DDEQP_TARGET=opencl_icd']),
431 generator = DEFAULT_WIN64_GENERATOR,
432 binaryName = "modules/opencl/deqp-opencl",
433 args = ['--deqp-cl-platform-id=2 --deqp-cl-device-ids=1'] + BASE_ARGS,
434 testset = ["dEQP-CL.*"],
435 exclude = ["dEQP-CL.performance.*", "dEQP-CL.robustness.*", "dEQP-CL.stress.memory.*"],
437 "dEQP-CL.scheduler.random.*",
438 "dEQP-CL.language.set_kernel_arg.random_structs.*",
439 "dEQP-CL.language.builtin_function.work_item.invalid_get_global_offset",
440 "dEQP-CL.language.call_function.arguments.random_structs.*",
441 "dEQP-CL.language.call_kernel.random_structs.*",
442 "dEQP-CL.language.inf_nan.nan.frexp.float",
443 "dEQP-CL.language.inf_nan.nan.lgamma_r.float",
444 "dEQP-CL.language.inf_nan.nan.modf.float",
445 "dEQP-CL.language.inf_nan.nan.sqrt.float",
446 "dEQP-CL.api.multithread.*",
447 "dEQP-CL.api.callback.random.nested.*",
448 "dEQP-CL.api.memory_migration.out_of_order_host.image2d.single_device_kernel_migrate_validate_abb",
449 "dEQP-CL.api.memory_migration.out_of_order.image2d.single_device_kernel_migrate_kernel_validate_abbb",
450 "dEQP-CL.image.addressing_filtering12.1d_array.*",
451 "dEQP-CL.image.addressing_filtering12.2d_array.*"
456 if __name__ == "__main__":
459 if len(sys.argv) == 2:
460 cfgName = sys.argv[1]
461 for curCfg in CONFIGS:
462 if curCfg.name == cfgName:
467 isOk = runNightly(config)
471 print "%s: [config]" % sys.argv[0]
473 print " Available configs:"
474 for config in CONFIGS:
475 print " %s" % config.name