scripts/run_nightly.py

   1 # -*- coding: utf-8 -*-
   2
   3 from build.common import *
   4 from build.config import *
   5 from build.build import *
   6
   7 import os
   8 import sys
   9 import string
  10 import socket
  11 import fnmatch
  12 from datetime import datetime
  13
  14 BASE_NIGHTLY_DIR        = os.path.normpath(os.path.join(DEQP_DIR, "..", "deqp-nightly"))
  15 BASE_BUILD_DIR          = os.path.join(BASE_NIGHTLY_DIR, "build")
  16 BASE_LOGS_DIR           = os.path.join(BASE_NIGHTLY_DIR, "logs")
  17 BASE_REFS_DIR           = os.path.join(BASE_NIGHTLY_DIR, "refs")
  18
  19 EXECUTOR_PATH           = "executor/executor"
  20 LOG_TO_CSV_PATH         = "executor/testlog-to-csv"
  21 EXECSERVER_PATH         = "execserver/execserver"
  22
  23 CASELIST_PATH           = os.path.join(DEQP_DIR, "Candy", "Data")
  24
  25 COMPARE_NUM_RESULTS     = 4
  26 COMPARE_REPORT_NAME     = "nightly-report.html"
  27
  28 COMPARE_REPORT_TMPL = '''
  29 <html>
  30 <head>
  31 <title>${TITLE}</title>
  32 <style type="text/css">
  33 <!--
  34 body                            { font: serif; font-size: 1em; }
  35 table                           { border-spacing: 0; border-collapse: collapse; }
  36 td                                      { border-width: 1px; border-style: solid; border-color: #808080; }
  37 .Header                         { font-weight: bold; font-size: 1em; border-style: none; }
  38 .CasePath                       { }
  39 .Pass                           { background: #80ff80; }
  40 .Fail                           { background: #ff4040; }
  41 .QualityWarning         { background: #ffff00; }
  42 .CompabilityWarning     { background: #ffff00; }
  43 .Pending                        { background: #808080; }
  44 .Running                        { background: #d3d3d3; }
  45 .NotSupported           { background: #ff69b4; }
  46 .ResourceError          { background: #ff4040; }
  47 .InternalError          { background: #ff1493; }
  48 .Canceled                       { background: #808080; }
  49 .Crash                          { background: #ffa500; }
  50 .Timeout                        { background: #ffa500; }
  51 .Disabled                       { background: #808080; }
  52 .Missing                        { background: #808080; }
  53 .Ignored                        { opacity: 0.5; }
  54 -->
  55 </style>
  56 </head>
  57 <body>
  58 <h1>${TITLE}</h1>
  59 <table>
  60 ${RESULTS}
  61 </table>
  62 </body>
  63 </html>
  64 '''
  65
  66 class NightlyRunConfig:
  67         def __init__(self, name, buildConfig, generator, binaryName, testset, args = [], exclude = [], ignore = []):
  68                 self.name                       = name
  69                 self.buildConfig        = buildConfig
  70                 self.generator          = generator
  71                 self.binaryName         = binaryName
  72                 self.testset            = testset
  73                 self.args                       = args
  74                 self.exclude            = exclude
  75                 self.ignore                     = ignore
  76
  77         def getBinaryPath(self, basePath):
  78                 return os.path.join(self.buildConfig.getBuildDir(), self.generator.getBinaryPath(self.buildConfig.getBuildType(), basePath))
  79
  80 class NightlyBuildConfig(BuildConfig):
  81         def __init__(self, name, buildType, args):
  82                 BuildConfig.__init__(self, os.path.join(BASE_BUILD_DIR, name), buildType, args)
  83
  84 class TestCaseResult:
  85         def __init__ (self, name, statusCode):
  86                 self.name               = name
  87                 self.statusCode = statusCode
  88
  89 class MultiResult:
  90         def __init__ (self, name, statusCodes):
  91                 self.name                       = name
  92                 self.statusCodes        = statusCodes
  93
  94 class BatchResult:
  95         def __init__ (self, name):
  96                 self.name               = name
  97                 self.results    = []
  98
  99 def parseResultCsv (data):
 100         lines   = data.splitlines()[1:]
 101         results = []
 102
 103         for line in lines:
 104                 items = line.split(",")
 105                 results.append(TestCaseResult(items[0], items[1]))
 106
 107         return results
 108
 109 def readTestCaseResultsFromCSV (filename):
 110         return parseResultCsv(readFile(filename))
 111
 112 def readBatchResultFromCSV (filename, batchResultName = None):
 113         batchResult = BatchResult(batchResultName if batchResultName != None else os.path.basename(filename))
 114         batchResult.results = readTestCaseResultsFromCSV(filename)
 115         return batchResult
 116
 117 def getResultTimestamp ():
 118         return datetime.now().strftime("%Y-%m-%d-%H-%M")
 119
 120 def getCompareFilenames (logsDir):
 121         files = []
 122         for file in os.listdir(logsDir):
 123                 fullPath = os.path.join(logsDir, file)
 124                 if os.path.isfile(fullPath) and fnmatch.fnmatch(file, "*.csv"):
 125                         files.append(fullPath)
 126         files.sort()
 127
 128         return files[-COMPARE_NUM_RESULTS:]
 129
 130 def parseAsCSV (logPath, config):
 131         args = [config.getBinaryPath(LOG_TO_CSV_PATH), "--mode=all", "--format=csv", logPath]
 132         proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 133         out, err = proc.communicate()
 134         return out
 135
 136 def computeUnifiedTestCaseList (batchResults):
 137         caseList        = []
 138         caseSet         = set()
 139
 140         for batchResult in batchResults:
 141                 for result in batchResult.results:
 142                         if not result.name in caseSet:
 143                                 caseList.append(result.name)
 144                                 caseSet.add(result.name)
 145
 146         return caseList
 147
 148 def computeUnifiedResults (batchResults):
 149
 150         def genResultMap (batchResult):
 151                 resMap = {}
 152                 for result in batchResult.results:
 153                         resMap[result.name] = result
 154                 return resMap
 155
 156         resultMap       = [genResultMap(r) for r in batchResults]
 157         caseList        = computeUnifiedTestCaseList(batchResults)
 158         results         = []
 159
 160         for caseName in caseList:
 161                 statusCodes = []
 162
 163                 for i in range(0, len(batchResults)):
 164                         result          = resultMap[i][caseName] if caseName in resultMap[i] else None
 165                         statusCode      = result.statusCode if result != None else 'Missing'
 166                         statusCodes.append(statusCode)
 167
 168                 results.append(MultiResult(caseName, statusCodes))
 169
 170         return results
 171
 172 def allStatusCodesEqual (result):
 173         firstCode = result.statusCodes[0]
 174         for i in range(1, len(result.statusCodes)):
 175                 if result.statusCodes[i] != firstCode:
 176                         return False
 177         return True
 178
 179 def computeDiffResults (unifiedResults):
 180         diff = []
 181         for result in unifiedResults:
 182                 if not allStatusCodesEqual(result):
 183                         diff.append(result)
 184         return diff
 185
 186 def genCompareReport (batchResults, title, ignoreCases):
 187         class TableRow:
 188                 def __init__ (self, testCaseName, innerHTML):
 189                         self.testCaseName = testCaseName
 190                         self.innerHTML = innerHTML
 191
 192         unifiedResults  = computeUnifiedResults(batchResults)
 193         diffResults             = computeDiffResults(unifiedResults)
 194         rows                    = []
 195
 196         # header
 197         headerCol = '<td class="Header">Test case</td>\n'
 198         for batchResult in batchResults:
 199                 headerCol += '<td class="Header">%s</td>\n' % batchResult.name
 200         rows.append(TableRow(None, headerCol))
 201
 202         # results
 203         for result in diffResults:
 204                 col = '<td class="CasePath">%s</td>\n' % result.name
 205                 for statusCode in result.statusCodes:
 206                         col += '<td class="%s">%s</td>\n' % (statusCode, statusCode)
 207
 208                 rows.append(TableRow(result.name, col))
 209
 210         tableStr = ""
 211         for row in rows:
 212                 if row.testCaseName is not None and matchesAnyPattern(row.testCaseName, ignoreCases):
 213                         tableStr += '<tr class="Ignored">\n%s</tr>\n' % row.innerHTML
 214                 else:
 215                         tableStr += '<tr>\n%s</tr>\n' % row.innerHTML
 216
 217         html = COMPARE_REPORT_TMPL
 218         html = html.replace("${TITLE}", title)
 219         html = html.replace("${RESULTS}", tableStr)
 220
 221         return html
 222
 223 def matchesAnyPattern (name, patterns):
 224         for pattern in patterns:
 225                 if fnmatch.fnmatch(name, pattern):
 226                         return True
 227         return False
 228
 229 def statusCodesMatch (refResult, resResult):
 230         return refResult == 'Missing' or resResult == 'Missing' or refResult == resResult
 231
 232 def compareBatchResults (referenceBatch, resultBatch, ignoreCases):
 233         unifiedResults  = computeUnifiedResults([referenceBatch, resultBatch])
 234         failedCases             = []
 235
 236         for result in unifiedResults:
 237                 if not matchesAnyPattern(result.name, ignoreCases):
 238                         refResult               = result.statusCodes[0]
 239                         resResult               = result.statusCodes[1]
 240
 241                         if not statusCodesMatch(refResult, resResult):
 242                                 failedCases.append(result)
 243
 244         return failedCases
 245
 246 def getUnusedPort ():
 247         # \note Not 100%-proof method as other apps may grab this port before we launch execserver
 248         s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
 249         s.bind(('localhost', 0))
 250         addr, port = s.getsockname()
 251         s.close()
 252         return port
 253
 254 def runNightly (config):
 255         build(config.buildConfig, config.generator)
 256
 257         # Run parameters
 258         timestamp               = getResultTimestamp()
 259         logDir                  = os.path.join(BASE_LOGS_DIR, config.name)
 260         testLogPath             = os.path.join(logDir, timestamp + ".qpa")
 261         infoLogPath             = os.path.join(logDir, timestamp + ".txt")
 262         csvLogPath              = os.path.join(logDir, timestamp + ".csv")
 263         compareLogPath  = os.path.join(BASE_REFS_DIR, config.name + ".csv")
 264         port                    = getUnusedPort()
 265
 266         if not os.path.exists(logDir):
 267                 os.makedirs(logDir)
 268
 269         if os.path.exists(testLogPath) or os.path.exists(infoLogPath):
 270                 raise Exception("Result '%s' already exists", timestamp)
 271
 272         # Paths, etc.
 273         binaryName              = config.generator.getBinaryPath(config.buildConfig.getBuildType(), os.path.basename(config.binaryName))
 274         workingDir              = os.path.join(config.buildConfig.getBuildDir(), os.path.dirname(config.binaryName))
 275
 276         execArgs = [
 277                 config.getBinaryPath(EXECUTOR_PATH),
 278                 '--start-server=%s' % config.getBinaryPath(EXECSERVER_PATH),
 279                 '--port=%d' % port,
 280                 '--binaryname=%s' % binaryName,
 281                 '--cmdline=%s' % string.join([shellquote(arg) for arg in config.args], " "),
 282                 '--workdir=%s' % workingDir,
 283                 '--caselistdir=%s' % CASELIST_PATH,
 284                 '--testset=%s' % string.join(config.testset, ","),
 285                 '--out=%s' % testLogPath,
 286                 '--info=%s' % infoLogPath,
 287                 '--summary=no'
 288         ]
 289
 290         if len(config.exclude) > 0:
 291                 execArgs += ['--exclude=%s' % string.join(config.exclude, ",")]
 292
 293         execute(execArgs)
 294
 295         # Translate to CSV for comparison purposes
 296         lastResultCsv           = parseAsCSV(testLogPath, config)
 297         writeFile(csvLogPath, lastResultCsv)
 298
 299         if os.path.exists(compareLogPath):
 300                 refBatchResult = readBatchResultFromCSV(compareLogPath, "reference")
 301         else:
 302                 refBatchResult = None
 303
 304         # Generate comparison report
 305         compareFilenames        = getCompareFilenames(logDir)
 306         batchResults            = [readBatchResultFromCSV(filename) for filename in compareFilenames]
 307
 308         if refBatchResult != None:
 309                 batchResults = [refBatchResult] + batchResults
 310
 311         writeFile(COMPARE_REPORT_NAME, genCompareReport(batchResults, config.name, config.ignore))
 312         print "Comparison report written to %s" % COMPARE_REPORT_NAME
 313
 314         # Compare to reference
 315         if refBatchResult != None:
 316                 curBatchResult          = BatchResult("current")
 317                 curBatchResult.results = parseResultCsv(lastResultCsv)
 318                 failedCases                     = compareBatchResults(refBatchResult, curBatchResult, config.ignore)
 319
 320                 print ""
 321                 for result in failedCases:
 322                         print "MISMATCH: %s: expected %s, got %s" % (result.name, result.statusCodes[0], result.statusCodes[1])
 323
 324                 print ""
 325                 print "%d / %d cases passed, run %s" % (len(curBatchResult.results)-len(failedCases), len(curBatchResult.results), "FAILED" if len(failedCases) > 0 else "passed")
 326
 327                 if len(failedCases) > 0:
 328                         return False
 329
 330         return True
 331
 332 # Configurations
 333
 334 DEFAULT_WIN32_GENERATOR                         = ANY_VS_X32_GENERATOR
 335 DEFAULT_WIN64_GENERATOR                         = ANY_VS_X64_GENERATOR
 336
 337 WGL_X64_RELEASE_BUILD_CFG                       = NightlyBuildConfig("wgl_x64_release", "Release", ['-DDEQP_TARGET=win32_wgl'])
 338 ARM_GLES3_EMU_X32_RELEASE_BUILD_CFG     = NightlyBuildConfig("arm_gles3_emu_release", "Release", ['-DDEQP_TARGET=arm_gles3_emu'])
 339
 340 BASE_ARGS                                                       = ['--deqp-visibility=hidden', '--deqp-watchdog=enable', '--deqp-crashhandler=enable']
 341
 342 CONFIGS = [
 343         NightlyRunConfig(
 344                 name                    = "wgl_x64_release_gles2",
 345                 buildConfig             = WGL_X64_RELEASE_BUILD_CFG,
 346                 generator               = DEFAULT_WIN64_GENERATOR,
 347                 binaryName              = "modules/gles2/deqp-gles2",
 348                 args                    = ['--deqp-gl-config-name=rgba8888d24s8ms0'] + BASE_ARGS,
 349                 testset                 = ["dEQP-GLES2.info.*", "dEQP-GLES2.functional.*", "dEQP-GLES2.usecases.*"],
 350                 exclude                 = [
 351                                 "dEQP-GLES2.functional.shaders.loops.*while*unconditional_continue*",
 352                                 "dEQP-GLES2.functional.shaders.loops.*while*only_continue*",
 353                                 "dEQP-GLES2.functional.shaders.loops.*while*double_continue*",
 354                         ],
 355                 ignore                  = []
 356                 ),
 357         NightlyRunConfig(
 358                 name                    = "wgl_x64_release_gles3",
 359                 buildConfig             = WGL_X64_RELEASE_BUILD_CFG,
 360                 generator               = DEFAULT_WIN64_GENERATOR,
 361                 binaryName              = "modules/gles3/deqp-gles3",
 362                 args                    = ['--deqp-gl-config-name=rgba8888d24s8ms0'] + BASE_ARGS,
 363                 testset                 = ["dEQP-GLES3.info.*", "dEQP-GLES3.functional.*", "dEQP-GLES3.usecases.*"],
 364                 exclude                 = [
 365                                 "dEQP-GLES3.functional.shaders.loops.*while*unconditional_continue*",
 366                                 "dEQP-GLES3.functional.shaders.loops.*while*only_continue*",
 367                                 "dEQP-GLES3.functional.shaders.loops.*while*double_continue*",
 368                         ],
 369                 ignore                  = [
 370                                 "dEQP-GLES3.functional.transform_feedback.*",
 371                                 "dEQP-GLES3.functional.occlusion_query.*",
 372                                 "dEQP-GLES3.functional.lifetime.*",
 373                                 "dEQP-GLES3.functional.fragment_ops.depth_stencil.stencil_ops",
 374                         ]
 375                 ),
 376         NightlyRunConfig(
 377                 name                    = "wgl_x64_release_gles31",
 378                 buildConfig             = WGL_X64_RELEASE_BUILD_CFG,
 379                 generator               = DEFAULT_WIN64_GENERATOR,
 380                 binaryName              = "modules/gles31/deqp-gles31",
 381                 args                    = ['--deqp-gl-config-name=rgba8888d24s8ms0'] + BASE_ARGS,
 382                 testset                 = ["dEQP-GLES31.*"],
 383                 exclude                 = [],
 384                 ignore                  = [
 385                                 "dEQP-GLES31.functional.draw_indirect.negative.command_bad_alignment_3",
 386                                 "dEQP-GLES31.functional.draw_indirect.negative.command_offset_not_in_buffer",
 387                                 "dEQP-GLES31.functional.vertex_attribute_binding.negative.bind_vertex_buffer_negative_offset",
 388                                 "dEQP-GLES31.functional.ssbo.layout.single_basic_type.packed.mediump_uint",
 389                                 "dEQP-GLES31.functional.blend_equation_advanced.basic.*",
 390                                 "dEQP-GLES31.functional.blend_equation_advanced.srgb.*",
 391                                 "dEQP-GLES31.functional.blend_equation_advanced.barrier.*",
 392                                 "dEQP-GLES31.functional.uniform_location.*",
 393                                 "dEQP-GLES31.functional.debug.negative_coverage.log.state.get_framebuffer_attachment_parameteriv",
 394                                 "dEQP-GLES31.functional.debug.negative_coverage.log.state.get_renderbuffer_parameteriv",
 395                                 "dEQP-GLES31.functional.debug.error_filters.case_0",
 396                                 "dEQP-GLES31.functional.debug.error_filters.case_2",
 397                         ]
 398                 ),
 399         NightlyRunConfig(
 400                 name                    = "wgl_x64_release_gl3",
 401                 buildConfig             = WGL_X64_RELEASE_BUILD_CFG,
 402                 generator               = DEFAULT_WIN64_GENERATOR,
 403                 binaryName              = "modules/gl3/deqp-gl3",
 404                 args                    = ['--deqp-gl-config-name=rgba8888d24s8ms0'] + BASE_ARGS,
 405                 testset                 = ["dEQP-GL3.info.*", "dEQP-GL3.functional.*"],
 406                 exclude                 = [
 407                                 "dEQP-GL3.functional.shaders.loops.*while*unconditional_continue*",
 408                                 "dEQP-GL3.functional.shaders.loops.*while*only_continue*",
 409                                 "dEQP-GL3.functional.shaders.loops.*while*double_continue*",
 410                         ],
 411                 ignore                  = [
 412                                 "dEQP-GL3.functional.transform_feedback.*"
 413                         ]
 414                 ),
 415         NightlyRunConfig(
 416                 name                    = "arm_gles3_emu_x32_egl",
 417                 buildConfig             = ARM_GLES3_EMU_X32_RELEASE_BUILD_CFG,
 418                 generator               = DEFAULT_WIN32_GENERATOR,
 419                 binaryName              = "modules/egl/deqp-egl",
 420                 args                    = BASE_ARGS,
 421                 testset                 = ["dEQP-EGL.info.*", "dEQP-EGL.functional.*"],
 422                 exclude                 = [
 423                                 "dEQP-EGL.functional.sharing.gles2.multithread.*",
 424                                 "dEQP-EGL.functional.multithread.*",
 425                         ],
 426                 ignore                  = []
 427                 ),
 428         NightlyRunConfig(
 429                 name                    = "opencl_x64_release",
 430                 buildConfig             = NightlyBuildConfig("opencl_x64_release", "Release", ['-DDEQP_TARGET=opencl_icd']),
 431                 generator               = DEFAULT_WIN64_GENERATOR,
 432                 binaryName              = "modules/opencl/deqp-opencl",
 433                 args                    = ['--deqp-cl-platform-id=2 --deqp-cl-device-ids=1'] + BASE_ARGS,
 434                 testset                 = ["dEQP-CL.*"],
 435                 exclude                 = ["dEQP-CL.performance.*", "dEQP-CL.robustness.*", "dEQP-CL.stress.memory.*"],
 436                 ignore                  = [
 437                                 "dEQP-CL.scheduler.random.*",
 438                                 "dEQP-CL.language.set_kernel_arg.random_structs.*",
 439                                 "dEQP-CL.language.builtin_function.work_item.invalid_get_global_offset",
 440                                 "dEQP-CL.language.call_function.arguments.random_structs.*",
 441                                 "dEQP-CL.language.call_kernel.random_structs.*",
 442                                 "dEQP-CL.language.inf_nan.nan.frexp.float",
 443                                 "dEQP-CL.language.inf_nan.nan.lgamma_r.float",
 444                                 "dEQP-CL.language.inf_nan.nan.modf.float",
 445                                 "dEQP-CL.language.inf_nan.nan.sqrt.float",
 446                                 "dEQP-CL.api.multithread.*",
 447                                 "dEQP-CL.api.callback.random.nested.*",
 448                                 "dEQP-CL.api.memory_migration.out_of_order_host.image2d.single_device_kernel_migrate_validate_abb",
 449                                 "dEQP-CL.api.memory_migration.out_of_order.image2d.single_device_kernel_migrate_kernel_validate_abbb",
 450                                 "dEQP-CL.image.addressing_filtering12.1d_array.*",
 451                                 "dEQP-CL.image.addressing_filtering12.2d_array.*"
 452                         ]
 453                 )
 454 ]
 455
 456 if __name__ == "__main__":
 457         config = None
 458
 459         if len(sys.argv) == 2:
 460                 cfgName = sys.argv[1]
 461                 for curCfg in CONFIGS:
 462                         if curCfg.name == cfgName:
 463                                 config = curCfg
 464                                 break
 465
 466         if config != None:
 467                 isOk = runNightly(config)
 468                 if not isOk:
 469                         sys.exit(-1)
 470         else:
 471                 print "%s: [config]" % sys.argv[0]
 472                 print ""
 473                 print "  Available configs:"
 474                 for config in CONFIGS:
 475                         print "    %s" % config.name
 476                 sys.exit(-1)