scripts/run_nightly.py

   1 # -*- coding: utf-8 -*-
   2
   3 #-------------------------------------------------------------------------
   4 # drawElements Quality Program utilities
   5 # --------------------------------------
   6 #
   7 # Copyright 2015 The Android Open Source Project
   8 #
   9 # Licensed under the Apache License, Version 2.0 (the "License");
  10 # you may not use this file except in compliance with the License.
  11 # You may obtain a copy of the License at
  12 #
  13 #      http://www.apache.org/licenses/LICENSE-2.0
  14 #
  15 # Unless required by applicable law or agreed to in writing, software
  16 # distributed under the License is distributed on an "AS IS" BASIS,
  17 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  18 # See the License for the specific language governing permissions and
  19 # limitations under the License.
  20 #
  21 #-------------------------------------------------------------------------
  22
  23 from build.common import *
  24 from build.config import *
  25 from build.build import *
  26
  27 import os
  28 import sys
  29 import string
  30 import socket
  31 import fnmatch
  32 from datetime import datetime
  33
  34 BASE_NIGHTLY_DIR        = os.path.normpath(os.path.join(DEQP_DIR, "..", "deqp-nightly"))
  35 BASE_BUILD_DIR          = os.path.join(BASE_NIGHTLY_DIR, "build")
  36 BASE_LOGS_DIR           = os.path.join(BASE_NIGHTLY_DIR, "logs")
  37 BASE_REFS_DIR           = os.path.join(BASE_NIGHTLY_DIR, "refs")
  38
  39 EXECUTOR_PATH           = "executor/executor"
  40 LOG_TO_CSV_PATH         = "executor/testlog-to-csv"
  41 EXECSERVER_PATH         = "execserver/execserver"
  42
  43 CASELIST_PATH           = os.path.join(DEQP_DIR, "Candy", "Data")
  44
  45 COMPARE_NUM_RESULTS     = 4
  46 COMPARE_REPORT_NAME     = "nightly-report.html"
  47
  48 COMPARE_REPORT_TMPL = '''
  49 <html>
  50 <head>
  51 <title>${TITLE}</title>
  52 <style type="text/css">
  53 <!--
  54 body                            { font: serif; font-size: 1em; }
  55 table                           { border-spacing: 0; border-collapse: collapse; }
  56 td                                      { border-width: 1px; border-style: solid; border-color: #808080; }
  57 .Header                         { font-weight: bold; font-size: 1em; border-style: none; }
  58 .CasePath                       { }
  59 .Pass                           { background: #80ff80; }
  60 .Fail                           { background: #ff4040; }
  61 .QualityWarning         { background: #ffff00; }
  62 .CompabilityWarning     { background: #ffff00; }
  63 .Pending                        { background: #808080; }
  64 .Running                        { background: #d3d3d3; }
  65 .NotSupported           { background: #ff69b4; }
  66 .ResourceError          { background: #ff4040; }
  67 .InternalError          { background: #ff1493; }
  68 .Canceled                       { background: #808080; }
  69 .Crash                          { background: #ffa500; }
  70 .Timeout                        { background: #ffa500; }
  71 .Disabled                       { background: #808080; }
  72 .Missing                        { background: #808080; }
  73 .Ignored                        { opacity: 0.5; }
  74 -->
  75 </style>
  76 </head>
  77 <body>
  78 <h1>${TITLE}</h1>
  79 <table>
  80 ${RESULTS}
  81 </table>
  82 </body>
  83 </html>
  84 '''
  85
  86 class NightlyRunConfig:
  87         def __init__(self, name, buildConfig, generator, binaryName, testset, args = [], exclude = [], ignore = []):
  88                 self.name                       = name
  89                 self.buildConfig        = buildConfig
  90                 self.generator          = generator
  91                 self.binaryName         = binaryName
  92                 self.testset            = testset
  93                 self.args                       = args
  94                 self.exclude            = exclude
  95                 self.ignore                     = ignore
  96
  97         def getBinaryPath(self, basePath):
  98                 return os.path.join(self.buildConfig.getBuildDir(), self.generator.getBinaryPath(self.buildConfig.getBuildType(), basePath))
  99
 100 class NightlyBuildConfig(BuildConfig):
 101         def __init__(self, name, buildType, args):
 102                 BuildConfig.__init__(self, os.path.join(BASE_BUILD_DIR, name), buildType, args)
 103
 104 class TestCaseResult:
 105         def __init__ (self, name, statusCode):
 106                 self.name               = name
 107                 self.statusCode = statusCode
 108
 109 class MultiResult:
 110         def __init__ (self, name, statusCodes):
 111                 self.name                       = name
 112                 self.statusCodes        = statusCodes
 113
 114 class BatchResult:
 115         def __init__ (self, name):
 116                 self.name               = name
 117                 self.results    = []
 118
 119 def parseResultCsv (data):
 120         lines   = data.splitlines()[1:]
 121         results = []
 122
 123         for line in lines:
 124                 items = line.split(",")
 125                 results.append(TestCaseResult(items[0], items[1]))
 126
 127         return results
 128
 129 def readTestCaseResultsFromCSV (filename):
 130         return parseResultCsv(readFile(filename))
 131
 132 def readBatchResultFromCSV (filename, batchResultName = None):
 133         batchResult = BatchResult(batchResultName if batchResultName != None else os.path.basename(filename))
 134         batchResult.results = readTestCaseResultsFromCSV(filename)
 135         return batchResult
 136
 137 def getResultTimestamp ():
 138         return datetime.now().strftime("%Y-%m-%d-%H-%M")
 139
 140 def getCompareFilenames (logsDir):
 141         files = []
 142         for file in os.listdir(logsDir):
 143                 fullPath = os.path.join(logsDir, file)
 144                 if os.path.isfile(fullPath) and fnmatch.fnmatch(file, "*.csv"):
 145                         files.append(fullPath)
 146         files.sort()
 147
 148         return files[-COMPARE_NUM_RESULTS:]
 149
 150 def parseAsCSV (logPath, config):
 151         args = [config.getBinaryPath(LOG_TO_CSV_PATH), "--mode=all", "--format=csv", logPath]
 152         proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 153         out, err = proc.communicate()
 154         return out
 155
 156 def computeUnifiedTestCaseList (batchResults):
 157         caseList        = []
 158         caseSet         = set()
 159
 160         for batchResult in batchResults:
 161                 for result in batchResult.results:
 162                         if not result.name in caseSet:
 163                                 caseList.append(result.name)
 164                                 caseSet.add(result.name)
 165
 166         return caseList
 167
 168 def computeUnifiedResults (batchResults):
 169
 170         def genResultMap (batchResult):
 171                 resMap = {}
 172                 for result in batchResult.results:
 173                         resMap[result.name] = result
 174                 return resMap
 175
 176         resultMap       = [genResultMap(r) for r in batchResults]
 177         caseList        = computeUnifiedTestCaseList(batchResults)
 178         results         = []
 179
 180         for caseName in caseList:
 181                 statusCodes = []
 182
 183                 for i in range(0, len(batchResults)):
 184                         result          = resultMap[i][caseName] if caseName in resultMap[i] else None
 185                         statusCode      = result.statusCode if result != None else 'Missing'
 186                         statusCodes.append(statusCode)
 187
 188                 results.append(MultiResult(caseName, statusCodes))
 189
 190         return results
 191
 192 def allStatusCodesEqual (result):
 193         firstCode = result.statusCodes[0]
 194         for i in range(1, len(result.statusCodes)):
 195                 if result.statusCodes[i] != firstCode:
 196                         return False
 197         return True
 198
 199 def computeDiffResults (unifiedResults):
 200         diff = []
 201         for result in unifiedResults:
 202                 if not allStatusCodesEqual(result):
 203                         diff.append(result)
 204         return diff
 205
 206 def genCompareReport (batchResults, title, ignoreCases):
 207         class TableRow:
 208                 def __init__ (self, testCaseName, innerHTML):
 209                         self.testCaseName = testCaseName
 210                         self.innerHTML = innerHTML
 211
 212         unifiedResults  = computeUnifiedResults(batchResults)
 213         diffResults             = computeDiffResults(unifiedResults)
 214         rows                    = []
 215
 216         # header
 217         headerCol = '<td class="Header">Test case</td>\n'
 218         for batchResult in batchResults:
 219                 headerCol += '<td class="Header">%s</td>\n' % batchResult.name
 220         rows.append(TableRow(None, headerCol))
 221
 222         # results
 223         for result in diffResults:
 224                 col = '<td class="CasePath">%s</td>\n' % result.name
 225                 for statusCode in result.statusCodes:
 226                         col += '<td class="%s">%s</td>\n' % (statusCode, statusCode)
 227
 228                 rows.append(TableRow(result.name, col))
 229
 230         tableStr = ""
 231         for row in rows:
 232                 if row.testCaseName is not None and matchesAnyPattern(row.testCaseName, ignoreCases):
 233                         tableStr += '<tr class="Ignored">\n%s</tr>\n' % row.innerHTML
 234                 else:
 235                         tableStr += '<tr>\n%s</tr>\n' % row.innerHTML
 236
 237         html = COMPARE_REPORT_TMPL
 238         html = html.replace("${TITLE}", title)
 239         html = html.replace("${RESULTS}", tableStr)
 240
 241         return html
 242
 243 def matchesAnyPattern (name, patterns):
 244         for pattern in patterns:
 245                 if fnmatch.fnmatch(name, pattern):
 246                         return True
 247         return False
 248
 249 def statusCodesMatch (refResult, resResult):
 250         return refResult == 'Missing' or resResult == 'Missing' or refResult == resResult
 251
 252 def compareBatchResults (referenceBatch, resultBatch, ignoreCases):
 253         unifiedResults  = computeUnifiedResults([referenceBatch, resultBatch])
 254         failedCases             = []
 255
 256         for result in unifiedResults:
 257                 if not matchesAnyPattern(result.name, ignoreCases):
 258                         refResult               = result.statusCodes[0]
 259                         resResult               = result.statusCodes[1]
 260
 261                         if not statusCodesMatch(refResult, resResult):
 262                                 failedCases.append(result)
 263
 264         return failedCases
 265
 266 def getUnusedPort ():
 267         # \note Not 100%-proof method as other apps may grab this port before we launch execserver
 268         s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
 269         s.bind(('localhost', 0))
 270         addr, port = s.getsockname()
 271         s.close()
 272         return port
 273
 274 def runNightly (config):
 275         build(config.buildConfig, config.generator)
 276
 277         # Run parameters
 278         timestamp               = getResultTimestamp()
 279         logDir                  = os.path.join(BASE_LOGS_DIR, config.name)
 280         testLogPath             = os.path.join(logDir, timestamp + ".qpa")
 281         infoLogPath             = os.path.join(logDir, timestamp + ".txt")
 282         csvLogPath              = os.path.join(logDir, timestamp + ".csv")
 283         compareLogPath  = os.path.join(BASE_REFS_DIR, config.name + ".csv")
 284         port                    = getUnusedPort()
 285
 286         if not os.path.exists(logDir):
 287                 os.makedirs(logDir)
 288
 289         if os.path.exists(testLogPath) or os.path.exists(infoLogPath):
 290                 raise Exception("Result '%s' already exists", timestamp)
 291
 292         # Paths, etc.
 293         binaryName              = config.generator.getBinaryPath(config.buildConfig.getBuildType(), os.path.basename(config.binaryName))
 294         workingDir              = os.path.join(config.buildConfig.getBuildDir(), os.path.dirname(config.binaryName))
 295
 296         execArgs = [
 297                 config.getBinaryPath(EXECUTOR_PATH),
 298                 '--start-server=%s' % config.getBinaryPath(EXECSERVER_PATH),
 299                 '--port=%d' % port,
 300                 '--binaryname=%s' % binaryName,
 301                 '--cmdline=%s' % string.join([shellquote(arg) for arg in config.args], " "),
 302                 '--workdir=%s' % workingDir,
 303                 '--caselistdir=%s' % CASELIST_PATH,
 304                 '--testset=%s' % string.join(config.testset, ","),
 305                 '--out=%s' % testLogPath,
 306                 '--info=%s' % infoLogPath,
 307                 '--summary=no'
 308         ]
 309
 310         if len(config.exclude) > 0:
 311                 execArgs += ['--exclude=%s' % string.join(config.exclude, ",")]
 312
 313         execute(execArgs)
 314
 315         # Translate to CSV for comparison purposes
 316         lastResultCsv           = parseAsCSV(testLogPath, config)
 317         writeFile(csvLogPath, lastResultCsv)
 318
 319         if os.path.exists(compareLogPath):
 320                 refBatchResult = readBatchResultFromCSV(compareLogPath, "reference")
 321         else:
 322                 refBatchResult = None
 323
 324         # Generate comparison report
 325         compareFilenames        = getCompareFilenames(logDir)
 326         batchResults            = [readBatchResultFromCSV(filename) for filename in compareFilenames]
 327
 328         if refBatchResult != None:
 329                 batchResults = [refBatchResult] + batchResults
 330
 331         writeFile(COMPARE_REPORT_NAME, genCompareReport(batchResults, config.name, config.ignore))
 332         print "Comparison report written to %s" % COMPARE_REPORT_NAME
 333
 334         # Compare to reference
 335         if refBatchResult != None:
 336                 curBatchResult          = BatchResult("current")
 337                 curBatchResult.results = parseResultCsv(lastResultCsv)
 338                 failedCases                     = compareBatchResults(refBatchResult, curBatchResult, config.ignore)
 339
 340                 print ""
 341                 for result in failedCases:
 342                         print "MISMATCH: %s: expected %s, got %s" % (result.name, result.statusCodes[0], result.statusCodes[1])
 343
 344                 print ""
 345                 print "%d / %d cases passed, run %s" % (len(curBatchResult.results)-len(failedCases), len(curBatchResult.results), "FAILED" if len(failedCases) > 0 else "passed")
 346
 347                 if len(failedCases) > 0:
 348                         return False
 349
 350         return True
 351
 352 # Configurations
 353
 354 DEFAULT_WIN32_GENERATOR                         = ANY_VS_X32_GENERATOR
 355 DEFAULT_WIN64_GENERATOR                         = ANY_VS_X64_GENERATOR
 356
 357 WGL_X64_RELEASE_BUILD_CFG                       = NightlyBuildConfig("wgl_x64_release", "Release", ['-DDEQP_TARGET=win32_wgl'])
 358 ARM_GLES3_EMU_X32_RELEASE_BUILD_CFG     = NightlyBuildConfig("arm_gles3_emu_release", "Release", ['-DDEQP_TARGET=arm_gles3_emu'])
 359
 360 BASE_ARGS                                                       = ['--deqp-visibility=hidden', '--deqp-watchdog=enable', '--deqp-crashhandler=enable']
 361
 362 CONFIGS = [
 363         NightlyRunConfig(
 364                 name                    = "wgl_x64_release_gles2",
 365                 buildConfig             = WGL_X64_RELEASE_BUILD_CFG,
 366                 generator               = DEFAULT_WIN64_GENERATOR,
 367                 binaryName              = "modules/gles2/deqp-gles2",
 368                 args                    = ['--deqp-gl-config-name=rgba8888d24s8ms0'] + BASE_ARGS,
 369                 testset                 = ["dEQP-GLES2.info.*", "dEQP-GLES2.functional.*", "dEQP-GLES2.usecases.*"],
 370                 exclude                 = [
 371                                 "dEQP-GLES2.functional.shaders.loops.*while*unconditional_continue*",
 372                                 "dEQP-GLES2.functional.shaders.loops.*while*only_continue*",
 373                                 "dEQP-GLES2.functional.shaders.loops.*while*double_continue*",
 374                         ],
 375                 ignore                  = []
 376                 ),
 377         NightlyRunConfig(
 378                 name                    = "wgl_x64_release_gles3",
 379                 buildConfig             = WGL_X64_RELEASE_BUILD_CFG,
 380                 generator               = DEFAULT_WIN64_GENERATOR,
 381                 binaryName              = "modules/gles3/deqp-gles3",
 382                 args                    = ['--deqp-gl-config-name=rgba8888d24s8ms0'] + BASE_ARGS,
 383                 testset                 = ["dEQP-GLES3.info.*", "dEQP-GLES3.functional.*", "dEQP-GLES3.usecases.*"],
 384                 exclude                 = [
 385                                 "dEQP-GLES3.functional.shaders.loops.*while*unconditional_continue*",
 386                                 "dEQP-GLES3.functional.shaders.loops.*while*only_continue*",
 387                                 "dEQP-GLES3.functional.shaders.loops.*while*double_continue*",
 388                         ],
 389                 ignore                  = [
 390                                 "dEQP-GLES3.functional.transform_feedback.*",
 391                                 "dEQP-GLES3.functional.occlusion_query.*",
 392                                 "dEQP-GLES3.functional.lifetime.*",
 393                                 "dEQP-GLES3.functional.fragment_ops.depth_stencil.stencil_ops",
 394                         ]
 395                 ),
 396         NightlyRunConfig(
 397                 name                    = "wgl_x64_release_gles31",
 398                 buildConfig             = WGL_X64_RELEASE_BUILD_CFG,
 399                 generator               = DEFAULT_WIN64_GENERATOR,
 400                 binaryName              = "modules/gles31/deqp-gles31",
 401                 args                    = ['--deqp-gl-config-name=rgba8888d24s8ms0'] + BASE_ARGS,
 402                 testset                 = ["dEQP-GLES31.*"],
 403                 exclude                 = [],
 404                 ignore                  = [
 405                                 "dEQP-GLES31.functional.draw_indirect.negative.command_bad_alignment_3",
 406                                 "dEQP-GLES31.functional.draw_indirect.negative.command_offset_not_in_buffer",
 407                                 "dEQP-GLES31.functional.vertex_attribute_binding.negative.bind_vertex_buffer_negative_offset",
 408                                 "dEQP-GLES31.functional.ssbo.layout.single_basic_type.packed.mediump_uint",
 409                                 "dEQP-GLES31.functional.blend_equation_advanced.basic.*",
 410                                 "dEQP-GLES31.functional.blend_equation_advanced.srgb.*",
 411                                 "dEQP-GLES31.functional.blend_equation_advanced.barrier.*",
 412                                 "dEQP-GLES31.functional.uniform_location.*",
 413                                 "dEQP-GLES31.functional.debug.negative_coverage.log.state.get_framebuffer_attachment_parameteriv",
 414                                 "dEQP-GLES31.functional.debug.negative_coverage.log.state.get_renderbuffer_parameteriv",
 415                                 "dEQP-GLES31.functional.debug.error_filters.case_0",
 416                                 "dEQP-GLES31.functional.debug.error_filters.case_2",
 417                         ]
 418                 ),
 419         NightlyRunConfig(
 420                 name                    = "wgl_x64_release_gl3",
 421                 buildConfig             = WGL_X64_RELEASE_BUILD_CFG,
 422                 generator               = DEFAULT_WIN64_GENERATOR,
 423                 binaryName              = "modules/gl3/deqp-gl3",
 424                 args                    = ['--deqp-gl-config-name=rgba8888d24s8ms0'] + BASE_ARGS,
 425                 testset                 = ["dEQP-GL3.info.*", "dEQP-GL3.functional.*"],
 426                 exclude                 = [
 427                                 "dEQP-GL3.functional.shaders.loops.*while*unconditional_continue*",
 428                                 "dEQP-GL3.functional.shaders.loops.*while*only_continue*",
 429                                 "dEQP-GL3.functional.shaders.loops.*while*double_continue*",
 430                         ],
 431                 ignore                  = [
 432                                 "dEQP-GL3.functional.transform_feedback.*"
 433                         ]
 434                 ),
 435         NightlyRunConfig(
 436                 name                    = "arm_gles3_emu_x32_egl",
 437                 buildConfig             = ARM_GLES3_EMU_X32_RELEASE_BUILD_CFG,
 438                 generator               = DEFAULT_WIN32_GENERATOR,
 439                 binaryName              = "modules/egl/deqp-egl",
 440                 args                    = BASE_ARGS,
 441                 testset                 = ["dEQP-EGL.info.*", "dEQP-EGL.functional.*"],
 442                 exclude                 = [
 443                                 "dEQP-EGL.functional.sharing.gles2.multithread.*",
 444                                 "dEQP-EGL.functional.multithread.*",
 445                         ],
 446                 ignore                  = []
 447                 ),
 448         NightlyRunConfig(
 449                 name                    = "opencl_x64_release",
 450                 buildConfig             = NightlyBuildConfig("opencl_x64_release", "Release", ['-DDEQP_TARGET=opencl_icd']),
 451                 generator               = DEFAULT_WIN64_GENERATOR,
 452                 binaryName              = "modules/opencl/deqp-opencl",
 453                 args                    = ['--deqp-cl-platform-id=2 --deqp-cl-device-ids=1'] + BASE_ARGS,
 454                 testset                 = ["dEQP-CL.*"],
 455                 exclude                 = ["dEQP-CL.performance.*", "dEQP-CL.robustness.*", "dEQP-CL.stress.memory.*"],
 456                 ignore                  = [
 457                                 "dEQP-CL.scheduler.random.*",
 458                                 "dEQP-CL.language.set_kernel_arg.random_structs.*",
 459                                 "dEQP-CL.language.builtin_function.work_item.invalid_get_global_offset",
 460                                 "dEQP-CL.language.call_function.arguments.random_structs.*",
 461                                 "dEQP-CL.language.call_kernel.random_structs.*",
 462                                 "dEQP-CL.language.inf_nan.nan.frexp.float",
 463                                 "dEQP-CL.language.inf_nan.nan.lgamma_r.float",
 464                                 "dEQP-CL.language.inf_nan.nan.modf.float",
 465                                 "dEQP-CL.language.inf_nan.nan.sqrt.float",
 466                                 "dEQP-CL.api.multithread.*",
 467                                 "dEQP-CL.api.callback.random.nested.*",
 468                                 "dEQP-CL.api.memory_migration.out_of_order_host.image2d.single_device_kernel_migrate_validate_abb",
 469                                 "dEQP-CL.api.memory_migration.out_of_order.image2d.single_device_kernel_migrate_kernel_validate_abbb",
 470                                 "dEQP-CL.image.addressing_filtering12.1d_array.*",
 471                                 "dEQP-CL.image.addressing_filtering12.2d_array.*"
 472                         ]
 473                 )
 474 ]
 475
 476 if __name__ == "__main__":
 477         config = None
 478
 479         if len(sys.argv) == 2:
 480                 cfgName = sys.argv[1]
 481                 for curCfg in CONFIGS:
 482                         if curCfg.name == cfgName:
 483                                 config = curCfg
 484                                 break
 485
 486         if config != None:
 487                 isOk = runNightly(config)
 488                 if not isOk:
 489                         sys.exit(-1)
 490         else:
 491                 print "%s: [config]" % sys.argv[0]
 492                 print ""
 493                 print "  Available configs:"
 494                 for config in CONFIGS:
 495                         print "    %s" % config.name
 496                 sys.exit(-1)