scripts/mustpass.py

   1 # -*- coding: utf-8 -*-
   2
   3 #-------------------------------------------------------------------------
   4 # drawElements Quality Program utilities
   5 # --------------------------------------
   6 #
   7 # Copyright 2016 The Android Open Source Project
   8 #
   9 # Licensed under the Apache License, Version 2.0 (the "License");
  10 # you may not use this file except in compliance with the License.
  11 # You may obtain a copy of the License at
  12 #
  13 #      http://www.apache.org/licenses/LICENSE-2.0
  14 #
  15 # Unless required by applicable law or agreed to in writing, software
  16 # distributed under the License is distributed on an "AS IS" BASIS,
  17 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  18 # See the License for the specific language governing permissions and
  19 # limitations under the License.
  20 #
  21 #-------------------------------------------------------------------------
  22
  23 from build.common import *
  24 from build.config import ANY_GENERATOR
  25 from build.build import build
  26 from build_caselists import Module, getModuleByName, getBuildConfig, genCaseList, getCaseListPath, DEFAULT_BUILD_DIR, DEFAULT_TARGET
  27 from fnmatch import fnmatch
  28 from copy import copy
  29
  30 import xml.etree.cElementTree as ElementTree
  31 import xml.dom.minidom as minidom
  32
  33 APK_NAME                = "com.drawelements.deqp.apk"
  34
  35 GENERATED_FILE_WARNING = """
  36      This file has been automatically generated. Edit with caution.
  37      """
  38
  39 class Project:
  40         def __init__ (self, path, copyright = None):
  41                 self.path               = path
  42                 self.copyright  = copyright
  43
  44 class Configuration:
  45         def __init__ (self, name, filters, glconfig = None, rotation = None, surfacetype = None, runtime = None):
  46                 self.name                        = name
  47                 self.glconfig            = glconfig
  48                 self.rotation            = rotation
  49                 self.surfacetype         = surfacetype
  50                 self.filters             = filters
  51                 self.expectedRuntime = runtime
  52
  53 class Package:
  54         def __init__ (self, module, configurations):
  55                 self.module                     = module
  56                 self.configurations     = configurations
  57
  58 class Mustpass:
  59         def __init__ (self, project, version, packages):
  60                 self.project    = project
  61                 self.version    = version
  62                 self.packages   = packages
  63
  64 class Filter:
  65         TYPE_INCLUDE = 0
  66         TYPE_EXCLUDE = 1
  67
  68         def __init__ (self, type, filename):
  69                 self.type               = type
  70                 self.filename   = filename
  71
  72 class TestRoot:
  73         def __init__ (self):
  74                 self.children   = []
  75
  76 class TestGroup:
  77         def __init__ (self, name):
  78                 self.name               = name
  79                 self.children   = []
  80
  81 class TestCase:
  82         def __init__ (self, name):
  83                 self.name                       = name
  84                 self.configurations     = []
  85
  86 class GLESVersion:
  87         def __init__(self, major, minor):
  88                 self.major = major
  89                 self.minor = minor
  90
  91         def encode (self):
  92                 return (self.major << 16) | (self.minor)
  93
  94 def getModuleGLESVersion (module):
  95         versions = {
  96                 'dEQP-EGL':             GLESVersion(2,0),
  97                 'dEQP-GLES2':   GLESVersion(2,0),
  98                 'dEQP-GLES3':   GLESVersion(3,0),
  99                 'dEQP-GLES31':  GLESVersion(3,1)
 100         }
 101         return versions[module.name] if module.name in versions else None
 102
 103 def getSrcDir (mustpass):
 104         return os.path.join(mustpass.project.path, mustpass.version, "src")
 105
 106 def getTmpDir (mustpass):
 107         return os.path.join(mustpass.project.path, mustpass.version, "tmp")
 108
 109 def getModuleShorthand (module):
 110         assert module.name[:5] == "dEQP-"
 111         return module.name[5:].lower()
 112
 113 def getCaseListFileName (package, configuration):
 114         return "%s-%s.txt" % (getModuleShorthand(package.module), configuration.name)
 115
 116 def getDstCaseListPath (mustpass, package, configuration):
 117         return os.path.join(mustpass.project.path, mustpass.version, getCaseListFileName(package, configuration))
 118
 119 def getCTSPackageName (package):
 120         return "com.drawelements.deqp." + getModuleShorthand(package.module)
 121
 122 def getCommandLine (config):
 123         cmdLine = ""
 124
 125         if config.glconfig != None:
 126                 cmdLine += "--deqp-gl-config-name=%s " % config.glconfig
 127
 128         if config.rotation != None:
 129                 cmdLine += "--deqp-screen-rotation=%s " % config.rotation
 130
 131         if config.surfacetype != None:
 132                 cmdLine += "--deqp-surface-type=%s " % config.surfacetype
 133
 134         cmdLine += "--deqp-watchdog=enable"
 135
 136         return cmdLine
 137
 138 def readCaseList (filename):
 139         cases = []
 140         with open(filename, 'rb') as f:
 141                 for line in f:
 142                         if line[:6] == "TEST: ":
 143                                 cases.append(line[6:].strip())
 144         return cases
 145
 146 def getCaseList (buildCfg, generator, module):
 147         build(buildCfg, generator, [module.binName])
 148         genCaseList(buildCfg, generator, module, "txt")
 149         return readCaseList(getCaseListPath(buildCfg, module, "txt"))
 150
 151 def readPatternList (filename):
 152         ptrns = []
 153         with open(filename, 'rb') as f:
 154                 for line in f:
 155                         line = line.strip()
 156                         if len(line) > 0 and line[0] != '#':
 157                                 ptrns.append(line)
 158         return ptrns
 159
 160 def applyPatterns (caseList, patterns, filename, op):
 161         matched                 = set()
 162         errors                  = []
 163         curList                 = copy(caseList)
 164         trivialPtrns    = [p for p in patterns if p.find('*') < 0]
 165         regularPtrns    = [p for p in patterns if p.find('*') >= 0]
 166
 167         # Apply trivial (just case paths)
 168         allCasesSet             = set(caseList)
 169         for path in trivialPtrns:
 170                 if path in allCasesSet:
 171                         if path in matched:
 172                                 errors.append((path, "Same case specified more than once"))
 173                         matched.add(path)
 174                 else:
 175                         errors.append((path, "Test case not found"))
 176
 177         curList = [c for c in curList if c not in matched]
 178
 179         for pattern in regularPtrns:
 180                 matchedThisPtrn = set()
 181
 182                 for case in curList:
 183                         if fnmatch(case, pattern):
 184                                 matchedThisPtrn.add(case)
 185
 186                 if len(matchedThisPtrn) == 0:
 187                         errors.append((pattern, "Pattern didn't match any cases"))
 188
 189                 matched = matched | matchedThisPtrn
 190                 curList = [c for c in curList if c not in matched]
 191
 192         for pattern, reason in errors:
 193                 print "ERROR: %s: %s" % (reason, pattern)
 194
 195         if len(errors) > 0:
 196                 die("Found %s invalid patterns while processing file %s" % (len(errors), filename))
 197
 198         return [c for c in caseList if op(c in matched)]
 199
 200 def applyInclude (caseList, patterns, filename):
 201         return applyPatterns(caseList, patterns, filename, lambda b: b)
 202
 203 def applyExclude (caseList, patterns, filename):
 204         return applyPatterns(caseList, patterns, filename, lambda b: not b)
 205
 206 def readPatternLists (mustpass):
 207         lists = {}
 208         for package in mustpass.packages:
 209                 for cfg in package.configurations:
 210                         for filter in cfg.filters:
 211                                 if not filter.filename in lists:
 212                                         lists[filter.filename] = readPatternList(os.path.join(getSrcDir(mustpass), filter.filename))
 213         return lists
 214
 215 def applyFilters (caseList, patternLists, filters):
 216         res = copy(caseList)
 217         for filter in filters:
 218                 ptrnList = patternLists[filter.filename]
 219                 if filter.type == Filter.TYPE_INCLUDE:
 220                         res = applyInclude(res, ptrnList, filter.filename)
 221                 else:
 222                         assert filter.type == Filter.TYPE_EXCLUDE
 223                         res = applyExclude(res, ptrnList, filter.filename)
 224         return res
 225
 226 def appendToHierarchy (root, casePath):
 227         def findChild (node, name):
 228                 for child in node.children:
 229                         if child.name == name:
 230                                 return child
 231                 return None
 232
 233         curNode         = root
 234         components      = casePath.split('.')
 235
 236         for component in components[:-1]:
 237                 nextNode = findChild(curNode, component)
 238                 if not nextNode:
 239                         nextNode = TestGroup(component)
 240                         curNode.children.append(nextNode)
 241                 curNode = nextNode
 242
 243         if not findChild(curNode, components[-1]):
 244                 curNode.children.append(TestCase(components[-1]))
 245
 246 def buildTestHierachy (caseList):
 247         root = TestRoot()
 248         for case in caseList:
 249                 appendToHierarchy(root, case)
 250         return root
 251
 252 def buildTestCaseMap (root):
 253         caseMap = {}
 254
 255         def recursiveBuild (curNode, prefix):
 256                 curPath = prefix + curNode.name
 257                 if isinstance(curNode, TestCase):
 258                         caseMap[curPath] = curNode
 259                 else:
 260                         for child in curNode.children:
 261                                 recursiveBuild(child, curPath + '.')
 262
 263         for child in root.children:
 264                 recursiveBuild(child, '')
 265
 266         return caseMap
 267
 268 def include (filename):
 269         return Filter(Filter.TYPE_INCLUDE, filename)
 270
 271 def exclude (filename):
 272         return Filter(Filter.TYPE_EXCLUDE, filename)
 273
 274 def insertXMLHeaders (mustpass, doc):
 275         if mustpass.project.copyright != None:
 276                 doc.insert(0, ElementTree.Comment(mustpass.project.copyright))
 277         doc.insert(1, ElementTree.Comment(GENERATED_FILE_WARNING))
 278
 279 def prettifyXML (doc):
 280         uglyString      = ElementTree.tostring(doc, 'utf-8')
 281         reparsed        = minidom.parseString(uglyString)
 282         return reparsed.toprettyxml(indent='\t', encoding='utf-8')
 283
 284 def genSpecXML (mustpass):
 285         mustpassElem = ElementTree.Element("Mustpass", version = mustpass.version)
 286         insertXMLHeaders(mustpass, mustpassElem)
 287
 288         for package in mustpass.packages:
 289                 packageElem = ElementTree.SubElement(mustpassElem, "TestPackage", name = package.module.name)
 290
 291                 for config in package.configurations:
 292                         configElem = ElementTree.SubElement(packageElem, "Configuration",
 293                                                                                                 name                    = config.name,
 294                                                                                                 caseListFile    = getCaseListFileName(package, config),
 295                                                                                                 commandLine             = getCommandLine(config))
 296
 297         return mustpassElem
 298
 299 def addOptionElement (parent, optionName, optionValue):
 300         ElementTree.SubElement(parent, "option", name=optionName, value=optionValue)
 301
 302 def genAndroidTestXml (mustpass):
 303         RUNNER_CLASS = "com.drawelements.deqp.runner.DeqpTestRunner"
 304         configElement = ElementTree.Element("configuration")
 305
 306         for package in mustpass.packages:
 307                 for config in package.configurations:
 308                         testElement = ElementTree.SubElement(configElement, "test")
 309                         testElement.set("class", RUNNER_CLASS)
 310                         addOptionElement(testElement, "deqp-package", package.module.name)
 311                         addOptionElement(testElement, "deqp-caselist-file", getCaseListFileName(package,config))
 312                         # \todo [2015-10-16 kalle]: Replace with just command line? - requires simplifications in the runner/tests as well.
 313                         if config.glconfig != None:
 314                                 addOptionElement(testElement, "deqp-gl-config-name", config.glconfig)
 315
 316                         if config.surfacetype != None:
 317                                 addOptionElement(testElement, "deqp-surface-type", config.surfacetype)
 318
 319                         if config.rotation != None:
 320                                 addOptionElement(testElement, "deqp-screen-rotation", config.rotation)
 321
 322                         if config.expectedRuntime != None:
 323                                 addOptionElement(testElement, "runtime-hint", config.expectedRuntime)
 324
 325         insertXMLHeaders(mustpass, configElement)
 326
 327         return configElement
 328
 329 def genMustpass (mustpass, moduleCaseLists):
 330         print "Generating mustpass '%s'" % mustpass.version
 331
 332         patternLists = readPatternLists(mustpass)
 333
 334         for package in mustpass.packages:
 335                 allCasesInPkg   = moduleCaseLists[package.module]
 336
 337                 for config in package.configurations:
 338                         filtered        = applyFilters(allCasesInPkg, patternLists, config.filters)
 339                         dstFile         = getDstCaseListPath(mustpass, package, config)
 340
 341                         print "  Writing deqp caselist: " + dstFile
 342                         writeFile(dstFile, "\n".join(filtered) + "\n")
 343
 344         specXML                 = genSpecXML(mustpass)
 345         specFilename    = os.path.join(mustpass.project.path, mustpass.version, "mustpass.xml")
 346
 347         print "  Writing spec: " + specFilename
 348         writeFile(specFilename, prettifyXML(specXML))
 349
 350         # TODO: Which is the best selector mechanism?
 351         if (mustpass.version == "master"):
 352                 androidTestXML          = genAndroidTestXml(mustpass)
 353                 androidTestFilename     = os.path.join(mustpass.project.path, "AndroidTest.xml")
 354
 355                 print "  Writing AndroidTest.xml: " + androidTestFilename
 356                 writeFile(androidTestFilename, prettifyXML(androidTestXML))
 357
 358         print "Done!"
 359
 360 def genMustpassLists (mustpassLists, generator, buildCfg):
 361         moduleCaseLists = {}
 362
 363         # Getting case lists involves invoking build, so we want to cache the results
 364         for mustpass in mustpassLists:
 365                 for package in mustpass.packages:
 366                         if not package.module in moduleCaseLists:
 367                                 moduleCaseLists[package.module] = getCaseList(buildCfg, generator, package.module)
 368
 369         for mustpass in mustpassLists:
 370                 genMustpass(mustpass, moduleCaseLists)