scripts/mustpass.py

   1 # -*- coding: utf-8 -*-
   2
   3 #-------------------------------------------------------------------------
   4 # drawElements Quality Program utilities
   5 # --------------------------------------
   6 #
   7 # Copyright 2016 The Android Open Source Project
   8 #
   9 # Licensed under the Apache License, Version 2.0 (the "License");
  10 # you may not use this file except in compliance with the License.
  11 # You may obtain a copy of the License at
  12 #
  13 #      http://www.apache.org/licenses/LICENSE-2.0
  14 #
  15 # Unless required by applicable law or agreed to in writing, software
  16 # distributed under the License is distributed on an "AS IS" BASIS,
  17 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  18 # See the License for the specific language governing permissions and
  19 # limitations under the License.
  20 #
  21 #-------------------------------------------------------------------------
  22
  23 from build.common import *
  24 from build.config import ANY_GENERATOR
  25 from build.build import build
  26 from build_caselists import Module, getModuleByName, getBuildConfig, genCaseList, getCaseListPath, DEFAULT_BUILD_DIR, DEFAULT_TARGET
  27 from fnmatch import fnmatch
  28 from copy import copy
  29
  30 import xml.etree.cElementTree as ElementTree
  31 import xml.dom.minidom as minidom
  32
  33 APK_NAME                = "com.drawelements.deqp.apk"
  34
  35 GENERATED_FILE_WARNING = """
  36      This file has been automatically generated. Edit with caution.
  37      """
  38
  39 class Project:
  40         def __init__ (self, path, copyright = None):
  41                 self.path               = path
  42                 self.copyright  = copyright
  43
  44 class Configuration:
  45         def __init__ (self, name, filters, glconfig = None, rotation = None, surfacetype = None, required = False, runtime = None):
  46                 self.name                               = name
  47                 self.glconfig                   = glconfig
  48                 self.rotation                   = rotation
  49                 self.surfacetype                = surfacetype
  50                 self.required                   = required
  51                 self.filters                    = filters
  52                 self.expectedRuntime    = runtime
  53
  54 class Package:
  55         def __init__ (self, module, configurations):
  56                 self.module                     = module
  57                 self.configurations     = configurations
  58
  59 class Mustpass:
  60         def __init__ (self, project, version, packages):
  61                 self.project    = project
  62                 self.version    = version
  63                 self.packages   = packages
  64
  65 class Filter:
  66         TYPE_INCLUDE = 0
  67         TYPE_EXCLUDE = 1
  68
  69         def __init__ (self, type, filename):
  70                 self.type               = type
  71                 self.filename   = filename
  72
  73 class TestRoot:
  74         def __init__ (self):
  75                 self.children   = []
  76
  77 class TestGroup:
  78         def __init__ (self, name):
  79                 self.name               = name
  80                 self.children   = []
  81
  82 class TestCase:
  83         def __init__ (self, name):
  84                 self.name                       = name
  85                 self.configurations     = []
  86
  87 class GLESVersion:
  88         def __init__(self, major, minor):
  89                 self.major = major
  90                 self.minor = minor
  91
  92         def encode (self):
  93                 return (self.major << 16) | (self.minor)
  94
  95 def getModuleGLESVersion (module):
  96         versions = {
  97                 'dEQP-EGL':             GLESVersion(2,0),
  98                 'dEQP-GLES2':   GLESVersion(2,0),
  99                 'dEQP-GLES3':   GLESVersion(3,0),
 100                 'dEQP-GLES31':  GLESVersion(3,1)
 101         }
 102         return versions[module.name] if module.name in versions else None
 103
 104 def getSrcDir (mustpass):
 105         return os.path.join(mustpass.project.path, mustpass.version, "src")
 106
 107 def getTmpDir (mustpass):
 108         return os.path.join(mustpass.project.path, mustpass.version, "tmp")
 109
 110 def getModuleShorthand (module):
 111         assert module.name[:5] == "dEQP-"
 112         return module.name[5:].lower()
 113
 114 def getCaseListFileName (package, configuration):
 115         return "%s-%s.txt" % (getModuleShorthand(package.module), configuration.name)
 116
 117 def getDstCaseListPath (mustpass, package, configuration):
 118         return os.path.join(mustpass.project.path, mustpass.version, getCaseListFileName(package, configuration))
 119
 120 def getCTSPackageName (package):
 121         return "com.drawelements.deqp." + getModuleShorthand(package.module)
 122
 123 def getCommandLine (config):
 124         cmdLine = ""
 125
 126         if config.glconfig != None:
 127                 cmdLine += "--deqp-gl-config-name=%s " % config.glconfig
 128
 129         if config.rotation != None:
 130                 cmdLine += "--deqp-screen-rotation=%s " % config.rotation
 131
 132         if config.surfacetype != None:
 133                 cmdLine += "--deqp-surface-type=%s " % config.surfacetype
 134
 135         cmdLine += "--deqp-watchdog=enable"
 136
 137         return cmdLine
 138
 139 def readCaseList (filename):
 140         cases = []
 141         with open(filename, 'rb') as f:
 142                 for line in f:
 143                         if line[:6] == "TEST: ":
 144                                 cases.append(line[6:].strip())
 145         return cases
 146
 147 def getCaseList (buildCfg, generator, module):
 148         build(buildCfg, generator, [module.binName])
 149         genCaseList(buildCfg, generator, module, "txt")
 150         return readCaseList(getCaseListPath(buildCfg, module, "txt"))
 151
 152 def readPatternList (filename):
 153         ptrns = []
 154         with open(filename, 'rb') as f:
 155                 for line in f:
 156                         line = line.strip()
 157                         if len(line) > 0 and line[0] != '#':
 158                                 ptrns.append(line)
 159         return ptrns
 160
 161 def applyPatterns (caseList, patterns, filename, op):
 162         matched                 = set()
 163         errors                  = []
 164         curList                 = copy(caseList)
 165         trivialPtrns    = [p for p in patterns if p.find('*') < 0]
 166         regularPtrns    = [p for p in patterns if p.find('*') >= 0]
 167
 168         # Apply trivial (just case paths)
 169         allCasesSet             = set(caseList)
 170         for path in trivialPtrns:
 171                 if path in allCasesSet:
 172                         if path in matched:
 173                                 errors.append((path, "Same case specified more than once"))
 174                         matched.add(path)
 175                 else:
 176                         errors.append((path, "Test case not found"))
 177
 178         curList = [c for c in curList if c not in matched]
 179
 180         for pattern in regularPtrns:
 181                 matchedThisPtrn = set()
 182
 183                 for case in curList:
 184                         if fnmatch(case, pattern):
 185                                 matchedThisPtrn.add(case)
 186
 187                 if len(matchedThisPtrn) == 0:
 188                         errors.append((pattern, "Pattern didn't match any cases"))
 189
 190                 matched = matched | matchedThisPtrn
 191                 curList = [c for c in curList if c not in matched]
 192
 193         for pattern, reason in errors:
 194                 print "ERROR: %s: %s" % (reason, pattern)
 195
 196         if len(errors) > 0:
 197                 die("Found %s invalid patterns while processing file %s" % (len(errors), filename))
 198
 199         return [c for c in caseList if op(c in matched)]
 200
 201 def applyInclude (caseList, patterns, filename):
 202         return applyPatterns(caseList, patterns, filename, lambda b: b)
 203
 204 def applyExclude (caseList, patterns, filename):
 205         return applyPatterns(caseList, patterns, filename, lambda b: not b)
 206
 207 def readPatternLists (mustpass):
 208         lists = {}
 209         for package in mustpass.packages:
 210                 for cfg in package.configurations:
 211                         for filter in cfg.filters:
 212                                 if not filter.filename in lists:
 213                                         lists[filter.filename] = readPatternList(os.path.join(getSrcDir(mustpass), filter.filename))
 214         return lists
 215
 216 def applyFilters (caseList, patternLists, filters):
 217         res = copy(caseList)
 218         for filter in filters:
 219                 ptrnList = patternLists[filter.filename]
 220                 if filter.type == Filter.TYPE_INCLUDE:
 221                         res = applyInclude(res, ptrnList, filter.filename)
 222                 else:
 223                         assert filter.type == Filter.TYPE_EXCLUDE
 224                         res = applyExclude(res, ptrnList, filter.filename)
 225         return res
 226
 227 def appendToHierarchy (root, casePath):
 228         def findChild (node, name):
 229                 for child in node.children:
 230                         if child.name == name:
 231                                 return child
 232                 return None
 233
 234         curNode         = root
 235         components      = casePath.split('.')
 236
 237         for component in components[:-1]:
 238                 nextNode = findChild(curNode, component)
 239                 if not nextNode:
 240                         nextNode = TestGroup(component)
 241                         curNode.children.append(nextNode)
 242                 curNode = nextNode
 243
 244         if not findChild(curNode, components[-1]):
 245                 curNode.children.append(TestCase(components[-1]))
 246
 247 def buildTestHierachy (caseList):
 248         root = TestRoot()
 249         for case in caseList:
 250                 appendToHierarchy(root, case)
 251         return root
 252
 253 def buildTestCaseMap (root):
 254         caseMap = {}
 255
 256         def recursiveBuild (curNode, prefix):
 257                 curPath = prefix + curNode.name
 258                 if isinstance(curNode, TestCase):
 259                         caseMap[curPath] = curNode
 260                 else:
 261                         for child in curNode.children:
 262                                 recursiveBuild(child, curPath + '.')
 263
 264         for child in root.children:
 265                 recursiveBuild(child, '')
 266
 267         return caseMap
 268
 269 def include (filename):
 270         return Filter(Filter.TYPE_INCLUDE, filename)
 271
 272 def exclude (filename):
 273         return Filter(Filter.TYPE_EXCLUDE, filename)
 274
 275 def insertXMLHeaders (mustpass, doc):
 276         if mustpass.project.copyright != None:
 277                 doc.insert(0, ElementTree.Comment(mustpass.project.copyright))
 278         doc.insert(1, ElementTree.Comment(GENERATED_FILE_WARNING))
 279
 280 def prettifyXML (doc):
 281         uglyString      = ElementTree.tostring(doc, 'utf-8')
 282         reparsed        = minidom.parseString(uglyString)
 283         return reparsed.toprettyxml(indent='\t', encoding='utf-8')
 284
 285 def genSpecXML (mustpass):
 286         mustpassElem = ElementTree.Element("Mustpass", version = mustpass.version)
 287         insertXMLHeaders(mustpass, mustpassElem)
 288
 289         for package in mustpass.packages:
 290                 packageElem = ElementTree.SubElement(mustpassElem, "TestPackage", name = package.module.name)
 291
 292                 for config in package.configurations:
 293                         configElem = ElementTree.SubElement(packageElem, "Configuration",
 294                                                                                                 name                    = config.name,
 295                                                                                                 caseListFile    = getCaseListFileName(package, config),
 296                                                                                                 commandLine             = getCommandLine(config))
 297
 298         return mustpassElem
 299
 300 def addOptionElement (parent, optionName, optionValue):
 301         ElementTree.SubElement(parent, "option", name=optionName, value=optionValue)
 302
 303 def genAndroidTestXml (mustpass):
 304         RUNNER_CLASS = "com.drawelements.deqp.runner.DeqpTestRunner"
 305         configElement = ElementTree.Element("configuration")
 306
 307         for package in mustpass.packages:
 308                 for config in package.configurations:
 309                         testElement = ElementTree.SubElement(configElement, "test")
 310                         testElement.set("class", RUNNER_CLASS)
 311                         addOptionElement(testElement, "deqp-package", package.module.name)
 312                         addOptionElement(testElement, "deqp-caselist-file", getCaseListFileName(package,config))
 313                         # \todo [2015-10-16 kalle]: Replace with just command line? - requires simplifications in the runner/tests as well.
 314                         if config.glconfig != None:
 315                                 addOptionElement(testElement, "deqp-gl-config-name", config.glconfig)
 316
 317                         if config.surfacetype != None:
 318                                 addOptionElement(testElement, "deqp-surface-type", config.surfacetype)
 319
 320                         if config.rotation != None:
 321                                 addOptionElement(testElement, "deqp-screen-rotation", config.rotation)
 322
 323                         if config.expectedRuntime != None:
 324                                 addOptionElement(testElement, "runtime-hint", config.expectedRuntime)
 325
 326                         if config.required:
 327                                 addOptionElement(testElement, "deqp-config-required", "true")
 328
 329         insertXMLHeaders(mustpass, configElement)
 330
 331         return configElement
 332
 333 def genMustpass (mustpass, moduleCaseLists):
 334         print "Generating mustpass '%s'" % mustpass.version
 335
 336         patternLists = readPatternLists(mustpass)
 337
 338         for package in mustpass.packages:
 339                 allCasesInPkg   = moduleCaseLists[package.module]
 340
 341                 for config in package.configurations:
 342                         filtered        = applyFilters(allCasesInPkg, patternLists, config.filters)
 343                         dstFile         = getDstCaseListPath(mustpass, package, config)
 344
 345                         print "  Writing deqp caselist: " + dstFile
 346                         writeFile(dstFile, "\n".join(filtered) + "\n")
 347
 348         specXML                 = genSpecXML(mustpass)
 349         specFilename    = os.path.join(mustpass.project.path, mustpass.version, "mustpass.xml")
 350
 351         print "  Writing spec: " + specFilename
 352         writeFile(specFilename, prettifyXML(specXML))
 353
 354         # TODO: Which is the best selector mechanism?
 355         if (mustpass.version == "master"):
 356                 androidTestXML          = genAndroidTestXml(mustpass)
 357                 androidTestFilename     = os.path.join(mustpass.project.path, "AndroidTest.xml")
 358
 359                 print "  Writing AndroidTest.xml: " + androidTestFilename
 360                 writeFile(androidTestFilename, prettifyXML(androidTestXML))
 361
 362         print "Done!"
 363
 364 def genMustpassLists (mustpassLists, generator, buildCfg):
 365         moduleCaseLists = {}
 366
 367         # Getting case lists involves invoking build, so we want to cache the results
 368         for mustpass in mustpassLists:
 369                 for package in mustpass.packages:
 370                         if not package.module in moduleCaseLists:
 371                                 moduleCaseLists[package.module] = getCaseList(buildCfg, generator, package.module)
 372
 373         for mustpass in mustpassLists:
 374                 genMustpass(mustpass, moduleCaseLists)