scripts/mustpass.py

   1 # -*- coding: utf-8 -*-
   2
   3 #-------------------------------------------------------------------------
   4 # drawElements Quality Program utilities
   5 # --------------------------------------
   6 #
   7 # Copyright 2016 The Android Open Source Project
   8 #
   9 # Licensed under the Apache License, Version 2.0 (the "License");
  10 # you may not use this file except in compliance with the License.
  11 # You may obtain a copy of the License at
  12 #
  13 #      http://www.apache.org/licenses/LICENSE-2.0
  14 #
  15 # Unless required by applicable law or agreed to in writing, software
  16 # distributed under the License is distributed on an "AS IS" BASIS,
  17 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  18 # See the License for the specific language governing permissions and
  19 # limitations under the License.
  20 #
  21 #-------------------------------------------------------------------------
  22
  23 from build.common import *
  24 from build.config import ANY_GENERATOR
  25 from build.build import build
  26 from build_caselists import Module, getModuleByName, getBuildConfig, genCaseList, getCaseListPath, DEFAULT_BUILD_DIR, DEFAULT_TARGET
  27 from fnmatch import fnmatch
  28 from copy import copy
  29
  30 import xml.etree.cElementTree as ElementTree
  31 import xml.dom.minidom as minidom
  32
  33 APK_NAME                = "com.drawelements.deqp.apk"
  34
  35 GENERATED_FILE_WARNING = """
  36      This file has been automatically generated. Edit with caution.
  37      """
  38
  39 class Project:
  40         def __init__ (self, path, copyright = None):
  41                 self.path               = path
  42                 self.copyright  = copyright
  43
  44 class Configuration:
  45         def __init__ (self, name, filters, glconfig = None, rotation = None, surfacetype = None, required = False, runtime = None):
  46                 self.name                               = name
  47                 self.glconfig                   = glconfig
  48                 self.rotation                   = rotation
  49                 self.surfacetype                = surfacetype
  50                 self.required                   = required
  51                 self.filters                    = filters
  52                 self.expectedRuntime    = runtime
  53
  54 class Package:
  55         def __init__ (self, module, configurations):
  56                 self.module                     = module
  57                 self.configurations     = configurations
  58
  59 class Mustpass:
  60         def __init__ (self, project, version, packages):
  61                 self.project    = project
  62                 self.version    = version
  63                 self.packages   = packages
  64
  65 class Filter:
  66         TYPE_INCLUDE = 0
  67         TYPE_EXCLUDE = 1
  68
  69         def __init__ (self, type, filename):
  70                 self.type               = type
  71                 self.filename   = filename
  72
  73 class TestRoot:
  74         def __init__ (self):
  75                 self.children   = []
  76
  77 class TestGroup:
  78         def __init__ (self, name):
  79                 self.name               = name
  80                 self.children   = []
  81
  82 class TestCase:
  83         def __init__ (self, name):
  84                 self.name                       = name
  85                 self.configurations     = []
  86
  87 class GLESVersion:
  88         def __init__(self, major, minor):
  89                 self.major = major
  90                 self.minor = minor
  91
  92         def encode (self):
  93                 return (self.major << 16) | (self.minor)
  94
  95 def getModuleGLESVersion (module):
  96         versions = {
  97                 'dEQP-EGL':             GLESVersion(2,0),
  98                 'dEQP-GLES2':   GLESVersion(2,0),
  99                 'dEQP-GLES3':   GLESVersion(3,0),
 100                 'dEQP-GLES31':  GLESVersion(3,1)
 101         }
 102         return versions[module.name] if module.name in versions else None
 103
 104 def getSrcDir (mustpass):
 105         return os.path.join(mustpass.project.path, mustpass.version, "src")
 106
 107 def getTmpDir (mustpass):
 108         return os.path.join(mustpass.project.path, mustpass.version, "tmp")
 109
 110 def getModuleShorthand (module):
 111         assert module.name[:5] == "dEQP-"
 112         return module.name[5:].lower()
 113
 114 def getCaseListFileName (package, configuration):
 115         return "%s-%s.txt" % (getModuleShorthand(package.module), configuration.name)
 116
 117 def getDstCaseListPath (mustpass, package, configuration):
 118         return os.path.join(mustpass.project.path, mustpass.version, getCaseListFileName(package, configuration))
 119
 120 def getCTSPackageName (package):
 121         return "com.drawelements.deqp." + getModuleShorthand(package.module)
 122
 123 def getCommandLine (config):
 124         cmdLine = ""
 125
 126         if config.glconfig != None:
 127                 cmdLine += "--deqp-gl-config-name=%s " % config.glconfig
 128
 129         if config.rotation != None:
 130                 cmdLine += "--deqp-screen-rotation=%s " % config.rotation
 131
 132         if config.surfacetype != None:
 133                 cmdLine += "--deqp-surface-type=%s " % config.surfacetype
 134
 135         cmdLine += "--deqp-watchdog=enable"
 136
 137         return cmdLine
 138
 139 def readCaseList (filename):
 140         cases = []
 141         with open(filename, 'rb') as f:
 142                 for line in f:
 143                         if line[:6] == "TEST: ":
 144                                 cases.append(line[6:].strip())
 145         return cases
 146
 147 def getCaseList (buildCfg, generator, module):
 148         build(buildCfg, generator, [module.binName])
 149         genCaseList(buildCfg, generator, module, "txt")
 150         return readCaseList(getCaseListPath(buildCfg, module, "txt"))
 151
 152 def readPatternList (filename):
 153         ptrns = []
 154         with open(filename, 'rb') as f:
 155                 for line in f:
 156                         line = line.strip()
 157                         if len(line) > 0 and line[0] != '#':
 158                                 ptrns.append(line)
 159         return ptrns
 160
 161 def applyPatterns (caseList, patterns, filename, op):
 162         matched                 = set()
 163         errors                  = []
 164         curList                 = copy(caseList)
 165         trivialPtrns    = [p for p in patterns if p.find('*') < 0]
 166         regularPtrns    = [p for p in patterns if p.find('*') >= 0]
 167
 168         # Apply trivial (just case paths)
 169         allCasesSet             = set(caseList)
 170         for path in trivialPtrns:
 171                 if path in allCasesSet:
 172                         if path in matched:
 173                                 errors.append((path, "Same case specified more than once"))
 174                         matched.add(path)
 175                 else:
 176                         errors.append((path, "Test case not found"))
 177
 178         curList = [c for c in curList if c not in matched]
 179
 180         for pattern in regularPtrns:
 181                 matchedThisPtrn = set()
 182
 183                 for case in curList:
 184                         if fnmatch(case, pattern):
 185                                 matchedThisPtrn.add(case)
 186
 187                 if len(matchedThisPtrn) == 0:
 188                         errors.append((pattern, "Pattern didn't match any cases"))
 189
 190                 matched = matched | matchedThisPtrn
 191                 curList = [c for c in curList if c not in matched]
 192
 193         for pattern, reason in errors:
 194                 print "ERROR: %s: %s" % (reason, pattern)
 195
 196         if len(errors) > 0:
 197                 die("Found %s invalid patterns while processing file %s" % (len(errors), filename))
 198
 199         return [c for c in caseList if op(c in matched)]
 200
 201 def applyInclude (caseList, patterns, filename):
 202         return applyPatterns(caseList, patterns, filename, lambda b: b)
 203
 204 def applyExclude (caseList, patterns, filename):
 205         return applyPatterns(caseList, patterns, filename, lambda b: not b)
 206
 207 def readPatternLists (mustpass):
 208         lists = {}
 209         for package in mustpass.packages:
 210                 for cfg in package.configurations:
 211                         for filter in cfg.filters:
 212                                 if not filter.filename in lists:
 213                                         lists[filter.filename] = readPatternList(os.path.join(getSrcDir(mustpass), filter.filename))
 214         return lists
 215
 216 def applyFilters (caseList, patternLists, filters):
 217         res = copy(caseList)
 218         for filter in filters:
 219                 ptrnList = patternLists[filter.filename]
 220                 if filter.type == Filter.TYPE_INCLUDE:
 221                         res = applyInclude(res, ptrnList, filter.filename)
 222                 else:
 223                         assert filter.type == Filter.TYPE_EXCLUDE
 224                         res = applyExclude(res, ptrnList, filter.filename)
 225         return res
 226
 227 def appendToHierarchy (root, casePath):
 228         def findChild (node, name):
 229                 for child in node.children:
 230                         if child.name == name:
 231                                 return child
 232                 return None
 233
 234         curNode         = root
 235         components      = casePath.split('.')
 236
 237         for component in components[:-1]:
 238                 nextNode = findChild(curNode, component)
 239                 if not nextNode:
 240                         nextNode = TestGroup(component)
 241                         curNode.children.append(nextNode)
 242                 curNode = nextNode
 243
 244         if not findChild(curNode, components[-1]):
 245                 curNode.children.append(TestCase(components[-1]))
 246
 247 def buildTestHierachy (caseList):
 248         root = TestRoot()
 249         for case in caseList:
 250                 appendToHierarchy(root, case)
 251         return root
 252
 253 def buildTestCaseMap (root):
 254         caseMap = {}
 255
 256         def recursiveBuild (curNode, prefix):
 257                 curPath = prefix + curNode.name
 258                 if isinstance(curNode, TestCase):
 259                         caseMap[curPath] = curNode
 260                 else:
 261                         for child in curNode.children:
 262                                 recursiveBuild(child, curPath + '.')
 263
 264         for child in root.children:
 265                 recursiveBuild(child, '')
 266
 267         return caseMap
 268
 269 def include (filename):
 270         return Filter(Filter.TYPE_INCLUDE, filename)
 271
 272 def exclude (filename):
 273         return Filter(Filter.TYPE_EXCLUDE, filename)
 274
 275 def insertXMLHeaders (mustpass, doc):
 276         if mustpass.project.copyright != None:
 277                 doc.insert(0, ElementTree.Comment(mustpass.project.copyright))
 278         doc.insert(1, ElementTree.Comment(GENERATED_FILE_WARNING))
 279
 280 def prettifyXML (doc):
 281         uglyString      = ElementTree.tostring(doc, 'utf-8')
 282         reparsed        = minidom.parseString(uglyString)
 283         return reparsed.toprettyxml(indent='\t', encoding='utf-8')
 284
 285 def genCTSPackageXML (mustpass, package, root):
 286         def isLeafGroup (testGroup):
 287                 numGroups       = 0
 288                 numTests        = 0
 289
 290                 for child in testGroup.children:
 291                         if isinstance(child, TestCase):
 292                                 numTests += 1
 293                         else:
 294                                 numGroups += 1
 295
 296                 assert numGroups + numTests > 0
 297
 298                 if numGroups > 0 and numTests > 0:
 299                         die("Mixed groups and cases in %s" % testGroup.name)
 300
 301                 return numGroups == 0
 302
 303         def makeConfiguration (parentElem, config):
 304                 attributes = {}
 305
 306                 if config.glconfig != None:
 307                         attributes['glconfig'] = config.glconfig
 308
 309                 if config.rotation != None:
 310                         attributes['rotation'] = config.rotation
 311
 312                 if config.surfacetype != None:
 313                         attributes['surfacetype'] = config.surfacetype
 314
 315                 return ElementTree.SubElement(parentElem, "TestInstance", attributes)
 316
 317         def makeTestCase (parentElem, testCase):
 318                 caseElem = ElementTree.SubElement(parentElem, "Test", name=testCase.name)
 319                 for config in testCase.configurations:
 320                         makeConfiguration(caseElem, config)
 321                 return caseElem
 322
 323         def makeTestGroup (parentElem, testGroup):
 324                 groupElem = ElementTree.SubElement(parentElem, "TestCase" if isLeafGroup(testGroup) else "TestSuite", name=testGroup.name)
 325                 for child in testGroup.children:
 326                         if isinstance(child, TestCase):
 327                                 makeTestCase(groupElem, child)
 328                         else:
 329                                 makeTestGroup(groupElem, child)
 330                 return groupElem
 331
 332         pkgElem = ElementTree.Element("TestPackage",
 333                                                                   name                          = package.module.name,
 334                                                                   appPackageName        = getCTSPackageName(package),
 335                                                                   testType                      = "deqpTest")
 336
 337         pkgElem.set("xmlns:deqp", "http://drawelements.com/deqp")
 338         insertXMLHeaders(mustpass, pkgElem)
 339
 340         glesVersion = getModuleGLESVersion(package.module)
 341
 342         if glesVersion != None:
 343                 pkgElem.set("deqp:glesVersion", str(glesVersion.encode()))
 344
 345         for child in root.children:
 346                 makeTestGroup(pkgElem, child)
 347
 348         return pkgElem
 349
 350 def genSpecXML (mustpass):
 351         mustpassElem = ElementTree.Element("Mustpass", version = mustpass.version)
 352         insertXMLHeaders(mustpass, mustpassElem)
 353
 354         for package in mustpass.packages:
 355                 packageElem = ElementTree.SubElement(mustpassElem, "TestPackage", name = package.module.name)
 356
 357                 for config in package.configurations:
 358                         configElem = ElementTree.SubElement(packageElem, "Configuration",
 359                                                                                                 name                    = config.name,
 360                                                                                                 caseListFile    = getCaseListFileName(package, config),
 361                                                                                                 commandLine             = getCommandLine(config))
 362
 363         return mustpassElem
 364
 365 def addOptionElement (parent, optionName, optionValue):
 366         ElementTree.SubElement(parent, "option", name=optionName, value=optionValue)
 367
 368 def genAndroidTestXml (mustpass):
 369         RUNNER_CLASS = "com.drawelements.deqp.runner.DeqpTestRunner"
 370         configElement = ElementTree.Element("configuration")
 371
 372         for package in mustpass.packages:
 373                 for config in package.configurations:
 374                         testElement = ElementTree.SubElement(configElement, "test")
 375                         testElement.set("class", RUNNER_CLASS)
 376                         addOptionElement(testElement, "deqp-package", package.module.name)
 377                         addOptionElement(testElement, "deqp-caselist-file", getCaseListFileName(package,config))
 378                         # \todo [2015-10-16 kalle]: Replace with just command line? - requires simplifications in the runner/tests as well.
 379                         if config.glconfig != None:
 380                                 addOptionElement(testElement, "deqp-gl-config-name", config.glconfig)
 381
 382                         if config.surfacetype != None:
 383                                 addOptionElement(testElement, "deqp-surface-type", config.surfacetype)
 384
 385                         if config.rotation != None:
 386                                 addOptionElement(testElement, "deqp-screen-rotation", config.rotation)
 387
 388                         if config.expectedRuntime != None:
 389                                 addOptionElement(testElement, "runtime-hint", config.expectedRuntime)
 390
 391                         if config.required:
 392                                 addOptionElement(testElement, "deqp-config-required", "true")
 393
 394         insertXMLHeaders(mustpass, configElement)
 395
 396         return configElement
 397
 398 def genMustpass (mustpass, moduleCaseLists):
 399         print "Generating mustpass '%s'" % mustpass.version
 400
 401         patternLists = readPatternLists(mustpass)
 402
 403         for package in mustpass.packages:
 404                 allCasesInPkg           = moduleCaseLists[package.module]
 405                 matchingByConfig        = {}
 406                 allMatchingSet          = set()
 407
 408                 for config in package.configurations:
 409                         filtered        = applyFilters(allCasesInPkg, patternLists, config.filters)
 410                         dstFile         = getDstCaseListPath(mustpass, package, config)
 411
 412                         print "  Writing deqp caselist: " + dstFile
 413                         writeFile(dstFile, "\n".join(filtered) + "\n")
 414
 415                         matchingByConfig[config]        = filtered
 416                         allMatchingSet                          = allMatchingSet | set(filtered)
 417
 418                 allMatchingCases        = [c for c in allCasesInPkg if c in allMatchingSet] # To preserve ordering
 419                 root                            = buildTestHierachy(allMatchingCases)
 420                 testCaseMap                     = buildTestCaseMap(root)
 421
 422                 for config in package.configurations:
 423                         for case in matchingByConfig[config]:
 424                                 testCaseMap[case].configurations.append(config)
 425
 426                 # NOTE: CTS v2 does not need package XML files. Remove when transition is complete.
 427                 packageXml      = genCTSPackageXML(mustpass, package, root)
 428                 xmlFilename     = os.path.join(mustpass.project.path, mustpass.version, getCTSPackageName(package) + ".xml")
 429
 430                 print "  Writing CTS caselist: " + xmlFilename
 431                 writeFile(xmlFilename, prettifyXML(packageXml))
 432
 433         specXML                 = genSpecXML(mustpass)
 434         specFilename    = os.path.join(mustpass.project.path, mustpass.version, "mustpass.xml")
 435
 436         print "  Writing spec: " + specFilename
 437         writeFile(specFilename, prettifyXML(specXML))
 438
 439         # TODO: Which is the best selector mechanism?
 440         if (mustpass.version == "master"):
 441                 androidTestXML          = genAndroidTestXml(mustpass)
 442                 androidTestFilename     = os.path.join(mustpass.project.path, "AndroidTest.xml")
 443
 444                 print "  Writing AndroidTest.xml: " + androidTestFilename
 445                 writeFile(androidTestFilename, prettifyXML(androidTestXML))
 446
 447         print "Done!"
 448
 449 def genMustpassLists (mustpassLists, generator, buildCfg):
 450         moduleCaseLists = {}
 451
 452         # Getting case lists involves invoking build, so we want to cache the results
 453         for mustpass in mustpassLists:
 454                 for package in mustpass.packages:
 455                         if not package.module in moduleCaseLists:
 456                                 moduleCaseLists[package.module] = getCaseList(buildCfg, generator, package.module)
 457
 458         for mustpass in mustpassLists:
 459                 genMustpass(mustpass, moduleCaseLists)