scripts/mustpass.py

   1 # -*- coding: utf-8 -*-
   2
   3 #-------------------------------------------------------------------------
   4 # drawElements Quality Program utilities
   5 # --------------------------------------
   6 #
   7 # Copyright 2016 The Android Open Source Project
   8 #
   9 # Licensed under the Apache License, Version 2.0 (the "License");
  10 # you may not use this file except in compliance with the License.
  11 # You may obtain a copy of the License at
  12 #
  13 #      http://www.apache.org/licenses/LICENSE-2.0
  14 #
  15 # Unless required by applicable law or agreed to in writing, software
  16 # distributed under the License is distributed on an "AS IS" BASIS,
  17 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  18 # See the License for the specific language governing permissions and
  19 # limitations under the License.
  20 #
  21 #-------------------------------------------------------------------------
  22
  23 from build.common import *
  24 from build.config import ANY_GENERATOR
  25 from build.build import build
  26 from build_caselists import Module, getModuleByName, getBuildConfig, genCaseList, getCaseListPath, DEFAULT_BUILD_DIR, DEFAULT_TARGET
  27 from fnmatch import fnmatch
  28 from copy import copy
  29
  30 import xml.etree.cElementTree as ElementTree
  31 import xml.dom.minidom as minidom
  32
  33 APK_NAME                = "com.drawelements.deqp.apk"
  34
  35 GENERATED_FILE_WARNING = """
  36      This file has been automatically generated. Edit with caution.
  37      """
  38
  39 class Project:
  40         def __init__ (self, path, copyright = None):
  41                 self.path               = path
  42                 self.copyright  = copyright
  43
  44 class Configuration:
  45         def __init__ (self, name, filters, glconfig = None, rotation = None, surfacetype = None, runtime = None):
  46                 self.name                        = name
  47                 self.glconfig            = glconfig
  48                 self.rotation            = rotation
  49                 self.surfacetype         = surfacetype
  50                 self.filters             = filters
  51                 self.expectedRuntime = runtime
  52
  53 class Package:
  54         def __init__ (self, module, configurations):
  55                 self.module                     = module
  56                 self.configurations     = configurations
  57
  58 class Mustpass:
  59         def __init__ (self, project, version, packages):
  60                 self.project    = project
  61                 self.version    = version
  62                 self.packages   = packages
  63
  64 class Filter:
  65         TYPE_INCLUDE = 0
  66         TYPE_EXCLUDE = 1
  67
  68         def __init__ (self, type, filename):
  69                 self.type               = type
  70                 self.filename   = filename
  71
  72 class TestRoot:
  73         def __init__ (self):
  74                 self.children   = []
  75
  76 class TestGroup:
  77         def __init__ (self, name):
  78                 self.name               = name
  79                 self.children   = []
  80
  81 class TestCase:
  82         def __init__ (self, name):
  83                 self.name                       = name
  84                 self.configurations     = []
  85
  86 class GLESVersion:
  87         def __init__(self, major, minor):
  88                 self.major = major
  89                 self.minor = minor
  90
  91         def encode (self):
  92                 return (self.major << 16) | (self.minor)
  93
  94 def getModuleGLESVersion (module):
  95         versions = {
  96                 'dEQP-EGL':             GLESVersion(2,0),
  97                 'dEQP-GLES2':   GLESVersion(2,0),
  98                 'dEQP-GLES3':   GLESVersion(3,0),
  99                 'dEQP-GLES31':  GLESVersion(3,1)
 100         }
 101         return versions[module.name] if module.name in versions else None
 102
 103 def getSrcDir (mustpass):
 104         return os.path.join(mustpass.project.path, mustpass.version, "src")
 105
 106 def getTmpDir (mustpass):
 107         return os.path.join(mustpass.project.path, mustpass.version, "tmp")
 108
 109 def getModuleShorthand (module):
 110         assert module.name[:5] == "dEQP-"
 111         return module.name[5:].lower()
 112
 113 def getCaseListFileName (package, configuration):
 114         return "%s-%s.txt" % (getModuleShorthand(package.module), configuration.name)
 115
 116 def getDstCaseListPath (mustpass, package, configuration):
 117         return os.path.join(mustpass.project.path, mustpass.version, getCaseListFileName(package, configuration))
 118
 119 def getCTSPackageName (package):
 120         return "com.drawelements.deqp." + getModuleShorthand(package.module)
 121
 122 def getCommandLine (config):
 123         cmdLine = ""
 124
 125         if config.glconfig != None:
 126                 cmdLine += "--deqp-gl-config-name=%s " % config.glconfig
 127
 128         if config.rotation != None:
 129                 cmdLine += "--deqp-screen-rotation=%s " % config.rotation
 130
 131         if config.surfacetype != None:
 132                 cmdLine += "--deqp-surface-type=%s " % config.surfacetype
 133
 134         cmdLine += "--deqp-watchdog=enable"
 135
 136         return cmdLine
 137
 138 def readCaseList (filename):
 139         cases = []
 140         with open(filename, 'rb') as f:
 141                 for line in f:
 142                         if line[:6] == "TEST: ":
 143                                 cases.append(line[6:].strip())
 144         return cases
 145
 146 def getCaseList (buildCfg, generator, module):
 147         build(buildCfg, generator, [module.binName])
 148         genCaseList(buildCfg, generator, module, "txt")
 149         return readCaseList(getCaseListPath(buildCfg, module, "txt"))
 150
 151 def readPatternList (filename):
 152         ptrns = []
 153         with open(filename, 'rb') as f:
 154                 for line in f:
 155                         line = line.strip()
 156                         if len(line) > 0 and line[0] != '#':
 157                                 ptrns.append(line)
 158         return ptrns
 159
 160 def applyPatterns (caseList, patterns, filename, op):
 161         matched                 = set()
 162         errors                  = []
 163         curList                 = copy(caseList)
 164         trivialPtrns    = [p for p in patterns if p.find('*') < 0]
 165         regularPtrns    = [p for p in patterns if p.find('*') >= 0]
 166
 167         # Apply trivial (just case paths)
 168         allCasesSet             = set(caseList)
 169         for path in trivialPtrns:
 170                 if path in allCasesSet:
 171                         if path in matched:
 172                                 errors.append((path, "Same case specified more than once"))
 173                         matched.add(path)
 174                 else:
 175                         errors.append((path, "Test case not found"))
 176
 177         curList = [c for c in curList if c not in matched]
 178
 179         for pattern in regularPtrns:
 180                 matchedThisPtrn = set()
 181
 182                 for case in curList:
 183                         if fnmatch(case, pattern):
 184                                 matchedThisPtrn.add(case)
 185
 186                 if len(matchedThisPtrn) == 0:
 187                         errors.append((pattern, "Pattern didn't match any cases"))
 188
 189                 matched = matched | matchedThisPtrn
 190                 curList = [c for c in curList if c not in matched]
 191
 192         for pattern, reason in errors:
 193                 print "ERROR: %s: %s" % (reason, pattern)
 194
 195         if len(errors) > 0:
 196                 die("Found %s invalid patterns while processing file %s" % (len(errors), filename))
 197
 198         return [c for c in caseList if op(c in matched)]
 199
 200 def applyInclude (caseList, patterns, filename):
 201         return applyPatterns(caseList, patterns, filename, lambda b: b)
 202
 203 def applyExclude (caseList, patterns, filename):
 204         return applyPatterns(caseList, patterns, filename, lambda b: not b)
 205
 206 def readPatternLists (mustpass):
 207         lists = {}
 208         for package in mustpass.packages:
 209                 for cfg in package.configurations:
 210                         for filter in cfg.filters:
 211                                 if not filter.filename in lists:
 212                                         lists[filter.filename] = readPatternList(os.path.join(getSrcDir(mustpass), filter.filename))
 213         return lists
 214
 215 def applyFilters (caseList, patternLists, filters):
 216         res = copy(caseList)
 217         for filter in filters:
 218                 ptrnList = patternLists[filter.filename]
 219                 if filter.type == Filter.TYPE_INCLUDE:
 220                         res = applyInclude(res, ptrnList, filter.filename)
 221                 else:
 222                         assert filter.type == Filter.TYPE_EXCLUDE
 223                         res = applyExclude(res, ptrnList, filter.filename)
 224         return res
 225
 226 def appendToHierarchy (root, casePath):
 227         def findChild (node, name):
 228                 for child in node.children:
 229                         if child.name == name:
 230                                 return child
 231                 return None
 232
 233         curNode         = root
 234         components      = casePath.split('.')
 235
 236         for component in components[:-1]:
 237                 nextNode = findChild(curNode, component)
 238                 if not nextNode:
 239                         nextNode = TestGroup(component)
 240                         curNode.children.append(nextNode)
 241                 curNode = nextNode
 242
 243         if not findChild(curNode, components[-1]):
 244                 curNode.children.append(TestCase(components[-1]))
 245
 246 def buildTestHierachy (caseList):
 247         root = TestRoot()
 248         for case in caseList:
 249                 appendToHierarchy(root, case)
 250         return root
 251
 252 def buildTestCaseMap (root):
 253         caseMap = {}
 254
 255         def recursiveBuild (curNode, prefix):
 256                 curPath = prefix + curNode.name
 257                 if isinstance(curNode, TestCase):
 258                         caseMap[curPath] = curNode
 259                 else:
 260                         for child in curNode.children:
 261                                 recursiveBuild(child, curPath + '.')
 262
 263         for child in root.children:
 264                 recursiveBuild(child, '')
 265
 266         return caseMap
 267
 268 def include (filename):
 269         return Filter(Filter.TYPE_INCLUDE, filename)
 270
 271 def exclude (filename):
 272         return Filter(Filter.TYPE_EXCLUDE, filename)
 273
 274 def insertXMLHeaders (mustpass, doc):
 275         if mustpass.project.copyright != None:
 276                 doc.insert(0, ElementTree.Comment(mustpass.project.copyright))
 277         doc.insert(1, ElementTree.Comment(GENERATED_FILE_WARNING))
 278
 279 def prettifyXML (doc):
 280         uglyString      = ElementTree.tostring(doc, 'utf-8')
 281         reparsed        = minidom.parseString(uglyString)
 282         return reparsed.toprettyxml(indent='\t', encoding='utf-8')
 283
 284 def genCTSPackageXML (mustpass, package, root):
 285         def isLeafGroup (testGroup):
 286                 numGroups       = 0
 287                 numTests        = 0
 288
 289                 for child in testGroup.children:
 290                         if isinstance(child, TestCase):
 291                                 numTests += 1
 292                         else:
 293                                 numGroups += 1
 294
 295                 assert numGroups + numTests > 0
 296
 297                 if numGroups > 0 and numTests > 0:
 298                         die("Mixed groups and cases in %s" % testGroup.name)
 299
 300                 return numGroups == 0
 301
 302         def makeConfiguration (parentElem, config):
 303                 attributes = {}
 304
 305                 if config.glconfig != None:
 306                         attributes['glconfig'] = config.glconfig
 307
 308                 if config.rotation != None:
 309                         attributes['rotation'] = config.rotation
 310
 311                 if config.surfacetype != None:
 312                         attributes['surfacetype'] = config.surfacetype
 313
 314                 return ElementTree.SubElement(parentElem, "TestInstance", attributes)
 315
 316         def makeTestCase (parentElem, testCase):
 317                 caseElem = ElementTree.SubElement(parentElem, "Test", name=testCase.name)
 318                 for config in testCase.configurations:
 319                         makeConfiguration(caseElem, config)
 320                 return caseElem
 321
 322         def makeTestGroup (parentElem, testGroup):
 323                 groupElem = ElementTree.SubElement(parentElem, "TestCase" if isLeafGroup(testGroup) else "TestSuite", name=testGroup.name)
 324                 for child in testGroup.children:
 325                         if isinstance(child, TestCase):
 326                                 makeTestCase(groupElem, child)
 327                         else:
 328                                 makeTestGroup(groupElem, child)
 329                 return groupElem
 330
 331         pkgElem = ElementTree.Element("TestPackage",
 332                                                                   name                          = package.module.name,
 333                                                                   appPackageName        = getCTSPackageName(package),
 334                                                                   testType                      = "deqpTest")
 335
 336         pkgElem.set("xmlns:deqp", "http://drawelements.com/deqp")
 337         insertXMLHeaders(mustpass, pkgElem)
 338
 339         glesVersion = getModuleGLESVersion(package.module)
 340
 341         if glesVersion != None:
 342                 pkgElem.set("deqp:glesVersion", str(glesVersion.encode()))
 343
 344         for child in root.children:
 345                 makeTestGroup(pkgElem, child)
 346
 347         return pkgElem
 348
 349 def genSpecXML (mustpass):
 350         mustpassElem = ElementTree.Element("Mustpass", version = mustpass.version)
 351         insertXMLHeaders(mustpass, mustpassElem)
 352
 353         for package in mustpass.packages:
 354                 packageElem = ElementTree.SubElement(mustpassElem, "TestPackage", name = package.module.name)
 355
 356                 for config in package.configurations:
 357                         configElem = ElementTree.SubElement(packageElem, "Configuration",
 358                                                                                                 name                    = config.name,
 359                                                                                                 caseListFile    = getCaseListFileName(package, config),
 360                                                                                                 commandLine             = getCommandLine(config))
 361
 362         return mustpassElem
 363
 364 def addOptionElement (parent, optionName, optionValue):
 365         ElementTree.SubElement(parent, "option", name=optionName, value=optionValue)
 366
 367 def genAndroidTestXml (mustpass):
 368         RUNNER_CLASS = "com.drawelements.deqp.runner.DeqpTestRunner"
 369         configElement = ElementTree.Element("configuration")
 370
 371         for package in mustpass.packages:
 372                 for config in package.configurations:
 373                         testElement = ElementTree.SubElement(configElement, "test")
 374                         testElement.set("class", RUNNER_CLASS)
 375                         addOptionElement(testElement, "deqp-package", package.module.name)
 376                         addOptionElement(testElement, "deqp-caselist-file", getCaseListFileName(package,config))
 377                         # \todo [2015-10-16 kalle]: Replace with just command line? - requires simplifications in the runner/tests as well.
 378                         if config.glconfig != None:
 379                                 addOptionElement(testElement, "deqp-gl-config-name", config.glconfig)
 380
 381                         if config.surfacetype != None:
 382                                 addOptionElement(testElement, "deqp-surface-type", config.surfacetype)
 383
 384                         if config.rotation != None:
 385                                 addOptionElement(testElement, "deqp-screen-rotation", config.rotation)
 386
 387                         if config.expectedRuntime != None:
 388                                 addOptionElement(testElement, "runtime-hint", config.expectedRuntime)
 389
 390         insertXMLHeaders(mustpass, configElement)
 391
 392         return configElement
 393
 394 def genMustpass (mustpass, moduleCaseLists):
 395         print "Generating mustpass '%s'" % mustpass.version
 396
 397         patternLists = readPatternLists(mustpass)
 398
 399         for package in mustpass.packages:
 400                 allCasesInPkg           = moduleCaseLists[package.module]
 401                 matchingByConfig        = {}
 402                 allMatchingSet          = set()
 403
 404                 for config in package.configurations:
 405                         filtered        = applyFilters(allCasesInPkg, patternLists, config.filters)
 406                         dstFile         = getDstCaseListPath(mustpass, package, config)
 407
 408                         print "  Writing deqp caselist: " + dstFile
 409                         writeFile(dstFile, "\n".join(filtered) + "\n")
 410
 411                         matchingByConfig[config]        = filtered
 412                         allMatchingSet                          = allMatchingSet | set(filtered)
 413
 414                 allMatchingCases        = [c for c in allCasesInPkg if c in allMatchingSet] # To preserve ordering
 415                 root                            = buildTestHierachy(allMatchingCases)
 416                 testCaseMap                     = buildTestCaseMap(root)
 417
 418                 for config in package.configurations:
 419                         for case in matchingByConfig[config]:
 420                                 testCaseMap[case].configurations.append(config)
 421
 422                 # NOTE: CTS v2 does not need package XML files. Remove when transition is complete.
 423                 packageXml      = genCTSPackageXML(mustpass, package, root)
 424                 xmlFilename     = os.path.join(mustpass.project.path, mustpass.version, getCTSPackageName(package) + ".xml")
 425
 426                 print "  Writing CTS caselist: " + xmlFilename
 427                 writeFile(xmlFilename, prettifyXML(packageXml))
 428
 429         specXML                 = genSpecXML(mustpass)
 430         specFilename    = os.path.join(mustpass.project.path, mustpass.version, "mustpass.xml")
 431
 432         print "  Writing spec: " + specFilename
 433         writeFile(specFilename, prettifyXML(specXML))
 434
 435         # TODO: Which is the best selector mechanism?
 436         if (mustpass.version == "master"):
 437                 androidTestXML          = genAndroidTestXml(mustpass)
 438                 androidTestFilename     = os.path.join(mustpass.project.path, "AndroidTest.xml")
 439
 440                 print "  Writing AndroidTest.xml: " + androidTestFilename
 441                 writeFile(androidTestFilename, prettifyXML(androidTestXML))
 442
 443         print "Done!"
 444
 445 def genMustpassLists (mustpassLists, generator, buildCfg):
 446         moduleCaseLists = {}
 447
 448         # Getting case lists involves invoking build, so we want to cache the results
 449         for mustpass in mustpassLists:
 450                 for package in mustpass.packages:
 451                         if not package.module in moduleCaseLists:
 452                                 moduleCaseLists[package.module] = getCaseList(buildCfg, generator, package.module)
 453
 454         for mustpass in mustpassLists:
 455                 genMustpass(mustpass, moduleCaseLists)