scripts/mustpass.py

   1 # -*- coding: utf-8 -*-
   2
   3 #-------------------------------------------------------------------------
   4 # drawElements Quality Program utilities
   5 # --------------------------------------
   6 #
   7 # Copyright 2016 The Android Open Source Project
   8 #
   9 # Licensed under the Apache License, Version 2.0 (the "License");
  10 # you may not use this file except in compliance with the License.
  11 # You may obtain a copy of the License at
  12 #
  13 #      http://www.apache.org/licenses/LICENSE-2.0
  14 #
  15 # Unless required by applicable law or agreed to in writing, software
  16 # distributed under the License is distributed on an "AS IS" BASIS,
  17 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  18 # See the License for the specific language governing permissions and
  19 # limitations under the License.
  20 #
  21 #-------------------------------------------------------------------------
  22
  23 from build.common import *
  24 from build.config import ANY_GENERATOR
  25 from build.build import build
  26 from build_caselists import Module, getModuleByName, getBuildConfig, genCaseList, getCaseListPath, DEFAULT_BUILD_DIR, DEFAULT_TARGET
  27 from fnmatch import fnmatch
  28 from copy import copy
  29
  30 import argparse
  31 import xml.etree.cElementTree as ElementTree
  32 import xml.dom.minidom as minidom
  33
  34 APK_NAME                = "com.drawelements.deqp.apk"
  35
  36 GENERATED_FILE_WARNING = """
  37      This file has been automatically generated. Edit with caution.
  38      """
  39
  40 class Project:
  41         def __init__ (self, path, copyright = None):
  42                 self.path               = path
  43                 self.copyright  = copyright
  44
  45 class Configuration:
  46         def __init__ (self, name, filters, glconfig = None, rotation = None, surfacetype = None, required = False, runtime = None):
  47                 self.name                               = name
  48                 self.glconfig                   = glconfig
  49                 self.rotation                   = rotation
  50                 self.surfacetype                = surfacetype
  51                 self.required                   = required
  52                 self.filters                    = filters
  53                 self.expectedRuntime    = runtime
  54
  55 class Package:
  56         def __init__ (self, module, configurations):
  57                 self.module                     = module
  58                 self.configurations     = configurations
  59
  60 class Mustpass:
  61         def __init__ (self, project, version, packages):
  62                 self.project    = project
  63                 self.version    = version
  64                 self.packages   = packages
  65
  66 class Filter:
  67         TYPE_INCLUDE = 0
  68         TYPE_EXCLUDE = 1
  69
  70         def __init__ (self, type, filename):
  71                 self.type               = type
  72                 self.filename   = filename
  73
  74 class TestRoot:
  75         def __init__ (self):
  76                 self.children   = []
  77
  78 class TestGroup:
  79         def __init__ (self, name):
  80                 self.name               = name
  81                 self.children   = []
  82
  83 class TestCase:
  84         def __init__ (self, name):
  85                 self.name                       = name
  86                 self.configurations     = []
  87
  88 class GLESVersion:
  89         def __init__(self, major, minor):
  90                 self.major = major
  91                 self.minor = minor
  92
  93         def encode (self):
  94                 return (self.major << 16) | (self.minor)
  95
  96 def getModuleGLESVersion (module):
  97         versions = {
  98                 'dEQP-EGL':             GLESVersion(2,0),
  99                 'dEQP-GLES2':   GLESVersion(2,0),
 100                 'dEQP-GLES3':   GLESVersion(3,0),
 101                 'dEQP-GLES31':  GLESVersion(3,1)
 102         }
 103         return versions[module.name] if module.name in versions else None
 104
 105 def getSrcDir (mustpass):
 106         return os.path.join(mustpass.project.path, mustpass.version, "src")
 107
 108 def getTmpDir (mustpass):
 109         return os.path.join(mustpass.project.path, mustpass.version, "tmp")
 110
 111 def getModuleShorthand (module):
 112         assert module.name[:5] == "dEQP-"
 113         return module.name[5:].lower()
 114
 115 def getCaseListFileName (package, configuration):
 116         return "%s-%s.txt" % (getModuleShorthand(package.module), configuration.name)
 117
 118 def getDstCaseListPath (mustpass, package, configuration):
 119         return os.path.join(mustpass.project.path, mustpass.version, getCaseListFileName(package, configuration))
 120
 121 def getCTSPackageName (package):
 122         return "com.drawelements.deqp." + getModuleShorthand(package.module)
 123
 124 def getCommandLine (config):
 125         cmdLine = ""
 126
 127         if config.glconfig != None:
 128                 cmdLine += "--deqp-gl-config-name=%s " % config.glconfig
 129
 130         if config.rotation != None:
 131                 cmdLine += "--deqp-screen-rotation=%s " % config.rotation
 132
 133         if config.surfacetype != None:
 134                 cmdLine += "--deqp-surface-type=%s " % config.surfacetype
 135
 136         cmdLine += "--deqp-watchdog=enable"
 137
 138         return cmdLine
 139
 140 def readCaseList (filename):
 141         cases = []
 142         with open(filename, 'rb') as f:
 143                 for line in f:
 144                         if line[:6] == "TEST: ":
 145                                 cases.append(line[6:].strip())
 146         return cases
 147
 148 def getCaseList (buildCfg, generator, module):
 149         build(buildCfg, generator, [module.binName])
 150         genCaseList(buildCfg, generator, module, "txt")
 151         return readCaseList(getCaseListPath(buildCfg, module, "txt"))
 152
 153 def readPatternList (filename):
 154         ptrns = []
 155         with open(filename, 'rb') as f:
 156                 for line in f:
 157                         line = line.strip()
 158                         if len(line) > 0 and line[0] != '#':
 159                                 ptrns.append(line)
 160         return ptrns
 161
 162 def applyPatterns (caseList, patterns, filename, op):
 163         matched                 = set()
 164         errors                  = []
 165         curList                 = copy(caseList)
 166         trivialPtrns    = [p for p in patterns if p.find('*') < 0]
 167         regularPtrns    = [p for p in patterns if p.find('*') >= 0]
 168
 169         # Apply trivial (just case paths)
 170         allCasesSet             = set(caseList)
 171         for path in trivialPtrns:
 172                 if path in allCasesSet:
 173                         if path in matched:
 174                                 errors.append((path, "Same case specified more than once"))
 175                         matched.add(path)
 176                 else:
 177                         errors.append((path, "Test case not found"))
 178
 179         curList = [c for c in curList if c not in matched]
 180
 181         for pattern in regularPtrns:
 182                 matchedThisPtrn = set()
 183
 184                 for case in curList:
 185                         if fnmatch(case, pattern):
 186                                 matchedThisPtrn.add(case)
 187
 188                 if len(matchedThisPtrn) == 0:
 189                         errors.append((pattern, "Pattern didn't match any cases"))
 190
 191                 matched = matched | matchedThisPtrn
 192                 curList = [c for c in curList if c not in matched]
 193
 194         for pattern, reason in errors:
 195                 print "ERROR: %s: %s" % (reason, pattern)
 196
 197         if len(errors) > 0:
 198                 die("Found %s invalid patterns while processing file %s" % (len(errors), filename))
 199
 200         return [c for c in caseList if op(c in matched)]
 201
 202 def applyInclude (caseList, patterns, filename):
 203         return applyPatterns(caseList, patterns, filename, lambda b: b)
 204
 205 def applyExclude (caseList, patterns, filename):
 206         return applyPatterns(caseList, patterns, filename, lambda b: not b)
 207
 208 def readPatternLists (mustpass):
 209         lists = {}
 210         for package in mustpass.packages:
 211                 for cfg in package.configurations:
 212                         for filter in cfg.filters:
 213                                 if not filter.filename in lists:
 214                                         lists[filter.filename] = readPatternList(os.path.join(getSrcDir(mustpass), filter.filename))
 215         return lists
 216
 217 def applyFilters (caseList, patternLists, filters):
 218         res = copy(caseList)
 219         for filter in filters:
 220                 ptrnList = patternLists[filter.filename]
 221                 if filter.type == Filter.TYPE_INCLUDE:
 222                         res = applyInclude(res, ptrnList, filter.filename)
 223                 else:
 224                         assert filter.type == Filter.TYPE_EXCLUDE
 225                         res = applyExclude(res, ptrnList, filter.filename)
 226         return res
 227
 228 def appendToHierarchy (root, casePath):
 229         def findChild (node, name):
 230                 for child in node.children:
 231                         if child.name == name:
 232                                 return child
 233                 return None
 234
 235         curNode         = root
 236         components      = casePath.split('.')
 237
 238         for component in components[:-1]:
 239                 nextNode = findChild(curNode, component)
 240                 if not nextNode:
 241                         nextNode = TestGroup(component)
 242                         curNode.children.append(nextNode)
 243                 curNode = nextNode
 244
 245         if not findChild(curNode, components[-1]):
 246                 curNode.children.append(TestCase(components[-1]))
 247
 248 def buildTestHierachy (caseList):
 249         root = TestRoot()
 250         for case in caseList:
 251                 appendToHierarchy(root, case)
 252         return root
 253
 254 def buildTestCaseMap (root):
 255         caseMap = {}
 256
 257         def recursiveBuild (curNode, prefix):
 258                 curPath = prefix + curNode.name
 259                 if isinstance(curNode, TestCase):
 260                         caseMap[curPath] = curNode
 261                 else:
 262                         for child in curNode.children:
 263                                 recursiveBuild(child, curPath + '.')
 264
 265         for child in root.children:
 266                 recursiveBuild(child, '')
 267
 268         return caseMap
 269
 270 def include (filename):
 271         return Filter(Filter.TYPE_INCLUDE, filename)
 272
 273 def exclude (filename):
 274         return Filter(Filter.TYPE_EXCLUDE, filename)
 275
 276 def insertXMLHeaders (mustpass, doc):
 277         if mustpass.project.copyright != None:
 278                 doc.insert(0, ElementTree.Comment(mustpass.project.copyright))
 279         doc.insert(1, ElementTree.Comment(GENERATED_FILE_WARNING))
 280
 281 def prettifyXML (doc):
 282         uglyString      = ElementTree.tostring(doc, 'utf-8')
 283         reparsed        = minidom.parseString(uglyString)
 284         return reparsed.toprettyxml(indent='\t', encoding='utf-8')
 285
 286 def genSpecXML (mustpass):
 287         mustpassElem = ElementTree.Element("Mustpass", version = mustpass.version)
 288         insertXMLHeaders(mustpass, mustpassElem)
 289
 290         for package in mustpass.packages:
 291                 packageElem = ElementTree.SubElement(mustpassElem, "TestPackage", name = package.module.name)
 292
 293                 for config in package.configurations:
 294                         configElem = ElementTree.SubElement(packageElem, "Configuration",
 295                                                                                                 name                    = config.name,
 296                                                                                                 caseListFile    = getCaseListFileName(package, config),
 297                                                                                                 commandLine             = getCommandLine(config))
 298
 299         return mustpassElem
 300
 301 def addOptionElement (parent, optionName, optionValue):
 302         ElementTree.SubElement(parent, "option", name=optionName, value=optionValue)
 303
 304 def genAndroidTestXml (mustpass):
 305         RUNNER_CLASS = "com.drawelements.deqp.runner.DeqpTestRunner"
 306         configElement = ElementTree.Element("configuration")
 307
 308         for package in mustpass.packages:
 309                 for config in package.configurations:
 310                         testElement = ElementTree.SubElement(configElement, "test")
 311                         testElement.set("class", RUNNER_CLASS)
 312                         addOptionElement(testElement, "deqp-package", package.module.name)
 313                         addOptionElement(testElement, "deqp-caselist-file", getCaseListFileName(package,config))
 314                         # \todo [2015-10-16 kalle]: Replace with just command line? - requires simplifications in the runner/tests as well.
 315                         if config.glconfig != None:
 316                                 addOptionElement(testElement, "deqp-gl-config-name", config.glconfig)
 317
 318                         if config.surfacetype != None:
 319                                 addOptionElement(testElement, "deqp-surface-type", config.surfacetype)
 320
 321                         if config.rotation != None:
 322                                 addOptionElement(testElement, "deqp-screen-rotation", config.rotation)
 323
 324                         if config.expectedRuntime != None:
 325                                 addOptionElement(testElement, "runtime-hint", config.expectedRuntime)
 326
 327                         if config.required:
 328                                 addOptionElement(testElement, "deqp-config-required", "true")
 329
 330         insertXMLHeaders(mustpass, configElement)
 331
 332         return configElement
 333
 334 def genMustpass (mustpass, moduleCaseLists):
 335         print "Generating mustpass '%s'" % mustpass.version
 336
 337         patternLists = readPatternLists(mustpass)
 338
 339         for package in mustpass.packages:
 340                 allCasesInPkg   = moduleCaseLists[package.module]
 341
 342                 for config in package.configurations:
 343                         filtered        = applyFilters(allCasesInPkg, patternLists, config.filters)
 344                         dstFile         = getDstCaseListPath(mustpass, package, config)
 345
 346                         print "  Writing deqp caselist: " + dstFile
 347                         writeFile(dstFile, "\n".join(filtered) + "\n")
 348
 349         specXML                 = genSpecXML(mustpass)
 350         specFilename    = os.path.join(mustpass.project.path, mustpass.version, "mustpass.xml")
 351
 352         print "  Writing spec: " + specFilename
 353         writeFile(specFilename, prettifyXML(specXML))
 354
 355         # TODO: Which is the best selector mechanism?
 356         if (mustpass.version == "master"):
 357                 androidTestXML          = genAndroidTestXml(mustpass)
 358                 androidTestFilename     = os.path.join(mustpass.project.path, "AndroidTest.xml")
 359
 360                 print "  Writing AndroidTest.xml: " + androidTestFilename
 361                 writeFile(androidTestFilename, prettifyXML(androidTestXML))
 362
 363         print "Done!"
 364
 365 def genMustpassLists (mustpassLists, generator, buildCfg):
 366         moduleCaseLists = {}
 367
 368         # Getting case lists involves invoking build, so we want to cache the results
 369         for mustpass in mustpassLists:
 370                 for package in mustpass.packages:
 371                         if not package.module in moduleCaseLists:
 372                                 moduleCaseLists[package.module] = getCaseList(buildCfg, generator, package.module)
 373
 374         for mustpass in mustpassLists:
 375                 genMustpass(mustpass, moduleCaseLists)
 376
 377 def parseCmdLineArgs ():
 378         parser = argparse.ArgumentParser(description = "Build Android CTS mustpass",
 379                                                                          formatter_class=argparse.ArgumentDefaultsHelpFormatter)
 380         parser.add_argument("-b",
 381                                                 "--build-dir",
 382                                                 dest="buildDir",
 383                                                 default=DEFAULT_BUILD_DIR,
 384                                                 help="Temporary build directory")
 385         parser.add_argument("-t",
 386                                                 "--build-type",
 387                                                 dest="buildType",
 388                                                 default="Debug",
 389                                                 help="Build type")
 390         parser.add_argument("-c",
 391                                                 "--deqp-target",
 392                                                 dest="targetName",
 393                                                 default=DEFAULT_TARGET,
 394                                                 help="dEQP build target")
 395         return parser.parse_args()
 396
 397 def parseBuildConfigFromCmdLineArgs ():
 398         args = parseCmdLineArgs()
 399         return getBuildConfig(args.buildDir, args.targetName, args.buildType)