scripts/mustpass.py

   1 # -*- coding: utf-8 -*-
   2
   3 #-------------------------------------------------------------------------
   4 # drawElements Quality Program utilities
   5 # --------------------------------------
   6 #
   7 # Copyright 2016 The Android Open Source Project
   8 #
   9 # Licensed under the Apache License, Version 2.0 (the "License");
  10 # you may not use this file except in compliance with the License.
  11 # You may obtain a copy of the License at
  12 #
  13 #      http://www.apache.org/licenses/LICENSE-2.0
  14 #
  15 # Unless required by applicable law or agreed to in writing, software
  16 # distributed under the License is distributed on an "AS IS" BASIS,
  17 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  18 # See the License for the specific language governing permissions and
  19 # limitations under the License.
  20 #
  21 #-------------------------------------------------------------------------
  22
  23 from build.common import *
  24 from build.config import ANY_GENERATOR
  25 from build.build import build
  26 from build_caselists import Module, getModuleByName, getBuildConfig, genCaseList, getCaseListPath, DEFAULT_BUILD_DIR, DEFAULT_TARGET
  27 from fnmatch import fnmatch
  28 from copy import copy
  29
  30 import xml.etree.cElementTree as ElementTree
  31 import xml.dom.minidom as minidom
  32
  33 APK_NAME                = "com.drawelements.deqp.apk"
  34
  35 GENERATED_FILE_WARNING = """
  36      This file has been automatically generated. Edit with caution.
  37      """
  38
  39 class Project:
  40         def __init__ (self, path, copyright = None):
  41                 self.path               = path
  42                 self.copyright  = copyright
  43
  44 class Configuration:
  45         def __init__ (self, name, filters, glconfig = None, rotation = None, surfacetype = None):
  46                 self.name                       = name
  47                 self.glconfig           = glconfig
  48                 self.rotation           = rotation
  49                 self.surfacetype        = surfacetype
  50                 self.filters            = filters
  51
  52 class Package:
  53         def __init__ (self, module, configurations):
  54                 self.module                     = module
  55                 self.configurations     = configurations
  56
  57 class Mustpass:
  58         def __init__ (self, project, version, packages):
  59                 self.project    = project
  60                 self.version    = version
  61                 self.packages   = packages
  62
  63 class Filter:
  64         TYPE_INCLUDE = 0
  65         TYPE_EXCLUDE = 1
  66
  67         def __init__ (self, type, filename):
  68                 self.type               = type
  69                 self.filename   = filename
  70
  71 class TestRoot:
  72         def __init__ (self):
  73                 self.children   = []
  74
  75 class TestGroup:
  76         def __init__ (self, name):
  77                 self.name               = name
  78                 self.children   = []
  79
  80 class TestCase:
  81         def __init__ (self, name):
  82                 self.name                       = name
  83                 self.configurations     = []
  84
  85 class GLESVersion:
  86         def __init__(self, major, minor):
  87                 self.major = major
  88                 self.minor = minor
  89
  90         def encode (self):
  91                 return (self.major << 16) | (self.minor)
  92
  93 def getModuleGLESVersion (module):
  94         versions = {
  95                 'dEQP-EGL':             GLESVersion(2,0),
  96                 'dEQP-GLES2':   GLESVersion(2,0),
  97                 'dEQP-GLES3':   GLESVersion(3,0),
  98                 'dEQP-GLES31':  GLESVersion(3,1)
  99         }
 100         return versions[module.name] if module.name in versions else None
 101
 102 def getSrcDir (mustpass):
 103         return os.path.join(mustpass.project.path, mustpass.version, "src")
 104
 105 def getTmpDir (mustpass):
 106         return os.path.join(mustpass.project.path, mustpass.version, "tmp")
 107
 108 def getModuleShorthand (module):
 109         assert module.name[:5] == "dEQP-"
 110         return module.name[5:].lower()
 111
 112 def getCaseListFileName (package, configuration):
 113         return "%s-%s.txt" % (getModuleShorthand(package.module), configuration.name)
 114
 115 def getDstCaseListPath (mustpass, package, configuration):
 116         return os.path.join(mustpass.project.path, mustpass.version, getCaseListFileName(package, configuration))
 117
 118 def getCTSPackageName (package):
 119         return "com.drawelements.deqp." + getModuleShorthand(package.module)
 120
 121 def getCommandLine (config):
 122         cmdLine = ""
 123
 124         if config.glconfig != None:
 125                 cmdLine += "--deqp-gl-config-name=%s " % config.glconfig
 126
 127         if config.rotation != None:
 128                 cmdLine += "--deqp-screen-rotation=%s " % config.rotation
 129
 130         if config.surfacetype != None:
 131                 cmdLine += "--deqp-surface-type=%s " % config.surfacetype
 132
 133         cmdLine += "--deqp-watchdog=enable"
 134
 135         return cmdLine
 136
 137 def readCaseList (filename):
 138         cases = []
 139         with open(filename, 'rb') as f:
 140                 for line in f:
 141                         if line[:6] == "TEST: ":
 142                                 cases.append(line[6:].strip())
 143         return cases
 144
 145 def getCaseList (buildCfg, generator, module):
 146         build(buildCfg, generator, [module.binName])
 147         genCaseList(buildCfg, generator, module, "txt")
 148         return readCaseList(getCaseListPath(buildCfg, module, "txt"))
 149
 150 def readPatternList (filename):
 151         ptrns = []
 152         with open(filename, 'rb') as f:
 153                 for line in f:
 154                         line = line.strip()
 155                         if len(line) > 0 and line[0] != '#':
 156                                 ptrns.append(line)
 157         return ptrns
 158
 159 def applyPatterns (caseList, patterns, filename, op):
 160         matched                 = set()
 161         errors                  = []
 162         curList                 = copy(caseList)
 163         trivialPtrns    = [p for p in patterns if p.find('*') < 0]
 164         regularPtrns    = [p for p in patterns if p.find('*') >= 0]
 165
 166         # Apply trivial (just case paths)
 167         allCasesSet             = set(caseList)
 168         for path in trivialPtrns:
 169                 if path in allCasesSet:
 170                         if path in matched:
 171                                 errors.append((path, "Same case specified more than once"))
 172                         matched.add(path)
 173                 else:
 174                         errors.append((path, "Test case not found"))
 175
 176         curList = [c for c in curList if c not in matched]
 177
 178         for pattern in regularPtrns:
 179                 matchedThisPtrn = set()
 180
 181                 for case in curList:
 182                         if fnmatch(case, pattern):
 183                                 matchedThisPtrn.add(case)
 184
 185                 if len(matchedThisPtrn) == 0:
 186                         errors.append((pattern, "Pattern didn't match any cases"))
 187
 188                 matched = matched | matchedThisPtrn
 189                 curList = [c for c in curList if c not in matched]
 190
 191         for pattern, reason in errors:
 192                 print "ERROR: %s: %s" % (reason, pattern)
 193
 194         if len(errors) > 0:
 195                 die("Found %s invalid patterns while processing file %s" % (len(errors), filename))
 196
 197         return [c for c in caseList if op(c in matched)]
 198
 199 def applyInclude (caseList, patterns, filename):
 200         return applyPatterns(caseList, patterns, filename, lambda b: b)
 201
 202 def applyExclude (caseList, patterns, filename):
 203         return applyPatterns(caseList, patterns, filename, lambda b: not b)
 204
 205 def readPatternLists (mustpass):
 206         lists = {}
 207         for package in mustpass.packages:
 208                 for cfg in package.configurations:
 209                         for filter in cfg.filters:
 210                                 if not filter.filename in lists:
 211                                         lists[filter.filename] = readPatternList(os.path.join(getSrcDir(mustpass), filter.filename))
 212         return lists
 213
 214 def applyFilters (caseList, patternLists, filters):
 215         res = copy(caseList)
 216         for filter in filters:
 217                 ptrnList = patternLists[filter.filename]
 218                 if filter.type == Filter.TYPE_INCLUDE:
 219                         res = applyInclude(res, ptrnList, filter.filename)
 220                 else:
 221                         assert filter.type == Filter.TYPE_EXCLUDE
 222                         res = applyExclude(res, ptrnList, filter.filename)
 223         return res
 224
 225 def appendToHierarchy (root, casePath):
 226         def findChild (node, name):
 227                 for child in node.children:
 228                         if child.name == name:
 229                                 return child
 230                 return None
 231
 232         curNode         = root
 233         components      = casePath.split('.')
 234
 235         for component in components[:-1]:
 236                 nextNode = findChild(curNode, component)
 237                 if not nextNode:
 238                         nextNode = TestGroup(component)
 239                         curNode.children.append(nextNode)
 240                 curNode = nextNode
 241
 242         if not findChild(curNode, components[-1]):
 243                 curNode.children.append(TestCase(components[-1]))
 244
 245 def buildTestHierachy (caseList):
 246         root = TestRoot()
 247         for case in caseList:
 248                 appendToHierarchy(root, case)
 249         return root
 250
 251 def buildTestCaseMap (root):
 252         caseMap = {}
 253
 254         def recursiveBuild (curNode, prefix):
 255                 curPath = prefix + curNode.name
 256                 if isinstance(curNode, TestCase):
 257                         caseMap[curPath] = curNode
 258                 else:
 259                         for child in curNode.children:
 260                                 recursiveBuild(child, curPath + '.')
 261
 262         for child in root.children:
 263                 recursiveBuild(child, '')
 264
 265         return caseMap
 266
 267 def include (filename):
 268         return Filter(Filter.TYPE_INCLUDE, filename)
 269
 270 def exclude (filename):
 271         return Filter(Filter.TYPE_EXCLUDE, filename)
 272
 273 def insertXMLHeaders (mustpass, doc):
 274         if mustpass.project.copyright != None:
 275                 doc.insert(0, ElementTree.Comment(mustpass.project.copyright))
 276         doc.insert(1, ElementTree.Comment(GENERATED_FILE_WARNING))
 277
 278 def prettifyXML (doc):
 279         uglyString      = ElementTree.tostring(doc, 'utf-8')
 280         reparsed        = minidom.parseString(uglyString)
 281         return reparsed.toprettyxml(indent='\t', encoding='utf-8')
 282
 283 def genCTSPackageXML (mustpass, package, root):
 284         def isLeafGroup (testGroup):
 285                 numGroups       = 0
 286                 numTests        = 0
 287
 288                 for child in testGroup.children:
 289                         if isinstance(child, TestCase):
 290                                 numTests += 1
 291                         else:
 292                                 numGroups += 1
 293
 294                 assert numGroups + numTests > 0
 295
 296                 if numGroups > 0 and numTests > 0:
 297                         die("Mixed groups and cases in %s" % testGroup.name)
 298
 299                 return numGroups == 0
 300
 301         def makeConfiguration (parentElem, config):
 302                 attributes = {}
 303
 304                 if config.glconfig != None:
 305                         attributes['glconfig'] = config.glconfig
 306
 307                 if config.rotation != None:
 308                         attributes['rotation'] = config.rotation
 309
 310                 if config.surfacetype != None:
 311                         attributes['surfacetype'] = config.surfacetype
 312
 313                 return ElementTree.SubElement(parentElem, "TestInstance", attributes)
 314
 315         def makeTestCase (parentElem, testCase):
 316                 caseElem = ElementTree.SubElement(parentElem, "Test", name=testCase.name)
 317                 for config in testCase.configurations:
 318                         makeConfiguration(caseElem, config)
 319                 return caseElem
 320
 321         def makeTestGroup (parentElem, testGroup):
 322                 groupElem = ElementTree.SubElement(parentElem, "TestCase" if isLeafGroup(testGroup) else "TestSuite", name=testGroup.name)
 323                 for child in testGroup.children:
 324                         if isinstance(child, TestCase):
 325                                 makeTestCase(groupElem, child)
 326                         else:
 327                                 makeTestGroup(groupElem, child)
 328                 return groupElem
 329
 330         pkgElem = ElementTree.Element("TestPackage",
 331                                                                   name                          = package.module.name,
 332                                                                   appPackageName        = getCTSPackageName(package),
 333                                                                   testType                      = "deqpTest")
 334
 335         pkgElem.set("xmlns:deqp", "http://drawelements.com/deqp")
 336         insertXMLHeaders(mustpass, pkgElem)
 337
 338         glesVersion = getModuleGLESVersion(package.module)
 339
 340         if glesVersion != None:
 341                 pkgElem.set("deqp:glesVersion", str(glesVersion.encode()))
 342
 343         for child in root.children:
 344                 makeTestGroup(pkgElem, child)
 345
 346         return pkgElem
 347
 348 def genSpecXML (mustpass):
 349         mustpassElem = ElementTree.Element("Mustpass", version = mustpass.version)
 350         insertXMLHeaders(mustpass, mustpassElem)
 351
 352         for package in mustpass.packages:
 353                 packageElem = ElementTree.SubElement(mustpassElem, "TestPackage", name = package.module.name)
 354
 355                 for config in package.configurations:
 356                         configElem = ElementTree.SubElement(packageElem, "Configuration",
 357                                                                                                 name                    = config.name,
 358                                                                                                 caseListFile    = getCaseListFileName(package, config),
 359                                                                                                 commandLine             = getCommandLine(config))
 360
 361         return mustpassElem
 362
 363 def addOptionElement (parent, optionName, optionValue):
 364         ElementTree.SubElement(parent, "option", name=optionName, value=optionValue)
 365
 366 def genAndroidTestXml (mustpass):
 367         INSTALLER_CLASS = "com.android.compatibility.common.tradefed.targetprep.ApkInstaller"
 368         RUNNER_CLASS = "com.drawelements.deqp.runner.DeqpTestRunner"
 369         configElement = ElementTree.Element("configuration")
 370         preparerElement = ElementTree.SubElement(configElement, "target_preparer")
 371         preparerElement.set("class", INSTALLER_CLASS)
 372         addOptionElement(preparerElement, "cleanup-apks", "true")
 373         addOptionElement(preparerElement, "test-file-name", APK_NAME)
 374
 375         for package in mustpass.packages:
 376                 for config in package.configurations:
 377                         testElement = ElementTree.SubElement(configElement, "test")
 378                         testElement.set("class", RUNNER_CLASS)
 379                         addOptionElement(testElement, "deqp-package", package.module.name)
 380                         addOptionElement(testElement, "deqp-caselist-file", getCaseListFileName(package,config))
 381                         # \todo [2015-10-16 kalle]: Replace with just command line? - requires simplifications in the runner/tests as well.
 382                         if config.glconfig != None:
 383                                 addOptionElement(testElement, "deqp-gl-config-name", config.glconfig)
 384
 385                         if config.surfacetype != None:
 386                                 addOptionElement(testElement, "deqp-surface-type", config.surfacetype)
 387
 388                         if config.rotation != None:
 389                                 addOptionElement(testElement, "deqp-screen-rotation", config.rotation)
 390
 391         insertXMLHeaders(mustpass, configElement)
 392
 393         return configElement
 394
 395 def genMustpass (mustpass, moduleCaseLists):
 396         print "Generating mustpass '%s'" % mustpass.version
 397
 398         patternLists = readPatternLists(mustpass)
 399
 400         for package in mustpass.packages:
 401                 allCasesInPkg           = moduleCaseLists[package.module]
 402                 matchingByConfig        = {}
 403                 allMatchingSet          = set()
 404
 405                 for config in package.configurations:
 406                         filtered        = applyFilters(allCasesInPkg, patternLists, config.filters)
 407                         dstFile         = getDstCaseListPath(mustpass, package, config)
 408
 409                         print "  Writing deqp caselist: " + dstFile
 410                         writeFile(dstFile, "\n".join(filtered) + "\n")
 411
 412                         matchingByConfig[config]        = filtered
 413                         allMatchingSet                          = allMatchingSet | set(filtered)
 414
 415                 allMatchingCases        = [c for c in allCasesInPkg if c in allMatchingSet] # To preserve ordering
 416                 root                            = buildTestHierachy(allMatchingCases)
 417                 testCaseMap                     = buildTestCaseMap(root)
 418
 419                 for config in package.configurations:
 420                         for case in matchingByConfig[config]:
 421                                 testCaseMap[case].configurations.append(config)
 422
 423                 # NOTE: CTS v2 does not need package XML files. Remove when transition is complete.
 424                 packageXml      = genCTSPackageXML(mustpass, package, root)
 425                 xmlFilename     = os.path.join(mustpass.project.path, mustpass.version, getCTSPackageName(package) + ".xml")
 426
 427                 print "  Writing CTS caselist: " + xmlFilename
 428                 writeFile(xmlFilename, prettifyXML(packageXml))
 429
 430         specXML                 = genSpecXML(mustpass)
 431         specFilename    = os.path.join(mustpass.project.path, mustpass.version, "mustpass.xml")
 432
 433         print "  Writing spec: " + specFilename
 434         writeFile(specFilename, prettifyXML(specXML))
 435
 436         # TODO: Which is the best selector mechanism?
 437         if (mustpass.version == "master"):
 438                 androidTestXML          = genAndroidTestXml(mustpass)
 439                 androidTestFilename     = os.path.join(mustpass.project.path, "AndroidTest.xml")
 440
 441                 print "  Writing AndroidTest.xml: " + androidTestFilename
 442                 writeFile(androidTestFilename, prettifyXML(androidTestXML))
 443
 444         print "Done!"
 445
 446 def genMustpassLists (mustpassLists, generator, buildCfg):
 447         moduleCaseLists = {}
 448
 449         # Getting case lists involves invoking build, so we want to cache the results
 450         for mustpass in mustpassLists:
 451                 for package in mustpass.packages:
 452                         if not package.module in moduleCaseLists:
 453                                 moduleCaseLists[package.module] = getCaseList(buildCfg, generator, package.module)
 454
 455         for mustpass in mustpassLists:
 456                 genMustpass(mustpass, moduleCaseLists)