scripts/mustpass.py

   1 # -*- coding: utf-8 -*-
   2
   3 #-------------------------------------------------------------------------
   4 # drawElements Quality Program utilities
   5 # --------------------------------------
   6 #
   7 # Copyright 2016 The Android Open Source Project
   8 #
   9 # Licensed under the Apache License, Version 2.0 (the "License");
  10 # you may not use this file except in compliance with the License.
  11 # You may obtain a copy of the License at
  12 #
  13 #      http://www.apache.org/licenses/LICENSE-2.0
  14 #
  15 # Unless required by applicable law or agreed to in writing, software
  16 # distributed under the License is distributed on an "AS IS" BASIS,
  17 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  18 # See the License for the specific language governing permissions and
  19 # limitations under the License.
  20 #
  21 #-------------------------------------------------------------------------
  22
  23 from build.common import *
  24 from build.config import ANY_GENERATOR
  25 from build.build import build
  26 from build_caselists import Module, getModuleByName, getBuildConfig, genCaseList, getCaseListPath, DEFAULT_BUILD_DIR, DEFAULT_TARGET
  27 from fnmatch import fnmatch
  28 from copy import copy
  29
  30 import argparse
  31 import xml.etree.cElementTree as ElementTree
  32 import xml.dom.minidom as minidom
  33
  34 APK_NAME                = "com.drawelements.deqp.apk"
  35
  36 GENERATED_FILE_WARNING = """
  37      This file has been automatically generated. Edit with caution.
  38      """
  39
  40 class Project:
  41         def __init__ (self, path, copyright = None):
  42                 self.path               = path
  43                 self.copyright  = copyright
  44
  45 class Configuration:
  46         def __init__ (self, name, filters, glconfig = None, rotation = None, surfacetype = None, required = False, runtime = None, runByDefault = True):
  47                 self.name                               = name
  48                 self.glconfig                   = glconfig
  49                 self.rotation                   = rotation
  50                 self.surfacetype                = surfacetype
  51                 self.required                   = required
  52                 self.filters                    = filters
  53                 self.expectedRuntime    = runtime
  54                 self.runByDefault               = runByDefault
  55
  56 class Package:
  57         def __init__ (self, module, configurations):
  58                 self.module                     = module
  59                 self.configurations     = configurations
  60
  61 class Mustpass:
  62         def __init__ (self, project, version, packages):
  63                 self.project    = project
  64                 self.version    = version
  65                 self.packages   = packages
  66
  67 class Filter:
  68         TYPE_INCLUDE = 0
  69         TYPE_EXCLUDE = 1
  70
  71         def __init__ (self, type, filename):
  72                 self.type               = type
  73                 self.filename   = filename
  74
  75 class TestRoot:
  76         def __init__ (self):
  77                 self.children   = []
  78
  79 class TestGroup:
  80         def __init__ (self, name):
  81                 self.name               = name
  82                 self.children   = []
  83
  84 class TestCase:
  85         def __init__ (self, name):
  86                 self.name                       = name
  87                 self.configurations     = []
  88
  89 class GLESVersion:
  90         def __init__(self, major, minor):
  91                 self.major = major
  92                 self.minor = minor
  93
  94         def encode (self):
  95                 return (self.major << 16) | (self.minor)
  96
  97 def getModuleGLESVersion (module):
  98         versions = {
  99                 'dEQP-EGL':             GLESVersion(2,0),
 100                 'dEQP-GLES2':   GLESVersion(2,0),
 101                 'dEQP-GLES3':   GLESVersion(3,0),
 102                 'dEQP-GLES31':  GLESVersion(3,1)
 103         }
 104         return versions[module.name] if module.name in versions else None
 105
 106 def getSrcDir (mustpass):
 107         return os.path.join(mustpass.project.path, mustpass.version, "src")
 108
 109 def getTmpDir (mustpass):
 110         return os.path.join(mustpass.project.path, mustpass.version, "tmp")
 111
 112 def getModuleShorthand (module):
 113         assert module.name[:5] == "dEQP-"
 114         return module.name[5:].lower()
 115
 116 def getCaseListFileName (package, configuration):
 117         return "%s-%s.txt" % (getModuleShorthand(package.module), configuration.name)
 118
 119 def getDstCaseListPath (mustpass, package, configuration):
 120         return os.path.join(mustpass.project.path, mustpass.version, getCaseListFileName(package, configuration))
 121
 122 def getCTSPackageName (package):
 123         return "com.drawelements.deqp." + getModuleShorthand(package.module)
 124
 125 def getCommandLine (config):
 126         cmdLine = ""
 127
 128         if config.glconfig != None:
 129                 cmdLine += "--deqp-gl-config-name=%s " % config.glconfig
 130
 131         if config.rotation != None:
 132                 cmdLine += "--deqp-screen-rotation=%s " % config.rotation
 133
 134         if config.surfacetype != None:
 135                 cmdLine += "--deqp-surface-type=%s " % config.surfacetype
 136
 137         cmdLine += "--deqp-watchdog=enable"
 138
 139         return cmdLine
 140
 141 def readCaseList (filename):
 142         cases = []
 143         with open(filename, 'rt') as f:
 144                 for line in f:
 145                         if line[:6] == "TEST: ":
 146                                 cases.append(line[6:].strip())
 147         return cases
 148
 149 def getCaseList (buildCfg, generator, module):
 150         build(buildCfg, generator, [module.binName])
 151         genCaseList(buildCfg, generator, module, "txt")
 152         return readCaseList(getCaseListPath(buildCfg, module, "txt"))
 153
 154 def readPatternList (filename):
 155         ptrns = []
 156         with open(filename, 'rt') as f:
 157                 for line in f:
 158                         line = line.strip()
 159                         if len(line) > 0 and line[0] != '#':
 160                                 ptrns.append(line)
 161         return ptrns
 162
 163 def applyPatterns (caseList, patterns, filename, op):
 164         matched                 = set()
 165         errors                  = []
 166         curList                 = copy(caseList)
 167         trivialPtrns    = [p for p in patterns if p.find('*') < 0]
 168         regularPtrns    = [p for p in patterns if p.find('*') >= 0]
 169
 170         # Apply trivial (just case paths)
 171         allCasesSet             = set(caseList)
 172         for path in trivialPtrns:
 173                 if path in allCasesSet:
 174                         if path in matched:
 175                                 errors.append((path, "Same case specified more than once"))
 176                         matched.add(path)
 177                 else:
 178                         errors.append((path, "Test case not found"))
 179
 180         curList = [c for c in curList if c not in matched]
 181
 182         for pattern in regularPtrns:
 183                 matchedThisPtrn = set()
 184
 185                 for case in curList:
 186                         if fnmatch(case, pattern):
 187                                 matchedThisPtrn.add(case)
 188
 189                 if len(matchedThisPtrn) == 0:
 190                         errors.append((pattern, "Pattern didn't match any cases"))
 191
 192                 matched = matched | matchedThisPtrn
 193                 curList = [c for c in curList if c not in matched]
 194
 195         for pattern, reason in errors:
 196                 print("ERROR: %s: %s" % (reason, pattern))
 197
 198         if len(errors) > 0:
 199                 die("Found %s invalid patterns while processing file %s" % (len(errors), filename))
 200
 201         return [c for c in caseList if op(c in matched)]
 202
 203 def applyInclude (caseList, patterns, filename):
 204         return applyPatterns(caseList, patterns, filename, lambda b: b)
 205
 206 def applyExclude (caseList, patterns, filename):
 207         return applyPatterns(caseList, patterns, filename, lambda b: not b)
 208
 209 def readPatternLists (mustpass):
 210         lists = {}
 211         for package in mustpass.packages:
 212                 for cfg in package.configurations:
 213                         for filter in cfg.filters:
 214                                 if not filter.filename in lists:
 215                                         lists[filter.filename] = readPatternList(os.path.join(getSrcDir(mustpass), filter.filename))
 216         return lists
 217
 218 def applyFilters (caseList, patternLists, filters):
 219         res = copy(caseList)
 220         for filter in filters:
 221                 ptrnList = patternLists[filter.filename]
 222                 if filter.type == Filter.TYPE_INCLUDE:
 223                         res = applyInclude(res, ptrnList, filter.filename)
 224                 else:
 225                         assert filter.type == Filter.TYPE_EXCLUDE
 226                         res = applyExclude(res, ptrnList, filter.filename)
 227         return res
 228
 229 def appendToHierarchy (root, casePath):
 230         def findChild (node, name):
 231                 for child in node.children:
 232                         if child.name == name:
 233                                 return child
 234                 return None
 235
 236         curNode         = root
 237         components      = casePath.split('.')
 238
 239         for component in components[:-1]:
 240                 nextNode = findChild(curNode, component)
 241                 if not nextNode:
 242                         nextNode = TestGroup(component)
 243                         curNode.children.append(nextNode)
 244                 curNode = nextNode
 245
 246         if not findChild(curNode, components[-1]):
 247                 curNode.children.append(TestCase(components[-1]))
 248
 249 def buildTestHierachy (caseList):
 250         root = TestRoot()
 251         for case in caseList:
 252                 appendToHierarchy(root, case)
 253         return root
 254
 255 def buildTestCaseMap (root):
 256         caseMap = {}
 257
 258         def recursiveBuild (curNode, prefix):
 259                 curPath = prefix + curNode.name
 260                 if isinstance(curNode, TestCase):
 261                         caseMap[curPath] = curNode
 262                 else:
 263                         for child in curNode.children:
 264                                 recursiveBuild(child, curPath + '.')
 265
 266         for child in root.children:
 267                 recursiveBuild(child, '')
 268
 269         return caseMap
 270
 271 def include (filename):
 272         return Filter(Filter.TYPE_INCLUDE, filename)
 273
 274 def exclude (filename):
 275         return Filter(Filter.TYPE_EXCLUDE, filename)
 276
 277 def insertXMLHeaders (mustpass, doc):
 278         if mustpass.project.copyright != None:
 279                 doc.insert(0, ElementTree.Comment(mustpass.project.copyright))
 280         doc.insert(1, ElementTree.Comment(GENERATED_FILE_WARNING))
 281
 282 def prettifyXML (doc):
 283         uglyString      = ElementTree.tostring(doc, 'utf-8')
 284         reparsed        = minidom.parseString(uglyString)
 285         return reparsed.toprettyxml(indent='\t', encoding='utf-8')
 286
 287 def genSpecXML (mustpass):
 288         mustpassElem = ElementTree.Element("Mustpass", version = mustpass.version)
 289         insertXMLHeaders(mustpass, mustpassElem)
 290
 291         for package in mustpass.packages:
 292                 packageElem = ElementTree.SubElement(mustpassElem, "TestPackage", name = package.module.name)
 293
 294                 for config in package.configurations:
 295                         configElem = ElementTree.SubElement(packageElem, "Configuration",
 296                                                                                                 caseListFile    = getCaseListFileName(package, config),
 297                                                                                                 commandLine             = getCommandLine(config),
 298                                                                                                 name                    = config.name)
 299
 300         return mustpassElem
 301
 302 def addOptionElement (parent, optionName, optionValue):
 303         ElementTree.SubElement(parent, "option", name=optionName, value=optionValue)
 304
 305 def genAndroidTestXml (mustpass):
 306         RUNNER_CLASS = "com.drawelements.deqp.runner.DeqpTestRunner"
 307         configElement = ElementTree.Element("configuration")
 308
 309         # have the deqp package installed on the device for us
 310         preparerElement = ElementTree.SubElement(configElement, "target_preparer")
 311         preparerElement.set("class", "com.android.tradefed.targetprep.suite.SuiteApkInstaller")
 312         addOptionElement(preparerElement, "cleanup-apks", "true")
 313         addOptionElement(preparerElement, "test-file-name", "com.drawelements.deqp.apk")
 314
 315         # add in metadata option for component name
 316         ElementTree.SubElement(configElement, "option", name="test-suite-tag", value="cts")
 317         ElementTree.SubElement(configElement, "option", key="component", name="config-descriptor:metadata", value="deqp")
 318         ElementTree.SubElement(configElement, "option", key="parameter", name="config-descriptor:metadata", value="not_instant_app")
 319         ElementTree.SubElement(configElement, "option", key="parameter", name="config-descriptor:metadata", value="multi_abi")
 320         ElementTree.SubElement(configElement, "option", key="parameter", name="config-descriptor:metadata", value="secondary_user")
 321         controllerElement = ElementTree.SubElement(configElement, "object")
 322         controllerElement.set("class", "com.android.tradefed.testtype.suite.module.TestFailureModuleController")
 323         controllerElement.set("type", "module_controller")
 324         addOptionElement(controllerElement, "screenshot-on-failure", "false")
 325
 326         for package in mustpass.packages:
 327                 for config in package.configurations:
 328                         if not config.runByDefault:
 329                                 continue
 330
 331                         testElement = ElementTree.SubElement(configElement, "test")
 332                         testElement.set("class", RUNNER_CLASS)
 333                         addOptionElement(testElement, "deqp-package", package.module.name)
 334                         addOptionElement(testElement, "deqp-caselist-file", getCaseListFileName(package,config))
 335                         # \todo [2015-10-16 kalle]: Replace with just command line? - requires simplifications in the runner/tests as well.
 336                         if config.glconfig != None:
 337                                 addOptionElement(testElement, "deqp-gl-config-name", config.glconfig)
 338
 339                         if config.surfacetype != None:
 340                                 addOptionElement(testElement, "deqp-surface-type", config.surfacetype)
 341
 342                         if config.rotation != None:
 343                                 addOptionElement(testElement, "deqp-screen-rotation", config.rotation)
 344
 345                         if config.expectedRuntime != None:
 346                                 addOptionElement(testElement, "runtime-hint", config.expectedRuntime)
 347
 348                         if config.required:
 349                                 addOptionElement(testElement, "deqp-config-required", "true")
 350
 351         insertXMLHeaders(mustpass, configElement)
 352
 353         return configElement
 354
 355 def genMustpass (mustpass, moduleCaseLists):
 356         print("Generating mustpass '%s'" % mustpass.version)
 357
 358         patternLists = readPatternLists(mustpass)
 359
 360         for package in mustpass.packages:
 361                 allCasesInPkg   = moduleCaseLists[package.module]
 362
 363                 for config in package.configurations:
 364                         filtered        = applyFilters(allCasesInPkg, patternLists, config.filters)
 365                         dstFile         = getDstCaseListPath(mustpass, package, config)
 366
 367                         print("  Writing deqp caselist: " + dstFile)
 368                         writeFile(dstFile, "\n".join(filtered) + "\n")
 369
 370         specXML                 = genSpecXML(mustpass)
 371         specFilename    = os.path.join(mustpass.project.path, mustpass.version, "mustpass.xml")
 372
 373         print("  Writing spec: " + specFilename)
 374         writeFile(specFilename, prettifyXML(specXML).decode())
 375
 376         # TODO: Which is the best selector mechanism?
 377         if (mustpass.version == "master"):
 378                 androidTestXML          = genAndroidTestXml(mustpass)
 379                 androidTestFilename     = os.path.join(mustpass.project.path, "AndroidTest.xml")
 380
 381                 print("  Writing AndroidTest.xml: " + androidTestFilename)
 382                 writeFile(androidTestFilename, prettifyXML(androidTestXML).decode())
 383
 384         print("Done!")
 385
 386 def genMustpassLists (mustpassLists, generator, buildCfg):
 387         moduleCaseLists = {}
 388
 389         # Getting case lists involves invoking build, so we want to cache the results
 390         for mustpass in mustpassLists:
 391                 for package in mustpass.packages:
 392                         if not package.module in moduleCaseLists:
 393                                 moduleCaseLists[package.module] = getCaseList(buildCfg, generator, package.module)
 394
 395         for mustpass in mustpassLists:
 396                 genMustpass(mustpass, moduleCaseLists)
 397
 398 def parseCmdLineArgs ():
 399         parser = argparse.ArgumentParser(description = "Build Android CTS mustpass",
 400                                                                          formatter_class=argparse.ArgumentDefaultsHelpFormatter)
 401         parser.add_argument("-b",
 402                                                 "--build-dir",
 403                                                 dest="buildDir",
 404                                                 default=DEFAULT_BUILD_DIR,
 405                                                 help="Temporary build directory")
 406         parser.add_argument("-t",
 407                                                 "--build-type",
 408                                                 dest="buildType",
 409                                                 default="Debug",
 410                                                 help="Build type")
 411         parser.add_argument("-c",
 412                                                 "--deqp-target",
 413                                                 dest="targetName",
 414                                                 default=DEFAULT_TARGET,
 415                                                 help="dEQP build target")
 416         return parser.parse_args()
 417
 418 def parseBuildConfigFromCmdLineArgs ():
 419         args = parseCmdLineArgs()
 420         return getBuildConfig(args.buildDir, args.targetName, args.buildType)