scripts/mustpass.py

   1 # -*- coding: utf-8 -*-
   2
   3 #-------------------------------------------------------------------------
   4 # drawElements Quality Program utilities
   5 # --------------------------------------
   6 #
   7 # Copyright 2016 The Android Open Source Project
   8 #
   9 # Licensed under the Apache License, Version 2.0 (the "License");
  10 # you may not use this file except in compliance with the License.
  11 # You may obtain a copy of the License at
  12 #
  13 #      http://www.apache.org/licenses/LICENSE-2.0
  14 #
  15 # Unless required by applicable law or agreed to in writing, software
  16 # distributed under the License is distributed on an "AS IS" BASIS,
  17 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  18 # See the License for the specific language governing permissions and
  19 # limitations under the License.
  20 #
  21 #-------------------------------------------------------------------------
  22
  23 from build.common import *
  24 from build.config import ANY_GENERATOR
  25 from build.build import build
  26 from build_caselists import Module, getModuleByName, getBuildConfig, genCaseList, getCaseListPath, DEFAULT_BUILD_DIR, DEFAULT_TARGET
  27 from fnmatch import fnmatch
  28 from copy import copy
  29 from collections import defaultdict
  30
  31 import argparse
  32 import xml.etree.cElementTree as ElementTree
  33 import xml.dom.minidom as minidom
  34
  35 APK_NAME                = "com.drawelements.deqp.apk"
  36
  37 GENERATED_FILE_WARNING = """
  38      This file has been automatically generated. Edit with caution.
  39      """
  40
  41 class Project:
  42         def __init__ (self, path, copyright = None):
  43                 self.path               = path
  44                 self.copyright  = copyright
  45
  46 class Configuration:
  47         def __init__ (self, name, filters, glconfig = None, rotation = None, surfacetype = None, required = False, runtime = None, runByDefault = True, splitToMultipleFiles = False):
  48                 self.name                                       = name
  49                 self.glconfig                           = glconfig
  50                 self.rotation                           = rotation
  51                 self.surfacetype                        = surfacetype
  52                 self.required                           = required
  53                 self.filters                            = filters
  54                 self.expectedRuntime            = runtime
  55                 self.runByDefault                       = runByDefault
  56                 self.splitToMultipleFiles       = splitToMultipleFiles
  57
  58 class Package:
  59         def __init__ (self, module, configurations):
  60                 self.module                     = module
  61                 self.configurations     = configurations
  62
  63 class Mustpass:
  64         def __init__ (self, project, version, packages):
  65                 self.project    = project
  66                 self.version    = version
  67                 self.packages   = packages
  68
  69 class Filter:
  70         TYPE_INCLUDE = 0
  71         TYPE_EXCLUDE = 1
  72
  73         def __init__ (self, type, filename):
  74                 self.type               = type
  75                 self.filename   = filename
  76
  77 class TestRoot:
  78         def __init__ (self):
  79                 self.children   = []
  80
  81 class TestGroup:
  82         def __init__ (self, name):
  83                 self.name               = name
  84                 self.children   = []
  85
  86 class TestCase:
  87         def __init__ (self, name):
  88                 self.name                       = name
  89                 self.configurations     = []
  90
  91 class GLESVersion:
  92         def __init__(self, major, minor):
  93                 self.major = major
  94                 self.minor = minor
  95
  96         def encode (self):
  97                 return (self.major << 16) | (self.minor)
  98
  99 def getModuleGLESVersion (module):
 100         versions = {
 101                 'dEQP-EGL':             GLESVersion(2,0),
 102                 'dEQP-GLES2':   GLESVersion(2,0),
 103                 'dEQP-GLES3':   GLESVersion(3,0),
 104                 'dEQP-GLES31':  GLESVersion(3,1)
 105         }
 106         return versions[module.name] if module.name in versions else None
 107
 108 def getSrcDir (mustpass):
 109         return os.path.join(mustpass.project.path, mustpass.version, "src")
 110
 111 def getTmpDir (mustpass):
 112         return os.path.join(mustpass.project.path, mustpass.version, "tmp")
 113
 114 def getModuleShorthand (module):
 115         assert module.name[:5] == "dEQP-"
 116         return module.name[5:].lower()
 117
 118 def getCaseListFileName (package, configuration):
 119         return "%s-%s.txt" % (getModuleShorthand(package.module), configuration.name)
 120
 121 def getDstCaseListPath (mustpass):
 122         return os.path.join(mustpass.project.path, mustpass.version)
 123
 124 def getCTSPackageName (package):
 125         return "com.drawelements.deqp." + getModuleShorthand(package.module)
 126
 127 def getCommandLine (config):
 128         cmdLine = ""
 129
 130         if config.glconfig != None:
 131                 cmdLine += "--deqp-gl-config-name=%s " % config.glconfig
 132
 133         if config.rotation != None:
 134                 cmdLine += "--deqp-screen-rotation=%s " % config.rotation
 135
 136         if config.surfacetype != None:
 137                 cmdLine += "--deqp-surface-type=%s " % config.surfacetype
 138
 139         cmdLine += "--deqp-watchdog=enable"
 140
 141         return cmdLine
 142
 143 def readCaseDict (filename):
 144         # cases are grouped per high-level test group
 145         # this is needed for chunked mustpass
 146         casesPerHighLevelGroup = {}
 147         currentHighLevelGroup = ""
 148         with open(filename, 'rt') as f:
 149                 for line in f:
 150                         entryType = line[:6]
 151                         if entryType == "TEST: ":
 152                                 assert currentHighLevelGroup != ""
 153                                 casesPerHighLevelGroup[currentHighLevelGroup].append(line[6:].strip())
 154                         # detect high-level group by number of dots in path
 155                         elif entryType == "GROUP:" and line.count('.') == 1:
 156                                 currentHighLevelGroup = line[line.find('.')+1:].rstrip().replace('_', '-')
 157                                 casesPerHighLevelGroup[currentHighLevelGroup] = []
 158         return casesPerHighLevelGroup
 159
 160 def getCaseDict (buildCfg, generator, module):
 161         build(buildCfg, generator, [module.binName])
 162         genCaseList(buildCfg, generator, module, "txt")
 163         return readCaseDict(getCaseListPath(buildCfg, module, "txt"))
 164
 165 def readPatternList (filename):
 166         ptrns = []
 167         with open(filename, 'rt') as f:
 168                 for line in f:
 169                         line = line.strip()
 170                         if len(line) > 0 and line[0] != '#':
 171                                 ptrns.append(line)
 172         return ptrns
 173
 174
 175 def constructNewDict(oldDict, listOfCases, op = lambda a: not a):
 176         # Helper function used to construct case dictionary without specific cases
 177         newDict = defaultdict(list)
 178         for topGroup in oldDict:
 179                 for c in oldDict[topGroup]:
 180                         if op(c in listOfCases):
 181                                 newDict[topGroup].append(c)
 182         return newDict
 183
 184 def applyPatterns (caseDict, patterns, filename, op):
 185         matched                 = set()
 186         errors                  = []
 187         trivialPtrns    = [p for p in patterns if p.find('*') < 0]
 188         regularPtrns    = [p for p in patterns if p.find('*') >= 0]
 189
 190         # Construct helper set that contains cases from all groups
 191         allCasesSet = set()
 192         for topGroup in caseDict:
 193                 allCasesSet = allCasesSet.union(set(caseDict[topGroup]))
 194
 195         # Apply trivial patterns - plain case paths without wildcard
 196         for path in trivialPtrns:
 197                 if path in allCasesSet:
 198                         if path in matched:
 199                                 errors.append((path, "Same case specified more than once"))
 200                         matched.add(path)
 201                 else:
 202                         errors.append((path, "Test case not found"))
 203
 204         # Construct new dictionary but without already matched paths
 205         curDict = constructNewDict(caseDict, matched)
 206
 207         # Apply regular patterns - paths with wildcard
 208         for pattern in regularPtrns:
 209                 matchedThisPtrn = set()
 210
 211                 for topGroup in curDict:
 212                         for c in curDict[topGroup]:
 213                                 if fnmatch(c, pattern):
 214                                         matchedThisPtrn.add(c)
 215
 216                 if len(matchedThisPtrn) == 0:
 217                         errors.append((pattern, "Pattern didn't match any cases"))
 218
 219                 matched = matched | matchedThisPtrn
 220
 221                 # To speed up search construct smaller case dictionary without already matched paths
 222                 curDict = constructNewDict(curDict, matched)
 223
 224         for pattern, reason in errors:
 225                 print("ERROR: %s: %s" % (reason, pattern))
 226
 227         if len(errors) > 0:
 228                 die("Found %s invalid patterns while processing file %s" % (len(errors), filename))
 229
 230         # Construct final dictionary using aproperiate operation
 231         return constructNewDict(caseDict, matched, op)
 232
 233 def applyInclude (caseDict, patterns, filename):
 234         return applyPatterns(caseDict, patterns, filename, lambda b: b)
 235
 236 def applyExclude (caseDict, patterns, filename):
 237         return applyPatterns(caseDict, patterns, filename, lambda b: not b)
 238
 239 def readPatternLists (mustpass):
 240         lists = {}
 241         for package in mustpass.packages:
 242                 for cfg in package.configurations:
 243                         for filter in cfg.filters:
 244                                 if not filter.filename in lists:
 245                                         lists[filter.filename] = readPatternList(os.path.join(getSrcDir(mustpass), filter.filename))
 246         return lists
 247
 248 def applyFilters (caseDict, patternLists, filters):
 249         res = copy(caseDict)
 250         for filter in filters:
 251                 ptrnList = patternLists[filter.filename]
 252                 if filter.type == Filter.TYPE_INCLUDE:
 253                         res = applyInclude(res, ptrnList, filter.filename)
 254                 else:
 255                         assert filter.type == Filter.TYPE_EXCLUDE
 256                         res = applyExclude(res, ptrnList, filter.filename)
 257         return res
 258
 259 def appendToHierarchy (root, casePath):
 260         def findChild (node, name):
 261                 for child in node.children:
 262                         if child.name == name:
 263                                 return child
 264                 return None
 265
 266         curNode         = root
 267         components      = casePath.split('.')
 268
 269         for component in components[:-1]:
 270                 nextNode = findChild(curNode, component)
 271                 if not nextNode:
 272                         nextNode = TestGroup(component)
 273                         curNode.children.append(nextNode)
 274                 curNode = nextNode
 275
 276         if not findChild(curNode, components[-1]):
 277                 curNode.children.append(TestCase(components[-1]))
 278
 279 def buildTestHierachy (caseList):
 280         root = TestRoot()
 281         for case in caseList:
 282                 appendToHierarchy(root, case)
 283         return root
 284
 285 def buildTestCaseMap (root):
 286         caseMap = {}
 287
 288         def recursiveBuild (curNode, prefix):
 289                 curPath = prefix + curNode.name
 290                 if isinstance(curNode, TestCase):
 291                         caseMap[curPath] = curNode
 292                 else:
 293                         for child in curNode.children:
 294                                 recursiveBuild(child, curPath + '.')
 295
 296         for child in root.children:
 297                 recursiveBuild(child, '')
 298
 299         return caseMap
 300
 301 def include (filename):
 302         return Filter(Filter.TYPE_INCLUDE, filename)
 303
 304 def exclude (filename):
 305         return Filter(Filter.TYPE_EXCLUDE, filename)
 306
 307 def insertXMLHeaders (mustpass, doc):
 308         if mustpass.project.copyright != None:
 309                 doc.insert(0, ElementTree.Comment(mustpass.project.copyright))
 310         doc.insert(1, ElementTree.Comment(GENERATED_FILE_WARNING))
 311
 312 def prettifyXML (doc):
 313         uglyString      = ElementTree.tostring(doc, 'utf-8')
 314         reparsed        = minidom.parseString(uglyString)
 315         return reparsed.toprettyxml(indent='\t', encoding='utf-8')
 316
 317 def genSpecXML (mustpass):
 318         mustpassElem = ElementTree.Element("Mustpass", version = mustpass.version)
 319         insertXMLHeaders(mustpass, mustpassElem)
 320
 321         for package in mustpass.packages:
 322                 packageElem = ElementTree.SubElement(mustpassElem, "TestPackage", name = package.module.name)
 323
 324                 for config in package.configurations:
 325                         configElem = ElementTree.SubElement(packageElem, "Configuration",
 326                                                                                                 caseListFile    = getCaseListFileName(package, config),
 327                                                                                                 commandLine             = getCommandLine(config),
 328                                                                                                 name                    = config.name)
 329
 330         return mustpassElem
 331
 332 def addOptionElement (parent, optionName, optionValue):
 333         ElementTree.SubElement(parent, "option", name=optionName, value=optionValue)
 334
 335 def genAndroidTestXml (mustpass):
 336         RUNNER_CLASS = "com.drawelements.deqp.runner.DeqpTestRunner"
 337         configElement = ElementTree.Element("configuration")
 338
 339         # have the deqp package installed on the device for us
 340         preparerElement = ElementTree.SubElement(configElement, "target_preparer")
 341         preparerElement.set("class", "com.android.tradefed.targetprep.suite.SuiteApkInstaller")
 342         addOptionElement(preparerElement, "cleanup-apks", "true")
 343         addOptionElement(preparerElement, "test-file-name", "com.drawelements.deqp.apk")
 344
 345         # add in metadata option for component name
 346         ElementTree.SubElement(configElement, "option", name="test-suite-tag", value="cts")
 347         ElementTree.SubElement(configElement, "option", key="component", name="config-descriptor:metadata", value="deqp")
 348         ElementTree.SubElement(configElement, "option", key="parameter", name="config-descriptor:metadata", value="not_instant_app")
 349         ElementTree.SubElement(configElement, "option", key="parameter", name="config-descriptor:metadata", value="multi_abi")
 350         ElementTree.SubElement(configElement, "option", key="parameter", name="config-descriptor:metadata", value="secondary_user")
 351         controllerElement = ElementTree.SubElement(configElement, "object")
 352         controllerElement.set("class", "com.android.tradefed.testtype.suite.module.TestFailureModuleController")
 353         controllerElement.set("type", "module_controller")
 354         addOptionElement(controllerElement, "screenshot-on-failure", "false")
 355
 356         for package in mustpass.packages:
 357                 for config in package.configurations:
 358                         if not config.runByDefault:
 359                                 continue
 360
 361                         testElement = ElementTree.SubElement(configElement, "test")
 362                         testElement.set("class", RUNNER_CLASS)
 363                         addOptionElement(testElement, "deqp-package", package.module.name)
 364                         addOptionElement(testElement, "deqp-caselist-file", getCaseListFileName(package,config))
 365                         # \todo [2015-10-16 kalle]: Replace with just command line? - requires simplifications in the runner/tests as well.
 366                         if config.glconfig != None:
 367                                 addOptionElement(testElement, "deqp-gl-config-name", config.glconfig)
 368
 369                         if config.surfacetype != None:
 370                                 addOptionElement(testElement, "deqp-surface-type", config.surfacetype)
 371
 372                         if config.rotation != None:
 373                                 addOptionElement(testElement, "deqp-screen-rotation", config.rotation)
 374
 375                         if config.expectedRuntime != None:
 376                                 addOptionElement(testElement, "runtime-hint", config.expectedRuntime)
 377
 378                         if config.required:
 379                                 addOptionElement(testElement, "deqp-config-required", "true")
 380
 381         insertXMLHeaders(mustpass, configElement)
 382
 383         return configElement
 384
 385 def genMustpass (mustpass, moduleCaseDicts):
 386         print("Generating mustpass '%s'" % mustpass.version)
 387
 388         patternLists = readPatternLists(mustpass)
 389
 390         for package in mustpass.packages:
 391                 allCasesInPkgDict       = moduleCaseDicts[package.module]
 392
 393                 for config in package.configurations:
 394
 395                         # construct dictionary with all filters applyed,
 396                         # key is top-level group name, value is list of all cases in that group
 397                         filteredCaseDict        = applyFilters(allCasesInPkgDict, patternLists, config.filters)
 398
 399                         # construct components of path to main destination file
 400                         mainDstFilePath         = getDstCaseListPath(mustpass)
 401                         mainDstFileName         = getCaseListFileName(package, config)
 402                         mainDstFile                     = os.path.join(mainDstFilePath, mainDstFileName)
 403                         gruopSubDir                     = mainDstFileName[:-4]
 404
 405                         # if case paths should be split to multiple files then main
 406                         # destination file will contain paths to individual files containing cases
 407                         if config.splitToMultipleFiles:
 408                                 groupPathsList = []
 409
 410                                 # make sure directory for group files exists
 411                                 groupPath = os.path.join(mainDstFilePath, gruopSubDir)
 412                                 if not os.path.exists(groupPath):
 413                                         os.makedirs(groupPath)
 414
 415                                 # iterate over all top-level groups and write files containing their cases
 416                                 print("  Writing top-level group caselists:")
 417                                 for tlGroup in filteredCaseDict:
 418                                         groupDstFileName    = tlGroup + ".txt"
 419                                         groupDstFileFullDir = os.path.join(groupPath, groupDstFileName)
 420                                         groupPathsList.append(gruopSubDir + "/" + groupDstFileName)
 421
 422                                         print("    " + groupDstFileFullDir)
 423                                         writeFile(groupDstFileFullDir, "\n".join(filteredCaseDict[tlGroup]) + "\n")
 424
 425                                 # write file containing names of all group files
 426                                 print("  Writing deqp top-level groups file list: " + mainDstFile)
 427                                 groupPathsList.sort()
 428                                 writeFile(mainDstFile, "\n".join(groupPathsList) + "\n")
 429                         else:
 430                                 # merge cases from all top level groups in to single case list
 431                                 filteredCaseList = []
 432                                 for tlGroup in filteredCaseDict:
 433                                         filteredCaseList.extend(filteredCaseDict[tlGroup])
 434
 435                                 # write file containing all cases
 436                                 print("  Writing deqp caselist: " + mainDstFile)
 437                                 writeFile(mainDstFile, "\n".join(filteredCaseList) + "\n")
 438
 439         specXML = genSpecXML(mustpass)
 440         specFilename = os.path.join(mustpass.project.path, mustpass.version, "mustpass.xml")
 441
 442         print("  Writing spec: " + specFilename)
 443         writeFile(specFilename, prettifyXML(specXML).decode())
 444
 445         # TODO: Which is the best selector mechanism?
 446         if (mustpass.version == "master"):
 447                 androidTestXML          = genAndroidTestXml(mustpass)
 448                 androidTestFilename     = os.path.join(mustpass.project.path, "AndroidTest.xml")
 449
 450                 print("  Writing AndroidTest.xml: " + androidTestFilename)
 451                 writeFile(androidTestFilename, prettifyXML(androidTestXML).decode())
 452
 453         print("Done!")
 454
 455 def genMustpassLists (mustpassLists, generator, buildCfg):
 456         moduleCaseDicts = {}
 457
 458         # Getting case lists involves invoking build, so we want to cache the results
 459         for mustpass in mustpassLists:
 460                 for package in mustpass.packages:
 461                         if not package.module in moduleCaseDicts:
 462                                 moduleCaseDicts[package.module] = getCaseDict(buildCfg, generator, package.module)
 463
 464         for mustpass in mustpassLists:
 465                 genMustpass(mustpass, moduleCaseDicts)
 466
 467 def parseCmdLineArgs ():
 468         parser = argparse.ArgumentParser(description = "Build Android CTS mustpass",
 469                                                                          formatter_class=argparse.ArgumentDefaultsHelpFormatter)
 470         parser.add_argument("-b",
 471                                                 "--build-dir",
 472                                                 dest="buildDir",
 473                                                 default=DEFAULT_BUILD_DIR,
 474                                                 help="Temporary build directory")
 475         parser.add_argument("-t",
 476                                                 "--build-type",
 477                                                 dest="buildType",
 478                                                 default="Debug",
 479                                                 help="Build type")
 480         parser.add_argument("-c",
 481                                                 "--deqp-target",
 482                                                 dest="targetName",
 483                                                 default=DEFAULT_TARGET,
 484                                                 help="dEQP build target")
 485         return parser.parse_args()
 486
 487 def parseBuildConfigFromCmdLineArgs ():
 488         args = parseCmdLineArgs()
 489         return getBuildConfig(args.buildDir, args.targetName, args.buildType)