scripts/mustpass.py

   1 # -*- coding: utf-8 -*-
   2
   3 #-------------------------------------------------------------------------
   4 # drawElements Quality Program utilities
   5 # --------------------------------------
   6 #
   7 # Copyright 2016 The Android Open Source Project
   8 #
   9 # Licensed under the Apache License, Version 2.0 (the "License");
  10 # you may not use this file except in compliance with the License.
  11 # You may obtain a copy of the License at
  12 #
  13 #      http://www.apache.org/licenses/LICENSE-2.0
  14 #
  15 # Unless required by applicable law or agreed to in writing, software
  16 # distributed under the License is distributed on an "AS IS" BASIS,
  17 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  18 # See the License for the specific language governing permissions and
  19 # limitations under the License.
  20 #
  21 #-------------------------------------------------------------------------
  22
  23 from build.common import *
  24 from build.config import ANY_GENERATOR
  25 from build.build import build
  26 from build_caselists import Module, getModuleByName, getBuildConfig, genCaseList, getCaseListPath, DEFAULT_BUILD_DIR, DEFAULT_TARGET
  27 from fnmatch import fnmatch
  28 from copy import copy
  29 from collections import defaultdict
  30
  31 import argparse
  32 import xml.etree.cElementTree as ElementTree
  33 import xml.dom.minidom as minidom
  34
  35 APK_NAME                = "com.drawelements.deqp.apk"
  36
  37 GENERATED_FILE_WARNING = """
  38      This file has been automatically generated. Edit with caution.
  39      """
  40
  41 class Project:
  42         def __init__ (self, path, copyright = None):
  43                 self.path               = path
  44                 self.copyright  = copyright
  45
  46 class Configuration:
  47         def __init__ (self, name, filters, glconfig = None, rotation = None, surfacetype = None, required = False, runtime = None, runByDefault = True, splitToMultipleFiles = False):
  48                 self.name                                       = name
  49                 self.glconfig                           = glconfig
  50                 self.rotation                           = rotation
  51                 self.surfacetype                        = surfacetype
  52                 self.required                           = required
  53                 self.filters                            = filters
  54                 self.expectedRuntime            = runtime
  55                 self.runByDefault                       = runByDefault
  56                 self.splitToMultipleFiles       = splitToMultipleFiles
  57
  58 class Package:
  59         def __init__ (self, module, configurations):
  60                 self.module                     = module
  61                 self.configurations     = configurations
  62
  63 class Mustpass:
  64         def __init__ (self, project, version, packages):
  65                 self.project    = project
  66                 self.version    = version
  67                 self.packages   = packages
  68
  69 class Filter:
  70         TYPE_INCLUDE = 0
  71         TYPE_EXCLUDE = 1
  72
  73         def __init__ (self, type, filename):
  74                 self.type               = type
  75                 self.filename   = filename
  76
  77 class TestRoot:
  78         def __init__ (self):
  79                 self.children   = []
  80
  81 class TestGroup:
  82         def __init__ (self, name):
  83                 self.name               = name
  84                 self.children   = []
  85
  86 class TestCase:
  87         def __init__ (self, name):
  88                 self.name                       = name
  89                 self.configurations     = []
  90
  91 class GLESVersion:
  92         def __init__(self, major, minor):
  93                 self.major = major
  94                 self.minor = minor
  95
  96         def encode (self):
  97                 return (self.major << 16) | (self.minor)
  98
  99 def getModuleGLESVersion (module):
 100         versions = {
 101                 'dEQP-EGL':             GLESVersion(2,0),
 102                 'dEQP-GLES2':   GLESVersion(2,0),
 103                 'dEQP-GLES3':   GLESVersion(3,0),
 104                 'dEQP-GLES31':  GLESVersion(3,1)
 105         }
 106         return versions[module.name] if module.name in versions else None
 107
 108 def getSrcDir (mustpass):
 109         return os.path.join(mustpass.project.path, mustpass.version, "src")
 110
 111 def getTmpDir (mustpass):
 112         return os.path.join(mustpass.project.path, mustpass.version, "tmp")
 113
 114 def getModuleShorthand (module):
 115         assert module.name[:5] == "dEQP-"
 116         return module.name[5:].lower()
 117
 118 def getCaseListFileName (package, configuration):
 119         return "%s-%s.txt" % (getModuleShorthand(package.module), configuration.name)
 120
 121 def getDstCaseListPath (mustpass):
 122         return os.path.join(mustpass.project.path, mustpass.version)
 123
 124 def getCTSPackageName (package):
 125         return "com.drawelements.deqp." + getModuleShorthand(package.module)
 126
 127 def getCommandLine (config):
 128         cmdLine = ""
 129
 130         if config.glconfig != None:
 131                 cmdLine += "--deqp-gl-config-name=%s " % config.glconfig
 132
 133         if config.rotation != None:
 134                 cmdLine += "--deqp-screen-rotation=%s " % config.rotation
 135
 136         if config.surfacetype != None:
 137                 cmdLine += "--deqp-surface-type=%s " % config.surfacetype
 138
 139         cmdLine += "--deqp-watchdog=enable"
 140
 141         return cmdLine
 142
 143 def readCaseDict (filename):
 144         # cases are grouped per high-level test group
 145         # this is needed for chunked mustpass
 146         casesPerHighLevelGroup = {}
 147         currentHighLevelGroup = ""
 148         with open(filename, 'rt') as f:
 149                 for line in f:
 150                         entryType = line[:6]
 151                         if entryType == "TEST: ":
 152                                 assert currentHighLevelGroup != ""
 153                                 casesPerHighLevelGroup[currentHighLevelGroup].append(line[6:].strip())
 154                         # detect high-level group by number of dots in path
 155                         elif entryType == "GROUP:" and line.count('.') == 1:
 156                                 currentHighLevelGroup = line[line.find('.')+1:].rstrip().replace('_', '-')
 157                                 casesPerHighLevelGroup[currentHighLevelGroup] = []
 158         return casesPerHighLevelGroup
 159
 160 def getCaseDict (buildCfg, generator, module):
 161         build(buildCfg, generator, [module.binName])
 162         genCaseList(buildCfg, generator, module, "txt")
 163         return readCaseDict(getCaseListPath(buildCfg, module, "txt"))
 164
 165 def readPatternList (filename):
 166         ptrns = []
 167         with open(filename, 'rt') as f:
 168                 for line in f:
 169                         line = line.strip()
 170                         if len(line) > 0 and line[0] != '#':
 171                                 ptrns.append(line)
 172         return ptrns
 173
 174
 175 def constructNewDict(oldDict, listOfCases, op = lambda a: not a):
 176         # Helper function used to construct case dictionary without specific cases
 177         newDict = defaultdict(list)
 178         for topGroup in oldDict:
 179                 for c in oldDict[topGroup]:
 180                         if op(c in listOfCases):
 181                                 newDict[topGroup].append(c)
 182         return newDict
 183
 184 def applyPatterns (caseDict, patterns, filename, op):
 185         matched                 = set()
 186         errors                  = []
 187         trivialPtrns    = [p for p in patterns if p.find('*') < 0]
 188         regularPtrns    = [p for p in patterns if p.find('*') >= 0]
 189
 190         # Construct helper set that contains cases from all groups
 191         allCasesSet = set()
 192         for topGroup in caseDict:
 193                 allCasesSet = allCasesSet.union(set(caseDict[topGroup]))
 194
 195         # Apply trivial patterns - plain case paths without wildcard
 196         for path in trivialPtrns:
 197                 if path in allCasesSet:
 198                         if path in matched:
 199                                 errors.append((path, "Same case specified more than once"))
 200                         matched.add(path)
 201                 else:
 202                         errors.append((path, "Test case not found"))
 203
 204         # Construct new dictionary but without already matched paths
 205         curDict = constructNewDict(caseDict, matched)
 206
 207         # Apply regular patterns - paths with wildcard
 208         for pattern in regularPtrns:
 209                 matchedThisPtrn = set()
 210
 211                 for topGroup in curDict:
 212                         for c in curDict[topGroup]:
 213                                 if fnmatch(c, pattern):
 214                                         matchedThisPtrn.add(c)
 215
 216                 if len(matchedThisPtrn) == 0:
 217                         errors.append((pattern, "Pattern didn't match any cases"))
 218
 219                 matched = matched | matchedThisPtrn
 220
 221                 # To speed up search construct smaller case dictionary without already matched paths
 222                 curDict = constructNewDict(curDict, matched)
 223
 224         for pattern, reason in errors:
 225                 print("ERROR: %s: %s" % (reason, pattern))
 226
 227         if len(errors) > 0:
 228                 die("Found %s invalid patterns while processing file %s" % (len(errors), filename))
 229
 230         # Construct final dictionary using aproperiate operation
 231         return constructNewDict(caseDict, matched, op)
 232
 233 def applyInclude (caseDict, patterns, filename):
 234         return applyPatterns(caseDict, patterns, filename, lambda b: b)
 235
 236 def applyExclude (caseDict, patterns, filename):
 237         return applyPatterns(caseDict, patterns, filename, lambda b: not b)
 238
 239 def readPatternLists (mustpass):
 240         lists = {}
 241         for package in mustpass.packages:
 242                 for cfg in package.configurations:
 243                         for filter in cfg.filters:
 244                                 if not filter.filename in lists:
 245                                         lists[filter.filename] = readPatternList(os.path.join(getSrcDir(mustpass), filter.filename))
 246         return lists
 247
 248 def applyFilters (caseDict, patternLists, filters):
 249         res = copy(caseDict)
 250         for filter in filters:
 251                 ptrnList = patternLists[filter.filename]
 252                 if filter.type == Filter.TYPE_INCLUDE:
 253                         res = applyInclude(res, ptrnList, filter.filename)
 254                 else:
 255                         assert filter.type == Filter.TYPE_EXCLUDE
 256                         res = applyExclude(res, ptrnList, filter.filename)
 257         return res
 258
 259 def appendToHierarchy (root, casePath):
 260         def findChild (node, name):
 261                 for child in node.children:
 262                         if child.name == name:
 263                                 return child
 264                 return None
 265
 266         curNode         = root
 267         components      = casePath.split('.')
 268
 269         for component in components[:-1]:
 270                 nextNode = findChild(curNode, component)
 271                 if not nextNode:
 272                         nextNode = TestGroup(component)
 273                         curNode.children.append(nextNode)
 274                 curNode = nextNode
 275
 276         if not findChild(curNode, components[-1]):
 277                 curNode.children.append(TestCase(components[-1]))
 278
 279 def buildTestHierachy (caseList):
 280         root = TestRoot()
 281         for case in caseList:
 282                 appendToHierarchy(root, case)
 283         return root
 284
 285 def buildTestCaseMap (root):
 286         caseMap = {}
 287
 288         def recursiveBuild (curNode, prefix):
 289                 curPath = prefix + curNode.name
 290                 if isinstance(curNode, TestCase):
 291                         caseMap[curPath] = curNode
 292                 else:
 293                         for child in curNode.children:
 294                                 recursiveBuild(child, curPath + '.')
 295
 296         for child in root.children:
 297                 recursiveBuild(child, '')
 298
 299         return caseMap
 300
 301 def include (filename):
 302         return Filter(Filter.TYPE_INCLUDE, filename)
 303
 304 def exclude (filename):
 305         return Filter(Filter.TYPE_EXCLUDE, filename)
 306
 307 def insertXMLHeaders (mustpass, doc):
 308         if mustpass.project.copyright != None:
 309                 doc.insert(0, ElementTree.Comment(mustpass.project.copyright))
 310         doc.insert(1, ElementTree.Comment(GENERATED_FILE_WARNING))
 311
 312 def prettifyXML (doc):
 313         uglyString      = ElementTree.tostring(doc, 'utf-8')
 314         reparsed        = minidom.parseString(uglyString)
 315         return reparsed.toprettyxml(indent='\t', encoding='utf-8')
 316
 317 def genSpecXML (mustpass):
 318         mustpassElem = ElementTree.Element("Mustpass", version = mustpass.version)
 319         insertXMLHeaders(mustpass, mustpassElem)
 320
 321         for package in mustpass.packages:
 322                 packageElem = ElementTree.SubElement(mustpassElem, "TestPackage", name = package.module.name)
 323
 324                 for config in package.configurations:
 325                         configElem = ElementTree.SubElement(packageElem, "Configuration",
 326                                                                                                 caseListFile    = getCaseListFileName(package, config),
 327                                                                                                 commandLine             = getCommandLine(config),
 328                                                                                                 name                    = config.name)
 329
 330         return mustpassElem
 331
 332 def addOptionElement (parent, optionName, optionValue):
 333         ElementTree.SubElement(parent, "option", name=optionName, value=optionValue)
 334
 335 def genAndroidTestXml (mustpass):
 336         RUNNER_CLASS = "com.drawelements.deqp.runner.DeqpTestRunner"
 337         configElement = ElementTree.Element("configuration")
 338
 339         # have the deqp package installed on the device for us
 340         preparerElement = ElementTree.SubElement(configElement, "target_preparer")
 341         preparerElement.set("class", "com.android.tradefed.targetprep.suite.SuiteApkInstaller")
 342         addOptionElement(preparerElement, "cleanup-apks", "true")
 343         addOptionElement(preparerElement, "test-file-name", "com.drawelements.deqp.apk")
 344
 345         # Target preparer for incremental dEQP
 346         preparerElement = ElementTree.SubElement(configElement, "target_preparer")
 347         preparerElement.set("class", "com.android.compatibility.common.tradefed.targetprep.IncrementalDeqpPreparer")
 348         addOptionElement(preparerElement, "disable", "true")
 349
 350         # add in metadata option for component name
 351         ElementTree.SubElement(configElement, "option", name="test-suite-tag", value="cts")
 352         ElementTree.SubElement(configElement, "option", key="component", name="config-descriptor:metadata", value="deqp")
 353         ElementTree.SubElement(configElement, "option", key="parameter", name="config-descriptor:metadata", value="not_instant_app")
 354         ElementTree.SubElement(configElement, "option", key="parameter", name="config-descriptor:metadata", value="multi_abi")
 355         ElementTree.SubElement(configElement, "option", key="parameter", name="config-descriptor:metadata", value="secondary_user")
 356         ElementTree.SubElement(configElement, "option", key="parameter", name="config-descriptor:metadata", value="no_foldable_states")
 357         controllerElement = ElementTree.SubElement(configElement, "object")
 358         controllerElement.set("class", "com.android.tradefed.testtype.suite.module.TestFailureModuleController")
 359         controllerElement.set("type", "module_controller")
 360         addOptionElement(controllerElement, "screenshot-on-failure", "false")
 361
 362         for package in mustpass.packages:
 363                 for config in package.configurations:
 364                         if not config.runByDefault:
 365                                 continue
 366
 367                         testElement = ElementTree.SubElement(configElement, "test")
 368                         testElement.set("class", RUNNER_CLASS)
 369                         addOptionElement(testElement, "deqp-package", package.module.name)
 370                         caseListFile = getCaseListFileName(package,config)
 371                         addOptionElement(testElement, "deqp-caselist-file", caseListFile)
 372                         if caseListFile.startswith("gles3"):
 373                                 addOptionElement(testElement, "incremental-deqp-include-file", "gles3-incremental-deqp.txt")
 374                         elif caseListFile.startswith("vk"):
 375                                 addOptionElement(testElement, "incremental-deqp-include-file", "vk-incremental-deqp.txt")
 376                         # \todo [2015-10-16 kalle]: Replace with just command line? - requires simplifications in the runner/tests as well.
 377                         if config.glconfig != None:
 378                                 addOptionElement(testElement, "deqp-gl-config-name", config.glconfig)
 379
 380                         if config.surfacetype != None:
 381                                 addOptionElement(testElement, "deqp-surface-type", config.surfacetype)
 382
 383                         if config.rotation != None:
 384                                 addOptionElement(testElement, "deqp-screen-rotation", config.rotation)
 385
 386                         if config.expectedRuntime != None:
 387                                 addOptionElement(testElement, "runtime-hint", config.expectedRuntime)
 388
 389                         if config.required:
 390                                 addOptionElement(testElement, "deqp-config-required", "true")
 391
 392         insertXMLHeaders(mustpass, configElement)
 393
 394         return configElement
 395
 396 def genMustpass (mustpass, moduleCaseDicts):
 397         print("Generating mustpass '%s'" % mustpass.version)
 398
 399         patternLists = readPatternLists(mustpass)
 400
 401         for package in mustpass.packages:
 402                 allCasesInPkgDict       = moduleCaseDicts[package.module]
 403
 404                 for config in package.configurations:
 405
 406                         # construct dictionary with all filters applyed,
 407                         # key is top-level group name, value is list of all cases in that group
 408                         filteredCaseDict        = applyFilters(allCasesInPkgDict, patternLists, config.filters)
 409
 410                         # construct components of path to main destination file
 411                         mainDstFilePath         = getDstCaseListPath(mustpass)
 412                         mainDstFileName         = getCaseListFileName(package, config)
 413                         mainDstFile                     = os.path.join(mainDstFilePath, mainDstFileName)
 414                         gruopSubDir                     = mainDstFileName[:-4]
 415
 416                         # if case paths should be split to multiple files then main
 417                         # destination file will contain paths to individual files containing cases
 418                         if config.splitToMultipleFiles:
 419                                 groupPathsList = []
 420
 421                                 # make sure directory for group files exists
 422                                 groupPath = os.path.join(mainDstFilePath, gruopSubDir)
 423                                 if not os.path.exists(groupPath):
 424                                         os.makedirs(groupPath)
 425
 426                                 # iterate over all top-level groups and write files containing their cases
 427                                 print("  Writing top-level group caselists:")
 428                                 for tlGroup in filteredCaseDict:
 429                                         groupDstFileName    = tlGroup + ".txt"
 430                                         groupDstFileFullDir = os.path.join(groupPath, groupDstFileName)
 431                                         groupPathsList.append(gruopSubDir + "/" + groupDstFileName)
 432
 433                                         print("    " + groupDstFileFullDir)
 434                                         writeFile(groupDstFileFullDir, "\n".join(filteredCaseDict[tlGroup]) + "\n")
 435
 436                                 # write file containing names of all group files
 437                                 print("  Writing deqp top-level groups file list: " + mainDstFile)
 438                                 groupPathsList.sort()
 439                                 writeFile(mainDstFile, "\n".join(groupPathsList) + "\n")
 440                         else:
 441                                 # merge cases from all top level groups in to single case list
 442                                 filteredCaseList = []
 443                                 for tlGroup in filteredCaseDict:
 444                                         filteredCaseList.extend(filteredCaseDict[tlGroup])
 445
 446                                 # write file containing all cases
 447                                 print("  Writing deqp caselist: " + mainDstFile)
 448                                 writeFile(mainDstFile, "\n".join(filteredCaseList) + "\n")
 449
 450         specXML = genSpecXML(mustpass)
 451         specFilename = os.path.join(mustpass.project.path, mustpass.version, "mustpass.xml")
 452
 453         print("  Writing spec: " + specFilename)
 454         writeFile(specFilename, prettifyXML(specXML).decode())
 455
 456         # TODO: Which is the best selector mechanism?
 457         if (mustpass.version == "master"):
 458                 androidTestXML          = genAndroidTestXml(mustpass)
 459                 androidTestFilename     = os.path.join(mustpass.project.path, "AndroidTest.xml")
 460
 461                 print("  Writing AndroidTest.xml: " + androidTestFilename)
 462                 writeFile(androidTestFilename, prettifyXML(androidTestXML).decode())
 463
 464         print("Done!")
 465
 466 def genMustpassLists (mustpassLists, generator, buildCfg):
 467         moduleCaseDicts = {}
 468
 469         # Getting case lists involves invoking build, so we want to cache the results
 470         for mustpass in mustpassLists:
 471                 for package in mustpass.packages:
 472                         if not package.module in moduleCaseDicts:
 473                                 moduleCaseDicts[package.module] = getCaseDict(buildCfg, generator, package.module)
 474
 475         for mustpass in mustpassLists:
 476                 genMustpass(mustpass, moduleCaseDicts)
 477
 478 def parseCmdLineArgs ():
 479         parser = argparse.ArgumentParser(description = "Build Android CTS mustpass",
 480                                                                          formatter_class=argparse.ArgumentDefaultsHelpFormatter)
 481         parser.add_argument("-b",
 482                                                 "--build-dir",
 483                                                 dest="buildDir",
 484                                                 default=DEFAULT_BUILD_DIR,
 485                                                 help="Temporary build directory")
 486         parser.add_argument("-t",
 487                                                 "--build-type",
 488                                                 dest="buildType",
 489                                                 default="Debug",
 490                                                 help="Build type")
 491         parser.add_argument("-c",
 492                                                 "--deqp-target",
 493                                                 dest="targetName",
 494                                                 default=DEFAULT_TARGET,
 495                                                 help="dEQP build target")
 496         return parser.parse_args()
 497
 498 def parseBuildConfigFromCmdLineArgs ():
 499         args = parseCmdLineArgs()
 500         return getBuildConfig(args.buildDir, args.targetName, args.buildType)