scripts/mustpass.py

   1 # -*- coding: utf-8 -*-
   2
   3 #-------------------------------------------------------------------------
   4 # drawElements Quality Program utilities
   5 # --------------------------------------
   6 #
   7 # Copyright 2016 The Android Open Source Project
   8 #
   9 # Licensed under the Apache License, Version 2.0 (the "License");
  10 # you may not use this file except in compliance with the License.
  11 # You may obtain a copy of the License at
  12 #
  13 #      http://www.apache.org/licenses/LICENSE-2.0
  14 #
  15 # Unless required by applicable law or agreed to in writing, software
  16 # distributed under the License is distributed on an "AS IS" BASIS,
  17 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  18 # See the License for the specific language governing permissions and
  19 # limitations under the License.
  20 #
  21 #-------------------------------------------------------------------------
  22
  23 from ctsbuild.common import *
  24 from ctsbuild.config import ANY_GENERATOR
  25 from ctsbuild.build import build
  26 from build_caselists import Module, getModuleByName, getBuildConfig, genCaseList, getCaseListPath, DEFAULT_BUILD_DIR, DEFAULT_TARGET
  27 from fnmatch import fnmatch
  28 from copy import copy
  29 from collections import defaultdict
  30
  31 import argparse
  32 import xml.etree.cElementTree as ElementTree
  33 import xml.dom.minidom as minidom
  34
  35 APK_NAME                = "com.drawelements.deqp.apk"
  36
  37 GENERATED_FILE_WARNING = """
  38      This file has been automatically generated. Edit with caution.
  39      """
  40
  41 class Project:
  42         def __init__ (self, path, copyright = None):
  43                 self.path               = path
  44                 self.copyright  = copyright
  45
  46 class Configuration:
  47         def __init__ (self, name, filters, glconfig = None, rotation = None, surfacetype = None, required = False, runtime = None, runByDefault = True, listOfGroupsToSplit = []):
  48                 self.name                                       = name
  49                 self.glconfig                           = glconfig
  50                 self.rotation                           = rotation
  51                 self.surfacetype                        = surfacetype
  52                 self.required                           = required
  53                 self.filters                            = filters
  54                 self.expectedRuntime            = runtime
  55                 self.runByDefault                       = runByDefault
  56                 self.listOfGroupsToSplit        = listOfGroupsToSplit
  57
  58 class Package:
  59         def __init__ (self, module, configurations):
  60                 self.module                     = module
  61                 self.configurations     = configurations
  62
  63 class Mustpass:
  64         def __init__ (self, project, version, packages):
  65                 self.project    = project
  66                 self.version    = version
  67                 self.packages   = packages
  68
  69 class Filter:
  70         TYPE_INCLUDE = 0
  71         TYPE_EXCLUDE = 1
  72
  73         def __init__ (self, type, filename):
  74                 self.type               = type
  75                 self.filename   = filename
  76
  77 class TestRoot:
  78         def __init__ (self):
  79                 self.children   = []
  80
  81 class TestGroup:
  82         def __init__ (self, name):
  83                 self.name               = name
  84                 self.children   = []
  85
  86 class TestCase:
  87         def __init__ (self, name):
  88                 self.name                       = name
  89                 self.configurations     = []
  90
  91 class GLESVersion:
  92         def __init__(self, major, minor):
  93                 self.major = major
  94                 self.minor = minor
  95
  96         def encode (self):
  97                 return (self.major << 16) | (self.minor)
  98
  99 def getModuleGLESVersion (module):
 100         versions = {
 101                 'dEQP-EGL':             GLESVersion(2,0),
 102                 'dEQP-GLES2':   GLESVersion(2,0),
 103                 'dEQP-GLES3':   GLESVersion(3,0),
 104                 'dEQP-GLES31':  GLESVersion(3,1)
 105         }
 106         return versions[module.name] if module.name in versions else None
 107
 108 def getSrcDir (mustpass):
 109         return os.path.join(mustpass.project.path, mustpass.version, "src")
 110
 111 def getTmpDir (mustpass):
 112         return os.path.join(mustpass.project.path, mustpass.version, "tmp")
 113
 114 def getModuleShorthand (module):
 115         assert module.name[:5] == "dEQP-"
 116         return module.name[5:].lower()
 117
 118 def getCaseListFileName (package, configuration):
 119         return "%s-%s.txt" % (getModuleShorthand(package.module), configuration.name)
 120
 121 def getDstCaseListPath (mustpass):
 122         return os.path.join(mustpass.project.path, mustpass.version)
 123
 124 def getCTSPackageName (package):
 125         return "com.drawelements.deqp." + getModuleShorthand(package.module)
 126
 127 def getCommandLine (config):
 128         cmdLine = ""
 129
 130         if config.glconfig != None:
 131                 cmdLine += "--deqp-gl-config-name=%s " % config.glconfig
 132
 133         if config.rotation != None:
 134                 cmdLine += "--deqp-screen-rotation=%s " % config.rotation
 135
 136         if config.surfacetype != None:
 137                 cmdLine += "--deqp-surface-type=%s " % config.surfacetype
 138
 139         cmdLine += "--deqp-watchdog=enable"
 140
 141         return cmdLine
 142
 143 def readCaseDict (filename):
 144         # read all cases and organize them in a tree; this is needed for chunked mustpass
 145         # groups are stored as dictionaries and cases as list of strings with full case paths
 146         groupTree = {}
 147         # limit how deep constructed tree should be - this later simplifies applying filters;
 148         # if in future we will need to split to separate .txt files deeper groups thet this value should be increased
 149         limitGroupTreeDepth = 3
 150         # create helper stack that will contain references to currently filled groups, from top to bottom
 151         groupStack = None
 152         # cretae variable that will hold currentlt processed line from the file
 153         processedLine = None
 154         with open(filename, 'rt') as f:
 155                 for nextLine in f:
 156                         # to be able to build tree structure we need to know what is the next line in the file this is
 157                         # why the first line read from the file will be actually processed during the second iteration
 158                         if processedLine is None:
 159                                 processedLine           = nextLine
 160                                 # to simplify code use this section to also extract root node name
 161                                 rootName                        = processedLine[7:processedLine.rfind('.')]
 162                                 groupTree[rootName]     = {}
 163                                 groupStack                      = [groupTree[rootName]]
 164                                 continue
 165                         # check if currently processed line is a test case or a group
 166                         processedEntryType = processedLine[:6]
 167                         if processedEntryType == "TEST: ":
 168                                 # append this test case to the last group on the stack
 169                                 groupStack[-1].append(processedLine[6:].strip())
 170                         elif processedEntryType == "GROUP:":
 171                                 # count number of dots in path to determine what is the depth of current group in the tree
 172                                 processedGroupDepth = processedLine.count('.')
 173                                 # limit tree construction just to specified level
 174                                 availableLimit = limitGroupTreeDepth - processedGroupDepth
 175                                 if availableLimit > 0:
 176                                         # check how deep is stack currently
 177                                         groupStackDepth = len(groupStack)
 178                                         # if stack is deeper then depth of current group then we need to pop number of items
 179                                         if processedGroupDepth < groupStackDepth:
 180                                                 groupStack = groupStack[:groupStackDepth-(groupStackDepth-processedGroupDepth)]
 181                                         # get group that will have new child - this is the last group on the stack
 182                                         parentGroup = groupStack[-1]
 183                                         # add new dict that will contain other groups or list of cases depending on the next line
 184                                         # and available depth limit (if are about to reach limit we won't add group dictionaries
 185                                         # but just add all cases from deeper groups to the group at this depth)
 186                                         processedGroupName = processedLine[7:-1]
 187                                         parentGroup[processedGroupName] = {} if (nextLine[:6] == "GROUP:") and (availableLimit > 1) else []
 188                                         # add new group to the stack (items in groupStack can be either list or dict)
 189                                         groupStack.append(parentGroup[processedGroupName])
 190                         # before going to the next line set procesedLine for the next iteration
 191                         processedLine = nextLine
 192         # handle last test case - we need to do it after the loop as in the loop we needed to know what is the next line
 193         assert(processedLine[:6] == "TEST: ")
 194         groupStack[-1].append(processedLine[6:].strip())
 195         return groupTree
 196
 197 def getCaseDict (buildCfg, generator, module):
 198         build(buildCfg, generator, [module.binName])
 199         genCaseList(buildCfg, generator, module, "txt")
 200         return readCaseDict(getCaseListPath(buildCfg, module, "txt"))
 201
 202 def readPatternList (filename):
 203         ptrns = []
 204         with open(filename, 'rt') as f:
 205                 for line in f:
 206                         line = line.strip()
 207                         if len(line) > 0 and line[0] != '#':
 208                                 ptrns.append(line)
 209         return ptrns
 210
 211
 212 def constructNewDict(oldDict, listOfCases, op = lambda a: not a):
 213         # Helper function used to construct case dictionary without specific cases
 214         rootName                = list(oldDict.keys())[0]
 215         newDict                 = {rootName : {}}
 216         newDictStack    = [newDict]
 217         oldDictStack    = [oldDict]
 218         while True:
 219                 # mak sure that both stacks have same number of items
 220                 assert(len(oldDictStack) == len(newDictStack))
 221                 # when all items from stack were processed then we can exit the loop
 222                 if len(oldDictStack) == 0:
 223                         break
 224                 # grab last item from both stacks
 225                 itemOnOldStack = oldDictStack.pop()
 226                 itemOnNewStack = newDictStack.pop()
 227                 # if item on stack is dictionary then it represents groups and
 228                 # we need to reconstruct them in new dictionary
 229                 if type(itemOnOldStack) is dict:
 230                         assert(type(itemOnNewStack) is dict)
 231                         listOfGroups = list(itemOnOldStack.keys())
 232                         for groupName in listOfGroups:
 233                                 # create list or dictionary depending on contnent of child group
 234                                 doesGroupsContainCases = type(itemOnOldStack[groupName]) is list
 235                                 itemOnNewStack[groupName] = [] if doesGroupsContainCases else {}
 236                                 # append groups on stacks
 237                                 assert(type(itemOnNewStack[groupName]) == type(itemOnOldStack[groupName]))
 238                                 newDictStack.append(itemOnNewStack[groupName])
 239                                 oldDictStack.append(itemOnOldStack[groupName])
 240                 else:
 241                         # if item on stack is list then it represents group that contain cases we need
 242                         # to apply filter on each of them to make sure only proper cases are appended
 243                         assert(type(itemOnOldStack) is list)
 244                         assert(type(itemOnNewStack) is list)
 245                         for caseName in itemOnOldStack:
 246                                 if op(caseName in listOfCases):
 247                                         itemOnNewStack.append(caseName)
 248         return newDict
 249
 250 def constructSet(caseDict, perGroupOperation):
 251         casesSet                = set()
 252         dictStack               = [caseDict]
 253         while True:
 254                 # when all items from stack were processed then we can exit the loop
 255                 if len(dictStack) == 0:
 256                         break
 257                 # grab last item from stack
 258                 itemOnStack = dictStack.pop()
 259                 # if item on stack is dictionary then it represents groups and we need to add them to stack
 260                 if type(itemOnStack) is dict:
 261                         for groupName in itemOnStack.keys():
 262                                 dictStack.append(itemOnStack[groupName])
 263                 else:
 264                         # if item on stack is a list of cases we can add them to set containing all cases
 265                         assert(type(itemOnStack) is list)
 266                         casesSet = perGroupOperation(casesSet, itemOnStack)
 267         return casesSet
 268
 269 def applyPatterns (caseDict, patterns, filename, op):
 270         matched                 = set()
 271         errors                  = []
 272         trivialPtrns    = [p for p in patterns if p.find('*') < 0]
 273         regularPtrns    = [p for p in patterns if p.find('*') >= 0]
 274
 275         # Construct helper set that contains cases from all groups
 276         unionOperation  = lambda resultCasesSet, groupCaseList: resultCasesSet.union(set(groupCaseList))
 277         allCasesSet             = constructSet(caseDict, unionOperation)
 278
 279         # Apply trivial patterns - plain case paths without wildcard
 280         for path in trivialPtrns:
 281                 if path in allCasesSet:
 282                         if path in matched:
 283                                 errors.append((path, "Same case specified more than once"))
 284                         matched.add(path)
 285                 else:
 286                         errors.append((path, "Test case not found"))
 287
 288         # Construct new dictionary but without already matched paths
 289         curDict = constructNewDict(caseDict, matched)
 290
 291         # Apply regular patterns - paths with wildcard
 292         for pattern in regularPtrns:
 293
 294                 # Helper function that checks if cases from case group match pattern
 295                 def matchOperation(resultCasesSet, groupCaseList):
 296                         for caseName in groupCaseList:
 297                                 if fnmatch(caseName, pattern):
 298                                         resultCasesSet.add(caseName)
 299                         return resultCasesSet
 300
 301                 matchedThisPtrn = constructSet(curDict, matchOperation)
 302
 303                 if len(matchedThisPtrn) == 0:
 304                         errors.append((pattern, "Pattern didn't match any cases"))
 305
 306                 matched = matched | matchedThisPtrn
 307
 308                 # To speed up search construct smaller case dictionary without already matched paths
 309                 curDict = constructNewDict(curDict, matched)
 310
 311         for pattern, reason in errors:
 312                 print("ERROR: %s: %s" % (reason, pattern))
 313
 314         if len(errors) > 0:
 315                 die("Found %s invalid patterns while processing file %s" % (len(errors), filename))
 316
 317         # Construct final dictionary using aproperiate operation
 318         return constructNewDict(caseDict, matched, op)
 319
 320 def applyInclude (caseDict, patterns, filename):
 321         return applyPatterns(caseDict, patterns, filename, lambda b: b)
 322
 323 def applyExclude (caseDict, patterns, filename):
 324         return applyPatterns(caseDict, patterns, filename, lambda b: not b)
 325
 326 def readPatternLists (mustpass):
 327         lists = {}
 328         for package in mustpass.packages:
 329                 for cfg in package.configurations:
 330                         for filter in cfg.filters:
 331                                 if not filter.filename in lists:
 332                                         lists[filter.filename] = readPatternList(os.path.join(getSrcDir(mustpass), filter.filename))
 333         return lists
 334
 335 def applyFilters (caseDict, patternLists, filters):
 336         res = copy(caseDict)
 337         for filter in filters:
 338                 ptrnList = patternLists[filter.filename]
 339                 if filter.type == Filter.TYPE_INCLUDE:
 340                         res = applyInclude(res, ptrnList, filter.filename)
 341                 else:
 342                         assert filter.type == Filter.TYPE_EXCLUDE
 343                         res = applyExclude(res, ptrnList, filter.filename)
 344         return res
 345
 346 def appendToHierarchy (root, casePath):
 347         def findChild (node, name):
 348                 for child in node.children:
 349                         if child.name == name:
 350                                 return child
 351                 return None
 352
 353         curNode         = root
 354         components      = casePath.split('.')
 355
 356         for component in components[:-1]:
 357                 nextNode = findChild(curNode, component)
 358                 if not nextNode:
 359                         nextNode = TestGroup(component)
 360                         curNode.children.append(nextNode)
 361                 curNode = nextNode
 362
 363         if not findChild(curNode, components[-1]):
 364                 curNode.children.append(TestCase(components[-1]))
 365
 366 def buildTestHierachy (caseList):
 367         root = TestRoot()
 368         for case in caseList:
 369                 appendToHierarchy(root, case)
 370         return root
 371
 372 def buildTestCaseMap (root):
 373         caseMap = {}
 374
 375         def recursiveBuild (curNode, prefix):
 376                 curPath = prefix + curNode.name
 377                 if isinstance(curNode, TestCase):
 378                         caseMap[curPath] = curNode
 379                 else:
 380                         for child in curNode.children:
 381                                 recursiveBuild(child, curPath + '.')
 382
 383         for child in root.children:
 384                 recursiveBuild(child, '')
 385
 386         return caseMap
 387
 388 def include (filename):
 389         return Filter(Filter.TYPE_INCLUDE, filename)
 390
 391 def exclude (filename):
 392         return Filter(Filter.TYPE_EXCLUDE, filename)
 393
 394 def insertXMLHeaders (mustpass, doc):
 395         if mustpass.project.copyright != None:
 396                 doc.insert(0, ElementTree.Comment(mustpass.project.copyright))
 397         doc.insert(1, ElementTree.Comment(GENERATED_FILE_WARNING))
 398
 399 def prettifyXML (doc):
 400         uglyString      = ElementTree.tostring(doc, 'utf-8')
 401         reparsed        = minidom.parseString(uglyString)
 402         return reparsed.toprettyxml(indent='\t', encoding='utf-8')
 403
 404 def genSpecXML (mustpass):
 405         mustpassElem = ElementTree.Element("Mustpass", version = mustpass.version)
 406         insertXMLHeaders(mustpass, mustpassElem)
 407
 408         for package in mustpass.packages:
 409                 packageElem = ElementTree.SubElement(mustpassElem, "TestPackage", name = package.module.name)
 410
 411                 for config in package.configurations:
 412                         configElem = ElementTree.SubElement(packageElem, "Configuration",
 413                                                                                                 caseListFile    = getCaseListFileName(package, config),
 414                                                                                                 commandLine             = getCommandLine(config),
 415                                                                                                 name                    = config.name)
 416
 417         return mustpassElem
 418
 419 def addOptionElement (parent, optionName, optionValue):
 420         ElementTree.SubElement(parent, "option", name=optionName, value=optionValue)
 421
 422 def genAndroidTestXml (mustpass):
 423         RUNNER_CLASS = "com.drawelements.deqp.runner.DeqpTestRunner"
 424         configElement = ElementTree.Element("configuration")
 425
 426         # have the deqp package installed on the device for us
 427         preparerElement = ElementTree.SubElement(configElement, "target_preparer")
 428         preparerElement.set("class", "com.android.tradefed.targetprep.suite.SuiteApkInstaller")
 429         addOptionElement(preparerElement, "cleanup-apks", "true")
 430         addOptionElement(preparerElement, "test-file-name", "com.drawelements.deqp.apk")
 431
 432         # Target preparer for incremental dEQP
 433         preparerElement = ElementTree.SubElement(configElement, "target_preparer")
 434         preparerElement.set("class", "com.android.compatibility.common.tradefed.targetprep.IncrementalDeqpPreparer")
 435         addOptionElement(preparerElement, "disable", "true")
 436
 437         # add in metadata option for component name
 438         ElementTree.SubElement(configElement, "option", name="test-suite-tag", value="cts")
 439         ElementTree.SubElement(configElement, "option", key="component", name="config-descriptor:metadata", value="deqp")
 440         ElementTree.SubElement(configElement, "option", key="parameter", name="config-descriptor:metadata", value="not_instant_app")
 441         ElementTree.SubElement(configElement, "option", key="parameter", name="config-descriptor:metadata", value="multi_abi")
 442         ElementTree.SubElement(configElement, "option", key="parameter", name="config-descriptor:metadata", value="secondary_user")
 443         ElementTree.SubElement(configElement, "option", key="parameter", name="config-descriptor:metadata", value="no_foldable_states")
 444         controllerElement = ElementTree.SubElement(configElement, "object")
 445         controllerElement.set("class", "com.android.tradefed.testtype.suite.module.TestFailureModuleController")
 446         controllerElement.set("type", "module_controller")
 447         addOptionElement(controllerElement, "screenshot-on-failure", "false")
 448
 449         for package in mustpass.packages:
 450                 for config in package.configurations:
 451                         if not config.runByDefault:
 452                                 continue
 453
 454                         testElement = ElementTree.SubElement(configElement, "test")
 455                         testElement.set("class", RUNNER_CLASS)
 456                         addOptionElement(testElement, "deqp-package", package.module.name)
 457                         caseListFile = getCaseListFileName(package,config)
 458                         addOptionElement(testElement, "deqp-caselist-file", caseListFile)
 459                         if caseListFile.startswith("gles3"):
 460                                 addOptionElement(testElement, "incremental-deqp-include-file", "gles3-incremental-deqp.txt")
 461                         elif caseListFile.startswith("vk"):
 462                                 addOptionElement(testElement, "incremental-deqp-include-file", "vk-incremental-deqp.txt")
 463                         # \todo [2015-10-16 kalle]: Replace with just command line? - requires simplifications in the runner/tests as well.
 464                         if config.glconfig != None:
 465                                 addOptionElement(testElement, "deqp-gl-config-name", config.glconfig)
 466
 467                         if config.surfacetype != None:
 468                                 addOptionElement(testElement, "deqp-surface-type", config.surfacetype)
 469
 470                         if config.rotation != None:
 471                                 addOptionElement(testElement, "deqp-screen-rotation", config.rotation)
 472
 473                         if config.expectedRuntime != None:
 474                                 addOptionElement(testElement, "runtime-hint", config.expectedRuntime)
 475
 476                         if config.required:
 477                                 addOptionElement(testElement, "deqp-config-required", "true")
 478
 479         insertXMLHeaders(mustpass, configElement)
 480
 481         return configElement
 482
 483 def genMustpass (mustpass, moduleCaseDicts):
 484         print("Generating mustpass '%s'" % mustpass.version)
 485
 486         patternLists = readPatternLists(mustpass)
 487
 488         for package in mustpass.packages:
 489                 allCasesInPkgDict       = moduleCaseDicts[package.module]
 490
 491                 for config in package.configurations:
 492
 493                         # construct dictionary with all filters applyed
 494                         filteredCaseDict        = applyFilters(allCasesInPkgDict, patternLists, config.filters)
 495
 496                         # construct components of path to main destination file
 497                         mainDstFilePath         = getDstCaseListPath(mustpass)
 498                         mainDstFileName         = getCaseListFileName(package, config)
 499                         mainDstFile                     = os.path.join(mainDstFilePath, mainDstFileName)
 500                         mainGruopSubDir         = mainDstFileName[:-4]
 501
 502                         # if case paths should be split to multiple files then main
 503                         # destination file will contain paths to individual files containing cases
 504                         if len(config.listOfGroupsToSplit) > 0:
 505                                 # make sure directory for group files exists
 506                                 rootGroupPath = os.path.join(mainDstFilePath, mainGruopSubDir)
 507                                 if not os.path.exists(rootGroupPath):
 508                                         os.makedirs(rootGroupPath)
 509
 510                                 # iterate over case dictionary and split it to .txt files acording to
 511                                 # groups that were specified in config.listOfGroupsToSplit
 512                                 splitedGroupsDict       = {}
 513                                 dictStack                       = [filteredCaseDict]
 514                                 helperListStack         = [ [] ]
 515                                 while True:
 516                                         # when all items from stack were processed then we can exit the loop
 517                                         if len(dictStack) == 0:
 518                                                 break
 519                                         assert(len(dictStack) == len(helperListStack))
 520                                         # grab last item from stack
 521                                         itemOnStack = dictStack.pop()
 522                                         caseListFromHelperStack = helperListStack.pop()
 523                                         # if item on stack is dictionary then it represents groups and we need to add them to stack
 524                                         if type(itemOnStack) is dict:
 525                                                 for groupName in sorted(itemOnStack):
 526
 527                                                         # check if this group should be split to multiple .txt files
 528                                                         if groupName in config.listOfGroupsToSplit:
 529                                                                 # we can split only groups that contain other groups,
 530                                                                 # listOfGroupsToSplit should not contain groups that contain test cases
 531                                                                 assert(type(itemOnStack[groupName]) is dict)
 532                                                                 # add child groups of this group to splitedGroupsDict
 533                                                                 for childGroupName in itemOnStack[groupName]:
 534                                                                         # make sure that child group should not be splited
 535                                                                         # (if it should then this will be handle in one of the next iterations)
 536                                                                         if childGroupName not in config.listOfGroupsToSplit:
 537                                                                                 splitedGroupsDict[childGroupName] = []
 538
 539                                                         # add this group to stack used for iteration over casses tree
 540                                                         dictStack.append(itemOnStack[groupName])
 541
 542                                                         # decide what list we should append to helperListStack;
 543                                                         # if this group represents one of individual .txt files then grab
 544                                                         # propper array of cases from splitedGroupsDict and add it to helper stack;
 545                                                         # if groupName is not in splitedGroupsDict then use the same list as was used
 546                                                         # by parent group (we are merging casses from those groups to single .txt file)
 547                                                         helperListStack.append(splitedGroupsDict.get(groupName, caseListFromHelperStack))
 548                                         else:
 549                                                 # if item on stack is a list of cases we can add them to proper list
 550                                                 assert(type(itemOnStack) is list)
 551                                                 caseListFromHelperStack.extend(itemOnStack)
 552
 553                                 print("  Writing separated caselists:")
 554                                 groupPathsList = []
 555                                 for groupPath in splitedGroupsDict:
 556                                         # skip groups that after filtering have no casses left
 557                                         if len(splitedGroupsDict[groupPath]) == 0:
 558                                                 continue
 559                                         # remove root node name from the beginning of group and replace all '_' with '-'
 560                                         processedGroupPath = groupPath[groupPath.find('.')+1:].replace('_', '-')
 561                                         # split group paths
 562                                         groupList = processedGroupPath.split('.')
 563                                         groupSubDir = '/'
 564                                         # create subdirectories if there is more then one group name in groupList
 565                                         path = rootGroupPath
 566                                         if len(groupList) > 1:
 567                                                 for groupName in groupList[:-1]:
 568                                                         # make sure directory for group files exists
 569                                                         groupSubDir     = groupSubDir + groupName + '/'
 570                                                         path            = os.path.join(path, groupName)
 571                                                         if not os.path.exists(path):
 572                                                                 os.makedirs(path)
 573                                         # construct path to .txt file and save all cases
 574                                         groupDstFileName        = groupList[-1] + ".txt"
 575                                         groupDstFileFullDir     = os.path.join(path, groupDstFileName)
 576                                         groupPathsList.append(mainGruopSubDir + groupSubDir + groupDstFileName)
 577                                         print("    " + groupDstFileFullDir)
 578                                         writeFile(groupDstFileFullDir, "\n".join(splitedGroupsDict[groupPath]) + "\n")
 579
 580                                 # write file containing names of all group files
 581                                 print("  Writing file containing list of separated case files: " + mainDstFile)
 582                                 groupPathsList.sort()
 583                                 writeFile(mainDstFile, "\n".join(groupPathsList) + "\n")
 584                         else:
 585                                 # merge all cases to single case list
 586                                 filteredCaseList        = []
 587                                 dictStack                       = [filteredCaseDict]
 588                                 while True:
 589                                         # when all items from stack were processed then we can exit the loop
 590                                         if len(dictStack) == 0:
 591                                                 break
 592                                         # grab last item from stack
 593                                         itemOnStack = dictStack.pop()
 594                                         # if item on stack is dictionary then it represents groups and we need to add them to stack
 595                                         if type(itemOnStack) is dict:
 596                                                 for groupName in itemOnStack.keys():
 597                                                         dictStack.append(itemOnStack[groupName])
 598                                         else:
 599                                                 # if item on stack is a list of cases we can add them to filteredCaseList
 600                                                 assert(type(itemOnStack) is list)
 601                                                 filteredCaseList.extend(itemOnStack)
 602                                 # write file containing all cases
 603                                 if len(filteredCaseList) > 0:
 604                                         print("  Writing deqp caselist: " + mainDstFile)
 605                                         writeFile(mainDstFile, "\n".join(filteredCaseList) + "\n")
 606
 607         specXML = genSpecXML(mustpass)
 608         specFilename = os.path.join(mustpass.project.path, mustpass.version, "mustpass.xml")
 609
 610         print("  Writing spec: " + specFilename)
 611         writeFile(specFilename, prettifyXML(specXML).decode())
 612
 613         # TODO: Which is the best selector mechanism?
 614         if (mustpass.version == "master"):
 615                 androidTestXML          = genAndroidTestXml(mustpass)
 616                 androidTestFilename     = os.path.join(mustpass.project.path, "AndroidTest.xml")
 617
 618                 print("  Writing AndroidTest.xml: " + androidTestFilename)
 619                 writeFile(androidTestFilename, prettifyXML(androidTestXML).decode())
 620
 621         print("Done!")
 622
 623 def genMustpassLists (mustpassLists, generator, buildCfg):
 624         moduleCaseDicts = {}
 625
 626         # Getting case lists involves invoking build, so we want to cache the results
 627         for mustpass in mustpassLists:
 628                 for package in mustpass.packages:
 629                         if not package.module in moduleCaseDicts:
 630                                 moduleCaseDicts[package.module] = getCaseDict(buildCfg, generator, package.module)
 631
 632         for mustpass in mustpassLists:
 633                 genMustpass(mustpass, moduleCaseDicts)
 634
 635 def parseCmdLineArgs ():
 636         parser = argparse.ArgumentParser(description = "Build Android CTS mustpass",
 637                                                                          formatter_class=argparse.ArgumentDefaultsHelpFormatter)
 638         parser.add_argument("-b",
 639                                                 "--build-dir",
 640                                                 dest="buildDir",
 641                                                 default=DEFAULT_BUILD_DIR,
 642                                                 help="Temporary build directory")
 643         parser.add_argument("-t",
 644                                                 "--build-type",
 645                                                 dest="buildType",
 646                                                 default="Debug",
 647                                                 help="Build type")
 648         parser.add_argument("-c",
 649                                                 "--deqp-target",
 650                                                 dest="targetName",
 651                                                 default=DEFAULT_TARGET,
 652                                                 help="dEQP build target")
 653         return parser.parse_args()
 654
 655 def parseBuildConfigFromCmdLineArgs ():
 656         args = parseCmdLineArgs()
 657         return getBuildConfig(args.buildDir, args.targetName, args.buildType)