1 # -*- coding: utf-8 -*-
3 #-------------------------------------------------------------------------
4 # drawElements Quality Program utilities
5 # --------------------------------------
7 # Copyright 2016 The Android Open Source Project
9 # Licensed under the Apache License, Version 2.0 (the "License");
10 # you may not use this file except in compliance with the License.
11 # You may obtain a copy of the License at
13 # http://www.apache.org/licenses/LICENSE-2.0
15 # Unless required by applicable law or agreed to in writing, software
16 # distributed under the License is distributed on an "AS IS" BASIS,
17 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18 # See the License for the specific language governing permissions and
19 # limitations under the License.
21 #-------------------------------------------------------------------------
23 from ctsbuild.common import *
24 from ctsbuild.config import ANY_GENERATOR
25 from ctsbuild.build import build
26 from build_caselists import Module, getModuleByName, getBuildConfig, genCaseList, getCaseListPath, DEFAULT_BUILD_DIR, DEFAULT_TARGET
27 from fnmatch import fnmatch
29 from collections import defaultdict
32 import xml.etree.cElementTree as ElementTree
33 import xml.dom.minidom as minidom
35 APK_NAME = "com.drawelements.deqp.apk"
37 GENERATED_FILE_WARNING = """
38 This file has been automatically generated. Edit with caution.
42 def __init__ (self, path, copyright = None):
44 self.copyright = copyright
47 def __init__ (self, name, filters, glconfig = None, rotation = None, surfacetype = None, required = False, runtime = None, runByDefault = True, listOfGroupsToSplit = []):
49 self.glconfig = glconfig
50 self.rotation = rotation
51 self.surfacetype = surfacetype
52 self.required = required
53 self.filters = filters
54 self.expectedRuntime = runtime
55 self.runByDefault = runByDefault
56 self.listOfGroupsToSplit = listOfGroupsToSplit
59 def __init__ (self, module, configurations):
61 self.configurations = configurations
64 def __init__ (self, project, version, packages):
65 self.project = project
66 self.version = version
67 self.packages = packages
73 def __init__ (self, type, filename):
75 self.filename = filename
82 def __init__ (self, name):
87 def __init__ (self, name):
89 self.configurations = []
92 def __init__(self, major, minor):
97 return (self.major << 16) | (self.minor)
99 def getModuleGLESVersion (module):
101 'dEQP-EGL': GLESVersion(2,0),
102 'dEQP-GLES2': GLESVersion(2,0),
103 'dEQP-GLES3': GLESVersion(3,0),
104 'dEQP-GLES31': GLESVersion(3,1)
106 return versions[module.name] if module.name in versions else None
108 def getSrcDir (mustpass):
109 return os.path.join(mustpass.project.path, mustpass.version, "src")
111 def getTmpDir (mustpass):
112 return os.path.join(mustpass.project.path, mustpass.version, "tmp")
114 def getModuleShorthand (module):
115 assert module.name[:5] == "dEQP-"
116 return module.name[5:].lower()
118 def getCaseListFileName (package, configuration):
119 return "%s-%s.txt" % (getModuleShorthand(package.module), configuration.name)
121 def getDstCaseListPath (mustpass):
122 return os.path.join(mustpass.project.path, mustpass.version)
124 def getCTSPackageName (package):
125 return "com.drawelements.deqp." + getModuleShorthand(package.module)
127 def getCommandLine (config):
130 if config.glconfig != None:
131 cmdLine += "--deqp-gl-config-name=%s " % config.glconfig
133 if config.rotation != None:
134 cmdLine += "--deqp-screen-rotation=%s " % config.rotation
136 if config.surfacetype != None:
137 cmdLine += "--deqp-surface-type=%s " % config.surfacetype
139 cmdLine += "--deqp-watchdog=enable"
143 def readCaseDict (filename):
144 # read all cases and organize them in a tree; this is needed for chunked mustpass
145 # groups are stored as dictionaries and cases as list of strings with full case paths
147 # limit how deep constructed tree should be - this later simplifies applying filters;
148 # if in future we will need to split to separate .txt files deeper groups thet this value should be increased
149 limitGroupTreeDepth = 3
150 # create helper stack that will contain references to currently filled groups, from top to bottom
152 # cretae variable that will hold currentlt processed line from the file
154 with open(filename, 'rt') as f:
156 # to be able to build tree structure we need to know what is the next line in the file this is
157 # why the first line read from the file will be actually processed during the second iteration
158 if processedLine is None:
159 processedLine = nextLine
160 # to simplify code use this section to also extract root node name
161 rootName = processedLine[7:processedLine.rfind('.')]
162 groupTree[rootName] = {}
163 groupStack = [groupTree[rootName]]
165 # check if currently processed line is a test case or a group
166 processedEntryType = processedLine[:6]
167 if processedEntryType == "TEST: ":
168 # append this test case to the last group on the stack
169 groupStack[-1].append(processedLine[6:].strip())
170 elif processedEntryType == "GROUP:":
171 # count number of dots in path to determine what is the depth of current group in the tree
172 processedGroupDepth = processedLine.count('.')
173 # limit tree construction just to specified level
174 availableLimit = limitGroupTreeDepth - processedGroupDepth
175 if availableLimit > 0:
176 # check how deep is stack currently
177 groupStackDepth = len(groupStack)
178 # if stack is deeper then depth of current group then we need to pop number of items
179 if processedGroupDepth < groupStackDepth:
180 groupStack = groupStack[:groupStackDepth-(groupStackDepth-processedGroupDepth)]
181 # get group that will have new child - this is the last group on the stack
182 parentGroup = groupStack[-1]
183 # add new dict that will contain other groups or list of cases depending on the next line
184 # and available depth limit (if are about to reach limit we won't add group dictionaries
185 # but just add all cases from deeper groups to the group at this depth)
186 processedGroupName = processedLine[7:-1]
187 parentGroup[processedGroupName] = {} if (nextLine[:6] == "GROUP:") and (availableLimit > 1) else []
188 # add new group to the stack (items in groupStack can be either list or dict)
189 groupStack.append(parentGroup[processedGroupName])
190 # before going to the next line set procesedLine for the next iteration
191 processedLine = nextLine
192 # handle last test case - we need to do it after the loop as in the loop we needed to know what is the next line
193 assert(processedLine[:6] == "TEST: ")
194 groupStack[-1].append(processedLine[6:].strip())
197 def getCaseDict (buildCfg, generator, module):
198 build(buildCfg, generator, [module.binName])
199 genCaseList(buildCfg, generator, module, "txt")
200 return readCaseDict(getCaseListPath(buildCfg, module, "txt"))
202 def readPatternList (filename):
204 with open(filename, 'rt') as f:
207 if len(line) > 0 and line[0] != '#':
212 def constructNewDict(oldDict, listOfCases, op = lambda a: not a):
213 # Helper function used to construct case dictionary without specific cases
214 rootName = list(oldDict.keys())[0]
215 newDict = {rootName : {}}
216 newDictStack = [newDict]
217 oldDictStack = [oldDict]
219 # mak sure that both stacks have same number of items
220 assert(len(oldDictStack) == len(newDictStack))
221 # when all items from stack were processed then we can exit the loop
222 if len(oldDictStack) == 0:
224 # grab last item from both stacks
225 itemOnOldStack = oldDictStack.pop()
226 itemOnNewStack = newDictStack.pop()
227 # if item on stack is dictionary then it represents groups and
228 # we need to reconstruct them in new dictionary
229 if type(itemOnOldStack) is dict:
230 assert(type(itemOnNewStack) is dict)
231 listOfGroups = list(itemOnOldStack.keys())
232 for groupName in listOfGroups:
233 # create list or dictionary depending on contnent of child group
234 doesGroupsContainCases = type(itemOnOldStack[groupName]) is list
235 itemOnNewStack[groupName] = [] if doesGroupsContainCases else {}
236 # append groups on stacks
237 assert(type(itemOnNewStack[groupName]) == type(itemOnOldStack[groupName]))
238 newDictStack.append(itemOnNewStack[groupName])
239 oldDictStack.append(itemOnOldStack[groupName])
241 # if item on stack is list then it represents group that contain cases we need
242 # to apply filter on each of them to make sure only proper cases are appended
243 assert(type(itemOnOldStack) is list)
244 assert(type(itemOnNewStack) is list)
245 for caseName in itemOnOldStack:
246 if op(caseName in listOfCases):
247 itemOnNewStack.append(caseName)
250 def constructSet(caseDict, perGroupOperation):
252 dictStack = [caseDict]
254 # when all items from stack were processed then we can exit the loop
255 if len(dictStack) == 0:
257 # grab last item from stack
258 itemOnStack = dictStack.pop()
259 # if item on stack is dictionary then it represents groups and we need to add them to stack
260 if type(itemOnStack) is dict:
261 for groupName in itemOnStack.keys():
262 dictStack.append(itemOnStack[groupName])
264 # if item on stack is a list of cases we can add them to set containing all cases
265 assert(type(itemOnStack) is list)
266 casesSet = perGroupOperation(casesSet, itemOnStack)
269 def applyPatterns (caseDict, patterns, filename, op):
272 trivialPtrns = [p for p in patterns if p.find('*') < 0]
273 regularPtrns = [p for p in patterns if p.find('*') >= 0]
275 # Construct helper set that contains cases from all groups
276 unionOperation = lambda resultCasesSet, groupCaseList: resultCasesSet.union(set(groupCaseList))
277 allCasesSet = constructSet(caseDict, unionOperation)
279 # Apply trivial patterns - plain case paths without wildcard
280 for path in trivialPtrns:
281 if path in allCasesSet:
283 errors.append((path, "Same case specified more than once"))
286 errors.append((path, "Test case not found"))
288 # Construct new dictionary but without already matched paths
289 curDict = constructNewDict(caseDict, matched)
291 # Apply regular patterns - paths with wildcard
292 for pattern in regularPtrns:
294 # Helper function that checks if cases from case group match pattern
295 def matchOperation(resultCasesSet, groupCaseList):
296 for caseName in groupCaseList:
297 if fnmatch(caseName, pattern):
298 resultCasesSet.add(caseName)
299 return resultCasesSet
301 matchedThisPtrn = constructSet(curDict, matchOperation)
303 if len(matchedThisPtrn) == 0:
304 errors.append((pattern, "Pattern didn't match any cases"))
306 matched = matched | matchedThisPtrn
308 # To speed up search construct smaller case dictionary without already matched paths
309 curDict = constructNewDict(curDict, matched)
311 for pattern, reason in errors:
312 print("ERROR: %s: %s" % (reason, pattern))
315 die("Found %s invalid patterns while processing file %s" % (len(errors), filename))
317 # Construct final dictionary using aproperiate operation
318 return constructNewDict(caseDict, matched, op)
320 def applyInclude (caseDict, patterns, filename):
321 return applyPatterns(caseDict, patterns, filename, lambda b: b)
323 def applyExclude (caseDict, patterns, filename):
324 return applyPatterns(caseDict, patterns, filename, lambda b: not b)
326 def readPatternLists (mustpass):
328 for package in mustpass.packages:
329 for cfg in package.configurations:
330 for filter in cfg.filters:
331 if not filter.filename in lists:
332 lists[filter.filename] = readPatternList(os.path.join(getSrcDir(mustpass), filter.filename))
335 def applyFilters (caseDict, patternLists, filters):
337 for filter in filters:
338 ptrnList = patternLists[filter.filename]
339 if filter.type == Filter.TYPE_INCLUDE:
340 res = applyInclude(res, ptrnList, filter.filename)
342 assert filter.type == Filter.TYPE_EXCLUDE
343 res = applyExclude(res, ptrnList, filter.filename)
346 def appendToHierarchy (root, casePath):
347 def findChild (node, name):
348 for child in node.children:
349 if child.name == name:
354 components = casePath.split('.')
356 for component in components[:-1]:
357 nextNode = findChild(curNode, component)
359 nextNode = TestGroup(component)
360 curNode.children.append(nextNode)
363 if not findChild(curNode, components[-1]):
364 curNode.children.append(TestCase(components[-1]))
366 def buildTestHierachy (caseList):
368 for case in caseList:
369 appendToHierarchy(root, case)
372 def buildTestCaseMap (root):
375 def recursiveBuild (curNode, prefix):
376 curPath = prefix + curNode.name
377 if isinstance(curNode, TestCase):
378 caseMap[curPath] = curNode
380 for child in curNode.children:
381 recursiveBuild(child, curPath + '.')
383 for child in root.children:
384 recursiveBuild(child, '')
388 def include (filename):
389 return Filter(Filter.TYPE_INCLUDE, filename)
391 def exclude (filename):
392 return Filter(Filter.TYPE_EXCLUDE, filename)
394 def insertXMLHeaders (mustpass, doc):
395 if mustpass.project.copyright != None:
396 doc.insert(0, ElementTree.Comment(mustpass.project.copyright))
397 doc.insert(1, ElementTree.Comment(GENERATED_FILE_WARNING))
399 def prettifyXML (doc):
400 uglyString = ElementTree.tostring(doc, 'utf-8')
401 reparsed = minidom.parseString(uglyString)
402 return reparsed.toprettyxml(indent='\t', encoding='utf-8')
404 def genSpecXML (mustpass):
405 mustpassElem = ElementTree.Element("Mustpass", version = mustpass.version)
406 insertXMLHeaders(mustpass, mustpassElem)
408 for package in mustpass.packages:
409 packageElem = ElementTree.SubElement(mustpassElem, "TestPackage", name = package.module.name)
411 for config in package.configurations:
412 configElem = ElementTree.SubElement(packageElem, "Configuration",
413 caseListFile = getCaseListFileName(package, config),
414 commandLine = getCommandLine(config),
419 def addOptionElement (parent, optionName, optionValue):
420 ElementTree.SubElement(parent, "option", name=optionName, value=optionValue)
422 def genAndroidTestXml (mustpass):
423 RUNNER_CLASS = "com.drawelements.deqp.runner.DeqpTestRunner"
424 configElement = ElementTree.Element("configuration")
426 # have the deqp package installed on the device for us
427 preparerElement = ElementTree.SubElement(configElement, "target_preparer")
428 preparerElement.set("class", "com.android.tradefed.targetprep.suite.SuiteApkInstaller")
429 addOptionElement(preparerElement, "cleanup-apks", "true")
430 addOptionElement(preparerElement, "test-file-name", "com.drawelements.deqp.apk")
432 # Target preparer for incremental dEQP
433 preparerElement = ElementTree.SubElement(configElement, "target_preparer")
434 preparerElement.set("class", "com.android.compatibility.common.tradefed.targetprep.IncrementalDeqpPreparer")
435 addOptionElement(preparerElement, "disable", "true")
437 # add in metadata option for component name
438 ElementTree.SubElement(configElement, "option", name="test-suite-tag", value="cts")
439 ElementTree.SubElement(configElement, "option", key="component", name="config-descriptor:metadata", value="deqp")
440 ElementTree.SubElement(configElement, "option", key="parameter", name="config-descriptor:metadata", value="not_instant_app")
441 ElementTree.SubElement(configElement, "option", key="parameter", name="config-descriptor:metadata", value="multi_abi")
442 ElementTree.SubElement(configElement, "option", key="parameter", name="config-descriptor:metadata", value="secondary_user")
443 ElementTree.SubElement(configElement, "option", key="parameter", name="config-descriptor:metadata", value="no_foldable_states")
444 controllerElement = ElementTree.SubElement(configElement, "object")
445 controllerElement.set("class", "com.android.tradefed.testtype.suite.module.TestFailureModuleController")
446 controllerElement.set("type", "module_controller")
447 addOptionElement(controllerElement, "screenshot-on-failure", "false")
449 for package in mustpass.packages:
450 for config in package.configurations:
451 if not config.runByDefault:
454 testElement = ElementTree.SubElement(configElement, "test")
455 testElement.set("class", RUNNER_CLASS)
456 addOptionElement(testElement, "deqp-package", package.module.name)
457 caseListFile = getCaseListFileName(package,config)
458 addOptionElement(testElement, "deqp-caselist-file", caseListFile)
459 if caseListFile.startswith("gles3"):
460 addOptionElement(testElement, "incremental-deqp-include-file", "gles3-incremental-deqp.txt")
461 elif caseListFile.startswith("vk"):
462 addOptionElement(testElement, "incremental-deqp-include-file", "vk-incremental-deqp.txt")
463 # \todo [2015-10-16 kalle]: Replace with just command line? - requires simplifications in the runner/tests as well.
464 if config.glconfig != None:
465 addOptionElement(testElement, "deqp-gl-config-name", config.glconfig)
467 if config.surfacetype != None:
468 addOptionElement(testElement, "deqp-surface-type", config.surfacetype)
470 if config.rotation != None:
471 addOptionElement(testElement, "deqp-screen-rotation", config.rotation)
473 if config.expectedRuntime != None:
474 addOptionElement(testElement, "runtime-hint", config.expectedRuntime)
477 addOptionElement(testElement, "deqp-config-required", "true")
479 insertXMLHeaders(mustpass, configElement)
483 def genMustpass (mustpass, moduleCaseDicts):
484 print("Generating mustpass '%s'" % mustpass.version)
486 patternLists = readPatternLists(mustpass)
488 for package in mustpass.packages:
489 allCasesInPkgDict = moduleCaseDicts[package.module]
491 for config in package.configurations:
493 # construct dictionary with all filters applyed
494 filteredCaseDict = applyFilters(allCasesInPkgDict, patternLists, config.filters)
496 # construct components of path to main destination file
497 mainDstFilePath = getDstCaseListPath(mustpass)
498 mainDstFileName = getCaseListFileName(package, config)
499 mainDstFile = os.path.join(mainDstFilePath, mainDstFileName)
500 mainGruopSubDir = mainDstFileName[:-4]
502 # if case paths should be split to multiple files then main
503 # destination file will contain paths to individual files containing cases
504 if len(config.listOfGroupsToSplit) > 0:
505 # make sure directory for group files exists
506 rootGroupPath = os.path.join(mainDstFilePath, mainGruopSubDir)
507 if not os.path.exists(rootGroupPath):
508 os.makedirs(rootGroupPath)
510 # iterate over case dictionary and split it to .txt files acording to
511 # groups that were specified in config.listOfGroupsToSplit
512 splitedGroupsDict = {}
513 dictStack = [filteredCaseDict]
514 helperListStack = [ [] ]
516 # when all items from stack were processed then we can exit the loop
517 if len(dictStack) == 0:
519 assert(len(dictStack) == len(helperListStack))
520 # grab last item from stack
521 itemOnStack = dictStack.pop()
522 caseListFromHelperStack = helperListStack.pop()
523 # if item on stack is dictionary then it represents groups and we need to add them to stack
524 if type(itemOnStack) is dict:
525 for groupName in sorted(itemOnStack):
527 # check if this group should be split to multiple .txt files
528 if groupName in config.listOfGroupsToSplit:
529 # we can split only groups that contain other groups,
530 # listOfGroupsToSplit should not contain groups that contain test cases
531 assert(type(itemOnStack[groupName]) is dict)
532 # add child groups of this group to splitedGroupsDict
533 for childGroupName in itemOnStack[groupName]:
534 # make sure that child group should not be splited
535 # (if it should then this will be handle in one of the next iterations)
536 if childGroupName not in config.listOfGroupsToSplit:
537 splitedGroupsDict[childGroupName] = []
539 # add this group to stack used for iteration over casses tree
540 dictStack.append(itemOnStack[groupName])
542 # decide what list we should append to helperListStack;
543 # if this group represents one of individual .txt files then grab
544 # propper array of cases from splitedGroupsDict and add it to helper stack;
545 # if groupName is not in splitedGroupsDict then use the same list as was used
546 # by parent group (we are merging casses from those groups to single .txt file)
547 helperListStack.append(splitedGroupsDict.get(groupName, caseListFromHelperStack))
549 # if item on stack is a list of cases we can add them to proper list
550 assert(type(itemOnStack) is list)
551 caseListFromHelperStack.extend(itemOnStack)
553 print(" Writing separated caselists:")
555 for groupPath in splitedGroupsDict:
556 # skip groups that after filtering have no casses left
557 if len(splitedGroupsDict[groupPath]) == 0:
559 # remove root node name from the beginning of group and replace all '_' with '-'
560 processedGroupPath = groupPath[groupPath.find('.')+1:].replace('_', '-')
562 groupList = processedGroupPath.split('.')
564 # create subdirectories if there is more then one group name in groupList
566 if len(groupList) > 1:
567 for groupName in groupList[:-1]:
568 # make sure directory for group files exists
569 groupSubDir = groupSubDir + groupName + '/'
570 path = os.path.join(path, groupName)
571 if not os.path.exists(path):
573 # construct path to .txt file and save all cases
574 groupDstFileName = groupList[-1] + ".txt"
575 groupDstFileFullDir = os.path.join(path, groupDstFileName)
576 groupPathsList.append(mainGruopSubDir + groupSubDir + groupDstFileName)
577 print(" " + groupDstFileFullDir)
578 writeFile(groupDstFileFullDir, "\n".join(splitedGroupsDict[groupPath]) + "\n")
580 # write file containing names of all group files
581 print(" Writing file containing list of separated case files: " + mainDstFile)
582 groupPathsList.sort()
583 writeFile(mainDstFile, "\n".join(groupPathsList) + "\n")
585 # merge all cases to single case list
586 filteredCaseList = []
587 dictStack = [filteredCaseDict]
589 # when all items from stack were processed then we can exit the loop
590 if len(dictStack) == 0:
592 # grab last item from stack
593 itemOnStack = dictStack.pop()
594 # if item on stack is dictionary then it represents groups and we need to add them to stack
595 if type(itemOnStack) is dict:
596 for groupName in itemOnStack.keys():
597 dictStack.append(itemOnStack[groupName])
599 # if item on stack is a list of cases we can add them to filteredCaseList
600 assert(type(itemOnStack) is list)
601 filteredCaseList.extend(itemOnStack)
602 # write file containing all cases
603 if len(filteredCaseList) > 0:
604 print(" Writing deqp caselist: " + mainDstFile)
605 writeFile(mainDstFile, "\n".join(filteredCaseList) + "\n")
607 specXML = genSpecXML(mustpass)
608 specFilename = os.path.join(mustpass.project.path, mustpass.version, "mustpass.xml")
610 print(" Writing spec: " + specFilename)
611 writeFile(specFilename, prettifyXML(specXML).decode())
613 # TODO: Which is the best selector mechanism?
614 if (mustpass.version == "master"):
615 androidTestXML = genAndroidTestXml(mustpass)
616 androidTestFilename = os.path.join(mustpass.project.path, "AndroidTest.xml")
618 print(" Writing AndroidTest.xml: " + androidTestFilename)
619 writeFile(androidTestFilename, prettifyXML(androidTestXML).decode())
623 def genMustpassLists (mustpassLists, generator, buildCfg):
626 # Getting case lists involves invoking build, so we want to cache the results
627 for mustpass in mustpassLists:
628 for package in mustpass.packages:
629 if not package.module in moduleCaseDicts:
630 moduleCaseDicts[package.module] = getCaseDict(buildCfg, generator, package.module)
632 for mustpass in mustpassLists:
633 genMustpass(mustpass, moduleCaseDicts)
635 def parseCmdLineArgs ():
636 parser = argparse.ArgumentParser(description = "Build Android CTS mustpass",
637 formatter_class=argparse.ArgumentDefaultsHelpFormatter)
638 parser.add_argument("-b",
641 default=DEFAULT_BUILD_DIR,
642 help="Temporary build directory")
643 parser.add_argument("-t",
648 parser.add_argument("-c",
651 default=DEFAULT_TARGET,
652 help="dEQP build target")
653 return parser.parse_args()
655 def parseBuildConfigFromCmdLineArgs ():
656 args = parseCmdLineArgs()
657 return getBuildConfig(args.buildDir, args.targetName, args.buildType)