Reduce scope of mustpass exclusions due to 22760309
[platform/upstream/VK-GL-CTS.git] / scripts / mustpass.py
1 # -*- coding: utf-8 -*-
2
3 #-------------------------------------------------------------------------
4 # drawElements Quality Program utilities
5 # --------------------------------------
6 #
7 # Copyright 2016 The Android Open Source Project
8 #
9 # Licensed under the Apache License, Version 2.0 (the "License");
10 # you may not use this file except in compliance with the License.
11 # You may obtain a copy of the License at
12 #
13 #      http://www.apache.org/licenses/LICENSE-2.0
14 #
15 # Unless required by applicable law or agreed to in writing, software
16 # distributed under the License is distributed on an "AS IS" BASIS,
17 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18 # See the License for the specific language governing permissions and
19 # limitations under the License.
20 #
21 #-------------------------------------------------------------------------
22
23 from build.common import *
24 from build.config import ANY_GENERATOR
25 from build.build import build
26 from build_caselists import Module, getModuleByName, getBuildConfig, genCaseList, getCaseListPath, DEFAULT_BUILD_DIR, DEFAULT_TARGET
27 from fnmatch import fnmatch
28 from copy import copy
29
30 import argparse
31 import xml.etree.cElementTree as ElementTree
32 import xml.dom.minidom as minidom
33
34 APK_NAME                = "com.drawelements.deqp.apk"
35
36 GENERATED_FILE_WARNING = """
37      This file has been automatically generated. Edit with caution.
38      """
39
40 class Project:
41         def __init__ (self, path, copyright = None):
42                 self.path               = path
43                 self.copyright  = copyright
44
45 class Configuration:
46         def __init__ (self, name, filters, glconfig = None, rotation = None, surfacetype = None, required = False, runtime = None):
47                 self.name                               = name
48                 self.glconfig                   = glconfig
49                 self.rotation                   = rotation
50                 self.surfacetype                = surfacetype
51                 self.required                   = required
52                 self.filters                    = filters
53                 self.expectedRuntime    = runtime
54
55 class Package:
56         def __init__ (self, module, configurations):
57                 self.module                     = module
58                 self.configurations     = configurations
59
60 class Mustpass:
61         def __init__ (self, project, version, packages):
62                 self.project    = project
63                 self.version    = version
64                 self.packages   = packages
65
66 class Filter:
67         TYPE_INCLUDE = 0
68         TYPE_EXCLUDE = 1
69
70         def __init__ (self, type, filename):
71                 self.type               = type
72                 self.filename   = filename
73
74 class TestRoot:
75         def __init__ (self):
76                 self.children   = []
77
78 class TestGroup:
79         def __init__ (self, name):
80                 self.name               = name
81                 self.children   = []
82
83 class TestCase:
84         def __init__ (self, name):
85                 self.name                       = name
86                 self.configurations     = []
87
88 class GLESVersion:
89         def __init__(self, major, minor):
90                 self.major = major
91                 self.minor = minor
92
93         def encode (self):
94                 return (self.major << 16) | (self.minor)
95
96 def getModuleGLESVersion (module):
97         versions = {
98                 'dEQP-EGL':             GLESVersion(2,0),
99                 'dEQP-GLES2':   GLESVersion(2,0),
100                 'dEQP-GLES3':   GLESVersion(3,0),
101                 'dEQP-GLES31':  GLESVersion(3,1)
102         }
103         return versions[module.name] if module.name in versions else None
104
105 def getSrcDir (mustpass):
106         return os.path.join(mustpass.project.path, mustpass.version, "src")
107
108 def getTmpDir (mustpass):
109         return os.path.join(mustpass.project.path, mustpass.version, "tmp")
110
111 def getModuleShorthand (module):
112         assert module.name[:5] == "dEQP-"
113         return module.name[5:].lower()
114
115 def getCaseListFileName (package, configuration):
116         return "%s-%s.txt" % (getModuleShorthand(package.module), configuration.name)
117
118 def getDstCaseListPath (mustpass, package, configuration):
119         return os.path.join(mustpass.project.path, mustpass.version, getCaseListFileName(package, configuration))
120
121 def getCTSPackageName (package):
122         return "com.drawelements.deqp." + getModuleShorthand(package.module)
123
124 def getCommandLine (config):
125         cmdLine = ""
126
127         if config.glconfig != None:
128                 cmdLine += "--deqp-gl-config-name=%s " % config.glconfig
129
130         if config.rotation != None:
131                 cmdLine += "--deqp-screen-rotation=%s " % config.rotation
132
133         if config.surfacetype != None:
134                 cmdLine += "--deqp-surface-type=%s " % config.surfacetype
135
136         cmdLine += "--deqp-watchdog=enable"
137
138         return cmdLine
139
140 def readCaseList (filename):
141         cases = []
142         with open(filename, 'rb') as f:
143                 for line in f:
144                         if line[:6] == "TEST: ":
145                                 cases.append(line[6:].strip())
146         return cases
147
148 def getCaseList (buildCfg, generator, module):
149         build(buildCfg, generator, [module.binName])
150         genCaseList(buildCfg, generator, module, "txt")
151         return readCaseList(getCaseListPath(buildCfg, module, "txt"))
152
153 def readPatternList (filename):
154         ptrns = []
155         with open(filename, 'rb') as f:
156                 for line in f:
157                         line = line.strip()
158                         if len(line) > 0 and line[0] != '#':
159                                 ptrns.append(line)
160         return ptrns
161
162 def applyPatterns (caseList, patterns, filename, op):
163         matched                 = set()
164         errors                  = []
165         curList                 = copy(caseList)
166         trivialPtrns    = [p for p in patterns if p.find('*') < 0]
167         regularPtrns    = [p for p in patterns if p.find('*') >= 0]
168
169         # Apply trivial (just case paths)
170         allCasesSet             = set(caseList)
171         for path in trivialPtrns:
172                 if path in allCasesSet:
173                         if path in matched:
174                                 errors.append((path, "Same case specified more than once"))
175                         matched.add(path)
176                 else:
177                         errors.append((path, "Test case not found"))
178
179         curList = [c for c in curList if c not in matched]
180
181         for pattern in regularPtrns:
182                 matchedThisPtrn = set()
183
184                 for case in curList:
185                         if fnmatch(case, pattern):
186                                 matchedThisPtrn.add(case)
187
188                 if len(matchedThisPtrn) == 0:
189                         errors.append((pattern, "Pattern didn't match any cases"))
190
191                 matched = matched | matchedThisPtrn
192                 curList = [c for c in curList if c not in matched]
193
194         for pattern, reason in errors:
195                 print "ERROR: %s: %s" % (reason, pattern)
196
197         if len(errors) > 0:
198                 die("Found %s invalid patterns while processing file %s" % (len(errors), filename))
199
200         return [c for c in caseList if op(c in matched)]
201
202 def applyInclude (caseList, patterns, filename):
203         return applyPatterns(caseList, patterns, filename, lambda b: b)
204
205 def applyExclude (caseList, patterns, filename):
206         return applyPatterns(caseList, patterns, filename, lambda b: not b)
207
208 def readPatternLists (mustpass):
209         lists = {}
210         for package in mustpass.packages:
211                 for cfg in package.configurations:
212                         for filter in cfg.filters:
213                                 if not filter.filename in lists:
214                                         lists[filter.filename] = readPatternList(os.path.join(getSrcDir(mustpass), filter.filename))
215         return lists
216
217 def applyFilters (caseList, patternLists, filters):
218         res = copy(caseList)
219         for filter in filters:
220                 ptrnList = patternLists[filter.filename]
221                 if filter.type == Filter.TYPE_INCLUDE:
222                         res = applyInclude(res, ptrnList, filter.filename)
223                 else:
224                         assert filter.type == Filter.TYPE_EXCLUDE
225                         res = applyExclude(res, ptrnList, filter.filename)
226         return res
227
228 def appendToHierarchy (root, casePath):
229         def findChild (node, name):
230                 for child in node.children:
231                         if child.name == name:
232                                 return child
233                 return None
234
235         curNode         = root
236         components      = casePath.split('.')
237
238         for component in components[:-1]:
239                 nextNode = findChild(curNode, component)
240                 if not nextNode:
241                         nextNode = TestGroup(component)
242                         curNode.children.append(nextNode)
243                 curNode = nextNode
244
245         if not findChild(curNode, components[-1]):
246                 curNode.children.append(TestCase(components[-1]))
247
248 def buildTestHierachy (caseList):
249         root = TestRoot()
250         for case in caseList:
251                 appendToHierarchy(root, case)
252         return root
253
254 def buildTestCaseMap (root):
255         caseMap = {}
256
257         def recursiveBuild (curNode, prefix):
258                 curPath = prefix + curNode.name
259                 if isinstance(curNode, TestCase):
260                         caseMap[curPath] = curNode
261                 else:
262                         for child in curNode.children:
263                                 recursiveBuild(child, curPath + '.')
264
265         for child in root.children:
266                 recursiveBuild(child, '')
267
268         return caseMap
269
270 def include (filename):
271         return Filter(Filter.TYPE_INCLUDE, filename)
272
273 def exclude (filename):
274         return Filter(Filter.TYPE_EXCLUDE, filename)
275
276 def insertXMLHeaders (mustpass, doc):
277         if mustpass.project.copyright != None:
278                 doc.insert(0, ElementTree.Comment(mustpass.project.copyright))
279         doc.insert(1, ElementTree.Comment(GENERATED_FILE_WARNING))
280
281 def prettifyXML (doc):
282         uglyString      = ElementTree.tostring(doc, 'utf-8')
283         reparsed        = minidom.parseString(uglyString)
284         return reparsed.toprettyxml(indent='\t', encoding='utf-8')
285
286 def genSpecXML (mustpass):
287         mustpassElem = ElementTree.Element("Mustpass", version = mustpass.version)
288         insertXMLHeaders(mustpass, mustpassElem)
289
290         for package in mustpass.packages:
291                 packageElem = ElementTree.SubElement(mustpassElem, "TestPackage", name = package.module.name)
292
293                 for config in package.configurations:
294                         configElem = ElementTree.SubElement(packageElem, "Configuration",
295                                                                                                 name                    = config.name,
296                                                                                                 caseListFile    = getCaseListFileName(package, config),
297                                                                                                 commandLine             = getCommandLine(config))
298
299         return mustpassElem
300
301 def addOptionElement (parent, optionName, optionValue):
302         ElementTree.SubElement(parent, "option", name=optionName, value=optionValue)
303
304 def genAndroidTestXml (mustpass):
305         RUNNER_CLASS = "com.drawelements.deqp.runner.DeqpTestRunner"
306         configElement = ElementTree.Element("configuration")
307
308         # add in metadata option for component name
309         ElementTree.SubElement(configElement, "option", name="config-descriptor:metadata", key="component", value="deqp")
310
311         for package in mustpass.packages:
312                 for config in package.configurations:
313                         testElement = ElementTree.SubElement(configElement, "test")
314                         testElement.set("class", RUNNER_CLASS)
315                         addOptionElement(testElement, "deqp-package", package.module.name)
316                         addOptionElement(testElement, "deqp-caselist-file", getCaseListFileName(package,config))
317                         # \todo [2015-10-16 kalle]: Replace with just command line? - requires simplifications in the runner/tests as well.
318                         if config.glconfig != None:
319                                 addOptionElement(testElement, "deqp-gl-config-name", config.glconfig)
320
321                         if config.surfacetype != None:
322                                 addOptionElement(testElement, "deqp-surface-type", config.surfacetype)
323
324                         if config.rotation != None:
325                                 addOptionElement(testElement, "deqp-screen-rotation", config.rotation)
326
327                         if config.expectedRuntime != None:
328                                 addOptionElement(testElement, "runtime-hint", config.expectedRuntime)
329
330                         if config.required:
331                                 addOptionElement(testElement, "deqp-config-required", "true")
332
333         insertXMLHeaders(mustpass, configElement)
334
335         return configElement
336
337 def genMustpass (mustpass, moduleCaseLists):
338         print "Generating mustpass '%s'" % mustpass.version
339
340         patternLists = readPatternLists(mustpass)
341
342         for package in mustpass.packages:
343                 allCasesInPkg   = moduleCaseLists[package.module]
344
345                 for config in package.configurations:
346                         filtered        = applyFilters(allCasesInPkg, patternLists, config.filters)
347                         dstFile         = getDstCaseListPath(mustpass, package, config)
348
349                         print "  Writing deqp caselist: " + dstFile
350                         writeFile(dstFile, "\n".join(filtered) + "\n")
351
352         specXML                 = genSpecXML(mustpass)
353         specFilename    = os.path.join(mustpass.project.path, mustpass.version, "mustpass.xml")
354
355         print "  Writing spec: " + specFilename
356         writeFile(specFilename, prettifyXML(specXML))
357
358         # TODO: Which is the best selector mechanism?
359         if (mustpass.version == "master"):
360                 androidTestXML          = genAndroidTestXml(mustpass)
361                 androidTestFilename     = os.path.join(mustpass.project.path, "AndroidTest.xml")
362
363                 print "  Writing AndroidTest.xml: " + androidTestFilename
364                 writeFile(androidTestFilename, prettifyXML(androidTestXML))
365
366         print "Done!"
367
368 def genMustpassLists (mustpassLists, generator, buildCfg):
369         moduleCaseLists = {}
370
371         # Getting case lists involves invoking build, so we want to cache the results
372         for mustpass in mustpassLists:
373                 for package in mustpass.packages:
374                         if not package.module in moduleCaseLists:
375                                 moduleCaseLists[package.module] = getCaseList(buildCfg, generator, package.module)
376
377         for mustpass in mustpassLists:
378                 genMustpass(mustpass, moduleCaseLists)
379
380 def parseCmdLineArgs ():
381         parser = argparse.ArgumentParser(description = "Build Android CTS mustpass",
382                                                                          formatter_class=argparse.ArgumentDefaultsHelpFormatter)
383         parser.add_argument("-b",
384                                                 "--build-dir",
385                                                 dest="buildDir",
386                                                 default=DEFAULT_BUILD_DIR,
387                                                 help="Temporary build directory")
388         parser.add_argument("-t",
389                                                 "--build-type",
390                                                 dest="buildType",
391                                                 default="Debug",
392                                                 help="Build type")
393         parser.add_argument("-c",
394                                                 "--deqp-target",
395                                                 dest="targetName",
396                                                 default=DEFAULT_TARGET,
397                                                 help="dEQP build target")
398         return parser.parse_args()
399
400 def parseBuildConfigFromCmdLineArgs ():
401         args = parseCmdLineArgs()
402         return getBuildConfig(args.buildDir, args.targetName, args.buildType)