Merge vulkan-cts-1.0 to master
[platform/upstream/VK-GL-CTS.git] / scripts / mustpass.py
1 # -*- coding: utf-8 -*-
2
3 #-------------------------------------------------------------------------
4 # drawElements Quality Program utilities
5 # --------------------------------------
6 #
7 # Copyright 2016 The Android Open Source Project
8 #
9 # Licensed under the Apache License, Version 2.0 (the "License");
10 # you may not use this file except in compliance with the License.
11 # You may obtain a copy of the License at
12 #
13 #      http://www.apache.org/licenses/LICENSE-2.0
14 #
15 # Unless required by applicable law or agreed to in writing, software
16 # distributed under the License is distributed on an "AS IS" BASIS,
17 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18 # See the License for the specific language governing permissions and
19 # limitations under the License.
20 #
21 #-------------------------------------------------------------------------
22
23 from build.common import *
24 from build.config import ANY_GENERATOR
25 from build.build import build
26 from build_caselists import Module, getModuleByName, getBuildConfig, genCaseList, getCaseListPath, DEFAULT_BUILD_DIR, DEFAULT_TARGET
27 from fnmatch import fnmatch
28 from copy import copy
29
30 import xml.etree.cElementTree as ElementTree
31 import xml.dom.minidom as minidom
32
33 APK_NAME                = "com.drawelements.deqp.apk"
34
35 GENERATED_FILE_WARNING = """
36      This file has been automatically generated. Edit with caution.
37      """
38
39 class Project:
40         def __init__ (self, path, copyright = None):
41                 self.path               = path
42                 self.copyright  = copyright
43
44 class Configuration:
45         def __init__ (self, name, filters, glconfig = None, rotation = None, surfacetype = None):
46                 self.name                       = name
47                 self.glconfig           = glconfig
48                 self.rotation           = rotation
49                 self.surfacetype        = surfacetype
50                 self.filters            = filters
51
52 class Package:
53         def __init__ (self, module, configurations):
54                 self.module                     = module
55                 self.configurations     = configurations
56
57 class Mustpass:
58         def __init__ (self, project, version, packages):
59                 self.project    = project
60                 self.version    = version
61                 self.packages   = packages
62
63 class Filter:
64         TYPE_INCLUDE = 0
65         TYPE_EXCLUDE = 1
66
67         def __init__ (self, type, filename):
68                 self.type               = type
69                 self.filename   = filename
70
71 class TestRoot:
72         def __init__ (self):
73                 self.children   = []
74
75 class TestGroup:
76         def __init__ (self, name):
77                 self.name               = name
78                 self.children   = []
79
80 class TestCase:
81         def __init__ (self, name):
82                 self.name                       = name
83                 self.configurations     = []
84
85 class GLESVersion:
86         def __init__(self, major, minor):
87                 self.major = major
88                 self.minor = minor
89
90         def encode (self):
91                 return (self.major << 16) | (self.minor)
92
93 def getModuleGLESVersion (module):
94         versions = {
95                 'dEQP-EGL':             GLESVersion(2,0),
96                 'dEQP-GLES2':   GLESVersion(2,0),
97                 'dEQP-GLES3':   GLESVersion(3,0),
98                 'dEQP-GLES31':  GLESVersion(3,1)
99         }
100         return versions[module.name] if module.name in versions else None
101
102 def getSrcDir (mustpass):
103         return os.path.join(mustpass.project.path, mustpass.version, "src")
104
105 def getTmpDir (mustpass):
106         return os.path.join(mustpass.project.path, mustpass.version, "tmp")
107
108 def getModuleShorthand (module):
109         assert module.name[:5] == "dEQP-"
110         return module.name[5:].lower()
111
112 def getCaseListFileName (package, configuration):
113         return "%s-%s.txt" % (getModuleShorthand(package.module), configuration.name)
114
115 def getDstCaseListPath (mustpass, package, configuration):
116         return os.path.join(mustpass.project.path, mustpass.version, getCaseListFileName(package, configuration))
117
118 def getCTSPackageName (package):
119         return "com.drawelements.deqp." + getModuleShorthand(package.module)
120
121 def getCommandLine (config):
122         cmdLine = ""
123
124         if config.glconfig != None:
125                 cmdLine += "--deqp-gl-config-name=%s " % config.glconfig
126
127         if config.rotation != None:
128                 cmdLine += "--deqp-screen-rotation=%s " % config.rotation
129
130         if config.surfacetype != None:
131                 cmdLine += "--deqp-surface-type=%s " % config.surfacetype
132
133         cmdLine += "--deqp-watchdog=enable"
134
135         return cmdLine
136
137 def readCaseList (filename):
138         cases = []
139         with open(filename, 'rb') as f:
140                 for line in f:
141                         if line[:6] == "TEST: ":
142                                 cases.append(line[6:].strip())
143         return cases
144
145 def getCaseList (buildCfg, generator, module):
146         build(buildCfg, generator, [module.binName])
147         genCaseList(buildCfg, generator, module, "txt")
148         return readCaseList(getCaseListPath(buildCfg, module, "txt"))
149
150 def readPatternList (filename):
151         ptrns = []
152         with open(filename, 'rb') as f:
153                 for line in f:
154                         line = line.strip()
155                         if len(line) > 0 and line[0] != '#':
156                                 ptrns.append(line)
157         return ptrns
158
159 def applyPatterns (caseList, patterns, filename, op):
160         matched                 = set()
161         errors                  = []
162         curList                 = copy(caseList)
163         trivialPtrns    = [p for p in patterns if p.find('*') < 0]
164         regularPtrns    = [p for p in patterns if p.find('*') >= 0]
165
166         # Apply trivial (just case paths)
167         allCasesSet             = set(caseList)
168         for path in trivialPtrns:
169                 if path in allCasesSet:
170                         if path in matched:
171                                 errors.append((path, "Same case specified more than once"))
172                         matched.add(path)
173                 else:
174                         errors.append((path, "Test case not found"))
175
176         curList = [c for c in curList if c not in matched]
177
178         for pattern in regularPtrns:
179                 matchedThisPtrn = set()
180
181                 for case in curList:
182                         if fnmatch(case, pattern):
183                                 matchedThisPtrn.add(case)
184
185                 if len(matchedThisPtrn) == 0:
186                         errors.append((pattern, "Pattern didn't match any cases"))
187
188                 matched = matched | matchedThisPtrn
189                 curList = [c for c in curList if c not in matched]
190
191         for pattern, reason in errors:
192                 print "ERROR: %s: %s" % (reason, pattern)
193
194         if len(errors) > 0:
195                 die("Found %s invalid patterns while processing file %s" % (len(errors), filename))
196
197         return [c for c in caseList if op(c in matched)]
198
199 def applyInclude (caseList, patterns, filename):
200         return applyPatterns(caseList, patterns, filename, lambda b: b)
201
202 def applyExclude (caseList, patterns, filename):
203         return applyPatterns(caseList, patterns, filename, lambda b: not b)
204
205 def readPatternLists (mustpass):
206         lists = {}
207         for package in mustpass.packages:
208                 for cfg in package.configurations:
209                         for filter in cfg.filters:
210                                 if not filter.filename in lists:
211                                         lists[filter.filename] = readPatternList(os.path.join(getSrcDir(mustpass), filter.filename))
212         return lists
213
214 def applyFilters (caseList, patternLists, filters):
215         res = copy(caseList)
216         for filter in filters:
217                 ptrnList = patternLists[filter.filename]
218                 if filter.type == Filter.TYPE_INCLUDE:
219                         res = applyInclude(res, ptrnList, filter.filename)
220                 else:
221                         assert filter.type == Filter.TYPE_EXCLUDE
222                         res = applyExclude(res, ptrnList, filter.filename)
223         return res
224
225 def appendToHierarchy (root, casePath):
226         def findChild (node, name):
227                 for child in node.children:
228                         if child.name == name:
229                                 return child
230                 return None
231
232         curNode         = root
233         components      = casePath.split('.')
234
235         for component in components[:-1]:
236                 nextNode = findChild(curNode, component)
237                 if not nextNode:
238                         nextNode = TestGroup(component)
239                         curNode.children.append(nextNode)
240                 curNode = nextNode
241
242         if not findChild(curNode, components[-1]):
243                 curNode.children.append(TestCase(components[-1]))
244
245 def buildTestHierachy (caseList):
246         root = TestRoot()
247         for case in caseList:
248                 appendToHierarchy(root, case)
249         return root
250
251 def buildTestCaseMap (root):
252         caseMap = {}
253
254         def recursiveBuild (curNode, prefix):
255                 curPath = prefix + curNode.name
256                 if isinstance(curNode, TestCase):
257                         caseMap[curPath] = curNode
258                 else:
259                         for child in curNode.children:
260                                 recursiveBuild(child, curPath + '.')
261
262         for child in root.children:
263                 recursiveBuild(child, '')
264
265         return caseMap
266
267 def include (filename):
268         return Filter(Filter.TYPE_INCLUDE, filename)
269
270 def exclude (filename):
271         return Filter(Filter.TYPE_EXCLUDE, filename)
272
273 def insertXMLHeaders (mustpass, doc):
274         if mustpass.project.copyright != None:
275                 doc.insert(0, ElementTree.Comment(mustpass.project.copyright))
276         doc.insert(1, ElementTree.Comment(GENERATED_FILE_WARNING))
277
278 def prettifyXML (doc):
279         uglyString      = ElementTree.tostring(doc, 'utf-8')
280         reparsed        = minidom.parseString(uglyString)
281         return reparsed.toprettyxml(indent='\t', encoding='utf-8')
282
283 def genCTSPackageXML (mustpass, package, root):
284         def isLeafGroup (testGroup):
285                 numGroups       = 0
286                 numTests        = 0
287
288                 for child in testGroup.children:
289                         if isinstance(child, TestCase):
290                                 numTests += 1
291                         else:
292                                 numGroups += 1
293
294                 assert numGroups + numTests > 0
295
296                 if numGroups > 0 and numTests > 0:
297                         die("Mixed groups and cases in %s" % testGroup.name)
298
299                 return numGroups == 0
300
301         def makeConfiguration (parentElem, config):
302                 attributes = {}
303
304                 if config.glconfig != None:
305                         attributes['glconfig'] = config.glconfig
306
307                 if config.rotation != None:
308                         attributes['rotation'] = config.rotation
309
310                 if config.surfacetype != None:
311                         attributes['surfacetype'] = config.surfacetype
312
313                 return ElementTree.SubElement(parentElem, "TestInstance", attributes)
314
315         def makeTestCase (parentElem, testCase):
316                 caseElem = ElementTree.SubElement(parentElem, "Test", name=testCase.name)
317                 for config in testCase.configurations:
318                         makeConfiguration(caseElem, config)
319                 return caseElem
320
321         def makeTestGroup (parentElem, testGroup):
322                 groupElem = ElementTree.SubElement(parentElem, "TestCase" if isLeafGroup(testGroup) else "TestSuite", name=testGroup.name)
323                 for child in testGroup.children:
324                         if isinstance(child, TestCase):
325                                 makeTestCase(groupElem, child)
326                         else:
327                                 makeTestGroup(groupElem, child)
328                 return groupElem
329
330         pkgElem = ElementTree.Element("TestPackage",
331                                                                   name                          = package.module.name,
332                                                                   appPackageName        = getCTSPackageName(package),
333                                                                   testType                      = "deqpTest")
334
335         pkgElem.set("xmlns:deqp", "http://drawelements.com/deqp")
336         insertXMLHeaders(mustpass, pkgElem)
337
338         glesVersion = getModuleGLESVersion(package.module)
339
340         if glesVersion != None:
341                 pkgElem.set("deqp:glesVersion", str(glesVersion.encode()))
342
343         for child in root.children:
344                 makeTestGroup(pkgElem, child)
345
346         return pkgElem
347
348 def genSpecXML (mustpass):
349         mustpassElem = ElementTree.Element("Mustpass", version = mustpass.version)
350         insertXMLHeaders(mustpass, mustpassElem)
351
352         for package in mustpass.packages:
353                 packageElem = ElementTree.SubElement(mustpassElem, "TestPackage", name = package.module.name)
354
355                 for config in package.configurations:
356                         configElem = ElementTree.SubElement(packageElem, "Configuration",
357                                                                                                 name                    = config.name,
358                                                                                                 caseListFile    = getCaseListFileName(package, config),
359                                                                                                 commandLine             = getCommandLine(config))
360
361         return mustpassElem
362
363 def addOptionElement (parent, optionName, optionValue):
364         ElementTree.SubElement(parent, "option", name=optionName, value=optionValue)
365
366 def genAndroidTestXml (mustpass):
367         INSTALLER_CLASS = "com.android.compatibility.common.tradefed.targetprep.ApkInstaller"
368         RUNNER_CLASS = "com.drawelements.deqp.runner.DeqpTestRunner"
369         configElement = ElementTree.Element("configuration")
370         preparerElement = ElementTree.SubElement(configElement, "target_preparer")
371         preparerElement.set("class", INSTALLER_CLASS)
372         addOptionElement(preparerElement, "cleanup-apks", "true")
373         addOptionElement(preparerElement, "test-file-name", APK_NAME)
374
375         for package in mustpass.packages:
376                 for config in package.configurations:
377                         testElement = ElementTree.SubElement(configElement, "test")
378                         testElement.set("class", RUNNER_CLASS)
379                         addOptionElement(testElement, "deqp-package", package.module.name)
380                         addOptionElement(testElement, "deqp-caselist-file", getCaseListFileName(package,config))
381                         # \todo [2015-10-16 kalle]: Replace with just command line? - requires simplifications in the runner/tests as well.
382                         if config.glconfig != None:
383                                 addOptionElement(testElement, "deqp-gl-config-name", config.glconfig)
384
385                         if config.surfacetype != None:
386                                 addOptionElement(testElement, "deqp-surface-type", config.surfacetype)
387
388                         if config.rotation != None:
389                                 addOptionElement(testElement, "deqp-screen-rotation", config.rotation)
390
391         insertXMLHeaders(mustpass, configElement)
392
393         return configElement
394
395 def genMustpass (mustpass, moduleCaseLists):
396         print "Generating mustpass '%s'" % mustpass.version
397
398         patternLists = readPatternLists(mustpass)
399
400         for package in mustpass.packages:
401                 allCasesInPkg           = moduleCaseLists[package.module]
402                 matchingByConfig        = {}
403                 allMatchingSet          = set()
404
405                 for config in package.configurations:
406                         filtered        = applyFilters(allCasesInPkg, patternLists, config.filters)
407                         dstFile         = getDstCaseListPath(mustpass, package, config)
408
409                         print "  Writing deqp caselist: " + dstFile
410                         writeFile(dstFile, "\n".join(filtered) + "\n")
411
412                         matchingByConfig[config]        = filtered
413                         allMatchingSet                          = allMatchingSet | set(filtered)
414
415                 allMatchingCases        = [c for c in allCasesInPkg if c in allMatchingSet] # To preserve ordering
416                 root                            = buildTestHierachy(allMatchingCases)
417                 testCaseMap                     = buildTestCaseMap(root)
418
419                 for config in package.configurations:
420                         for case in matchingByConfig[config]:
421                                 testCaseMap[case].configurations.append(config)
422
423                 # NOTE: CTS v2 does not need package XML files. Remove when transition is complete.
424                 packageXml      = genCTSPackageXML(mustpass, package, root)
425                 xmlFilename     = os.path.join(mustpass.project.path, mustpass.version, getCTSPackageName(package) + ".xml")
426
427                 print "  Writing CTS caselist: " + xmlFilename
428                 writeFile(xmlFilename, prettifyXML(packageXml))
429
430         specXML                 = genSpecXML(mustpass)
431         specFilename    = os.path.join(mustpass.project.path, mustpass.version, "mustpass.xml")
432
433         print "  Writing spec: " + specFilename
434         writeFile(specFilename, prettifyXML(specXML))
435
436         # TODO: Which is the best selector mechanism?
437         if (mustpass.version == "master"):
438                 androidTestXML          = genAndroidTestXml(mustpass)
439                 androidTestFilename     = os.path.join(mustpass.project.path, "AndroidTest.xml")
440
441                 print "  Writing AndroidTest.xml: " + androidTestFilename
442                 writeFile(androidTestFilename, prettifyXML(androidTestXML))
443
444         print "Done!"
445
446 def genMustpassLists (mustpassLists, generator, buildCfg):
447         moduleCaseLists = {}
448
449         # Getting case lists involves invoking build, so we want to cache the results
450         for mustpass in mustpassLists:
451                 for package in mustpass.packages:
452                         if not package.module in moduleCaseLists:
453                                 moduleCaseLists[package.module] = getCaseList(buildCfg, generator, package.module)
454
455         for mustpass in mustpassLists:
456                 genMustpass(mustpass, moduleCaseLists)