GetDevices: Also allow `\.` to be part of the host name am: dbf9715509 am: a44c013aaf...
[platform/upstream/VK-GL-CTS.git] / scripts / mustpass.py
1 # -*- coding: utf-8 -*-
2
3 #-------------------------------------------------------------------------
4 # drawElements Quality Program utilities
5 # --------------------------------------
6 #
7 # Copyright 2016 The Android Open Source Project
8 #
9 # Licensed under the Apache License, Version 2.0 (the "License");
10 # you may not use this file except in compliance with the License.
11 # You may obtain a copy of the License at
12 #
13 #      http://www.apache.org/licenses/LICENSE-2.0
14 #
15 # Unless required by applicable law or agreed to in writing, software
16 # distributed under the License is distributed on an "AS IS" BASIS,
17 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18 # See the License for the specific language governing permissions and
19 # limitations under the License.
20 #
21 #-------------------------------------------------------------------------
22
23 from build.common import *
24 from build.config import ANY_GENERATOR
25 from build.build import build
26 from build_caselists import Module, getModuleByName, getBuildConfig, genCaseList, getCaseListPath, DEFAULT_BUILD_DIR, DEFAULT_TARGET
27 from fnmatch import fnmatch
28 from copy import copy
29
30 import xml.etree.cElementTree as ElementTree
31 import xml.dom.minidom as minidom
32
33 APK_NAME                = "com.drawelements.deqp.apk"
34
35 GENERATED_FILE_WARNING = """
36      This file has been automatically generated. Edit with caution.
37      """
38
39 class Project:
40         def __init__ (self, path, copyright = None):
41                 self.path               = path
42                 self.copyright  = copyright
43
44 class Configuration:
45         def __init__ (self, name, filters, glconfig = None, rotation = None, surfacetype = None, required = False, runtime = None):
46                 self.name                               = name
47                 self.glconfig                   = glconfig
48                 self.rotation                   = rotation
49                 self.surfacetype                = surfacetype
50                 self.required                   = required
51                 self.filters                    = filters
52                 self.expectedRuntime    = runtime
53
54 class Package:
55         def __init__ (self, module, configurations):
56                 self.module                     = module
57                 self.configurations     = configurations
58
59 class Mustpass:
60         def __init__ (self, project, version, packages):
61                 self.project    = project
62                 self.version    = version
63                 self.packages   = packages
64
65 class Filter:
66         TYPE_INCLUDE = 0
67         TYPE_EXCLUDE = 1
68
69         def __init__ (self, type, filename):
70                 self.type               = type
71                 self.filename   = filename
72
73 class TestRoot:
74         def __init__ (self):
75                 self.children   = []
76
77 class TestGroup:
78         def __init__ (self, name):
79                 self.name               = name
80                 self.children   = []
81
82 class TestCase:
83         def __init__ (self, name):
84                 self.name                       = name
85                 self.configurations     = []
86
87 class GLESVersion:
88         def __init__(self, major, minor):
89                 self.major = major
90                 self.minor = minor
91
92         def encode (self):
93                 return (self.major << 16) | (self.minor)
94
95 def getModuleGLESVersion (module):
96         versions = {
97                 'dEQP-EGL':             GLESVersion(2,0),
98                 'dEQP-GLES2':   GLESVersion(2,0),
99                 'dEQP-GLES3':   GLESVersion(3,0),
100                 'dEQP-GLES31':  GLESVersion(3,1)
101         }
102         return versions[module.name] if module.name in versions else None
103
104 def getSrcDir (mustpass):
105         return os.path.join(mustpass.project.path, mustpass.version, "src")
106
107 def getTmpDir (mustpass):
108         return os.path.join(mustpass.project.path, mustpass.version, "tmp")
109
110 def getModuleShorthand (module):
111         assert module.name[:5] == "dEQP-"
112         return module.name[5:].lower()
113
114 def getCaseListFileName (package, configuration):
115         return "%s-%s.txt" % (getModuleShorthand(package.module), configuration.name)
116
117 def getDstCaseListPath (mustpass, package, configuration):
118         return os.path.join(mustpass.project.path, mustpass.version, getCaseListFileName(package, configuration))
119
120 def getCTSPackageName (package):
121         return "com.drawelements.deqp." + getModuleShorthand(package.module)
122
123 def getCommandLine (config):
124         cmdLine = ""
125
126         if config.glconfig != None:
127                 cmdLine += "--deqp-gl-config-name=%s " % config.glconfig
128
129         if config.rotation != None:
130                 cmdLine += "--deqp-screen-rotation=%s " % config.rotation
131
132         if config.surfacetype != None:
133                 cmdLine += "--deqp-surface-type=%s " % config.surfacetype
134
135         cmdLine += "--deqp-watchdog=enable"
136
137         return cmdLine
138
139 def readCaseList (filename):
140         cases = []
141         with open(filename, 'rb') as f:
142                 for line in f:
143                         if line[:6] == "TEST: ":
144                                 cases.append(line[6:].strip())
145         return cases
146
147 def getCaseList (buildCfg, generator, module):
148         build(buildCfg, generator, [module.binName])
149         genCaseList(buildCfg, generator, module, "txt")
150         return readCaseList(getCaseListPath(buildCfg, module, "txt"))
151
152 def readPatternList (filename):
153         ptrns = []
154         with open(filename, 'rb') as f:
155                 for line in f:
156                         line = line.strip()
157                         if len(line) > 0 and line[0] != '#':
158                                 ptrns.append(line)
159         return ptrns
160
161 def applyPatterns (caseList, patterns, filename, op):
162         matched                 = set()
163         errors                  = []
164         curList                 = copy(caseList)
165         trivialPtrns    = [p for p in patterns if p.find('*') < 0]
166         regularPtrns    = [p for p in patterns if p.find('*') >= 0]
167
168         # Apply trivial (just case paths)
169         allCasesSet             = set(caseList)
170         for path in trivialPtrns:
171                 if path in allCasesSet:
172                         if path in matched:
173                                 errors.append((path, "Same case specified more than once"))
174                         matched.add(path)
175                 else:
176                         errors.append((path, "Test case not found"))
177
178         curList = [c for c in curList if c not in matched]
179
180         for pattern in regularPtrns:
181                 matchedThisPtrn = set()
182
183                 for case in curList:
184                         if fnmatch(case, pattern):
185                                 matchedThisPtrn.add(case)
186
187                 if len(matchedThisPtrn) == 0:
188                         errors.append((pattern, "Pattern didn't match any cases"))
189
190                 matched = matched | matchedThisPtrn
191                 curList = [c for c in curList if c not in matched]
192
193         for pattern, reason in errors:
194                 print "ERROR: %s: %s" % (reason, pattern)
195
196         if len(errors) > 0:
197                 die("Found %s invalid patterns while processing file %s" % (len(errors), filename))
198
199         return [c for c in caseList if op(c in matched)]
200
201 def applyInclude (caseList, patterns, filename):
202         return applyPatterns(caseList, patterns, filename, lambda b: b)
203
204 def applyExclude (caseList, patterns, filename):
205         return applyPatterns(caseList, patterns, filename, lambda b: not b)
206
207 def readPatternLists (mustpass):
208         lists = {}
209         for package in mustpass.packages:
210                 for cfg in package.configurations:
211                         for filter in cfg.filters:
212                                 if not filter.filename in lists:
213                                         lists[filter.filename] = readPatternList(os.path.join(getSrcDir(mustpass), filter.filename))
214         return lists
215
216 def applyFilters (caseList, patternLists, filters):
217         res = copy(caseList)
218         for filter in filters:
219                 ptrnList = patternLists[filter.filename]
220                 if filter.type == Filter.TYPE_INCLUDE:
221                         res = applyInclude(res, ptrnList, filter.filename)
222                 else:
223                         assert filter.type == Filter.TYPE_EXCLUDE
224                         res = applyExclude(res, ptrnList, filter.filename)
225         return res
226
227 def appendToHierarchy (root, casePath):
228         def findChild (node, name):
229                 for child in node.children:
230                         if child.name == name:
231                                 return child
232                 return None
233
234         curNode         = root
235         components      = casePath.split('.')
236
237         for component in components[:-1]:
238                 nextNode = findChild(curNode, component)
239                 if not nextNode:
240                         nextNode = TestGroup(component)
241                         curNode.children.append(nextNode)
242                 curNode = nextNode
243
244         if not findChild(curNode, components[-1]):
245                 curNode.children.append(TestCase(components[-1]))
246
247 def buildTestHierachy (caseList):
248         root = TestRoot()
249         for case in caseList:
250                 appendToHierarchy(root, case)
251         return root
252
253 def buildTestCaseMap (root):
254         caseMap = {}
255
256         def recursiveBuild (curNode, prefix):
257                 curPath = prefix + curNode.name
258                 if isinstance(curNode, TestCase):
259                         caseMap[curPath] = curNode
260                 else:
261                         for child in curNode.children:
262                                 recursiveBuild(child, curPath + '.')
263
264         for child in root.children:
265                 recursiveBuild(child, '')
266
267         return caseMap
268
269 def include (filename):
270         return Filter(Filter.TYPE_INCLUDE, filename)
271
272 def exclude (filename):
273         return Filter(Filter.TYPE_EXCLUDE, filename)
274
275 def insertXMLHeaders (mustpass, doc):
276         if mustpass.project.copyright != None:
277                 doc.insert(0, ElementTree.Comment(mustpass.project.copyright))
278         doc.insert(1, ElementTree.Comment(GENERATED_FILE_WARNING))
279
280 def prettifyXML (doc):
281         uglyString      = ElementTree.tostring(doc, 'utf-8')
282         reparsed        = minidom.parseString(uglyString)
283         return reparsed.toprettyxml(indent='\t', encoding='utf-8')
284
285 def genCTSPackageXML (mustpass, package, root):
286         def isLeafGroup (testGroup):
287                 numGroups       = 0
288                 numTests        = 0
289
290                 for child in testGroup.children:
291                         if isinstance(child, TestCase):
292                                 numTests += 1
293                         else:
294                                 numGroups += 1
295
296                 assert numGroups + numTests > 0
297
298                 if numGroups > 0 and numTests > 0:
299                         die("Mixed groups and cases in %s" % testGroup.name)
300
301                 return numGroups == 0
302
303         def makeConfiguration (parentElem, config):
304                 attributes = {}
305
306                 if config.glconfig != None:
307                         attributes['glconfig'] = config.glconfig
308
309                 if config.rotation != None:
310                         attributes['rotation'] = config.rotation
311
312                 if config.surfacetype != None:
313                         attributes['surfacetype'] = config.surfacetype
314
315                 return ElementTree.SubElement(parentElem, "TestInstance", attributes)
316
317         def makeTestCase (parentElem, testCase):
318                 caseElem = ElementTree.SubElement(parentElem, "Test", name=testCase.name)
319                 for config in testCase.configurations:
320                         makeConfiguration(caseElem, config)
321                 return caseElem
322
323         def makeTestGroup (parentElem, testGroup):
324                 groupElem = ElementTree.SubElement(parentElem, "TestCase" if isLeafGroup(testGroup) else "TestSuite", name=testGroup.name)
325                 for child in testGroup.children:
326                         if isinstance(child, TestCase):
327                                 makeTestCase(groupElem, child)
328                         else:
329                                 makeTestGroup(groupElem, child)
330                 return groupElem
331
332         pkgElem = ElementTree.Element("TestPackage",
333                                                                   name                          = package.module.name,
334                                                                   appPackageName        = getCTSPackageName(package),
335                                                                   testType                      = "deqpTest")
336
337         pkgElem.set("xmlns:deqp", "http://drawelements.com/deqp")
338         insertXMLHeaders(mustpass, pkgElem)
339
340         glesVersion = getModuleGLESVersion(package.module)
341
342         if glesVersion != None:
343                 pkgElem.set("deqp:glesVersion", str(glesVersion.encode()))
344
345         for child in root.children:
346                 makeTestGroup(pkgElem, child)
347
348         return pkgElem
349
350 def genSpecXML (mustpass):
351         mustpassElem = ElementTree.Element("Mustpass", version = mustpass.version)
352         insertXMLHeaders(mustpass, mustpassElem)
353
354         for package in mustpass.packages:
355                 packageElem = ElementTree.SubElement(mustpassElem, "TestPackage", name = package.module.name)
356
357                 for config in package.configurations:
358                         configElem = ElementTree.SubElement(packageElem, "Configuration",
359                                                                                                 name                    = config.name,
360                                                                                                 caseListFile    = getCaseListFileName(package, config),
361                                                                                                 commandLine             = getCommandLine(config))
362
363         return mustpassElem
364
365 def addOptionElement (parent, optionName, optionValue):
366         ElementTree.SubElement(parent, "option", name=optionName, value=optionValue)
367
368 def genAndroidTestXml (mustpass):
369         RUNNER_CLASS = "com.drawelements.deqp.runner.DeqpTestRunner"
370         configElement = ElementTree.Element("configuration")
371
372         for package in mustpass.packages:
373                 for config in package.configurations:
374                         testElement = ElementTree.SubElement(configElement, "test")
375                         testElement.set("class", RUNNER_CLASS)
376                         addOptionElement(testElement, "deqp-package", package.module.name)
377                         addOptionElement(testElement, "deqp-caselist-file", getCaseListFileName(package,config))
378                         # \todo [2015-10-16 kalle]: Replace with just command line? - requires simplifications in the runner/tests as well.
379                         if config.glconfig != None:
380                                 addOptionElement(testElement, "deqp-gl-config-name", config.glconfig)
381
382                         if config.surfacetype != None:
383                                 addOptionElement(testElement, "deqp-surface-type", config.surfacetype)
384
385                         if config.rotation != None:
386                                 addOptionElement(testElement, "deqp-screen-rotation", config.rotation)
387
388                         if config.expectedRuntime != None:
389                                 addOptionElement(testElement, "runtime-hint", config.expectedRuntime)
390
391                         if config.required:
392                                 addOptionElement(testElement, "deqp-config-required", "true")
393
394         insertXMLHeaders(mustpass, configElement)
395
396         return configElement
397
398 def genMustpass (mustpass, moduleCaseLists):
399         print "Generating mustpass '%s'" % mustpass.version
400
401         patternLists = readPatternLists(mustpass)
402
403         for package in mustpass.packages:
404                 allCasesInPkg           = moduleCaseLists[package.module]
405                 matchingByConfig        = {}
406                 allMatchingSet          = set()
407
408                 for config in package.configurations:
409                         filtered        = applyFilters(allCasesInPkg, patternLists, config.filters)
410                         dstFile         = getDstCaseListPath(mustpass, package, config)
411
412                         print "  Writing deqp caselist: " + dstFile
413                         writeFile(dstFile, "\n".join(filtered) + "\n")
414
415                         matchingByConfig[config]        = filtered
416                         allMatchingSet                          = allMatchingSet | set(filtered)
417
418                 allMatchingCases        = [c for c in allCasesInPkg if c in allMatchingSet] # To preserve ordering
419                 root                            = buildTestHierachy(allMatchingCases)
420                 testCaseMap                     = buildTestCaseMap(root)
421
422                 for config in package.configurations:
423                         for case in matchingByConfig[config]:
424                                 testCaseMap[case].configurations.append(config)
425
426                 # NOTE: CTS v2 does not need package XML files. Remove when transition is complete.
427                 packageXml      = genCTSPackageXML(mustpass, package, root)
428                 xmlFilename     = os.path.join(mustpass.project.path, mustpass.version, getCTSPackageName(package) + ".xml")
429
430                 print "  Writing CTS caselist: " + xmlFilename
431                 writeFile(xmlFilename, prettifyXML(packageXml))
432
433         specXML                 = genSpecXML(mustpass)
434         specFilename    = os.path.join(mustpass.project.path, mustpass.version, "mustpass.xml")
435
436         print "  Writing spec: " + specFilename
437         writeFile(specFilename, prettifyXML(specXML))
438
439         # TODO: Which is the best selector mechanism?
440         if (mustpass.version == "master"):
441                 androidTestXML          = genAndroidTestXml(mustpass)
442                 androidTestFilename     = os.path.join(mustpass.project.path, "AndroidTest.xml")
443
444                 print "  Writing AndroidTest.xml: " + androidTestFilename
445                 writeFile(androidTestFilename, prettifyXML(androidTestXML))
446
447         print "Done!"
448
449 def genMustpassLists (mustpassLists, generator, buildCfg):
450         moduleCaseLists = {}
451
452         # Getting case lists involves invoking build, so we want to cache the results
453         for mustpass in mustpassLists:
454                 for package in mustpass.packages:
455                         if not package.module in moduleCaseLists:
456                                 moduleCaseLists[package.module] = getCaseList(buildCfg, generator, package.module)
457
458         for mustpass in mustpassLists:
459                 genMustpass(mustpass, moduleCaseLists)