scripts/build_android_mustpass.py

   1 # -*- coding: utf-8 -*-
   2
   3 #-------------------------------------------------------------------------
   4 # drawElements Quality Program utilities
   5 # --------------------------------------
   6 #
   7 # Copyright 2015 The Android Open Source Project
   8 #
   9 # Licensed under the Apache License, Version 2.0 (the "License");
  10 # you may not use this file except in compliance with the License.
  11 # You may obtain a copy of the License at
  12 #
  13 #      http://www.apache.org/licenses/LICENSE-2.0
  14 #
  15 # Unless required by applicable law or agreed to in writing, software
  16 # distributed under the License is distributed on an "AS IS" BASIS,
  17 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  18 # See the License for the specific language governing permissions and
  19 # limitations under the License.
  20 #
  21 #-------------------------------------------------------------------------
  22
  23 from build.common import *
  24 from build.config import ANY_GENERATOR
  25 from build.build import build
  26 from build_caselists import Module, getBuildConfig, genCaseList, getCaseListPath, DEFAULT_BUILD_DIR, DEFAULT_TARGET
  27 from fnmatch import fnmatch
  28 from copy import copy
  29
  30 import xml.etree.cElementTree as ElementTree
  31 import xml.dom.minidom as minidom
  32
  33 CTS_DATA_DIR    = os.path.join(DEQP_DIR, "android", "cts")
  34
  35 class Configuration:
  36         def __init__ (self, name, glconfig, rotation, surfacetype, filters):
  37                 self.name                       = name
  38                 self.glconfig           = glconfig
  39                 self.rotation           = rotation
  40                 self.surfacetype        = surfacetype
  41                 self.filters            = filters
  42
  43 class Package:
  44         def __init__ (self, module, configurations):
  45                 self.module                     = module
  46                 self.configurations     = configurations
  47
  48 class Mustpass:
  49         def __init__ (self, version, packages):
  50                 self.version    = version
  51                 self.packages   = packages
  52
  53 class Filter:
  54         TYPE_INCLUDE = 0
  55         TYPE_EXCLUDE = 1
  56
  57         def __init__ (self, type, filename):
  58                 self.type               = type
  59                 self.filename   = filename
  60
  61 class TestRoot:
  62         def __init__ (self):
  63                 self.children   = []
  64
  65 class TestGroup:
  66         def __init__ (self, name):
  67                 self.name               = name
  68                 self.children   = []
  69
  70 class TestCase:
  71         def __init__ (self, name):
  72                 self.name                       = name
  73                 self.configurations     = []
  74
  75 class GLESVersion:
  76         def __init__(self, major, minor):
  77                 self.major = major
  78                 self.minor = minor
  79
  80         def encode (self):
  81                 return (self.major << 16) | (self.minor)
  82
  83 def getModuleGLESVersion (module):
  84         versions = {
  85                 'dEQP-EGL':             GLESVersion(2,0),
  86                 'dEQP-GLES2':   GLESVersion(2,0),
  87                 'dEQP-GLES3':   GLESVersion(3,0),
  88                 'dEQP-GLES31':  GLESVersion(3,1)
  89         }
  90         return versions[module.name]
  91
  92 def getSrcDir (mustpass):
  93         return os.path.join(CTS_DATA_DIR, mustpass.version, "src")
  94
  95 def getTmpDir (mustpass):
  96         return os.path.join(CTS_DATA_DIR, mustpass.version, "tmp")
  97
  98 def getModuleShorthand (module):
  99         assert module.name[:5] == "dEQP-"
 100         return module.name[5:].lower()
 101
 102 def getCaseListFileName (package, configuration):
 103         return "%s-%s.txt" % (getModuleShorthand(package.module), configuration.name)
 104
 105 def getDstCaseListPath (mustpass, package, configuration):
 106         return os.path.join(CTS_DATA_DIR, mustpass.version, getCaseListFileName(package, configuration))
 107
 108 def getCTSPackageName (package):
 109         return "com.drawelements.deqp." + getModuleShorthand(package.module)
 110
 111 def getCommandLine (config):
 112         return "--deqp-gl-config-name=%s --deqp-screen-rotation=%s --deqp-surface-type=%s --deqp-watchdog=enable" % (config.glconfig, config.rotation, config.surfacetype)
 113
 114 def readCaseList (filename):
 115         cases = []
 116         with open(filename, 'rb') as f:
 117                 for line in f:
 118                         if line[:6] == "TEST: ":
 119                                 cases.append(line[6:].strip())
 120         return cases
 121
 122 def getCaseList (mustpass, module):
 123         generator       = ANY_GENERATOR
 124         buildCfg        = getBuildConfig(DEFAULT_BUILD_DIR, DEFAULT_TARGET, "Debug")
 125
 126         build(buildCfg, generator, [module.binName])
 127         genCaseList(buildCfg, generator, module, "txt")
 128
 129         return readCaseList(getCaseListPath(buildCfg, module, "txt"))
 130
 131 def readPatternList (filename):
 132         ptrns = []
 133         with open(filename, 'rb') as f:
 134                 for line in f:
 135                         line = line.strip()
 136                         if len(line) > 0 and line[0] != '#':
 137                                 ptrns.append(line)
 138         return ptrns
 139
 140 def applyPatterns (caseList, patterns, op):
 141         matched                 = set()
 142         errors                  = []
 143         curList                 = copy(caseList)
 144         trivialPtrns    = [p for p in patterns if p.find('*') < 0]
 145         regularPtrns    = [p for p in patterns if p.find('*') >= 0]
 146
 147         # Apply trivial (just case paths)
 148         allCasesSet             = set(caseList)
 149         for path in trivialPtrns:
 150                 if path in allCasesSet:
 151                         if path in matched:
 152                                 errors.append((path, "Same case specified more than once"))
 153                         matched.add(path)
 154                 else:
 155                         errors.append((path, "Test case not found"))
 156
 157         curList = [c for c in curList if c not in matched]
 158
 159         for pattern in regularPtrns:
 160                 matchedThisPtrn = set()
 161
 162                 for case in curList:
 163                         if fnmatch(case, pattern):
 164                                 matchedThisPtrn.add(case)
 165
 166                 if len(matchedThisPtrn) == 0:
 167                         errors.append((pattern, "Pattern didn't match any cases"))
 168
 169                 matched = matched | matchedThisPtrn
 170                 curList = [c for c in curList if c not in matched]
 171
 172         for pattern, reason in errors:
 173                 print "ERROR: %s: %s" % (reason, pattern)
 174
 175         if len(errors) > 0:
 176                 die("Found %s invalid patterns" % len(errors))
 177
 178         return [c for c in caseList if op(c in matched)]
 179
 180 def applyInclude (caseList, patterns):
 181         return applyPatterns(caseList, patterns, lambda b: b)
 182
 183 def applyExclude (caseList, patterns):
 184         return applyPatterns(caseList, patterns, lambda b: not b)
 185
 186 def readPatternLists (mustpass):
 187         lists = {}
 188         for package in mustpass.packages:
 189                 for cfg in package.configurations:
 190                         for filter in cfg.filters:
 191                                 if not filter.filename in lists:
 192                                         lists[filter.filename] = readPatternList(os.path.join(getSrcDir(mustpass), filter.filename))
 193         return lists
 194
 195 def applyFilters (caseList, patternLists, filters):
 196         res = copy(caseList)
 197         for filter in filters:
 198                 ptrnList = patternLists[filter.filename]
 199                 if filter.type == Filter.TYPE_INCLUDE:
 200                         res = applyInclude(res, ptrnList)
 201                 else:
 202                         assert filter.type == Filter.TYPE_EXCLUDE
 203                         res = applyExclude(res, ptrnList)
 204         return res
 205
 206 def appendToHierarchy (root, casePath):
 207         def findChild (node, name):
 208                 for child in node.children:
 209                         if child.name == name:
 210                                 return child
 211                 return None
 212
 213         curNode         = root
 214         components      = casePath.split('.')
 215
 216         for component in components[:-1]:
 217                 nextNode = findChild(curNode, component)
 218                 if not nextNode:
 219                         nextNode = TestGroup(component)
 220                         curNode.children.append(nextNode)
 221                 curNode = nextNode
 222
 223         if not findChild(curNode, components[-1]):
 224                 curNode.children.append(TestCase(components[-1]))
 225
 226 def buildTestHierachy (caseList):
 227         root = TestRoot()
 228         for case in caseList:
 229                 appendToHierarchy(root, case)
 230         return root
 231
 232 def buildTestCaseMap (root):
 233         caseMap = {}
 234
 235         def recursiveBuild (curNode, prefix):
 236                 curPath = prefix + curNode.name
 237                 if isinstance(curNode, TestCase):
 238                         caseMap[curPath] = curNode
 239                 else:
 240                         for child in curNode.children:
 241                                 recursiveBuild(child, curPath + '.')
 242
 243         for child in root.children:
 244                 recursiveBuild(child, '')
 245
 246         return caseMap
 247
 248 def include (filename):
 249         return Filter(Filter.TYPE_INCLUDE, filename)
 250
 251 def exclude (filename):
 252         return Filter(Filter.TYPE_EXCLUDE, filename)
 253
 254 def prettifyXML (doc):
 255         uglyString      = ElementTree.tostring(doc, 'utf-8')
 256         reparsed        = minidom.parseString(uglyString)
 257         return reparsed.toprettyxml(indent='\t', encoding='utf-8')
 258
 259 def genCTSPackageXML (package, root):
 260         def isLeafGroup (testGroup):
 261                 numGroups       = 0
 262                 numTests        = 0
 263
 264                 for child in testGroup.children:
 265                         if isinstance(child, TestCase):
 266                                 numTests += 1
 267                         else:
 268                                 numGroups += 1
 269
 270                 assert numGroups + numTests > 0
 271
 272                 if numGroups > 0 and numTests > 0:
 273                         die("Mixed groups and cases in %s" % testGroup.name)
 274
 275                 return numGroups == 0
 276
 277         def makeConfiguration (parentElem, configuration):
 278                 return ElementTree.SubElement(parentElem, "TestInstance", glconfig=configuration.glconfig, rotation=configuration.rotation, surfacetype=configuration.surfacetype)
 279
 280         def makeTestCase (parentElem, testCase):
 281                 caseElem = ElementTree.SubElement(parentElem, "Test", name=testCase.name)
 282                 for config in testCase.configurations:
 283                         makeConfiguration(caseElem, config)
 284                 return caseElem
 285
 286         def makeTestGroup (parentElem, testGroup):
 287                 groupElem = ElementTree.SubElement(parentElem, "TestCase" if isLeafGroup(testGroup) else "TestSuite", name=testGroup.name)
 288                 for child in testGroup.children:
 289                         if isinstance(child, TestCase):
 290                                 makeTestCase(groupElem, child)
 291                         else:
 292                                 makeTestGroup(groupElem, child)
 293                 return groupElem
 294
 295         pkgElem = ElementTree.Element("TestPackage",
 296                                                                   name                          = package.module.name,
 297                                                                   appPackageName        = getCTSPackageName(package),
 298                                                                   testType                      = "deqpTest")
 299
 300         pkgElem.set("xmlns:deqp", "http://drawelements.com/deqp")
 301         pkgElem.set("deqp:glesVersion", str(getModuleGLESVersion(package.module).encode()))
 302
 303         for child in root.children:
 304                 makeTestGroup(pkgElem, child)
 305
 306         return pkgElem
 307
 308 def genSpecXML (mustpass):
 309         mustpassElem = ElementTree.Element("Mustpass", version = mustpass.version)
 310
 311         for package in mustpass.packages:
 312                 packageElem = ElementTree.SubElement(mustpassElem, "TestPackage", name = package.module.name)
 313
 314                 for config in package.configurations:
 315                         configElem = ElementTree.SubElement(packageElem, "Configuration",
 316                                                                                                 name                    = config.name,
 317                                                                                                 caseListFile    = getCaseListFileName(package, config),
 318                                                                                                 commandLine             = getCommandLine(config))
 319
 320         return mustpassElem
 321
 322 def genMustpass (mustpass, moduleCaseLists):
 323         print "Generating mustpass '%s'" % mustpass.version
 324
 325         patternLists = readPatternLists(mustpass)
 326
 327         for package in mustpass.packages:
 328                 allCasesInPkg           = moduleCaseLists[package.module]
 329                 matchingByConfig        = {}
 330                 allMatchingSet          = set()
 331
 332                 for config in package.configurations:
 333                         filtered        = applyFilters(allCasesInPkg, patternLists, config.filters)
 334                         dstFile         = getDstCaseListPath(mustpass, package, config)
 335
 336                         print "  Writing deqp caselist: " + dstFile
 337                         writeFile(dstFile, "\n".join(filtered) + "\n")
 338
 339                         matchingByConfig[config]        = filtered
 340                         allMatchingSet                          = allMatchingSet | set(filtered)
 341
 342                 allMatchingCases        = [c for c in allCasesInPkg if c in allMatchingSet] # To preserve ordering
 343                 root                            = buildTestHierachy(allMatchingCases)
 344                 testCaseMap                     = buildTestCaseMap(root)
 345
 346                 for config in package.configurations:
 347                         for case in matchingByConfig[config]:
 348                                 testCaseMap[case].configurations.append(config)
 349
 350                 packageXml      = genCTSPackageXML(package, root)
 351                 xmlFilename     = os.path.join(CTS_DATA_DIR, mustpass.version, getCTSPackageName(package) + ".xml")
 352
 353                 print "  Writing CTS caselist: " + xmlFilename
 354                 writeFile(xmlFilename, prettifyXML(packageXml))
 355
 356         specXML                 = genSpecXML(mustpass)
 357         specFilename    = os.path.join(CTS_DATA_DIR, mustpass.version, "mustpass.xml")
 358
 359         print "  Writing spec: " + specFilename
 360         writeFile(specFilename, prettifyXML(specXML))
 361
 362         print "Done!"
 363
 364 def genMustpassLists (mustpassLists):
 365         moduleCaseLists = {}
 366
 367         # Getting case lists involves invoking build, so we want to cache the results
 368         for mustpass in mustpassLists:
 369                 for package in mustpass.packages:
 370                         if not package.module in moduleCaseLists:
 371                                 moduleCaseLists[package.module] = getCaseList(mustpass, package.module)
 372
 373         for mustpass in mustpassLists:
 374                 genMustpass(mustpass, moduleCaseLists)
 375
 376 EGL_MODULE                                              = Module(name = "dEQP-EGL", dirName = "egl", binName = "deqp-egl")
 377 GLES2_MODULE                                    = Module(name = "dEQP-GLES2", dirName = "gles2", binName = "deqp-gles2")
 378 GLES3_MODULE                                    = Module(name = "dEQP-GLES3", dirName = "gles3", binName = "deqp-gles3")
 379 GLES31_MODULE                                   = Module(name = "dEQP-GLES31", dirName = "gles31", binName = "deqp-gles31")
 380
 381 LMP_GLES3_PKG                                   = Package(module = GLES3_MODULE, configurations = [
 382                 Configuration(name                      = "master",
 383                                           glconfig              = "rgba8888d24s8ms0",
 384                                           rotation              = "unspecified",
 385                                           surfacetype   = "window",
 386                                           filters               = [include("es30-lmp.txt")]),
 387         ])
 388 LMP_GLES31_PKG                                  = Package(module = GLES31_MODULE, configurations = [
 389                 Configuration(name                      = "master",
 390                                           glconfig              = "rgba8888d24s8ms0",
 391                                           rotation              = "unspecified",
 392                                           surfacetype   = "window",
 393                                           filters               = [include("es31-lmp.txt")]),
 394         ])
 395
 396 LMP_MR1_GLES3_PKG                               = Package(module = GLES3_MODULE, configurations = [
 397                 Configuration(name                      = "master",
 398                                           glconfig              = "rgba8888d24s8ms0",
 399                                           rotation              = "unspecified",
 400                                           surfacetype   = "window",
 401                                           filters               = [include("es30-lmp-mr1.txt")]),
 402         ])
 403 LMP_MR1_GLES31_PKG                              = Package(module = GLES31_MODULE, configurations = [
 404                 Configuration(name                      = "master",
 405                                           glconfig              = "rgba8888d24s8ms0",
 406                                           rotation              = "unspecified",
 407                                           surfacetype   = "window",
 408                                           filters               = [include("es31-lmp-mr1.txt")]),
 409         ])
 410
 411 MASTER_EGL_COMMON_FILTERS               = [include("egl-master.txt"), exclude("egl-failures.txt")]
 412 MASTER_EGL_PKG                                  = Package(module = EGL_MODULE, configurations = [
 413                 # Master
 414                 Configuration(name                      = "master",
 415                                           glconfig              = "rgba8888d24s8ms0",
 416                                           rotation              = "unspecified",
 417                                           surfacetype   = "window",
 418                                           filters               = MASTER_EGL_COMMON_FILTERS),
 419         ])
 420
 421 MASTER_GLES2_COMMON_FILTERS             = [include("gles2-master.txt"), exclude("gles2-failures.txt")]
 422 MASTER_GLES2_PKG                                = Package(module = GLES2_MODULE, configurations = [
 423                 # Master
 424                 Configuration(name                      = "master",
 425                                           glconfig              = "rgba8888d24s8ms0",
 426                                           rotation              = "unspecified",
 427                                           surfacetype   = "window",
 428                                           filters               = MASTER_GLES2_COMMON_FILTERS),
 429         ])
 430
 431 MASTER_GLES3_COMMON_FILTERS             = [
 432                 include("gles3-master.txt"),
 433                 exclude("gles3-hw-issues.txt"),
 434                 exclude("gles3-test-issues.txt"),
 435                 exclude("gles3-spec-issues.txt")
 436         ]
 437 MASTER_GLES3_PKG                                = Package(module = GLES3_MODULE, configurations = [
 438                 # Master
 439                 Configuration(name                      = "master",
 440                                           glconfig              = "rgba8888d24s8ms0",
 441                                           rotation              = "unspecified",
 442                                           surfacetype   = "window",
 443                                           filters               = MASTER_GLES3_COMMON_FILTERS),
 444                 # Rotations
 445                 Configuration(name                      = "rotate-portrait",
 446                                           glconfig              = "rgba8888d24s8ms0",
 447                                           rotation              = "0",
 448                                           surfacetype   = "window",
 449                                           filters               = MASTER_GLES3_COMMON_FILTERS + [include("gles3-rotation.txt")]),
 450                 Configuration(name                      = "rotate-landscape",
 451                                           glconfig              = "rgba8888d24s8ms0",
 452                                           rotation              = "90",
 453                                           surfacetype   = "window",
 454                                           filters               = MASTER_GLES3_COMMON_FILTERS + [include("gles3-rotation.txt")]),
 455                 Configuration(name                      = "rotate-reverse-portrait",
 456                                           glconfig              = "rgba8888d24s8ms0",
 457                                           rotation              = "180",
 458                                           surfacetype   = "window",
 459                                           filters               = MASTER_GLES3_COMMON_FILTERS + [include("gles3-rotation.txt")]),
 460                 Configuration(name                      = "rotate-reverse-landscape",
 461                                           glconfig              = "rgba8888d24s8ms0",
 462                                           rotation              = "270",
 463                                           surfacetype   = "window",
 464                                           filters               = MASTER_GLES3_COMMON_FILTERS + [include("gles3-rotation.txt")]),
 465
 466                 # MSAA
 467                 Configuration(name                      = "multisample",
 468                                           glconfig              = "rgba8888d24s8ms4",
 469                                           rotation              = "unspecified",
 470                                           surfacetype   = "window",
 471                                           filters               = MASTER_GLES3_COMMON_FILTERS + [include("gles3-multisample.txt")]),
 472
 473                 # Pixel format
 474                 Configuration(name                      = "565-no-depth-no-stencil",
 475                                           glconfig              = "rgb565d0s0ms0",
 476                                           rotation              = "unspecified",
 477                                           surfacetype   = "window",
 478                                           filters               = MASTER_GLES3_COMMON_FILTERS + [include("gles3-pixelformat.txt")]),
 479         ])
 480
 481 MASTER_GLES31_COMMON_FILTERS    = [
 482                 include("gles31-master.txt"),
 483                 exclude("gles31-hw-issues.txt"),
 484                 exclude("gles31-test-issues.txt"),
 485         ]
 486 MASTER_GLES31_PKG                               = Package(module = GLES31_MODULE, configurations = [
 487                 # Master
 488                 Configuration(name                      = "master",
 489                                           glconfig              = "rgba8888d24s8ms0",
 490                                           rotation              = "unspecified",
 491                                           surfacetype   = "window",
 492                                           filters               = MASTER_GLES31_COMMON_FILTERS),
 493
 494                 # Rotations
 495                 Configuration(name                      = "rotate-portrait",
 496                                           glconfig              = "rgba8888d24s8ms0",
 497                                           rotation              = "0",
 498                                           surfacetype   = "window",
 499                                           filters               = MASTER_GLES31_COMMON_FILTERS + [include("gles31-rotation.txt")]),
 500                 Configuration(name                      = "rotate-landscape",
 501                                           glconfig              = "rgba8888d24s8ms0",
 502                                           rotation              = "90",
 503                                           surfacetype   = "window",
 504                                           filters               = MASTER_GLES31_COMMON_FILTERS + [include("gles31-rotation.txt")]),
 505                 Configuration(name                      = "rotate-reverse-portrait",
 506                                           glconfig              = "rgba8888d24s8ms0",
 507                                           rotation              = "180",
 508                                           surfacetype   = "window",
 509                                           filters               = MASTER_GLES31_COMMON_FILTERS + [include("gles31-rotation.txt")]),
 510                 Configuration(name                      = "rotate-reverse-landscape",
 511                                           glconfig              = "rgba8888d24s8ms0",
 512                                           rotation              = "270",
 513                                           surfacetype   = "window",
 514                                           filters               = MASTER_GLES31_COMMON_FILTERS + [include("gles31-rotation.txt")]),
 515
 516                 # MSAA
 517                 Configuration(name                      = "multisample",
 518                                           glconfig              = "rgba8888d24s8ms4",
 519                                           rotation              = "unspecified",
 520                                           surfacetype   = "window",
 521                                           filters               = MASTER_GLES31_COMMON_FILTERS + [include("gles31-multisample.txt")]),
 522
 523                 # Pixel format
 524                 Configuration(name                      = "565-no-depth-no-stencil",
 525                                           glconfig              = "rgb565d0s0ms0",
 526                                           rotation              = "unspecified",
 527                                           surfacetype   = "window",
 528                                           filters               = MASTER_GLES31_COMMON_FILTERS + [include("gles31-pixelformat.txt")]),
 529         ])
 530
 531 MUSTPASS_LISTS                          = [
 532                 Mustpass(version = "lmp",               packages = [LMP_GLES3_PKG, LMP_GLES31_PKG]),
 533                 Mustpass(version = "lmp-mr1",   packages = [LMP_MR1_GLES3_PKG, LMP_MR1_GLES31_PKG]),
 534                 Mustpass(version = "master",    packages = [MASTER_EGL_PKG, MASTER_GLES2_PKG, MASTER_GLES3_PKG, MASTER_GLES31_PKG])
 535         ]
 536
 537 if __name__ == "__main__":
 538         genMustpassLists(MUSTPASS_LISTS)