--- /dev/null
+# -*- coding: utf-8 -*-
+
+#-------------------------------------------------------------------------
+# Khronos OpenGL CTS
+# ------------------
+#
+# Copyright (c) 2016 The Khronos Group Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#-------------------------------------------------------------------------
+
+import xml.dom.minidom
+
+class TestRunSummary:
+ def __init__ (self, type, isConformant, configLogFilename, runLogFilenames, runLogAndCaselist):
+ self.type = type
+ self.isConformant = isConformant
+ self.configLogFilename = configLogFilename
+ self.runLogFilenames = runLogFilenames
+ self.runLogAndCaselist = runLogAndCaselist
+
+def parseRunSummary (filename):
+ doc = xml.dom.minidom.parse(filename)
+ summary = doc.documentElement
+ if summary.localName != "Summary":
+ raise Exception("Document element is not <Summmary>")
+
+ type = summary.getAttributeNode("Type").nodeValue
+ isConformant = summary.getAttributeNode("Conformant").nodeValue == "True"
+
+ configRuns = doc.getElementsByTagName("Configs")
+ if len(configRuns) != 1:
+ raise Exception("Excepted one <Configs> element, found %d" % len(configRuns))
+
+ runLogFilenames = []
+ runLogAndCaselist = {}
+ runFiles = doc.getElementsByTagName("TestRun")
+ for n in runFiles:
+ runLog = n.getAttributeNode("FileName").nodeValue
+ runLogFilenames.append(runLog)
+ cmdLine = n.getAttributeNode("CmdLine").nodeValue
+ caseList = None
+ for words in cmdLine.split():
+ if "deqp-caselist" in words:
+ caseList = words.split("=")[1]
+ caseList = caseList[len("gl_cts/"):]
+ runLogAndCaselist[runLog] = caseList
+
+ return TestRunSummary(type, isConformant, configRuns[0].getAttributeNode("FileName").nodeValue, runLogFilenames, runLogAndCaselist)
+
--- /dev/null
+# -*- coding: utf-8 -*-
+
+#-------------------------------------------------------------------------
+# Khronos OpenGL CTS
+# ------------------
+#
+# Copyright (c) 2016 The Khronos Group Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#-------------------------------------------------------------------------
+
+import os
+import sys
+import xml.dom.minidom
+import re
+
+ROOT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..", ".."))
+sys.path.append(os.path.join(ROOT_DIR, "scripts", "verify"))
+sys.path.append(os.path.join(ROOT_DIR, "scripts", "build"))
+sys.path.append(os.path.join(ROOT_DIR, "scripts", "log"))
+
+from package import getPackageDescription
+from verify import *
+from message import *
+from common import *
+from log_parser import *
+from summary import *
+
+def getConfigCaseName (type):
+ configs = { "es32" : ["CTS-Configs.es32", "CTS-Configs.es31", "CTS-Configs.es3", "CTS-Configs.es2"],
+ "es31" : ["CTS-Configs.es31", "CTS-Configs.es3", "CTS-Configs.es2"],
+ "es3" : ["CTS-Configs.es3", "CTS-Configs.es2"],
+ "es2" : ["CTS-Configs.es2"]}
+ return configs[type]
+
+def retrieveReportedConfigs(caseName, log):
+ doc = xml.dom.minidom.parseString(log)
+ sectionItems = doc.getElementsByTagName('Section')
+ sectionName = None
+
+ configs = []
+ for sectionItem in sectionItems:
+ sectionName = sectionItem.getAttributeNode('Name').nodeValue
+ if sectionName == "Configs":
+ assert len(configs) == 0
+ textItems = sectionItem.getElementsByTagName('Text')
+ for textItem in textItems:
+ configs.append(getNodeText(textItem))
+ res = {caseName : configs}
+ return res
+
+def compareConfigs(filename, baseConfigs, cmpConfigs):
+ messages = []
+ assert len(list(baseConfigs.keys())) == 1
+ assert len(list(cmpConfigs.keys())) == 1
+ baseKey = list(baseConfigs.keys())[0]
+ cmpKey = list(cmpConfigs.keys())[0]
+
+ if cmp(baseConfigs[baseKey], cmpConfigs[cmpKey]) != 0:
+ messages.append(error(filename, "Confomant configs reported for %s and %s do not match" % (baseKey,cmpKey)))
+
+ return messages
+
+def verifyConfigFile (filename, type):
+ messages = []
+ caseNames = getConfigCaseName(type)
+
+ parser = BatchResultParser()
+ results = parser.parseFile(filename)
+ baseConfigs = None
+
+ for caseName in caseNames:
+ caseResult = None
+ print "Verifying %s in %s" % (caseName, filename)
+ for result in results:
+ if result.name == caseName:
+ caseResult = result
+ break;
+ if caseResult == None:
+ messages.append(error(filename, "Missing %s" % caseName))
+ else:
+ configs = retrieveReportedConfigs(caseName, result.log)
+ if baseConfigs == None:
+ baseConfigs = configs
+ else:
+ messages += compareConfigs(filename, baseConfigs, configs)
+ if not caseResult.statusCode in ALLOWED_STATUS_CODES:
+ messages.append(error(filename, "%s failed" % caseResult))
+
+ return messages
+
+def verifyMustpassCases(package, mustpassCases):
+ messages = []
+ for mustpass in mustpassCases:
+ mustpassXML = os.path.join(mustpass, "mustpass.xml")
+ doc = xml.dom.minidom.parse(mustpassXML)
+ testConfigs = doc.getElementsByTagName("Configuration")
+ for testConfig in testConfigs:
+ caseListFile = testConfig.getAttributeNode("caseListFile").nodeValue
+ pattern = "config-" + os.path.splitext(caseListFile)[0] + "-cfg-[0-9]*"+"-run-[0-9]*"
+ cmdLine = testConfig.getAttributeNode("commandLine").nodeValue
+ cfgItems = {'height':None, 'width':None, 'seed':None, 'rotation':None}
+ for arg in cmdLine.split():
+ val = arg.split('=')[1]
+ if "deqp-surface-height" in arg:
+ cfgItems['height'] = val
+ elif "deqp-surface-width" in arg:
+ cfgItems['width'] = val
+ elif "deqp-base-seed" in arg:
+ cfgItems['seed'] = val
+ elif "deqp-screen-rotation" in arg:
+ cfgItems['rotation'] = val
+ pattern += "-width-" + cfgItems['width'] + "-height-" + cfgItems['height']
+ if cfgItems['seed'] != None:
+ pattern += "-seed-" + cfgItems['seed']
+ pattern += ".qpa"
+ p = re.compile(pattern)
+ matches = [m for l in mustpassCases[mustpass] for m in (p.match(l),) if m]
+ if len(matches) == 0:
+ conformOs = testConfig.getAttributeNode("os").nodeValue
+ txt = "Configuration %s %s was not executed" % (caseListFile, cmdLine)
+ if conformOs == "any" or (package.conformOs != None and conformOs in package.conformOs.lower()):
+ msg = error(mustpassXML, txt)
+ else:
+ msg = warning(mustpassXML, txt)
+ messages.append(msg)
+
+ return messages
+
+def verifyTestLogs (package):
+ messages = []
+
+ try:
+ execute(['git', 'checkout', '--quiet', package.conformVersion])
+ except Exception, e:
+ print str(e)
+ print "Failed to checkout release tag %s." % package.conformVersion
+ return messages
+
+ messages = []
+ summary = parseRunSummary(os.path.join(package.basePath, package.summary))
+ mustpassDirs = []
+
+ # Check Conformant attribute
+ if not summary.isConformant:
+ messages.append(error(package.summary, "Runner reported conformance failure (Conformant=\"False\" in <Summary>)"))
+
+ # Verify config list
+ messages += verifyConfigFile(os.path.join(package.basePath, summary.configLogFilename), summary.type)
+
+ mustpassCases = {}
+ # Verify that all run files passed
+ for runLog in summary.runLogAndCaselist:
+ sys.stdout.write("Verifying %s -" % runLog)
+ sys.stdout.flush()
+
+ mustpassFile = os.path.join(ROOT_DIR, "external", "openglcts", summary.runLogAndCaselist[runLog])
+ key = os.path.dirname(mustpassFile)
+ if key in mustpassCases:
+ mpCase = mustpassCases[key]
+ else:
+ mpCase = []
+ mpCase.append(runLog)
+ mustpassCases[os.path.dirname(mustpassFile)] = mpCase
+ mustpass = readMustpass(mustpassFile)
+ messages_log = verifyTestLog(os.path.join(package.basePath, runLog), mustpass)
+
+ errors = [m for m in messages_log if m.type == ValidationMessage.TYPE_ERROR]
+ warnings = [m for m in messages_log if m.type == ValidationMessage.TYPE_WARNING]
+ if len(errors) > 0:
+ sys.stdout.write(" finished with ERRRORS")
+ if len(warnings) > 0:
+ sys.stdout.write(" finished with WARNINGS")
+ if len(errors) == 0 and len(warnings) == 0:
+ sys.stdout.write(" OK")
+ sys.stdout.write("\n")
+ sys.stdout.flush()
+
+ messages += messages_log
+
+ messages += verifyMustpassCases(package, mustpassCases)
+
+ return messages
+
+def verifyGitStatusFiles (package):
+ messages = []
+
+ if len(package.gitStatus) != 2:
+ messages.append(error(package.basePath, "Exactly two git status files must be present, found %s" % len(package.gitStatus)))
+
+ messages += verifyGitStatus(package)
+
+ return messages
+
+def verifyGitLogFiles (package):
+ messages = []
+
+ if len(package.gitLog) != 2:
+ messages.append(error(package.basePath, "Exactly two git log file must be present, found %s" % len(package.gitLog)))
+
+ messages += verifyGitLog(package)
+
+ return messages
+
+def verifyPackage (package):
+ messages = []
+
+ messages += verifyStatement(package)
+ messages += verifyGitStatusFiles(package)
+ messages += verifyGitLogFiles(package)
+ messages += verifyPatches(package)
+
+ for item in package.otherItems:
+ messages.append(warning(os.path.join(package.basePath, item), "Unknown file"))
+
+ return messages
+
+def verifyESSubmission(argv):
+ if len(argv) != 2:
+ print "%s: [extracted submission package directory]" % sys.argv[0]
+ sys.exit(-1)
+ try:
+ execute(['git', 'ls-remote', 'origin', '--quiet'])
+ except Exception, e:
+ print str(e)
+ print "This script must be executed inside VK-GL-CTS directory."
+ sys.exit(-1)
+
+ packagePath = os.path.normpath(sys.argv[1])
+ package = getPackageDescription(packagePath)
+ messages = verifyPackage(package)
+ messages += verifyTestLogs(package)
+
+ errors = [m for m in messages if m.type == ValidationMessage.TYPE_ERROR]
+ warnings = [m for m in messages if m.type == ValidationMessage.TYPE_WARNING]
+
+ for message in messages:
+ print str(message)
+
+ print ""
+
+ if len(errors) > 0:
+ print "Found %d validation errors and %d warnings!" % (len(errors), len(warnings))
+ sys.exit(-2)
+ elif len(warnings) > 0:
+ print "Found %d warnings, manual review required" % len(warnings)
+ sys.exit(-1)
+ else:
+ print "All validation checks passed"
+
+if __name__ == "__main__":
+ verifyESSubmission(sys.argv)
--- /dev/null
+# -*- coding: utf-8 -*-
+
+#-------------------------------------------------------------------------
+# Khronos OpenGL CTS
+# ------------------
+#
+# Copyright (c) 2016 The Khronos Group Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#-------------------------------------------------------------------------
+
+import os
+import sys
+import xml.dom.minidom
+
+sys.path.append(os.path.join(os.path.dirname(__file__), "..", "..", "..", "scripts", "log"))
+
+from log_parser import BatchResultParser, StatusCode
+
+from summary import *
+
+VALID_STATUS_CODES = set([
+ StatusCode.PASS,
+ StatusCode.COMPATIBILITY_WARNING,
+ StatusCode.QUALITY_WARNING,
+ StatusCode.NOT_SUPPORTED
+ ])
+
+def isStatusCodeOk (code):
+ return code in VALID_STATUS_CODES
+
+def getConfigCaseName (type):
+ return "CTS-Configs.%s" % type
+
+def verifyConfigFile (filename, type):
+ caseName = getConfigCaseName(type)
+
+ print "Verifying %s in %s" % (caseName, filename)
+
+ parser = BatchResultParser()
+ results = parser.parseFile(filename)
+ caseResult = None
+
+ for result in results:
+ if result.name == caseName:
+ caseResult = result
+ break
+
+ if caseResult == None:
+ print "FAIL: %s not found" % caseName
+ return False
+
+ if not isStatusCodeOk(caseResult.statusCode):
+ print "FAIL: %s" % caseResult
+ return False
+
+ return True
+
+def verifySubmission (dirname):
+ summary = parseRunSummary(os.path.join(dirname, "cts-run-summary.xml"))
+ allOk = True
+
+ # Check Conformant attribute
+ if not summary.isConformant:
+ print "FAIL: Runner reported conformance failure (Conformant=\"False\" in <Summary>)"
+
+ # Verify config list
+ if not verifyConfigFile(os.path.join(dirname, summary.configLogFilename), summary.type):
+ allOk = False
+
+ # Verify that all run files passed
+ for runFilename in summary.runLogFilenames:
+ print "Verifying %s" % runFilename
+
+ logParser = BatchResultParser()
+ batchResult = logParser.parseFile(os.path.join(dirname, runFilename))
+
+ for result in batchResult:
+ if not isStatusCodeOk(result.statusCode):
+ print "FAIL: %s" % str(result)
+ allOk = False
+
+ return allOk
+
+def verifyGLSubmission(argv):
+ if len(argv) != 2:
+ print "%s: [extracted submission package directory]" % argv[0]
+ sys.exit(-1)
+
+ try:
+ isOk = verifySubmission(argv[1])
+ print "Verification %s" % ("PASSED" if isOk else "FAILED!")
+ sys.exit(0 if isOk else 1)
+ except Exception, e:
+ print str(e)
+ print "Error occurred during verification"
+ sys.exit(-1)
+
+if __name__ == "__main__":
+ verifyGLSubmission(sys.argv)
+
import os
import sys
-import xml.dom.minidom
-sys.path.append(os.path.join(os.path.dirname(__file__), "..", "..", "..", "scripts", "log"))
+sys.path.append(os.path.join(os.path.dirname(__file__), "verify"))
-from log_parser import BatchResultParser, StatusCode
-
-VALID_STATUS_CODES = set([
- StatusCode.PASS,
- StatusCode.COMPATIBILITY_WARNING,
- StatusCode.QUALITY_WARNING,
- StatusCode.NOT_SUPPORTED
- ])
-
-def isStatusCodeOk (code):
- return code in VALID_STATUS_CODES
-
-def getConfigCaseName (type):
- return "CTS-Configs.%s" % type
-
-class TestRunSummary:
- def __init__ (self, type, isConformant, configLogFilename, runLogFilenames):
- self.type = type
- self.isConformant = isConformant
- self.configLogFilename = configLogFilename
- self.runLogFilenames = runLogFilenames
-
-def parseRunSummary (filename):
- doc = xml.dom.minidom.parse(filename)
- summary = doc.documentElement
- if summary.localName != "Summary":
- raise Exception("Document element is not <Summmary>")
-
- type = summary.getAttributeNode("Type").nodeValue
- isConformant = summary.getAttributeNode("Conformant").nodeValue == "True"
-
- configRuns = doc.getElementsByTagName("Configs")
- if len(configRuns) != 1:
- raise Exception("Excepted one <Configs> element, found %d" % len(configRuns))
-
- runFiles = doc.getElementsByTagName("TestRun")
-
- return TestRunSummary(type, isConformant, configRuns[0].getAttributeNode("FileName").nodeValue, [n.getAttributeNode("FileName").nodeValue for n in runFiles])
-
-def verifyConfigFile (filename, type):
- caseName = getConfigCaseName(type)
-
- print "Verifying %s in %s" % (caseName, filename)
-
- parser = BatchResultParser()
- results = parser.parseFile(filename)
- caseResult = None
-
- for result in results:
- if result.name == caseName:
- caseResult = result
- break
-
- if caseResult == None:
- print "FAIL: %s not found" % caseName
- return False
-
- if not isStatusCodeOk(caseResult.statusCode):
- print "FAIL: %s" % caseResult
- return False
-
- return True
-
-def verifySubmission (dirname):
- summary = parseRunSummary(os.path.join(dirname, "cts-run-summary.xml"))
- allOk = True
-
- # Check Conformant attribute
- if not summary.isConformant:
- print "FAIL: Runner reported conformance failure (Conformant=\"False\" in <Summary>)"
-
- # Verify config list
- if not verifyConfigFile(os.path.join(dirname, summary.configLogFilename), summary.type):
- allOk = False
-
- # Verify that all run files passed
- for runFilename in summary.runLogFilenames:
- print "Verifying %s" % runFilename
-
- logParser = BatchResultParser()
- batchResult = logParser.parseFile(os.path.join(dirname, runFilename))
-
- for result in batchResult:
- if not isStatusCodeOk(result.statusCode):
- print "FAIL: %s" % str(result)
- allOk = False
-
- return allOk
+from summary import *
+from verify_es import verifyESSubmission
+from verify_gl import verifyGLSubmission
if __name__ == "__main__":
if len(sys.argv) != 2:
print "%s: [directory]" % sys.argv[0]
sys.exit(-1)
- try:
- isOk = verifySubmission(sys.argv[1])
- print "Verification %s" % ("PASSED" if isOk else "FAILED!")
- sys.exit(0 if isOk else 1)
- except Exception, e:
- print str(e)
- print "Error occurred during verification"
- sys.exit(-1)
+ summary = parseRunSummary(os.path.join(sys.argv[1], "cts-run-summary.xml"))
+ if "es" in summary.type:
+ verifyESSubmission(sys.argv)
+ else:
+ assert "gl" in summary.type
+ verifyGLSubmission(sys.argv)
+
def parseTestCaseResult (self, name, log):
try:
+ # The XML parser has troubles with invalid characters deliberately included in the shaders.
+ # This line removes such characters before calling the parser
+ log = log.decode('utf-8','ignore').encode("utf-8")
doc = xml.dom.minidom.parseString(log)
resultItems = doc.getElementsByTagName('Result')
if len(resultItems) != 1:
GIT_STATUS_PATTERN = "*git-status.txt"
GIT_LOG_PATTERN = "*git-log.txt"
PATCH_PATTERN = "*.patch"
+SUMMARY_PATTERN = "cts-run-summary.xml"
class PackageDescription:
- def __init__ (self, basePath, statement, testLogs, gitStatus, gitLog, patches, conformVersion, otherItems):
+ def __init__ (self, basePath, statement, testLogs, gitStatus, gitLog, patches, summary, conformVersion, conformOs, otherItems):
self.basePath = basePath
self.statement = statement
self.testLogs = testLogs
self.gitStatus = gitStatus
self.gitLog = gitLog
self.patches = patches
+ self.summary = summary
self.otherItems = otherItems
self.conformVersion = conformVersion
+ self.conformOs = conformOs
def getPackageDescription (packagePath):
allItems = os.listdir(packagePath)
gitStatus = []
gitLog = []
patches = []
+ summary = None
otherItems = []
conformVersion = None
+ conformOs = None
for item in allItems:
if fnmatch(item, STATEMENT_PATTERN):
gitLog.append(item)
elif fnmatch(item, PATCH_PATTERN):
patches.append(item)
+ elif fnmatch(item, SUMMARY_PATTERN):
+ assert summary == None
+ summary = item
else:
otherItems.append(item)
- return PackageDescription(packagePath, statement, testLogs, gitStatus, gitLog, patches, conformVersion, otherItems)
+ return PackageDescription(packagePath, statement, testLogs, gitStatus, gitLog, patches, summary, conformVersion, conformOs, otherItems)
if hasVersion:
messages.append(error(statementPath, "Multiple CONFORM_VERSIONs"))
else:
+ assert len(line.split()) >= 2
+ package.conformVersion = line.split()[1]
hasVersion = True
elif beginsWith(line, "PRODUCT:"):
hasProduct = True # Multiple products allowed
if hasOs:
messages.append(error(statementPath, "Multiple OSes"))
else:
+ assert len(line.split()) >= 2
+ package.conformOs = line.split()[1]
hasOs = True
if not hasVersion:
projects / platform / upstream / VK-GL-CTS.git / commitdiff