From b90ed7c95140b4518dc17e32f6f12bb744ce71dc Mon Sep 17 00:00:00 2001 From: Alexander Galazin Date: Wed, 14 Dec 2016 17:35:11 +0100 Subject: [PATCH] Updates to submission verification script Change-Id: I4c2a3c897ceeecdcac586fad022a34ca18041b0d --- external/openglcts/scripts/verify/summary.py | 61 ++++++ external/openglcts/scripts/verify/verify_es.py | 263 ++++++++++++++++++++++++ external/openglcts/scripts/verify/verify_gl.py | 112 ++++++++++ external/openglcts/scripts/verify_submission.py | 109 +--------- scripts/log/log_parser.py | 3 + scripts/verify/package.py | 12 +- scripts/verify/verify.py | 4 + 7 files changed, 464 insertions(+), 100 deletions(-) create mode 100644 external/openglcts/scripts/verify/summary.py create mode 100644 external/openglcts/scripts/verify/verify_es.py create mode 100644 external/openglcts/scripts/verify/verify_gl.py diff --git a/external/openglcts/scripts/verify/summary.py b/external/openglcts/scripts/verify/summary.py new file mode 100644 index 0000000..377b503 --- /dev/null +++ b/external/openglcts/scripts/verify/summary.py @@ -0,0 +1,61 @@ +# -*- coding: utf-8 -*- + +#------------------------------------------------------------------------- +# Khronos OpenGL CTS +# ------------------ +# +# Copyright (c) 2016 The Khronos Group Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +#------------------------------------------------------------------------- + +import xml.dom.minidom + +class TestRunSummary: + def __init__ (self, type, isConformant, configLogFilename, runLogFilenames, runLogAndCaselist): + self.type = type + self.isConformant = isConformant + self.configLogFilename = configLogFilename + self.runLogFilenames = runLogFilenames + self.runLogAndCaselist = runLogAndCaselist + +def parseRunSummary (filename): + doc = xml.dom.minidom.parse(filename) + summary = doc.documentElement + if summary.localName != "Summary": + raise Exception("Document element is not ") + + type = summary.getAttributeNode("Type").nodeValue + isConformant = summary.getAttributeNode("Conformant").nodeValue == "True" + + configRuns = doc.getElementsByTagName("Configs") + if len(configRuns) != 1: + raise Exception("Excepted one element, found %d" % len(configRuns)) + + runLogFilenames = [] + runLogAndCaselist = {} + runFiles = doc.getElementsByTagName("TestRun") + for n in runFiles: + runLog = n.getAttributeNode("FileName").nodeValue + runLogFilenames.append(runLog) + cmdLine = n.getAttributeNode("CmdLine").nodeValue + caseList = None + for words in cmdLine.split(): + if "deqp-caselist" in words: + caseList = words.split("=")[1] + caseList = caseList[len("gl_cts/"):] + runLogAndCaselist[runLog] = caseList + + return TestRunSummary(type, isConformant, configRuns[0].getAttributeNode("FileName").nodeValue, runLogFilenames, runLogAndCaselist) + diff --git a/external/openglcts/scripts/verify/verify_es.py b/external/openglcts/scripts/verify/verify_es.py new file mode 100644 index 0000000..cbe6860 --- /dev/null +++ b/external/openglcts/scripts/verify/verify_es.py @@ -0,0 +1,263 @@ +# -*- coding: utf-8 -*- + +#------------------------------------------------------------------------- +# Khronos OpenGL CTS +# ------------------ +# +# Copyright (c) 2016 The Khronos Group Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +#------------------------------------------------------------------------- + +import os +import sys +import xml.dom.minidom +import re + +ROOT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..", "..")) +sys.path.append(os.path.join(ROOT_DIR, "scripts", "verify")) +sys.path.append(os.path.join(ROOT_DIR, "scripts", "build")) +sys.path.append(os.path.join(ROOT_DIR, "scripts", "log")) + +from package import getPackageDescription +from verify import * +from message import * +from common import * +from log_parser import * +from summary import * + +def getConfigCaseName (type): + configs = { "es32" : ["CTS-Configs.es32", "CTS-Configs.es31", "CTS-Configs.es3", "CTS-Configs.es2"], + "es31" : ["CTS-Configs.es31", "CTS-Configs.es3", "CTS-Configs.es2"], + "es3" : ["CTS-Configs.es3", "CTS-Configs.es2"], + "es2" : ["CTS-Configs.es2"]} + return configs[type] + +def retrieveReportedConfigs(caseName, log): + doc = xml.dom.minidom.parseString(log) + sectionItems = doc.getElementsByTagName('Section') + sectionName = None + + configs = [] + for sectionItem in sectionItems: + sectionName = sectionItem.getAttributeNode('Name').nodeValue + if sectionName == "Configs": + assert len(configs) == 0 + textItems = sectionItem.getElementsByTagName('Text') + for textItem in textItems: + configs.append(getNodeText(textItem)) + res = {caseName : configs} + return res + +def compareConfigs(filename, baseConfigs, cmpConfigs): + messages = [] + assert len(list(baseConfigs.keys())) == 1 + assert len(list(cmpConfigs.keys())) == 1 + baseKey = list(baseConfigs.keys())[0] + cmpKey = list(cmpConfigs.keys())[0] + + if cmp(baseConfigs[baseKey], cmpConfigs[cmpKey]) != 0: + messages.append(error(filename, "Confomant configs reported for %s and %s do not match" % (baseKey,cmpKey))) + + return messages + +def verifyConfigFile (filename, type): + messages = [] + caseNames = getConfigCaseName(type) + + parser = BatchResultParser() + results = parser.parseFile(filename) + baseConfigs = None + + for caseName in caseNames: + caseResult = None + print "Verifying %s in %s" % (caseName, filename) + for result in results: + if result.name == caseName: + caseResult = result + break; + if caseResult == None: + messages.append(error(filename, "Missing %s" % caseName)) + else: + configs = retrieveReportedConfigs(caseName, result.log) + if baseConfigs == None: + baseConfigs = configs + else: + messages += compareConfigs(filename, baseConfigs, configs) + if not caseResult.statusCode in ALLOWED_STATUS_CODES: + messages.append(error(filename, "%s failed" % caseResult)) + + return messages + +def verifyMustpassCases(package, mustpassCases): + messages = [] + for mustpass in mustpassCases: + mustpassXML = os.path.join(mustpass, "mustpass.xml") + doc = xml.dom.minidom.parse(mustpassXML) + testConfigs = doc.getElementsByTagName("Configuration") + for testConfig in testConfigs: + caseListFile = testConfig.getAttributeNode("caseListFile").nodeValue + pattern = "config-" + os.path.splitext(caseListFile)[0] + "-cfg-[0-9]*"+"-run-[0-9]*" + cmdLine = testConfig.getAttributeNode("commandLine").nodeValue + cfgItems = {'height':None, 'width':None, 'seed':None, 'rotation':None} + for arg in cmdLine.split(): + val = arg.split('=')[1] + if "deqp-surface-height" in arg: + cfgItems['height'] = val + elif "deqp-surface-width" in arg: + cfgItems['width'] = val + elif "deqp-base-seed" in arg: + cfgItems['seed'] = val + elif "deqp-screen-rotation" in arg: + cfgItems['rotation'] = val + pattern += "-width-" + cfgItems['width'] + "-height-" + cfgItems['height'] + if cfgItems['seed'] != None: + pattern += "-seed-" + cfgItems['seed'] + pattern += ".qpa" + p = re.compile(pattern) + matches = [m for l in mustpassCases[mustpass] for m in (p.match(l),) if m] + if len(matches) == 0: + conformOs = testConfig.getAttributeNode("os").nodeValue + txt = "Configuration %s %s was not executed" % (caseListFile, cmdLine) + if conformOs == "any" or (package.conformOs != None and conformOs in package.conformOs.lower()): + msg = error(mustpassXML, txt) + else: + msg = warning(mustpassXML, txt) + messages.append(msg) + + return messages + +def verifyTestLogs (package): + messages = [] + + try: + execute(['git', 'checkout', '--quiet', package.conformVersion]) + except Exception, e: + print str(e) + print "Failed to checkout release tag %s." % package.conformVersion + return messages + + messages = [] + summary = parseRunSummary(os.path.join(package.basePath, package.summary)) + mustpassDirs = [] + + # Check Conformant attribute + if not summary.isConformant: + messages.append(error(package.summary, "Runner reported conformance failure (Conformant=\"False\" in )")) + + # Verify config list + messages += verifyConfigFile(os.path.join(package.basePath, summary.configLogFilename), summary.type) + + mustpassCases = {} + # Verify that all run files passed + for runLog in summary.runLogAndCaselist: + sys.stdout.write("Verifying %s -" % runLog) + sys.stdout.flush() + + mustpassFile = os.path.join(ROOT_DIR, "external", "openglcts", summary.runLogAndCaselist[runLog]) + key = os.path.dirname(mustpassFile) + if key in mustpassCases: + mpCase = mustpassCases[key] + else: + mpCase = [] + mpCase.append(runLog) + mustpassCases[os.path.dirname(mustpassFile)] = mpCase + mustpass = readMustpass(mustpassFile) + messages_log = verifyTestLog(os.path.join(package.basePath, runLog), mustpass) + + errors = [m for m in messages_log if m.type == ValidationMessage.TYPE_ERROR] + warnings = [m for m in messages_log if m.type == ValidationMessage.TYPE_WARNING] + if len(errors) > 0: + sys.stdout.write(" finished with ERRRORS") + if len(warnings) > 0: + sys.stdout.write(" finished with WARNINGS") + if len(errors) == 0 and len(warnings) == 0: + sys.stdout.write(" OK") + sys.stdout.write("\n") + sys.stdout.flush() + + messages += messages_log + + messages += verifyMustpassCases(package, mustpassCases) + + return messages + +def verifyGitStatusFiles (package): + messages = [] + + if len(package.gitStatus) != 2: + messages.append(error(package.basePath, "Exactly two git status files must be present, found %s" % len(package.gitStatus))) + + messages += verifyGitStatus(package) + + return messages + +def verifyGitLogFiles (package): + messages = [] + + if len(package.gitLog) != 2: + messages.append(error(package.basePath, "Exactly two git log file must be present, found %s" % len(package.gitLog))) + + messages += verifyGitLog(package) + + return messages + +def verifyPackage (package): + messages = [] + + messages += verifyStatement(package) + messages += verifyGitStatusFiles(package) + messages += verifyGitLogFiles(package) + messages += verifyPatches(package) + + for item in package.otherItems: + messages.append(warning(os.path.join(package.basePath, item), "Unknown file")) + + return messages + +def verifyESSubmission(argv): + if len(argv) != 2: + print "%s: [extracted submission package directory]" % sys.argv[0] + sys.exit(-1) + try: + execute(['git', 'ls-remote', 'origin', '--quiet']) + except Exception, e: + print str(e) + print "This script must be executed inside VK-GL-CTS directory." + sys.exit(-1) + + packagePath = os.path.normpath(sys.argv[1]) + package = getPackageDescription(packagePath) + messages = verifyPackage(package) + messages += verifyTestLogs(package) + + errors = [m for m in messages if m.type == ValidationMessage.TYPE_ERROR] + warnings = [m for m in messages if m.type == ValidationMessage.TYPE_WARNING] + + for message in messages: + print str(message) + + print "" + + if len(errors) > 0: + print "Found %d validation errors and %d warnings!" % (len(errors), len(warnings)) + sys.exit(-2) + elif len(warnings) > 0: + print "Found %d warnings, manual review required" % len(warnings) + sys.exit(-1) + else: + print "All validation checks passed" + +if __name__ == "__main__": + verifyESSubmission(sys.argv) diff --git a/external/openglcts/scripts/verify/verify_gl.py b/external/openglcts/scripts/verify/verify_gl.py new file mode 100644 index 0000000..8148a96 --- /dev/null +++ b/external/openglcts/scripts/verify/verify_gl.py @@ -0,0 +1,112 @@ +# -*- coding: utf-8 -*- + +#------------------------------------------------------------------------- +# Khronos OpenGL CTS +# ------------------ +# +# Copyright (c) 2016 The Khronos Group Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +#------------------------------------------------------------------------- + +import os +import sys +import xml.dom.minidom + +sys.path.append(os.path.join(os.path.dirname(__file__), "..", "..", "..", "scripts", "log")) + +from log_parser import BatchResultParser, StatusCode + +from summary import * + +VALID_STATUS_CODES = set([ + StatusCode.PASS, + StatusCode.COMPATIBILITY_WARNING, + StatusCode.QUALITY_WARNING, + StatusCode.NOT_SUPPORTED + ]) + +def isStatusCodeOk (code): + return code in VALID_STATUS_CODES + +def getConfigCaseName (type): + return "CTS-Configs.%s" % type + +def verifyConfigFile (filename, type): + caseName = getConfigCaseName(type) + + print "Verifying %s in %s" % (caseName, filename) + + parser = BatchResultParser() + results = parser.parseFile(filename) + caseResult = None + + for result in results: + if result.name == caseName: + caseResult = result + break + + if caseResult == None: + print "FAIL: %s not found" % caseName + return False + + if not isStatusCodeOk(caseResult.statusCode): + print "FAIL: %s" % caseResult + return False + + return True + +def verifySubmission (dirname): + summary = parseRunSummary(os.path.join(dirname, "cts-run-summary.xml")) + allOk = True + + # Check Conformant attribute + if not summary.isConformant: + print "FAIL: Runner reported conformance failure (Conformant=\"False\" in )" + + # Verify config list + if not verifyConfigFile(os.path.join(dirname, summary.configLogFilename), summary.type): + allOk = False + + # Verify that all run files passed + for runFilename in summary.runLogFilenames: + print "Verifying %s" % runFilename + + logParser = BatchResultParser() + batchResult = logParser.parseFile(os.path.join(dirname, runFilename)) + + for result in batchResult: + if not isStatusCodeOk(result.statusCode): + print "FAIL: %s" % str(result) + allOk = False + + return allOk + +def verifyGLSubmission(argv): + if len(argv) != 2: + print "%s: [extracted submission package directory]" % argv[0] + sys.exit(-1) + + try: + isOk = verifySubmission(argv[1]) + print "Verification %s" % ("PASSED" if isOk else "FAILED!") + sys.exit(0 if isOk else 1) + except Exception, e: + print str(e) + print "Error occurred during verification" + sys.exit(-1) + +if __name__ == "__main__": + verifyGLSubmission(sys.argv) + diff --git a/external/openglcts/scripts/verify_submission.py b/external/openglcts/scripts/verify_submission.py index 2e8ec32..acb2989 100644 --- a/external/openglcts/scripts/verify_submission.py +++ b/external/openglcts/scripts/verify_submission.py @@ -22,109 +22,22 @@ import os import sys -import xml.dom.minidom -sys.path.append(os.path.join(os.path.dirname(__file__), "..", "..", "..", "scripts", "log")) +sys.path.append(os.path.join(os.path.dirname(__file__), "verify")) -from log_parser import BatchResultParser, StatusCode - -VALID_STATUS_CODES = set([ - StatusCode.PASS, - StatusCode.COMPATIBILITY_WARNING, - StatusCode.QUALITY_WARNING, - StatusCode.NOT_SUPPORTED - ]) - -def isStatusCodeOk (code): - return code in VALID_STATUS_CODES - -def getConfigCaseName (type): - return "CTS-Configs.%s" % type - -class TestRunSummary: - def __init__ (self, type, isConformant, configLogFilename, runLogFilenames): - self.type = type - self.isConformant = isConformant - self.configLogFilename = configLogFilename - self.runLogFilenames = runLogFilenames - -def parseRunSummary (filename): - doc = xml.dom.minidom.parse(filename) - summary = doc.documentElement - if summary.localName != "Summary": - raise Exception("Document element is not ") - - type = summary.getAttributeNode("Type").nodeValue - isConformant = summary.getAttributeNode("Conformant").nodeValue == "True" - - configRuns = doc.getElementsByTagName("Configs") - if len(configRuns) != 1: - raise Exception("Excepted one element, found %d" % len(configRuns)) - - runFiles = doc.getElementsByTagName("TestRun") - - return TestRunSummary(type, isConformant, configRuns[0].getAttributeNode("FileName").nodeValue, [n.getAttributeNode("FileName").nodeValue for n in runFiles]) - -def verifyConfigFile (filename, type): - caseName = getConfigCaseName(type) - - print "Verifying %s in %s" % (caseName, filename) - - parser = BatchResultParser() - results = parser.parseFile(filename) - caseResult = None - - for result in results: - if result.name == caseName: - caseResult = result - break - - if caseResult == None: - print "FAIL: %s not found" % caseName - return False - - if not isStatusCodeOk(caseResult.statusCode): - print "FAIL: %s" % caseResult - return False - - return True - -def verifySubmission (dirname): - summary = parseRunSummary(os.path.join(dirname, "cts-run-summary.xml")) - allOk = True - - # Check Conformant attribute - if not summary.isConformant: - print "FAIL: Runner reported conformance failure (Conformant=\"False\" in )" - - # Verify config list - if not verifyConfigFile(os.path.join(dirname, summary.configLogFilename), summary.type): - allOk = False - - # Verify that all run files passed - for runFilename in summary.runLogFilenames: - print "Verifying %s" % runFilename - - logParser = BatchResultParser() - batchResult = logParser.parseFile(os.path.join(dirname, runFilename)) - - for result in batchResult: - if not isStatusCodeOk(result.statusCode): - print "FAIL: %s" % str(result) - allOk = False - - return allOk +from summary import * +from verify_es import verifyESSubmission +from verify_gl import verifyGLSubmission if __name__ == "__main__": if len(sys.argv) != 2: print "%s: [directory]" % sys.argv[0] sys.exit(-1) - try: - isOk = verifySubmission(sys.argv[1]) - print "Verification %s" % ("PASSED" if isOk else "FAILED!") - sys.exit(0 if isOk else 1) - except Exception, e: - print str(e) - print "Error occurred during verification" - sys.exit(-1) + summary = parseRunSummary(os.path.join(sys.argv[1], "cts-run-summary.xml")) + if "es" in summary.type: + verifyESSubmission(sys.argv) + else: + assert "gl" in summary.type + verifyGLSubmission(sys.argv) + diff --git a/scripts/log/log_parser.py b/scripts/log/log_parser.py index 17e1311..dd90d2b 100644 --- a/scripts/log/log_parser.py +++ b/scripts/log/log_parser.py @@ -162,6 +162,9 @@ class BatchResultParser: def parseTestCaseResult (self, name, log): try: + # The XML parser has troubles with invalid characters deliberately included in the shaders. + # This line removes such characters before calling the parser + log = log.decode('utf-8','ignore').encode("utf-8") doc = xml.dom.minidom.parseString(log) resultItems = doc.getElementsByTagName('Result') if len(resultItems) != 1: diff --git a/scripts/verify/package.py b/scripts/verify/package.py index e178b48..d73c17f 100644 --- a/scripts/verify/package.py +++ b/scripts/verify/package.py @@ -28,17 +28,20 @@ TEST_LOG_PATTERN = "*.qpa" GIT_STATUS_PATTERN = "*git-status.txt" GIT_LOG_PATTERN = "*git-log.txt" PATCH_PATTERN = "*.patch" +SUMMARY_PATTERN = "cts-run-summary.xml" class PackageDescription: - def __init__ (self, basePath, statement, testLogs, gitStatus, gitLog, patches, conformVersion, otherItems): + def __init__ (self, basePath, statement, testLogs, gitStatus, gitLog, patches, summary, conformVersion, conformOs, otherItems): self.basePath = basePath self.statement = statement self.testLogs = testLogs self.gitStatus = gitStatus self.gitLog = gitLog self.patches = patches + self.summary = summary self.otherItems = otherItems self.conformVersion = conformVersion + self.conformOs = conformOs def getPackageDescription (packagePath): allItems = os.listdir(packagePath) @@ -47,8 +50,10 @@ def getPackageDescription (packagePath): gitStatus = [] gitLog = [] patches = [] + summary = None otherItems = [] conformVersion = None + conformOs = None for item in allItems: if fnmatch(item, STATEMENT_PATTERN): @@ -62,7 +67,10 @@ def getPackageDescription (packagePath): gitLog.append(item) elif fnmatch(item, PATCH_PATTERN): patches.append(item) + elif fnmatch(item, SUMMARY_PATTERN): + assert summary == None + summary = item else: otherItems.append(item) - return PackageDescription(packagePath, statement, testLogs, gitStatus, gitLog, patches, conformVersion, otherItems) + return PackageDescription(packagePath, statement, testLogs, gitStatus, gitLog, patches, summary, conformVersion, conformOs, otherItems) diff --git a/scripts/verify/verify.py b/scripts/verify/verify.py index 663bba5..e23f7f0 100644 --- a/scripts/verify/verify.py +++ b/scripts/verify/verify.py @@ -110,6 +110,8 @@ def verifyStatement (package): if hasVersion: messages.append(error(statementPath, "Multiple CONFORM_VERSIONs")) else: + assert len(line.split()) >= 2 + package.conformVersion = line.split()[1] hasVersion = True elif beginsWith(line, "PRODUCT:"): hasProduct = True # Multiple products allowed @@ -122,6 +124,8 @@ def verifyStatement (package): if hasOs: messages.append(error(statementPath, "Multiple OSes")) else: + assert len(line.split()) >= 2 + package.conformOs = line.split()[1] hasOs = True if not hasVersion: -- 2.7.4