From: Alexander Galazin Date: Fri, 19 Mar 2021 11:12:55 +0000 (+0300) Subject: Remove in-repo verify_submission scripts X-Git-Tag: upstream/1.3.5~436^2~9^2~5^2 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=07bc5346e2dddd4656f4ae9435fc5136a62620f7;p=platform%2Fupstream%2FVK-GL-CTS.git Remove in-repo verify_submission scripts These scripts are now replaced by the CTS Tools repo: https://github.com/KhronosGroup/VK-GL-CTS-Tools Components: Vulkan, OpenGL Change-Id: If0f47b4186ed10f9946852e8d37956d7328625a8 (cherry picked from commit bdfb5d67b268ab2dba5178618cc1515f68087be3) --- diff --git a/external/openglcts/scripts/verify/summary.py b/external/openglcts/scripts/verify/summary.py deleted file mode 100644 index 377b503..0000000 --- a/external/openglcts/scripts/verify/summary.py +++ /dev/null @@ -1,61 +0,0 @@ -# -*- coding: utf-8 -*- - -#------------------------------------------------------------------------- -# Khronos OpenGL CTS -# ------------------ -# -# Copyright (c) 2016 The Khronos Group Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -#------------------------------------------------------------------------- - -import xml.dom.minidom - -class TestRunSummary: - def __init__ (self, type, isConformant, configLogFilename, runLogFilenames, runLogAndCaselist): - self.type = type - self.isConformant = isConformant - self.configLogFilename = configLogFilename - self.runLogFilenames = runLogFilenames - self.runLogAndCaselist = runLogAndCaselist - -def parseRunSummary (filename): - doc = xml.dom.minidom.parse(filename) - summary = doc.documentElement - if summary.localName != "Summary": - raise Exception("Document element is not ") - - type = summary.getAttributeNode("Type").nodeValue - isConformant = summary.getAttributeNode("Conformant").nodeValue == "True" - - configRuns = doc.getElementsByTagName("Configs") - if len(configRuns) != 1: - raise Exception("Excepted one element, found %d" % len(configRuns)) - - runLogFilenames = [] - runLogAndCaselist = {} - runFiles = doc.getElementsByTagName("TestRun") - for n in runFiles: - runLog = n.getAttributeNode("FileName").nodeValue - runLogFilenames.append(runLog) - cmdLine = n.getAttributeNode("CmdLine").nodeValue - caseList = None - for words in cmdLine.split(): - if "deqp-caselist" in words: - caseList = words.split("=")[1] - caseList = caseList[len("gl_cts/"):] - runLogAndCaselist[runLog] = caseList - - return TestRunSummary(type, isConformant, configRuns[0].getAttributeNode("FileName").nodeValue, runLogFilenames, runLogAndCaselist) - diff --git a/external/openglcts/scripts/verify/verify_es.py b/external/openglcts/scripts/verify/verify_es.py deleted file mode 100644 index 559718a..0000000 --- a/external/openglcts/scripts/verify/verify_es.py +++ /dev/null @@ -1,343 +0,0 @@ -# -*- coding: utf-8 -*- - -#------------------------------------------------------------------------- -# Khronos OpenGL CTS -# ------------------ -# -# Copyright (c) 2017 The Khronos Group Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -#------------------------------------------------------------------------- - -import os -import sys -import xml.dom.minidom -import re -import subprocess - -ROOT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..", "..")) -sys.path.append(os.path.join(ROOT_DIR, "scripts", "verify")) -sys.path.append(os.path.join(ROOT_DIR, "scripts", "build")) -sys.path.append(os.path.join(ROOT_DIR, "scripts", "log")) - -from package import getPackageDescription -from verify import * -from message import * -from common import * -from log_parser import * -from summary import * - -def getConfigCaseName (type): - configs = { "es32" : ["CTS-Configs.es32", "CTS-Configs.es31", "CTS-Configs.es3", "CTS-Configs.es2"], - "es31" : ["CTS-Configs.es31", "CTS-Configs.es3", "CTS-Configs.es2"], - "es3" : ["CTS-Configs.es3", "CTS-Configs.es2"], - "es2" : ["CTS-Configs.es2"]} - return configs[type] - -def retrieveReportedConfigs(caseName, log): - doc = xml.dom.minidom.parseString(log) - sectionItems = doc.getElementsByTagName('Section') - sectionName = None - - configs = [] - for sectionItem in sectionItems: - sectionName = sectionItem.getAttributeNode('Name').nodeValue - if sectionName == "Configs": - assert len(configs) == 0 - textItems = sectionItem.getElementsByTagName('Text') - for textItem in textItems: - configs.append(getNodeText(textItem)) - res = {caseName : configs} - return res - -def compareConfigs(filename, baseConfigs, cmpConfigs): - messages = [] - assert len(list(baseConfigs.keys())) == 1 - assert len(list(cmpConfigs.keys())) == 1 - baseKey = list(baseConfigs.keys())[0] - cmpKey = list(cmpConfigs.keys())[0] - - if cmp(baseConfigs[baseKey], cmpConfigs[cmpKey]) != 0: - messages.append(error(filename, "Confomant configs reported for %s and %s do not match" % (baseKey,cmpKey))) - - return messages - -def numGitLogStatusFile (releaseTag): - KC_CTS_RELEASE = ["opengl-es-cts-3\.2\.[2-3]\.[0-9]*", "opengl-cts-4\.6\.[0-9]*\.[0-9]*"] - for r in KC_CTS_RELEASE: - if re.match(r, releaseTag): - return 2 - - return 1 - -def verifyConfigFile (filename, type): - messages = [] - caseNames = getConfigCaseName(type) - - parser = BatchResultParser() - results = parser.parseFile(filename) - baseConfigs = None - - for caseName in caseNames: - caseResult = None - print "Verifying %s in %s" % (caseName, filename) - for result in results: - if result.name == caseName: - caseResult = result - break; - if caseResult == None: - messages.append(error(filename, "Missing %s" % caseName)) - else: - configs = retrieveReportedConfigs(caseName, result.log) - if baseConfigs == None: - baseConfigs = configs - else: - messages += compareConfigs(filename, baseConfigs, configs) - if not caseResult.statusCode in ALLOWED_STATUS_CODES: - messages.append(error(filename, "%s failed" % caseResult)) - - return messages - -def verifyMustpassCases(package, mustpassCases, type): - messages = [] - apiToTest = { "es32" : ["gles32", "gles31", "gles3", "gles2", "egl"], - "es31" : ["gles31", "gles3", "gles2", "egl"], - "es3" : ["gles3", "gles2", "egl"], - "es2" : ["gles2", "egl"]} - - for mustpass in mustpassCases: - mustpassXML = os.path.join(mustpass, "mustpass.xml") - doc = xml.dom.minidom.parse(mustpassXML) - testConfigs = doc.getElementsByTagName("Configuration") - # check that all configs that must be tested are present - for testConfig in testConfigs: - caseListFile = testConfig.getAttributeNode("caseListFile").nodeValue - # identify APIs that must be tested for the given type - apis = apiToTest[type] - # identify API tested by the current config - configAPI = caseListFile.split('-')[0] - if configAPI in apis: - # the API in this config is expected to be tested - mustTest = True - else: - mustTest = False - pattern = "config-" + os.path.splitext(caseListFile)[0] + "-cfg-[0-9]*"+"-run-[0-9]*" - cmdLine = testConfig.getAttributeNode("commandLine").nodeValue - cfgItems = {'height':None, 'width':None, 'seed':None, 'rotation':None} - for arg in cmdLine.split(): - val = arg.split('=')[1] - if "deqp-surface-height" in arg: - cfgItems['height'] = val - elif "deqp-surface-width" in arg: - cfgItems['width'] = val - elif "deqp-base-seed" in arg: - cfgItems['seed'] = val - elif "deqp-screen-rotation" in arg: - cfgItems['rotation'] = val - pattern += "-width-" + cfgItems['width'] + "-height-" + cfgItems['height'] - if cfgItems['seed'] != None: - pattern += "-seed-" + cfgItems['seed'] - pattern += ".qpa" - p = re.compile(pattern) - matches = [m for l in mustpassCases[mustpass] for m in (p.match(l),) if m] - if len(matches) == 0 and mustTest == True: - conformOs = testConfig.getAttributeNode("os").nodeValue - txt = "Configuration %s %s was not executed" % (caseListFile, cmdLine) - if conformOs == "any" or (package.conformOs != None and conformOs in package.conformOs.lower()): - msg = error(mustpassXML, txt) - else: - msg = warning(mustpassXML, txt) - messages.append(msg) - elif len(matches) != 0 and mustTest == False: - messages.append(error(mustpassXML, "Configuration %s %s was not expected to be tested but present in cts-run-summary.xml" % (caseListFile, cmdLine))) - - return messages - -def verifyTestLogs (package): - messages = [] - - try: - execute(['git', 'checkout', '--quiet', package.conformVersion]) - except Exception, e: - print str(e) - print "Failed to checkout release tag %s." % package.conformVersion - return messages - - messages = [] - summary = parseRunSummary(os.path.join(package.basePath, package.summary)) - mustpassDirs = [] - - # Check Conformant attribute - if not summary.isConformant: - messages.append(error(package.summary, "Runner reported conformance failure (Conformant=\"False\" in )")) - - # Verify config list - messages += verifyConfigFile(os.path.join(package.basePath, summary.configLogFilename), summary.type) - - mustpassCases = {} - # Verify that all run files passed - for runLog in summary.runLogAndCaselist: - sys.stdout.write("Verifying %s -" % runLog) - sys.stdout.flush() - - mustpassFile = os.path.join(ROOT_DIR, "external", "openglcts", summary.runLogAndCaselist[runLog]) - key = os.path.dirname(mustpassFile) - if key in mustpassCases: - mpCase = mustpassCases[key] - else: - mpCase = [] - mpCase.append(runLog) - mustpassCases[os.path.dirname(mustpassFile)] = mpCase - mustpass = readMustpass(mustpassFile) - messages_log = verifyTestLog(os.path.join(package.basePath, runLog), mustpass) - - errors = [m for m in messages_log if m.type == ValidationMessage.TYPE_ERROR] - warnings = [m for m in messages_log if m.type == ValidationMessage.TYPE_WARNING] - if len(errors) > 0: - sys.stdout.write(" finished with ERRRORS") - if len(warnings) > 0: - sys.stdout.write(" finished with WARNINGS") - if len(errors) == 0 and len(warnings) == 0: - sys.stdout.write(" OK") - sys.stdout.write("\n") - sys.stdout.flush() - - messages += messages_log - - messages += verifyMustpassCases(package, mustpassCases, summary.type) - - return messages - -def verifyGitStatusFiles (package): - messages = [] - - errorDict = {1 : 'one git status file', 2 : 'two git status files'} - numFiles = numGitLogStatusFile(package.conformVersion) - - if len(package.gitStatus) != numFiles: - messages.append(error(package.basePath, "Exactly %s must be present, found %s" % (errorDict[numFiles], len(package.gitStatus)))) - - messages += verifyGitStatus(package) - - return messages - -def isGitLogFileEmpty (package, modulePath, gitLog): - logPath = os.path.join(package.basePath, gitLog) - log = readFile(logPath) - - args = ['git', 'log', '-1', package.conformVersion] - process = subprocess.Popen(args, cwd=modulePath, stdout=subprocess.PIPE) - output = process.communicate()[0] - if process.returncode != 0: - raise Exception("Failed to execute '%s', got %d" % (str(args), process.returncode)) - - return log == output - -def verifyGitLogFile (package): - messages = [] - - if len(package.gitLog) > 0: - for log, path in package.gitLog: - try: - isEmpty = isGitLogFileEmpty(package, path, log) - except Exception, e: - print str(e) - isEmpty = False - - if not isEmpty: - messages.append(warning(os.path.join(package.basePath, log), "Log is not empty")) - else: - messages.append(error(package.basePath, "Missing git log files")) - - return messages - -def verifyPatchFiles (package): - messages = [] - hasPatches = len(package.patches) - logEmpty = True - for log, path in package.gitLog: - logEmpty &= isGitLogFileEmpty(package, path, log) - - if hasPatches and logEmpty: - messages.append(error(package.basePath, "Package includes patches but log is empty")) - elif not hasPatches and not logEmpty: - messages.append(error(package.basePath, "Test log is not empty but package doesn't contain patches")) - - return messages - -def verifyGitLogFiles (package): - messages = [] - - errorDict = {1 : 'one git log file', 2 : 'two git log files'} - numFiles = numGitLogStatusFile(package.conformVersion) - - if len(package.gitLog) != numFiles: - messages.append(error(package.basePath, "Exactly %s must be present, found %s" % (errorDict[numFiles], len(package.gitLog)))) - - for i, gitLog in enumerate(package.gitLog): - if "kc-cts" in gitLog[0] and numFiles > 1: - package.gitLog[i] = gitLog[:1] + ("external/kc-cts/src",) + gitLog[2:] - - messages += verifyGitLogFile(package) - - return messages - -def verifyPackage (package): - messages = [] - - messages += verifyStatement(package) - messages += verifyGitStatusFiles(package) - messages += verifyGitLogFiles(package) - messages += verifyPatchFiles(package) - - for item in package.otherItems: - messages.append(warning(os.path.join(package.basePath, item), "Unknown file")) - - return messages - -def verifyESSubmission(argv): - if len(argv) != 2: - print "%s: [extracted submission package directory]" % sys.argv[0] - sys.exit(-1) - try: - execute(['git', 'ls-remote', 'origin', '--quiet']) - except Exception, e: - print str(e) - print "This script must be executed inside VK-GL-CTS directory." - sys.exit(-1) - - packagePath = os.path.normpath(sys.argv[1]) - package = getPackageDescription(packagePath) - messages = verifyPackage(package) - messages += verifyTestLogs(package) - - errors = [m for m in messages if m.type == ValidationMessage.TYPE_ERROR] - warnings = [m for m in messages if m.type == ValidationMessage.TYPE_WARNING] - - for message in messages: - print str(message) - - print "" - - if len(errors) > 0: - print "Found %d validation errors and %d warnings!" % (len(errors), len(warnings)) - sys.exit(-2) - elif len(warnings) > 0: - print "Found %d warnings, manual review required" % len(warnings) - sys.exit(-1) - else: - print "All validation checks passed" - -if __name__ == "__main__": - verifyESSubmission(sys.argv) diff --git a/external/openglcts/scripts/verify/verify_gl.py b/external/openglcts/scripts/verify/verify_gl.py deleted file mode 100644 index 8148a96..0000000 --- a/external/openglcts/scripts/verify/verify_gl.py +++ /dev/null @@ -1,112 +0,0 @@ -# -*- coding: utf-8 -*- - -#------------------------------------------------------------------------- -# Khronos OpenGL CTS -# ------------------ -# -# Copyright (c) 2016 The Khronos Group Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -#------------------------------------------------------------------------- - -import os -import sys -import xml.dom.minidom - -sys.path.append(os.path.join(os.path.dirname(__file__), "..", "..", "..", "scripts", "log")) - -from log_parser import BatchResultParser, StatusCode - -from summary import * - -VALID_STATUS_CODES = set([ - StatusCode.PASS, - StatusCode.COMPATIBILITY_WARNING, - StatusCode.QUALITY_WARNING, - StatusCode.NOT_SUPPORTED - ]) - -def isStatusCodeOk (code): - return code in VALID_STATUS_CODES - -def getConfigCaseName (type): - return "CTS-Configs.%s" % type - -def verifyConfigFile (filename, type): - caseName = getConfigCaseName(type) - - print "Verifying %s in %s" % (caseName, filename) - - parser = BatchResultParser() - results = parser.parseFile(filename) - caseResult = None - - for result in results: - if result.name == caseName: - caseResult = result - break - - if caseResult == None: - print "FAIL: %s not found" % caseName - return False - - if not isStatusCodeOk(caseResult.statusCode): - print "FAIL: %s" % caseResult - return False - - return True - -def verifySubmission (dirname): - summary = parseRunSummary(os.path.join(dirname, "cts-run-summary.xml")) - allOk = True - - # Check Conformant attribute - if not summary.isConformant: - print "FAIL: Runner reported conformance failure (Conformant=\"False\" in )" - - # Verify config list - if not verifyConfigFile(os.path.join(dirname, summary.configLogFilename), summary.type): - allOk = False - - # Verify that all run files passed - for runFilename in summary.runLogFilenames: - print "Verifying %s" % runFilename - - logParser = BatchResultParser() - batchResult = logParser.parseFile(os.path.join(dirname, runFilename)) - - for result in batchResult: - if not isStatusCodeOk(result.statusCode): - print "FAIL: %s" % str(result) - allOk = False - - return allOk - -def verifyGLSubmission(argv): - if len(argv) != 2: - print "%s: [extracted submission package directory]" % argv[0] - sys.exit(-1) - - try: - isOk = verifySubmission(argv[1]) - print "Verification %s" % ("PASSED" if isOk else "FAILED!") - sys.exit(0 if isOk else 1) - except Exception, e: - print str(e) - print "Error occurred during verification" - sys.exit(-1) - -if __name__ == "__main__": - verifyGLSubmission(sys.argv) - diff --git a/external/openglcts/scripts/verify_submission.py b/external/openglcts/scripts/verify_submission.py deleted file mode 100644 index acb2989..0000000 --- a/external/openglcts/scripts/verify_submission.py +++ /dev/null @@ -1,43 +0,0 @@ -# -*- coding: utf-8 -*- - -#------------------------------------------------------------------------- -# Khronos OpenGL CTS -# ------------------ -# -# Copyright (c) 2016 The Khronos Group Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -#------------------------------------------------------------------------- - -import os -import sys - -sys.path.append(os.path.join(os.path.dirname(__file__), "verify")) - -from summary import * -from verify_es import verifyESSubmission -from verify_gl import verifyGLSubmission - -if __name__ == "__main__": - if len(sys.argv) != 2: - print "%s: [directory]" % sys.argv[0] - sys.exit(-1) - - summary = parseRunSummary(os.path.join(sys.argv[1], "cts-run-summary.xml")) - if "es" in summary.type: - verifyESSubmission(sys.argv) - else: - assert "gl" in summary.type - verifyGLSubmission(sys.argv) - diff --git a/external/vulkancts/scripts/verify_submission.py b/external/vulkancts/scripts/verify_submission.py deleted file mode 100644 index 4a73047..0000000 --- a/external/vulkancts/scripts/verify_submission.py +++ /dev/null @@ -1,103 +0,0 @@ -# -*- coding: utf-8 -*- - -#------------------------------------------------------------------------- -# Vulkan CTS -# ---------- -# -# Copyright (c) 2016 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -#------------------------------------------------------------------------- - -import os -import sys - -sys.path.append(os.path.join(os.path.dirname(__file__), "..", "..", "..", "scripts", "verify")) - -from package import getPackageDescription -from verify import * -from message import * - -def verifyGitStatusFiles (package): - messages = [] - - if len(package.gitStatus) > 1: - messages.append(error(package.basePath, "Exactly one git status file must be present, found %s" % len(package.gitStatus))) - - messages += verifyGitStatus(package) - - return messages - -def verifyGitLogFiles (package): - messages = [] - - if len(package.gitLog) > 1: - messages.append(error(package.basePath, "Exactly one git log file must be present, found %s" % len(package.gitLog))) - - messages += verifyGitLog(package) - - return messages - -def verifyTestLogs (package, mustpass): - messages = [] - - for testLogFile in package.testLogs: - messages += verifyTestLog(os.path.join(package.basePath, testLogFile), mustpass) - - if len(package.testLogs) == 0: - messages.append(error(package.basePath, "No test log files found")) - - return messages - -def verifyPackage (package, mustpass): - messages = [] - - messages += verifyStatement(package) - messages += verifyGitStatusFiles(package) - messages += verifyGitLogFiles(package) - messages += verifyPatches(package) - messages += verifyTestLogs(package, mustpass) - - for item in package.otherItems: - messages.append(warning(os.path.join(package.basePath, item), "Unknown file")) - - return messages - -if __name__ == "__main__": - if len(sys.argv) != 3: - print "%s: [extracted submission package] [mustpass]" % sys.argv[0] - sys.exit(-1) - - packagePath = os.path.normpath(sys.argv[1]) - mustpassPath = sys.argv[2] - package = getPackageDescription(packagePath) - mustpass = readMustpass(mustpassPath) - messages = verifyPackage(package, mustpass) - - errors = [m for m in messages if m.type == ValidationMessage.TYPE_ERROR] - warnings = [m for m in messages if m.type == ValidationMessage.TYPE_WARNING] - - for message in messages: - print str(message) - - print "" - - if len(errors) > 0: - print "Found %d validation errors and %d warnings!" % (len(errors), len(warnings)) - sys.exit(-2) - elif len(warnings) > 0: - print "Found %d warnings, manual review required" % len(warnings) - sys.exit(-1) - else: - print "All validation checks passed"