Unicode and Python3 fixes for verification scripts

Fix several issues related to running verification scripts under
different versions of Python, making sure the scripts are compatible
with both Python 2 and Python 3 and do not choke on invalid characters,
which are sometimes present in test shaders.

Components: OpenGL, Framework
VK-GL-CTS issue: 2756

Change-Id: Id472b842cb8fd6536e61489f3c01e28c462dbebd

diff --git a/external/openglcts/scripts/verify/verify_es.py b/external/openglcts/scripts/verify/verify_es.py
index c14b8ab..6cc363f 100644 (file)
--- a/external/openglcts/scripts/verify/verify_es.py
+++ b/external/openglcts/scripts/verify/verify_es.py
@@ -61,6 +61,9 @@ def retrieveReportedConfigs(caseName, log):
        res = {caseName : configs}
        return res
 
+def cmpElements(a, b):
+       return ((a > b) - (a < b))
+
 def compareConfigs(filename, baseConfigs, cmpConfigs):
        messages = []
        assert len(list(baseConfigs.keys())) == 1
@@ -68,7 +71,7 @@ def compareConfigs(filename, baseConfigs, cmpConfigs):
        baseKey = list(baseConfigs.keys())[0]
        cmpKey = list(cmpConfigs.keys())[0]
 
-       if cmp(baseConfigs[baseKey], cmpConfigs[cmpKey]) != 0:
+       if cmpElements(baseConfigs[baseKey], cmpConfigs[cmpKey]) != 0:
                messages.append(error(filename, "Confomant configs reported for %s and %s do not match" % (baseKey,cmpKey)))
 
        return messages
@@ -242,6 +245,9 @@ def isGitLogFileEmpty (package, modulePath, gitLog):
        if process.returncode != 0:
                raise Exception("Failed to execute '%s', got %d" % (str(args), process.returncode))
 
+       # Python 3 returns process output as bytes, while reading the log file gives a string.
+       if sys.version_info > (3, 0):
+               output = output.decode('utf-8', 'ignore')
        return log == output
 
 def verifyGitLogFile (package):

diff --git a/scripts/log/log_parser.py b/scripts/log/log_parser.py
index b77af3f..c25c235 100644 (file)
--- a/scripts/log/log_parser.py
+++ b/scripts/log/log_parser.py
@@ -21,6 +21,7 @@
 #-------------------------------------------------------------------------
 
 import shlex
+import sys
 import xml.dom.minidom
 
 class StatusCode:
@@ -73,7 +74,12 @@ class ParseError(Exception):
                return "%s:%d: %s" % (self.filename, self.line, self.message)
 
 def splitContainerLine (line):
-       return shlex.split(line)
+       if sys.version_info > (3, 0):
+               # In Python 3, shlex works better with unicode.
+               return shlex.split(line)
+       else:
+               # In Python 2, shlex works better with bytes, so encode and decode again upon return.
+               return [w.decode('utf-8') for w in shlex.split(line.encode('utf-8'))]
 
 def getNodeText (node):
        rc = []
@@ -131,7 +137,8 @@ class BatchResultParser:
                self.filename                   = filename
 
        def parseLine (self, line):
-               text = line.decode('utf-8')
+               # Some test shaders contain invalid characters.
+               text = line.decode('utf-8', 'ignore')
                if len(text) > 0 and text[0] == '#':
                        return self.parseContainerLine(line)
                elif self.curResultText != None:
@@ -141,7 +148,8 @@ class BatchResultParser:
 
        def parseContainerLine (self, line):
                isTestCaseResult = False
-               text = line.decode('utf-8')
+               # Some test shaders contain invalid characters.
+               text = line.decode('utf-8', 'ignore')
                args = splitContainerLine(text)
                if args[0] == "#sessionInfo":
                        if len(args) < 3: