From 0686cb1047466f033dc66657fd2dfd3749884e5f Mon Sep 17 00:00:00 2001
From: Ricardo Garcia <rgarcia@igalia.com>
Date: Mon, 25 Jan 2021 16:02:01 +0100
Subject: [PATCH] Unicode and Python3 fixes for verification scripts

Fix several issues related to running verification scripts under
different versions of Python, making sure the scripts are compatible
with both Python 2 and Python 3 and do not choke on invalid characters,
which are sometimes present in test shaders.

Components: OpenGL, Framework
VK-GL-CTS issue: 2756

Change-Id: Id472b842cb8fd6536e61489f3c01e28c462dbebd
---
 external/openglcts/scripts/verify/verify_es.py |  8 +++++++-
 scripts/log/log_parser.py                      | 14 +++++++++++---
 2 files changed, 18 insertions(+), 4 deletions(-)

diff --git a/external/openglcts/scripts/verify/verify_es.py b/external/openglcts/scripts/verify/verify_es.py
index c14b8ab..6cc363f 100644
--- a/external/openglcts/scripts/verify/verify_es.py
+++ b/external/openglcts/scripts/verify/verify_es.py
@@ -61,6 +61,9 @@ def retrieveReportedConfigs(caseName, log):
 	res = {caseName : configs}
 	return res
 
+def cmpElements(a, b):
+	return ((a > b) - (a < b))
+
 def compareConfigs(filename, baseConfigs, cmpConfigs):
 	messages = []
 	assert len(list(baseConfigs.keys())) == 1
@@ -68,7 +71,7 @@ def compareConfigs(filename, baseConfigs, cmpConfigs):
 	baseKey = list(baseConfigs.keys())[0]
 	cmpKey = list(cmpConfigs.keys())[0]
 
-	if cmp(baseConfigs[baseKey], cmpConfigs[cmpKey]) != 0:
+	if cmpElements(baseConfigs[baseKey], cmpConfigs[cmpKey]) != 0:
 		messages.append(error(filename, "Confomant configs reported for %s and %s do not match" % (baseKey,cmpKey)))
 
 	return messages
@@ -242,6 +245,9 @@ def isGitLogFileEmpty (package, modulePath, gitLog):
 	if process.returncode != 0:
 		raise Exception("Failed to execute '%s', got %d" % (str(args), process.returncode))
 
+	# Python 3 returns process output as bytes, while reading the log file gives a string.
+	if sys.version_info > (3, 0):
+		output = output.decode('utf-8', 'ignore')
 	return log == output
 
 def verifyGitLogFile (package):
diff --git a/scripts/log/log_parser.py b/scripts/log/log_parser.py
index b77af3f..c25c235 100644
--- a/scripts/log/log_parser.py
+++ b/scripts/log/log_parser.py
@@ -21,6 +21,7 @@
 #-------------------------------------------------------------------------
 
 import shlex
+import sys
 import xml.dom.minidom
 
 class StatusCode:
@@ -73,7 +74,12 @@ class ParseError(Exception):
 		return "%s:%d: %s" % (self.filename, self.line, self.message)
 
 def splitContainerLine (line):
-	return shlex.split(line)
+	if sys.version_info > (3, 0):
+		# In Python 3, shlex works better with unicode.
+		return shlex.split(line)
+	else:
+		# In Python 2, shlex works better with bytes, so encode and decode again upon return.
+		return [w.decode('utf-8') for w in shlex.split(line.encode('utf-8'))]
 
 def getNodeText (node):
 	rc = []
@@ -131,7 +137,8 @@ class BatchResultParser:
 		self.filename			= filename
 
 	def parseLine (self, line):
-		text = line.decode('utf-8')
+		# Some test shaders contain invalid characters.
+		text = line.decode('utf-8', 'ignore')
 		if len(text) > 0 and text[0] == '#':
 			return self.parseContainerLine(line)
 		elif self.curResultText != None:
@@ -141,7 +148,8 @@ class BatchResultParser:
 
 	def parseContainerLine (self, line):
 		isTestCaseResult = False
-		text = line.decode('utf-8')
+		# Some test shaders contain invalid characters.
+		text = line.decode('utf-8', 'ignore')
 		args = splitContainerLine(text)
 		if args[0] == "#sessionInfo":
 			if len(args) < 3:
-- 
2.7.4