Reduce memory consumption in log parsing scripts

Updated .qpa to .xml conversion scripts to reduce memory consumption.

Before, the script was reading the whole .qpa file and storing it all in
memory. On top of that, it created xml.dom.minidom objects for each
testcase in the .qpa which were also stored in memory.

With this change, we create the .xml file at the same time that we parse
the .qpa which results in only one xml.dom.minidom object being created at
a time.

Components: Scripts

VK-GL-CTS issue: 1813

Change-Id: Iba4cb3bf6a94f57c481eb98a324b28638924075e

diff --git a/scripts/log/log_parser.py b/scripts/log/log_parser.py
index b0fd64d..2b3eb7b 100644 (file)
--- a/scripts/log/log_parser.py
+++ b/scripts/log/log_parser.py
@@ -97,6 +97,26 @@ class BatchResultParser:
 
                return self.testCaseResults
 
+       def getNextTestCaseResult (self, file):
+               try:
+                       del self.testCaseResults[:]
+                       self.curResultText = None
+
+                       isNextResult = self.parseLine(file.next())
+                       while not isNextResult:
+                               isNextResult = self.parseLine(file.next())
+
+                       # Return the next TestCaseResult
+                       return self.testCaseResults.pop()
+
+               except StopIteration:
+                       # If end of file was reached and there is no log left, the parsing finished successful (return None).
+                       # Otherwise, if there is still log to be parsed, it means that there was a crash.
+                       if self.curResultText:
+                               return TestCaseResult(self.curCaseName, StatusCode.CRASH, StatusCode.CRASH, self.curResultText)
+                       else:
+                               return None
+
        def init (self, filename):
                # Results
                self.sessionInfo                = []
@@ -112,12 +132,14 @@ class BatchResultParser:
 
        def parseLine (self, line):
                if len(line) > 0 and line[0] == '#':
-                       self.parseContainerLine(line)
+                       return self.parseContainerLine(line)
                elif self.curResultText != None:
                        self.curResultText += line
+                       return None
                # else: just ignored
 
        def parseContainerLine (self, line):
+               isTestCaseResult = False
                args = splitContainerLine(line)
                if args[0] == "#sessionInfo":
                        if len(args) < 3:
@@ -137,6 +159,7 @@ class BatchResultParser:
                        self.parseTestCaseResult(self.curCaseName, self.curResultText)
                        self.curCaseName        = None
                        self.curResultText      = None
+                       isTestCaseResult        = True
                elif args[0] == "#terminateTestCaseResult":
                        if len(args) < 2 or self.curCaseName == None:
                                self.parseError("Invalid #terminateTestCaseResult")
@@ -155,11 +178,14 @@ class BatchResultParser:
 
                        self.curCaseName        = None
                        self.curResultText      = None
+                       isTestCaseResult        = True
                else:
                        # Assume this is result text
                        if self.curResultText != None:
                                self.curResultText += line
 
+               return isTestCaseResult
+
        def parseTestCaseResult (self, name, log):
                try:
                        # The XML parser has troubles with invalid characters deliberately included in the shaders.

diff --git a/scripts/log/log_to_xml.py b/scripts/log/log_to_xml.py
index 1038953..281bd0c 100644 (file)
--- a/scripts/log/log_to_xml.py
+++ b/scripts/log/log_to_xml.py
@@ -137,49 +137,60 @@ def normalizeToXml (result, doc):
 
        return rootNodes
 
-def logToXml (inFile, outFile):
-       parser  = BatchResultParser()
-       results = parser.parseFile(inFile)
-
-       dstDoc                  = xml.dom.minidom.Document()
+def logToXml (logFilePath, outFilePath):
+       # Initialize Xml Document
+       dstDoc = xml.dom.minidom.Document()
        batchResultNode = dstDoc.createElement('BatchResult')
-       batchResultNode.setAttribute("FileName", os.path.basename(inFile))
-
+       batchResultNode.setAttribute("FileName", os.path.basename(logFilePath))
        dstDoc.appendChild(batchResultNode)
 
-       for result in results:
-               # Normalize log to valid XML
-               rootNodes = normalizeToXml(result, dstDoc)
-               for node in rootNodes:
-                       batchResultNode.appendChild(node)
-
-       # Summary
+       # Initialize dictionary for counting status codes
        countByStatusCode = {}
        for code in StatusCode.STATUS_CODES:
                countByStatusCode[code] = 0
 
-       for result in results:
-               countByStatusCode[result.statusCode] += 1
+       # Write custom headers
+       out = codecs.open(outFilePath, "wb", encoding="utf-8")
+       out.write("<?xml version=\"1.0\"?>\n")
+       out.write("<?xml-stylesheet href=\"%s\" type=\"text/xsl\"?>\n" % STYLESHEET_FILENAME)
 
        summaryElem = dstDoc.createElement('ResultTotals')
-       for code in StatusCode.STATUS_CODES:
-               summaryElem.setAttribute(code, "%d" % countByStatusCode[code])
-       summaryElem.setAttribute('All', "%d" % len(results))
        batchResultNode.appendChild(summaryElem)
 
-       text = dstDoc.toprettyxml()
+       # Print the first line manually <BatchResult FileName=something.xml>
+       out.write(dstDoc.toprettyxml().splitlines()[1])
+       out.write("\n")
 
-       out = codecs.open(outFile, "wb", encoding="utf-8")
+       parser = BatchResultParser()
+       parser.init(logFilePath)
+       logFile = open(logFilePath, 'rb')
 
-       # Write custom headers
-       out.write("<?xml version=\"1.0\"?>\n")
-       out.write("<?xml-stylesheet href=\"%s\" type=\"text/xsl\"?>\n" % STYLESHEET_FILENAME)
+       result = parser.getNextTestCaseResult(logFile)
+       while result is not None:
+
+               countByStatusCode[result.statusCode] += 1
+               rootNodes = normalizeToXml(result, dstDoc)
+
+               for node in rootNodes:
+
+                       # Do not append TestResults to dstDoc to save memory.
+                       # Instead print them directly to the file and add tabs manually.
+                       for line in node.toprettyxml().splitlines():
+                               out.write("\t" + line + "\n")
+
+               result = parser.getNextTestCaseResult(logFile)
+
+       # Calculate the totals to add at the end of the Xml file
+       for code in StatusCode.STATUS_CODES:
+               summaryElem.setAttribute(code, "%d" % countByStatusCode[code])
+       summaryElem.setAttribute('All', "%d" % sum(countByStatusCode.values()))
 
-       for line in text.splitlines()[1:]:
-               out.write(line)
-               out.write("\n")
+       # Print the test totals and finish the Xml Document"
+       for line in dstDoc.toprettyxml().splitlines()[2:]:
+               out.write(line + "\n")
 
        out.close()
+       logFile.close()
 
 if __name__ == "__main__":
        if len(sys.argv) != 3: