From: Fabio Date: Tue, 4 Jun 2019 15:09:17 +0000 (+0200) Subject: Reduce memory consumption in log parsing scripts X-Git-Tag: upstream/1.3.5~2033 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=65e3d090e4318d7ba2253511ac88d38ddfbc74da;p=platform%2Fupstream%2FVK-GL-CTS.git Reduce memory consumption in log parsing scripts Updated .qpa to .xml conversion scripts to reduce memory consumption. Before, the script was reading the whole .qpa file and storing it all in memory. On top of that, it created xml.dom.minidom objects for each testcase in the .qpa which were also stored in memory. With this change, we create the .xml file at the same time that we parse the .qpa which results in only one xml.dom.minidom object being created at a time. Components: Scripts VK-GL-CTS issue: 1813 Change-Id: Iba4cb3bf6a94f57c481eb98a324b28638924075e --- diff --git a/scripts/log/log_parser.py b/scripts/log/log_parser.py index b0fd64d..2b3eb7b 100644 --- a/scripts/log/log_parser.py +++ b/scripts/log/log_parser.py @@ -97,6 +97,26 @@ class BatchResultParser: return self.testCaseResults + def getNextTestCaseResult (self, file): + try: + del self.testCaseResults[:] + self.curResultText = None + + isNextResult = self.parseLine(file.next()) + while not isNextResult: + isNextResult = self.parseLine(file.next()) + + # Return the next TestCaseResult + return self.testCaseResults.pop() + + except StopIteration: + # If end of file was reached and there is no log left, the parsing finished successful (return None). + # Otherwise, if there is still log to be parsed, it means that there was a crash. + if self.curResultText: + return TestCaseResult(self.curCaseName, StatusCode.CRASH, StatusCode.CRASH, self.curResultText) + else: + return None + def init (self, filename): # Results self.sessionInfo = [] @@ -112,12 +132,14 @@ class BatchResultParser: def parseLine (self, line): if len(line) > 0 and line[0] == '#': - self.parseContainerLine(line) + return self.parseContainerLine(line) elif self.curResultText != None: self.curResultText += line + return None # else: just ignored def parseContainerLine (self, line): + isTestCaseResult = False args = splitContainerLine(line) if args[0] == "#sessionInfo": if len(args) < 3: @@ -137,6 +159,7 @@ class BatchResultParser: self.parseTestCaseResult(self.curCaseName, self.curResultText) self.curCaseName = None self.curResultText = None + isTestCaseResult = True elif args[0] == "#terminateTestCaseResult": if len(args) < 2 or self.curCaseName == None: self.parseError("Invalid #terminateTestCaseResult") @@ -155,11 +178,14 @@ class BatchResultParser: self.curCaseName = None self.curResultText = None + isTestCaseResult = True else: # Assume this is result text if self.curResultText != None: self.curResultText += line + return isTestCaseResult + def parseTestCaseResult (self, name, log): try: # The XML parser has troubles with invalid characters deliberately included in the shaders. diff --git a/scripts/log/log_to_xml.py b/scripts/log/log_to_xml.py index 1038953..281bd0c 100644 --- a/scripts/log/log_to_xml.py +++ b/scripts/log/log_to_xml.py @@ -137,49 +137,60 @@ def normalizeToXml (result, doc): return rootNodes -def logToXml (inFile, outFile): - parser = BatchResultParser() - results = parser.parseFile(inFile) - - dstDoc = xml.dom.minidom.Document() +def logToXml (logFilePath, outFilePath): + # Initialize Xml Document + dstDoc = xml.dom.minidom.Document() batchResultNode = dstDoc.createElement('BatchResult') - batchResultNode.setAttribute("FileName", os.path.basename(inFile)) - + batchResultNode.setAttribute("FileName", os.path.basename(logFilePath)) dstDoc.appendChild(batchResultNode) - for result in results: - # Normalize log to valid XML - rootNodes = normalizeToXml(result, dstDoc) - for node in rootNodes: - batchResultNode.appendChild(node) - - # Summary + # Initialize dictionary for counting status codes countByStatusCode = {} for code in StatusCode.STATUS_CODES: countByStatusCode[code] = 0 - for result in results: - countByStatusCode[result.statusCode] += 1 + # Write custom headers + out = codecs.open(outFilePath, "wb", encoding="utf-8") + out.write("\n") + out.write("\n" % STYLESHEET_FILENAME) summaryElem = dstDoc.createElement('ResultTotals') - for code in StatusCode.STATUS_CODES: - summaryElem.setAttribute(code, "%d" % countByStatusCode[code]) - summaryElem.setAttribute('All', "%d" % len(results)) batchResultNode.appendChild(summaryElem) - text = dstDoc.toprettyxml() + # Print the first line manually + out.write(dstDoc.toprettyxml().splitlines()[1]) + out.write("\n") - out = codecs.open(outFile, "wb", encoding="utf-8") + parser = BatchResultParser() + parser.init(logFilePath) + logFile = open(logFilePath, 'rb') - # Write custom headers - out.write("\n") - out.write("\n" % STYLESHEET_FILENAME) + result = parser.getNextTestCaseResult(logFile) + while result is not None: + + countByStatusCode[result.statusCode] += 1 + rootNodes = normalizeToXml(result, dstDoc) + + for node in rootNodes: + + # Do not append TestResults to dstDoc to save memory. + # Instead print them directly to the file and add tabs manually. + for line in node.toprettyxml().splitlines(): + out.write("\t" + line + "\n") + + result = parser.getNextTestCaseResult(logFile) + + # Calculate the totals to add at the end of the Xml file + for code in StatusCode.STATUS_CODES: + summaryElem.setAttribute(code, "%d" % countByStatusCode[code]) + summaryElem.setAttribute('All', "%d" % sum(countByStatusCode.values())) - for line in text.splitlines()[1:]: - out.write(line) - out.write("\n") + # Print the test totals and finish the Xml Document" + for line in dstDoc.toprettyxml().splitlines()[2:]: + out.write(line + "\n") out.close() + logFile.close() if __name__ == "__main__": if len(sys.argv) != 3: