Fixed aspell issues. 28/64228/2
authorTomasz Olszak <t.olszak@samsung.com>
Wed, 30 Mar 2016 12:03:38 +0000 (14:03 +0200)
committerTomasz Olszak <t.olszak@samsung.com>
Thu, 31 Mar 2016 09:54:39 +0000 (11:54 +0200)
Now aspell is more accurate and produces better gerrit comments.

Change-Id: I4af8c9d39e453cfc496baa1123b60aa75f861901

tool/development/buildbot/nativesamples.py

index b705303..dc1c466 100644 (file)
@@ -187,7 +187,7 @@ class NativeSampleGerritManager:
                else:
                        return None
        def addCommentToChange(self, nativeSampleChange, commentText):
-               command = subprocess.list2cmdline(["review", "--message="+ commentText[:20000], nativeSampleChange.revisionId])
+               command = subprocess.list2cmdline(["review", "-m", commentText[:20000].replace("\n", "\n "), nativeSampleChange.revisionId])
                result = self.client.run_gerrit_command(command)
                print result.stdout.read(), result.stderr.read()
 
@@ -555,45 +555,62 @@ class NativeSamples:
                                                file = open(fullFilePath)
                                                lines = file.readlines()
                                                file.close()
-                                               aspell = subprocess.Popen("bash -c 'set -o pipefail; aspell pipe list --mode=ccpp --run-together --lang=en_US " + aspellIgnoreDictPathArg + "< " +
+
+                                               for i, line in enumerate(lines):
+                                                       if line.startswith("*"):
+                                                               return False, "Please fix line number:" + str(i+1) + " in file:" + relativeFilePath + ". It shouldn't start with '*' because it breaks aspell"
+
+                                               aspell = subprocess.Popen("bash -c 'set -o pipefail; aspell pipe list --encoding=utf-8 --mode=ccpp --run-together --lang=en_US " + aspellIgnoreDictPathArg + "< " +
                                                                 fullFilePath + " | grep \"\w\+ [0-9]\+ [0-9]\+:\" || true'", shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
                                                stdoutOutput,stderrOutput = aspell.communicate()
 
-                                               if aspell.returncode != 0:
+                                               if aspell.returncode != 0 or stderrOutput:
                                                        return False, "Error from aspell (error code:" + str(aspell.returncode) + "):\n" + stderrOutput
 
                                                #remove empty lines
                                                outputLines = filter(None, stdoutOutput.split("\n"))
 
                                                if outputLines is not None and len(outputLines) > 0:
-                                                       words = []
-                                                       hints = []
+                                                       class WordInfo:
+                                                               def __init__(self, wordStr, hintsStr, positionInLine):
+                                                                       self.wordStr = wordStr
+                                                                       self.hintsStr = hintsStr
+                                                                       self.position = positionInLine
+                                                                       self.lineNumber = None
+                                                                       self.lineStr = None
+                                                               def __eq__(self, other):
+                                                                       return (isinstance(other, self.__class__) and self.__dict__ == other.__dict__)
+
+                                                               def __ne__(self, other):
+                                                                       return not self.__eq__(other)
+
+                                                       wordsInfo = []
                                                        for outputLine in outputLines:
-                                                               tmpWord = re.search("[\w']+(?= )", outputLine).group(0)
-                                                               if tmpWord not in words:
-                                                                       words.append(tmpWord)
-                                                                       hints.append(outputLine.split(":")[1])
-                                                       longestOutputLine = len(max(words, key=len))
-                                                       checkResult = []
-                                                       for i,word in enumerate(words):
+                                                               wordParts = re.search("[\w']+ [0-9]+ [0-9]+", outputLine).group(0).split(" ")
+                                                               tmpWord = WordInfo(wordStr = wordParts[0], hintsStr = outputLine.split(":")[1], positionInLine = int(wordParts[2]))
+                                                               wordsInfo.append(tmpWord)
+                                                       longestOutputLine = len(max(wordsInfo, key=lambda wordInfo: len(wordInfo.wordStr)).wordStr)
+                                                       for i, wordInfo in enumerate(wordsInfo):
+                                                               wordInfoTmp = wordInfo
                                                                for lineNum, inputFileLine in enumerate(lines):
                                                                        #standard regex \b doesn't include _ but aspell treats it as beginning of a word
-                                                                       reRes = re.search("(\\b{0}\\b)|(_+{0}\\b)|(_+{0}_+)|(\\b{0}_+)".format(word), inputFileLine)
+                                                                       reRes = re.search("^((\\b{0}\\b)|(\\b{0}_+))".format(wordInfo.wordStr), inputFileLine[wordInfo.position:])
                                                                        if reRes is not None:
-                                                                               checkResult.append(str(lineNum + 1).rjust(5, " ") + ": " + word.ljust(longestOutputLine + 1, " ") + " -> " + re.sub("\\b"+word+"\\b", word.upper(),inputFileLine.strip()) +
-                                                                                                               "\n" + "(".rjust(5 + 2 + longestOutputLine + 1 + 4, " ") + hints[i] + ")")
-                                                       if len(words) > len(checkResult):
-                                                               return False, "There is some problem in regular expression (words set is bigger than found set)"
-
-                                                       if len(checkResult) > 0:
-                                                               #sort by line number - the output is like LINU_NUM:WORD
-                                                               #   12: someunknownword
-                                                               checkResult.sort(key=lambda line: int(line.split(":")[0].strip()))
-
-                                                               aspellSummary += "\nASPELL CHECK FILE: " + relativeFilePath + "\n"
-                                                               if stderrOutput:
-                                                                       aspellSummary += "Something appeared on stderr: " + stderrOutput + "\n"
-                                                               aspellSummary += "\n".join(checkResult)
+                                                                               wordInfoTmp.lineNumber, wordInfoTmp.lineStr = lineNum, inputFileLine
+                                                                               if wordsInfo[i].lineNumber is None:
+                                                                                       wordsInfo[i] = wordInfoTmp
+                                                                               elif wordsInfo[i].lineNumber < wordInfoTmp.lineNumber:
+                                                                                       wordsInfo.append(wordInfoTmp)
+                                                       #let's sort by line numbers now
+                                                       wordsInfo.sort(key=lambda wordInfo: wordInfo.lineNumber)
+
+                                                       aspellSummary += "\nASPELL CHECK FILE: " + relativeFilePath + "\n"
+                                                       if stderrOutput:
+                                                               aspellSummary += "Something appeared on stderr: " + stderrOutput + "\n"
+                                                       for wordInfo in wordsInfo:
+                                                               lineWithWordUppercased = wordInfo.lineStr[:wordInfo.position] + wordInfo.lineStr[wordInfo.position:].replace(wordInfo.wordStr, wordInfo.wordStr.upper(), 1).strip()
+                                                               aspellSummary += (str(wordInfo.lineNumber + 1).rjust(5, " ") + ": " + wordInfo.wordStr.ljust(longestOutputLine + 1, " ") + str(wordInfo.position) + "-> " + lineWithWordUppercased +
+                                                                                                               "\n" + "(".rjust(5 + 2 + longestOutputLine + 1 + 4, " ") + wordInfo.hintsStr + ")" + "\n")
                        return len(aspellSummary) == 0, aspellSummary
                except subprocess.CalledProcessError as err:
                        return False, traceback.format_exc() + "\noutput:" + err.output
@@ -732,7 +749,6 @@ class NativeSamples:
                        #now we are on tpk branch - let's get first change
                        print '=======> TPK_BUILD for ', nativeSample.projectName
                        ret["TPK_BUILD"] = convertResult(self.buildTpk())
-
                finally:
                        os.chdir(curDir)
                return ret