From 11d56f9f029fa22925406862b8b54238221eb55a Mon Sep 17 00:00:00 2001 From: Tomasz Olszak Date: Fri, 25 Mar 2016 11:26:54 +0100 Subject: [PATCH] Added aspell check to buildbot: daily build and commit check. Additionally some small fixes which were encountered during tests. Change-Id: Ice91083d5347d2b19da2ceda9ce013eab210495c --- tool/development/buildbot/nativesamples.py | 117 +++++++++++++++++++++++++---- 1 file changed, 104 insertions(+), 13 deletions(-) diff --git a/tool/development/buildbot/nativesamples.py b/tool/development/buildbot/nativesamples.py index e18d3df..96f9891 100644 --- a/tool/development/buildbot/nativesamples.py +++ b/tool/development/buildbot/nativesamples.py @@ -187,8 +187,8 @@ class NativeSampleGerritManager: else: return None def addCommentToChange(self, nativeSampleChange, commentText): - commentText = commentText.replace("'", " ").replace('"', " ") - result = self.client.run_gerrit_command("review -m '\""+commentText+"\"' " + nativeSampleChange.revisionId) + command = subprocess.list2cmdline(["review", "--message="+ commentText, nativeSampleChange.revisionId]) + result = self.client.run_gerrit_command(command) print result.stdout.read(), result.stderr.read() class EmailSender: @@ -537,6 +537,65 @@ class NativeSamples: return True, checkPatchTizenOutput except subprocess.CalledProcessError as err: return False, traceback.format_exc() + "\noutput:" + err.output + def invokeAspell(self): + try: + aspellSummary = "" + aspellIgnoreDictPath = "" + aspellIgnoreDictPathArg = "" + sourceType = self._currentSourceType() + if sourceType == self.SourceTypeTpkProject: + aspellIgnoreDictPath = os.path.join(os.getcwd(), "sample-project-src/aspell.ignore.txt") + elif sourceType == self.SourceTypeSampleProject: + aspellIgnoreDictPath = os.path.join(os.getcwd(), "aspell.ignore.txt") + if os.path.exists(aspellIgnoreDictPath): + aspellIgnoreDictPathArg = " --personal=" + aspellIgnoreDictPath + " " + for root, dirs, files in os.walk(os.getcwd()): + for fileName in files: + if fileName.endswith(".c") or fileName.endswith(".h"): + + fullFilePath = os.path.join(root, fileName) + relativeFilePath = os.path.relpath(fullFilePath) + file = open(fullFilePath) + lines = file.readlines() + file.close() + aspell = subprocess.Popen("bash -c 'set -o pipefail; aspell pipe list --mode=ccpp --run-together --lang=en_US " + aspellIgnoreDictPathArg + "< " + + fullFilePath + " | grep \"\w\+ [0-9]\+ [0-9]\+:\" || true'", shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdoutOutput,stderrOutput = aspell.communicate() + if aspell.returncode != 0 or stderrOutput: + return False, "Error from aspell (error code:" + str(aspell.returncode) + "):\n" + stderrOutput + #remove empty lines + outputLines = filter(None, stdoutOutput.split("\n")) + + if outputLines is not None and len(outputLines) > 0: + words = [] + hints = [] + for outputLine in outputLines: + tmpWord = re.search("[\w']+(?= )", outputLine).group(0) + if tmpWord not in words: + words.append(tmpWord) + hints.append(outputLine.split(":")[1]) + longestOutputLine = len(max(words, key=len)) + checkResult = [] + for i,word in enumerate(words): + for lineNum, inputFileLine in enumerate(lines): + #standard regex \b doesn't include _ but aspell treats it as beginning of a word + reRes = re.search("(\\b{0}\\b)|(_+{0}\\b)|(_+{0}_+)|(\\b{0}_+)".format(word), inputFileLine) + if reRes is not None: + checkResult.append(str(lineNum + 1).rjust(5, " ") + ": " + word.ljust(longestOutputLine + 1, " ") + " -> " + re.sub("\\b"+word+"\\b", word.upper(),inputFileLine.strip()) + + "\n" + "(".rjust(5 + 2 + longestOutputLine + 1 + 4, " ") + hints[i] + ")") + if len(words) > len(checkResult): + return False, "There is some problem in regular expression (words set is bigger than found set)" + + if len(checkResult) > 0: + #sort by line number - the output is like LINU_NUM:WORD + # 12: someunknownword + checkResult.sort(key=lambda line: int(line.split(":")[0].strip())) + + aspellSummary += "\nASPELL CHECK FILE: " + relativeFilePath + "\n" + aspellSummary += "\n".join(checkResult) + return len(aspellSummary) == 0, aspellSummary + except subprocess.CalledProcessError as err: + return False, traceback.format_exc() + "\noutput:" + err.output def checkSampleUsingCheckpatchTizen(self, nativeSampleChange, changeInfo): print "========> CHECKPATCH_TIZEN for ", nativeSampleChange.nativeSample.projectName, " and changeId:", nativeSampleChange.changeId @@ -559,7 +618,26 @@ class NativeSamples: else: print "change already " + changeInfo['status'] finally: + os.chdir(curDir) + return True, "" + def checkSampleUsingAspell(self, nativeSampleChange, changeInfo): + print "========> ASPELL_CHECK for ", nativeSampleChange.nativeSample.projectName, " and changeId:", nativeSampleChange.changeId + curDir = os.getcwd() + try: + if changeInfo is None: + changeInfo = self.gerrit.getChangeInfo(nativeSampleChange) + + os.chdir(nativeSampleChange.nativeSample.projectPath) self._cleanRepoAndCheckoutToRevision(sampleChange = nativeSampleChange) + res, output = self.invokeAspell() + if res: + print output + print "SUCCESS: sources checked with aspell for " + nativeSampleChange.changeId + " from project " + nativeSampleChange.nativeSample.projectName + else: + print "ERROR:", output + print "FAIL: sources checked failed with aspell for " + nativeSampleChange.changeId + " from project " + nativeSampleChange.nativeSample.projectName + return False, output + finally: os.chdir(curDir) return True, "" def evaluateChange(self, nativeSampleChange): @@ -579,25 +657,32 @@ class NativeSamples: if result: self.gerrit.addCommentToChange(nativeSampleChange, "BuildBot: Compilation successful") else: - self.gerrit.addCommentToChange(nativeSampleChange, "BuildBot: Compilation failed:"+message) + self.gerrit.addCommentToChange(nativeSampleChange, "BuildBot: Compilation failed:\n"+message) elif self._currentSourceType() == self.SourceTypeSampleProject: result, message = self.buildSampleFromProjectBranch(nativeSampleChange, changeInfo) if result: self.gerrit.addCommentToChange(nativeSampleChange, "BuildBot: Project creation and compilation successful") else: - self.gerrit.addCommentToChange(nativeSampleChange, "BuildBot: Project creation and/or compilation failed:"+message) + self.gerrit.addCommentToChange(nativeSampleChange, "BuildBot: Project creation and/or compilation failed:\n"+message) result, message = self.checkSampleUsingTcc(nativeSampleChange, changeInfo) if result: self.gerrit.addCommentToChange(nativeSampleChange, "BuildBot: TCC Check successful") else: - self.gerrit.addCommentToChange(nativeSampleChange, "BuildBot: TCC Check failed:"+message) + self.gerrit.addCommentToChange(nativeSampleChange, "BuildBot: TCC Check failed:\n"+message) result, message = self.checkSampleUsingCheckpatchTizen(nativeSampleChange, changeInfo) if result: self.gerrit.addCommentToChange(nativeSampleChange, "BuildBot: checkpatch_tizen successful") else: - self.gerrit.addCommentToChange(nativeSampleChange, "BuildBot: checkpatch_tizen failed:"+message) + self.gerrit.addCommentToChange(nativeSampleChange, "BuildBot: checkpatch_tizen failed:\n"+message) + + result, message = self.checkSampleUsingAspell(nativeSampleChange, changeInfo) + if result: + self.gerrit.addCommentToChange(nativeSampleChange, "BuildBot: spelling check successful") + else: + self.gerrit.addCommentToChange(nativeSampleChange, "BuildBot: fix spelling check errors or add words to ignore list:\n"+message) + else: print "Change already " + changeInfo['status'] else: @@ -624,10 +709,11 @@ class NativeSamples: ret = { - 'SAMPLE_PROJECT_BUILD': {'result': False, 'comment': None}, - 'TPK_BUILD': {'result': False, 'comment': None}, - 'TCC_CHECK': {'result': False, 'comment': None}, - 'CHECKPATCH_TIZEN': {'result': False, 'comment': None} + 'SAMPLE_PROJECT_BUILD': {'result': False, 'comment': ""}, + 'TPK_BUILD': {'result': False, 'comment': ""}, + 'TCC_CHECK': {'result': False, 'comment': ""}, + 'CHECKPATCH_TIZEN': {'result': False, 'comment': ""}, + 'ASPELL_CHECK': {'result': False, 'comment': ""} } self._cleanRepoAndCheckoutToRevision(repoPath = nativeSample.projectPath, revision="origin/tizen_2.4") @@ -641,6 +727,9 @@ class NativeSamples: print '=======> CHECKPATCH_TIZEN ', nativeSample.projectName ret["CHECKPATCH_TIZEN"] = convertResult(self.invokeCheckpatchTizen()) + print '=======> ASPELL_CHECK ', nativeSample.projectName + ret["ASPELL_CHECK"] = convertResult(self.invokeAspell()) + self._cleanRepoAndCheckoutToRevision(repoPath = nativeSample.projectPath, revision="origin/tpk") #now we are on tpk branch - let's get first change print '=======> TPK_BUILD for ', nativeSample.projectName @@ -667,19 +756,21 @@ class NativeSamples: print "Evaluating next change" def dailyRegressionCheck(self): htmlText = "" + def escapeHtml(s): + return s.replace("&", "&").replace("<", "<").replace(">", ">") for sampleProjectName in self.samplesList: try: nativeSample = self._getNativeSample(sampleProjectName) if not os.path.exists(nativeSample.projectPath): self._cloneSampleFromGerrit(nativeSample) res = self.evaluateProject(nativeSample) - spanText = "" % sampleProjectName + spanText = "" % sampleProjectName res.keys().sort() for i, checkStep in enumerate(res.keys()): htmlText += "" if i == 0: htmlText += spanText - htmlText += ("") % (checkStep, res[checkStep]['result'], res[checkStep]['comment'].replace("\n", "
")) + htmlText += ("") % (checkStep, res[checkStep]['result'], escapeHtml(res[checkStep]['comment'])) except KeyboardInterrupt: raise except: @@ -688,5 +779,5 @@ class NativeSamples: traceback.print_exc() self.emailSender.send(mailSubject = subject, mailText = stacktrace) print "Evaluating next project" - self.emailSender.send(mailSubject = 'DAILY REGRESSION TESTS SUMMARY (' + str(datetime.date.today())+")", mailText = htmlText, mimeType='html') htmlText += "
Project NameCheck stepResultComment
%s%s
%s%i%s
%s%i
%s
" + self.emailSender.send(mailSubject = 'DAILY REGRESSION TESTS SUMMARY (' + str(datetime.date.today())+")", mailText = htmlText, mimeType='html') -- 2.7.4