Added daily build regression check feature to sample buildbot.

Change-Id: Ie0306c26d23ee21f3f523bb3cafb774846d6ecf4

diff --git a/tool/development/buildbot/daily_regression_test.py b/tool/development/buildbot/daily_regression_test.py
new file mode 100755 (executable)
index 0000000..98690b4
--- /dev/null
+++ b/tool/development/buildbot/daily_regression_test.py
@@ -0,0 +1,33 @@
+#!/usr/bin/python
+
+import sys, os
+import json
+import argparse
+import subprocess
+import traceback
+import nativesamples as ns
+
+reload(sys)
+sys.setdefaultencoding('utf8')
+
+if ns.numberOfProcesses(".*python.*" + os.path.basename(__file__)) > 1:
+       print "process already running"
+       sys.exit(0)
+
+if len(sys.argv) < 2:
+       print "Please provide samples root build path"
+       sys.exit(0)
+
+rootBuildDir =  os.path.abspath(sys.argv[1])
+
+nativeSamples = ns.NativeSamples(rootBuildDir)
+
+emailSender = ns.EmailSender(nativeSamples.config)
+
+try:
+       nativeSamples.dailyRegressionCheck()
+except:
+       subject = 'Tizen SAMPLE DAILY BUILD TEST error: Something unexpected happened during daily build check'
+       stacktrace = "Exception Info:\n\n" + traceback.format_exc()
+       traceback.print_exc()
+       emailSender.send(mailSubject = subject, mailText = stacktrace)

diff --git a/tool/development/buildbot/nativesamples.py b/tool/development/buildbot/nativesamples.py
index d59dd12..d320470 100644 (file)
--- a/tool/development/buildbot/nativesamples.py
+++ b/tool/development/buildbot/nativesamples.py
@@ -9,6 +9,7 @@ from pygerrit.ssh import GerritSSHClient
 import smtplib
 import base64
 import traceback
+import datetime
 
 # Import the email modules we'll need
 from email.mime.text import MIMEText
@@ -205,8 +206,8 @@ class EmailSender:
                self.mailTo = config.emailNotificationsMailTo
                self.mailCc = config.emailNotificationsMailCc
                self.mailBcc = config.emailNotificationsMailBcc
-       def send(self, mailSubject, mailText = None, mailTo = None, mailFrom = None, mailCc = None, mailBcc = None):
-               msg = MIMEText(mailText, 'plain', 'utf-8')
+       def send(self, mailSubject, mailText = None, mailTo = None, mailFrom = None, mailCc = None, mailBcc = None, mimeType = None):
+               msg = MIMEText(mailText, mimeType or 'plain', 'utf-8')
                msg['Subject'] = mailSubject
                msg['From'] = self.mailFrom
                msg['To'] = ",".join(mailTo or self.mailTo)
@@ -258,7 +259,13 @@ class NativeSamples:
                        for changeId in changeIds:
                                revisionId = subprocess.check_output(gitCommandList + [ "log", "--pretty=format:%H", "-1", changeId])
                                self.databaseModel.saveNativeSampleChange(NativeSampleChange(nativeSample, revisionId))
-
+       def buildTpk(self):
+               subprocess.check_call(["rm", "-rf", "Debug"], stderr=subprocess.STDOUT)
+               try:
+                       output = subprocess.check_output([self.config.tizenBinary, "build-native", "-a", "arm"], stderr=subprocess.STDOUT)
+                       return True, output
+               except subprocess.CalledProcessError as err:
+                       return False, err.output
        def buildSampleFromTpkBranch(self, nativeSampleChange, changeInfo = None):
                print "building tpk for ", nativeSampleChange.nativeSample.projectName, " and changeId:", nativeSampleChange.changeId
                curDir = os.getcwd()
@@ -270,18 +277,14 @@ class NativeSamples:
                                if changeInfo['branch'] == 'tpk':
                                        print 'building tpk branch'
                                        if os.path.exists(os.path.join(nativeSampleChange.nativeSample.projectPath, "Build")):
-                                               subprocess.check_call(["rm", "-rf", "Debug"], stderr=subprocess.STDOUT)
-                                               try:
-                                                       subprocess.check_call([self.config.tizenBinary, "clean"], stderr=subprocess.STDOUT)
-                                                       output = subprocess.check_output([self.config.tizenBinary, "build-native", "-a", "arm"], stderr=subprocess.STDOUT)
+                                               result, output = self.buildTpk()
+                                               if result:
                                                        print "SUCCESS: built change " + nativeSampleChange.changeId + " from project " + nativeSampleChange.nativeSample.projectName
                                                        print output
-                                               except subprocess.CalledProcessError as err:
+                                               else:
                                                        print "FAIL: built change " + nativeSampleChange.changeId + " from project " + nativeSampleChange.nativeSample.projectName
-                                                       print "ERROR:", err.output
-                                                       return False, err.output
-                                               finally:
-                                                       os.chdir(curDir)
+                                                       print "ERROR:", output
+                                                       return False, output
                                        else:
                                                print 'No Tizen CLI configuration in ' + nativeSampleChange.nativeSample.projectPath
                                else:
@@ -291,7 +294,22 @@ class NativeSamples:
                finally:
                        os.chdir(curDir)
                return True, ""
-
+       def invokeConvert(self, revision):
+               subprocess.call(["git", "checkout", "."], stderr=subprocess.STDOUT)
+               subprocess.check_call(["git", "clean", "-fdx", "."], stderr=subprocess.STDOUT)
+               subprocess.check_call(["git", "checkout", "tizen_2.4"], stderr=subprocess.STDOUT)
+               subprocess.check_call([self.config.convertToSampleProjectBinary, "-v", "-r", revision], stderr=subprocess.STDOUT)
+
+       def invokeTcc(self, templateType):
+               templatePath = os.path.join(self.config.sampleCoreComponentsPath, "rule/" + templateType)
+               tccOutput = subprocess.check_output([self.config.tccBinary, "-c", templatePath, "."], stderr=subprocess.STDOUT)
+               result = re.search("[0-9]+(?= code sections are modified)", tccOutput)
+               if int(result.group(0)) > 0:
+                       return False, tccOutput
+               result = re.search("[0-9]+(?= template files are removed)", tccOutput)
+               if int(result.group(0)) > 0:
+                       return False, tccOutput
+               return True, tccOutput
        def checkSampleUsingTcc(self, nativeSampleChange, changeInfo = None):
                print "checking using tcc tool for ", nativeSampleChange.nativeSample.projectName, " and changeId:", nativeSampleChange.changeId
                curDir = os.getcwd()
@@ -307,22 +325,14 @@ class NativeSamples:
                                        if not os.path.exists(tccTemplateFile):
                                                return False, "No file " + str(tccTemplateFile)
 
-                                       templateType = open("sample-project-src/.tcc_template_type").readline()
-                                       templatePath = os.path.join(self.config.sampleCoreComponentsPath, "rule/" + templateType)
+                                       templateType = open(tccTemplateFile).readline()
 
-                                       subprocess.call(["git", "checkout", "."], stderr=subprocess.STDOUT)
-                                       subprocess.check_call(["git", "clean", "-fdx", "."], stderr=subprocess.STDOUT)
-                                       subprocess.check_call(["git", "checkout", "tizen_2.4"], stderr=subprocess.STDOUT)
-                                       subprocess.check_call([self.config.convertToSampleProjectBinary, "-v", "-r", nativeSampleChange.revisionId], stderr=subprocess.STDOUT)
+                                       self.invokeConvert(nativeSampleChange.revisionId)
 
-                                       tccOutput = subprocess.check_output([self.config.tccBinary, "-c", templatePath, "."], stderr=subprocess.STDOUT)
+                                       result, tccOutput = self.invokeTcc(templateType)
                                        print tccOutput
-                                       result = re.search("[0-9]+(?= code sections are modified)", tccOutput)
-                                       if int(result.group(0)) > 0:
-                                               return False, result.group(0) + " code sections modified: " + tccOutput
-                                       result = re.search("[0-9]+(?= template files are removed)", tccOutput)
-                                       if int(result.group(0)) > 0:
-                                               return False, result.group(0) + " template files are removed: " + tccOutput
+                                       if not result:
+                                               return False, tccOutput
                                else:
                                        print "Can't check for branch:" + changeInfo['branch'] + " only tpk"
                        else:
@@ -333,6 +343,18 @@ class NativeSamples:
                        subprocess.check_call(["git", "checkout", nativeSampleChange.revisionId])
                        os.chdir(curDir)
                return True, ""
+       def invokeCheckpatchTizen(self):
+               try:
+                       filesToCheck = []
+                       for root, dirs, files in os.walk(os.getcwd()):
+                               for fileName in files:
+                                       if fileName.endswith(".c") or fileName.endswith(".h"):
+                                               filesToCheck.append(os.path.join(root, fileName))
+                       checkPatchTizenOutput = subprocess.check_output(["perl", self.config.checkpatchTizenBinary] + filesToCheck, stderr=subprocess.STDOUT)
+                       return True, checkPatchTizenOutput
+               except subprocess.CalledProcessError as err:
+                       return False, err.output
+
        def checkSampleUsingCheckpatchTizen(self, nativeSampleChange, changeInfo):
                print "checking using checkpatch_tizen.pl", nativeSampleChange.nativeSample.projectName, " and changeId:", nativeSampleChange.changeId
                curDir = os.getcwd()
@@ -346,22 +368,18 @@ class NativeSamples:
                                        subprocess.check_call(["git", "checkout", "-f", nativeSampleChange.revisionId])
 
                                        try:
-                                               filesToCheck = []
-                                               for root, dirs, files in os.walk(os.getcwd()):
-                                                       for fileName in files:
-                                                               if fileName.endswith(".c") or fileName.endswith(".h"):
-                                                                       filesToCheck.append(os.path.join(root, fileName))
-                                               checkPatchTizenOutput = subprocess.check_output(["perl", self.config.checkpatchTizenBinary] + filesToCheck, stderr=subprocess.STDOUT)
-                                               print "================================"
-                                               print "SUCCESS: built change " + nativeSampleChange.changeId + " from project " + nativeSampleChange.nativeSample.projectName
-                                               print checkPatchTizenOutput
-                                               print "================================"
-                                       except subprocess.CalledProcessError as err:
-                                               print "================================"
-                                               print "FAIL: checkpatch_tizen.pl of change " + nativeSampleChange.changeId + " from project " + nativeSampleChange.nativeSample.projectName
-                                               print "ERROR:", err.output
-                                               print "================================"
-                                               return False, err.output
+                                               res, output = self.invokeCheckpatchTizen()
+                                               if res:
+                                                       print "================================"
+                                                       print "SUCCESS: built change " + nativeSampleChange.changeId + " from project " + nativeSampleChange.nativeSample.projectName
+                                                       print output
+                                                       print "================================"
+                                               else:
+                                                       print "================================"
+                                                       print "FAIL: checkpatch_tizen.pl of change " + nativeSampleChange.changeId + " from project " + nativeSampleChange.nativeSample.projectName
+                                                       print "ERROR:", output
+                                                       print "================================"
+                                                       return False, output
                                        finally:
                                                os.chdir(curDir)
 
@@ -413,7 +431,51 @@ class NativeSamples:
                        subject = "Can't find change id for change with revision : " + nativeSampleChange.revisionId + " of project:" + nativeSampleChange.nativeSample.projectName
                        print subject
                        self.emailSender.send(mailSubject = subject)
+               os.chdir(curDir)
 
+       def evaluateProject(self, nativeSample):
+               print "evaluating project", nativeSample.projectName
+               curDir = os.getcwd()
+               os.chdir(nativeSample.projectPath)
+               subprocess.call(["git", "fetch", "origin"], stderr=subprocess.STDOUT)
+               subprocess.call(["git", "checkout", "."], stderr=subprocess.STDOUT)
+               subprocess.check_call(["git", "clean", "-fdx", "."], stderr=subprocess.STDOUT)
+               subprocess.check_call(["git", "checkout", "-f", "-B", "tpk", "origin/tpk"])
+
+               #now we are on tpk branch - let's get first change
+               ret = {
+                       'TPK_BUILD': {'result': False, 'comment': None},
+                       'TCC_CHECK': {'result': False, 'comment': None},
+                       'CHECKPATCH_TIZEN': {'result': False, 'comment': None}
+                       }
+
+               print 'invoking tpk build'
+               ret["TPK_BUILD"]['result'], ret["TPK_BUILD"]['comment'] = self.buildTpk()
+               print ret["TPK_BUILD"]['comment']
+               if ret["TPK_BUILD"]['result']:
+                       ret["TPK_BUILD"]['comment'] = "OK"
+
+               print 'invoking checkpatchTizen'
+               ret["CHECKPATCH_TIZEN"]['result'], ret["CHECKPATCH_TIZEN"]['comment'] = self.invokeCheckpatchTizen()
+               print ret["CHECKPATCH_TIZEN"]['comment']
+               if ret["CHECKPATCH_TIZEN"]['result']:
+                       ret["CHECKPATCH_TIZEN"]['comment'] = "OK"
+
+               tccTemplateFile = os.path.join(nativeSample.projectPath, "sample-project-src/.tcc_template_type")
+
+               if os.path.exists(tccTemplateFile):
+                       print 'invoking tcc check'
+                       templateType = open(tccTemplateFile).readline()
+                       subprocess.check_call(["git", "checkout", "-f", "-B", "tizen_2.4", "origin/tizen_2.4"])
+                       self.invokeConvert("tpk")
+                       ret["TCC_CHECK"]['result'], ret["TCC_CHECK"]['comment'] = self.invokeTcc(templateType)
+                       print ret["TCC_CHECK"]['comment']
+                       if ret["TCC_CHECK"]['result']:
+                               ret["TCC_CHECK"]['comment'] = "OK"
+               else:
+                       ret["TCC_CHECK"]['result'], ret["TCC_CHECK"]['comment'] = False, "No file " + str(tccTemplateFile)
+               os.chdir(curDir)
+               return ret
        def evaluatePendingChanges(self):
                changesList = self.databaseModel.getPendingNativeSampleChanges(self)
 
@@ -428,3 +490,26 @@ class NativeSamples:
                                traceback.print_exc()
                                self.emailSender.send(mailSubject = subject, mailText = stacktrace)
                                print "Evaluating next change"
+       def dailyRegressionCheck(self):
+               htmlText = "<HTML><TABLE border=\"1\" style=\"width:80%\"><TR><TH>Project Name</TH><TH>Check step</TH><TH>Result</TH><TH>Comment</TH></TR>"
+               for sampleProjectName in self.samplesList:
+                       try:
+                               nativeSample = self._getNativeSample(sampleProjectName)
+                               if not os.path.exists(nativeSample.projectPath):
+                                       self._cloneSampleFromGerrit(nativeSample)
+                               res = self.evaluateProject(nativeSample)
+                               spanText = "<TD rowspan=\"3\">%s</TD>" % sampleProjectName
+                               res.keys().sort()
+                               for i, checkStep in enumerate(res.keys()):
+                                       htmlText += "<TR>"
+                                       if i == 0:
+                                               htmlText += spanText
+                                       htmlText += ("<TD>%s</TD><TD>%i</TD><TD>%s</TD></TR>") % (checkStep, res[checkStep]['result'], res[checkStep]['comment'])
+                       except:
+                               subject = 'Tizen DAILY REGRESSION BUILD SYSTEM error: Something unexpected happened during daily build process'
+                               stacktrace = "Exception Info:\n\n" + traceback.format_exc()
+                               traceback.print_exc()
+                               self.emailSender.send(mailSubject = subject, mailText = stacktrace)
+                               print "Evaluating next project"
+               self.emailSender.send(mailSubject = 'DAILY REGRESSION TESTS SUMMARY (' + str(datetime.date.today())+")", mailText = htmlText, mimeType='html')
+               htmlText += "</TABLE></HTML>"