tool/development/buildbot/nativesamples.py

   1 # -*- coding: utf-8 -*-
   2 import sys, os
   3 import sqlite3
   4 import ConfigParser
   5 import subprocess
   6 import re
   7 import json
   8 from pygerrit.ssh import GerritSSHClient
   9 import smtplib
  10 import base64
  11 import traceback
  12 import datetime
  13 import tempfile
  14 import xml.etree.ElementTree as ET
  15
  16 # Import the email modules we'll need
  17 from email.mime.text import MIMEText
  18 from email.mime.multipart import MIMEMultipart
  19
  20 reload(sys)
  21 sys.setdefaultencoding('utf-8')
  22
  23 def numberOfProcesses(pgrepArg):
  24         numberOfPollingProcesses = "0"
  25         try:
  26                 numberOfPollingProcesses = subprocess.check_output(["pgrep", "-c", "-f", pgrepArg])
  27         except subprocess.CalledProcessError as err:
  28                 numberOfPollingProcesses = err.output
  29         return int(numberOfPollingProcesses)
  30
  31
  32 class NativeSamplesConfig:
  33         configFilePath = None
  34         gerritServerName = None
  35         gerritUser = None
  36         gerritPort = None
  37         gerritSshPrefixUrl = None
  38         tizenSdkPath = None
  39         sampleCoreComponentsPath = None
  40         tizenBinary = None
  41         tccBinary = None
  42         checkpatchTizenBinary = None
  43         convertToSampleProjectBinary = None
  44         nativeSampleProjectList = None
  45
  46         emailNotificationsSmtpServerName = None
  47         emailNotificationsSmtpPort = None
  48         emailNotificationsSmtpUser = None
  49         emailNotificationsSmtpB64EncodedPassword = None
  50         emailNotificationsMailFrom = None
  51         emailNotificationsMailTo = None
  52         emailNotificationsMailCc = None
  53         emailNotificationsMailBcc = None
  54
  55         def __init__(self, configFilePath = None):
  56                 if configFilePath is not None:
  57                         self.readConfigFile(configFilePath)
  58         def readConfigFile(self, configFilePath):
  59                 if os.path.exists(configFilePath):
  60                         self.configFilePath = configFilePath
  61                         config = ConfigParser.SafeConfigParser({'GerritServerName': None,
  62                                                                                                         'GerritUser': None,
  63                                                                                                         'GerritPort': None,
  64                                                                                                         'TizenSdkPath': None,
  65                                                                                                         'SampleCoreComponentsPath': None,
  66                                                                                                         'SmtpServerName': None,
  67                                                                                                         'SmtpPort': None,
  68                                                                                                         'SmtpUser': None,
  69                                                                                                         'SmtpB64EncodedPassword': None,
  70                                                                                                         'MailFrom': None,
  71                                                                                                         'MailTo': None,
  72                                                                                                         'MailCc': None,
  73                                                                                                         'MailBcc': None})
  74                         config.read(configFilePath)
  75                         self.gerritServerName =  config.get('Gerrit', 'GerritServerName')
  76                         self.gerritUser = config.get('Gerrit', 'GerritUser')
  77                         self.gerritPort = config.get('Gerrit', 'GerritPort')
  78                         self.tizenSdkPath = config.get('BuildTools', 'TizenSdkPath')
  79                         self.sampleCoreComponentsPath =  config.get('BuildTools', 'SampleCoreComponentsPath')
  80                         #ssh://[username@]servername[:port]/
  81                         self.gerritSshPrefixUrl = "ssh://"
  82                         if self.gerritUser is not None:
  83                                 self.gerritSshPrefixUrl += self.gerritUser + "@"
  84                         self.gerritSshPrefixUrl += self.gerritServerName
  85                         if self.gerritPort is not None:
  86                                 self.gerritSshPrefixUrl += ":" + self.gerritPort
  87                         self.gerritSshPrefixUrl += "/"
  88                         self.nativeSampleProjectList = config.get('Gerrit', 'RepositoryList').split(",")
  89                         for i, prj in enumerate(self.nativeSampleProjectList):
  90                                 self.nativeSampleProjectList[i] = prj.strip("\n\t")
  91                         if os.path.exists(self.tizenSdkPath):
  92                                 self.tizenBinary = os.path.join(self.tizenSdkPath, "tools/ide/bin/tizen")
  93                         if os.path.exists(self.sampleCoreComponentsPath):
  94                                 self.tccBinary = os.path.join(self.sampleCoreComponentsPath, "tool/tcc.py")
  95                                 self.convertToSampleProjectBinary = os.path.join(self.sampleCoreComponentsPath, "tool/development/convert-tpk-to-sample-project.sh")
  96                                 self.checkpatchTizenBinary = os.path.join(self.sampleCoreComponentsPath, "tool/checkpatch_tizen.pl")
  97                         self.emailNotificationsSmtpServerName = config.get('EmailNotifications', 'SmtpServerName')
  98                         self.emailNotificationsSmtpPort = int(config.get('EmailNotifications', 'SmtpPort'))
  99                         self.emailNotificationsSmtpUser = config.get('EmailNotifications', 'SmtpUser')
 100                         self.emailNotificationsSmtpB64EncodedPassword = config.get('EmailNotifications', 'SmtpB64EncodedPassword')
 101                         self.emailNotificationsMailFrom = config.get('EmailNotifications', 'MailFrom')
 102                         self.emailNotificationsMailTo = config.get('EmailNotifications', 'MailTo').split(",")
 103                         self.emailNotificationsMailCc = config.get('EmailNotifications', 'MailCc').split(",")
 104                         self.emailNotificationsMailBcc = config.get('EmailNotifications', 'MailBcc').split(",")
 105                 else:
 106                         raise ValueError('Config file name:' + configFilePath + " does not exist")
 107
 108 class NativeSample:
 109         projectName = None
 110         projectPath = None
 111         repositoryUrl = None
 112         def __init__(self, projectName, projectPath, repositoryUrl):
 113                 self.projectName = projectName
 114                 self.projectPath = projectPath
 115                 self.repositoryUrl = repositoryUrl
 116         def __repr__(self):
 117                 return  ('NativeSample(projectName: %s, ' \
 118                                  'projectPath: %s, '\
 119                                  'repositoryUrl = %s)') % (
 120                                 self.projectName,
 121                                 self.projectPath,
 122                                 self.repositoryUrl)
 123
 124 class NativeSampleChange:
 125         revisionId = None
 126         changeId = None
 127         nativeSample = None
 128         def __init__(self, nativeSample, revId, changeId = None):
 129                 self.nativeSample = nativeSample
 130                 self.revisionId = revId
 131                 self.changeId = changeId
 132         def __repr__(self):
 133                 return  ('NativeSampleChange(nativeSample: %s, ' \
 134                                  'revisionId: %s, '\
 135                                  'changeId = %s)') % (
 136                                 self.nativeSample,
 137                                 self.revisionId,
 138                                 self.changeId)
 139
 140 class NativeSamplesDatabase:
 141         createDatabaseScript = """
 142         CREATE TABLE IF NOT EXISTS changes_to_evaluate (
 143                 id INTEGER NOT NULL PRIMARY KEY,
 144                 project_name TEXT NOT NULL,
 145                 revision_id TEXT NOT NULL
 146         );
 147         """
 148         insertChangeStatement = """
 149         INSERT INTO changes_to_evaluate(project_name, revision_id) VALUES(:project_name, :revision_id)
 150         """
 151         conn = None
 152         databaseFile = None
 153         def __init__(self, dbfile):
 154                 self.databaseFile = dbfile
 155                 self.conn = sqlite3.connect(dbfile, isolation_level = None)
 156                 self.conn.execute("PRAGMA busy_timeout = 30000")
 157                 self.cur = self.conn.cursor()
 158                 self.cur.executescript(self.createDatabaseScript)
 159                 self.conn.commit()
 160         def saveNativeSampleChange(self, nativeSampleChange):
 161                 self.cur.execute(self.insertChangeStatement, {'project_name':nativeSampleChange.nativeSample.projectName, 'revision_id': nativeSampleChange.revisionId})
 162         def saveNativeSampleChange(self, nativeSampleChange):
 163                 self.cur.execute(self.insertChangeStatement, {'project_name':nativeSampleChange.nativeSample.projectName, 'revision_id': nativeSampleChange.revisionId})
 164         def deleteNativeSampleChange(self, nativeSampleChange):
 165                 self.cur.execute("DELETE FROM changes_to_evaluate WHERE project_name = :project_name AND revision_id = :revision_id", {'project_name':nativeSampleChange.nativeSample.projectName, 'revision_id': nativeSampleChange.revisionId})
 166         def getPendingNativeSampleChanges(self, nativeSamples):
 167                 ret = []
 168                 for row in self.cur.execute("SELECT id, project_name, revision_id FROM changes_to_evaluate ORDER BY id ASC"):
 169                         ret.append(NativeSampleChange(NativeSample(row[1], None, None), row[2]))
 170                 return ret
 171
 172 class NativeSampleGerritManager:
 173         client = None
 174         version = None
 175         def __init__(self, config):
 176                 self.client = GerritSSHClient(config.gerritServerName, username=config.gerritUser, port = int(config.gerritPort), keepalive=True)
 177         def getVersion(self):
 178                 return self.client.run_gerrit_command("version").stdout.read()
 179         def getChangeInfo(self, nativeSampleChange):
 180                 if nativeSampleChange.revisionId is None:
 181                         raise ValueError('Native sample revision Id cannot be None for: ' + str(nativeSampleChange))
 182                 jsonChangeIdStr = self.client.run_gerrit_command("query --format=JSON --current-patch-set=" + nativeSampleChange.revisionId +" project:" + nativeSampleChange.nativeSample.projectName + " limit:1").stdout.read()
 183                 changeInfo = json.loads(jsonChangeIdStr.split("\n")[0])
 184                 if 'id' in changeInfo.keys() and 'currentPatchSet' in changeInfo.keys():
 185                         return changeInfo
 186                 else:
 187                         return None
 188         def addCommentToChange(self, nativeSampleChange, commentText):
 189                 commentText = commentText.replace("'", " ").replace('"', " ")
 190                 result = self.client.run_gerrit_command("review -m '\""+commentText+"\"' " + nativeSampleChange.revisionId)
 191                 print result.stdout.read(), result.stderr.read()
 192
 193 class EmailSender:
 194         smtpServer = None
 195         smtpPort = None
 196         smtpUser = None
 197         smtpPassword = None
 198         mailFrom = None
 199         mailTo = None
 200         mailCc = None
 201         mailBcc = None
 202         def __init__(self, config):
 203                 self.smtpServer = config.emailNotificationsSmtpServerName
 204                 self.smtpPort = config.emailNotificationsSmtpPort
 205                 self.smtpUser = config.emailNotificationsSmtpUser
 206                 self.smtpPassword = config.emailNotificationsSmtpB64EncodedPassword
 207                 self.mailFrom = config.emailNotificationsMailFrom
 208                 self.mailTo = config.emailNotificationsMailTo
 209                 self.mailCc = config.emailNotificationsMailCc
 210                 self.mailBcc = config.emailNotificationsMailBcc
 211         def send(self, mailSubject, mailText = None, mailTo = None, mailFrom = None, mailCc = None, mailBcc = None, mimeType = None):
 212                 msg = MIMEText(mailText, mimeType or 'plain', 'utf-8')
 213                 msg['Subject'] = mailSubject
 214                 msg['From'] = self.mailFrom
 215                 msg['To'] = ",".join(mailTo or self.mailTo)
 216                 msg['CC'] = ",".join(mailCc or self.mailCc)
 217                 msg['BCC'] = ",".join(mailBcc or self.mailBcc)
 218
 219                 s = smtplib.SMTP(host = self.smtpServer, port = self.smtpPort)
 220
 221                 s.login(self.smtpUser, base64.b64decode(self.smtpPassword))
 222                 s.sendmail(msg['From'], msg['To'].split(","), msg.as_string())
 223                 s.quit()
 224
 225 class NativeSamples:
 226         #enum like source type
 227         SourceTypeSampleProject = 0
 228         SourceTypeTpkProject = 1
 229         SampleTemplateTypeMobile = 0
 230         SampleTemplateTypeWearable = 1
 231         SampleTemplateTypeWatchface = 2
 232         SampleTemplateTypeService = 3
 233         samplesList = []
 234         config = None
 235         nativeSamplesRootPath = None
 236         databaseModel = None
 237         gerrit = None
 238         emailSender = None
 239         def __init__(self, rootBuildPath):
 240                 self.nativeSamplesRootPath = rootBuildPath
 241                 self.databaseModel = NativeSamplesDatabase(os.path.join(rootBuildPath, ".native-samples.db"))
 242                 self.config = NativeSamplesConfig(os.path.join(rootBuildPath, ".native-samples.cfg"))
 243                 self.samplesList = self.config.nativeSampleProjectList
 244                 self.gerrit = NativeSampleGerritManager(self.config)
 245                 self.emailSender = EmailSender(self.config)
 246         def _cloneSampleFromGerrit(self, nativeSample):
 247                 if os.path.exists(nativeSample.projectPath):
 248                         raise ValueError('Path ' + projectPath + ' of project ' + projectName + ' clone already exist')
 249                 print "Cloning repository of ", nativeSample
 250                 subprocess.check_call(["git", "clone", nativeSample.repositoryUrl, nativeSample.projectPath])
 251         def _getNativeSample(self, projectName):
 252                 return NativeSample(projectName, os.path.join(self.nativeSamplesRootPath, projectName), self.config.gerritSshPrefixUrl + projectName)
 253
 254         def pollForChanges(self, projectList = None):
 255                 if projectList is None:
 256                         projectList = self.samplesList
 257                 for nativeSampleProjectName in projectList:
 258                         nativeSample = self._getNativeSample(nativeSampleProjectName)
 259                         if not os.path.exists(nativeSample.projectPath):
 260                                 self._cloneSampleFromGerrit(nativeSample)
 261                         gitCommandList = ["git", "--git-dir=" + os.path.join(nativeSample.projectPath, ".git")]
 262                         fetchOutput = ""
 263                         try:
 264                                 fetchOutput = subprocess.check_output(gitCommandList + ["fetch", "origin", "refs/changes/*:refs/remotes/origin/gerrit/*"], stderr=subprocess.STDOUT)
 265                         except subprocess.CalledProcessError as error:
 266                                 if error.returncode == 128:
 267                                         print 'network or server error - trying to fetch next project'
 268                                 else:
 269                                         raise
 270                         if len(fetchOutput) > 0:
 271                                 print fetchOutput
 272                         changeIds = re.findall("(?<=-> ).*", fetchOutput)
 273                         if changeIds is None:
 274                                 continue
 275                         for changeId in changeIds:
 276                                 revisionId = subprocess.check_output(gitCommandList + [ "log", "--pretty=format:%H", "-1", changeId])
 277                                 self.databaseModel.saveNativeSampleChange(NativeSampleChange(nativeSample, revisionId))
 278         def _cleanGitRepo(self, repoDir):
 279                 curDir = os.getcwd()
 280                 try:
 281                         os.chdir(repoDir)
 282                         subprocess.call(["git", "checkout", "-q", "."], stderr=subprocess.STDOUT)
 283                         subprocess.check_call(["git", "clean", "-fdxq", "."], stderr=subprocess.STDOUT)
 284                 finally:
 285                         os.chdir(curDir)
 286         def _cleanCurrentGitRepo(self):
 287                 self._cleanGitRepo(os.getcwd())
 288         def _cleanRepoAndCheckoutToRevision(self, sampleChange = None, repoPath = None, revision = None):
 289                 curDir = os.getcwd()
 290                 try:
 291                         os.chdir(repoPath or sampleChange.nativeSample.projectPath or curDir)
 292                         self._cleanCurrentGitRepo()
 293                         subprocess.check_call(["git", "checkout", "-qf", revision or sampleChange.revisionId])
 294                 finally:
 295                         os.chdir(curDir)
 296         def _sourceType(self, sampleDirectoryPath = None):
 297                 curDir = os.getcwd()
 298                 try:
 299                         if sampleDirectoryPath is not None:
 300                                 os.chdir(sampleDirectoryPath)
 301
 302                         #if sample contains sample.xml then we assume that it is sample project type
 303                         if os.path.exists("sample.xml"):
 304                                 return self.SourceTypeSampleProject
 305                         if os.path.exists("tizen-manifest.xml"):
 306                                 return self.SourceTypeTpkProject
 307                         raise ValueError("Can't determine source project type in path:" + os.getcwd())
 308                 finally:
 309                         os.chdir(curDir)
 310         def _currentSourceType(self):
 311                 return self._sourceType(os.getcwd())
 312         def _directoryContainsFileWithString(self, directory, stringToFind):
 313                 try:
 314                         subprocess.check_call(["grep", "-qr", stringToFind, directory])
 315                         return True
 316                 except subprocess.CalledProcessError:
 317                         return False
 318
 319         def _sampleTemplateType(self, sampleDirectoryPath = None):
 320                 if sampleDirectoryPath is None:
 321                         sampleDirectoryPath = os.getcwd()
 322                 ret = None
 323                 #if it is service type then profile doesn't matter
 324                 if self._directoryContainsFileWithString(sampleDirectoryPath, "service_app_lifecycle_callback_s"):
 325                         ret = self.SampleTemplateTypeService
 326                 elif self._directoryContainsFileWithString(sampleDirectoryPath, "watch_app_lifecycle_callback_s"):
 327                         ret = self.SampleTemplateTypeWatchface
 328                 if ret is None:
 329                         sourceType = self._sourceType()
 330                         tizenXmlFilePath = None
 331                         descriptionXmlFilePath = None
 332                         if sourceType == self.SourceTypeSampleProject:
 333                                 descriptionXmlFilePath = os.path.join(os.getcwd(), "description.xml")
 334                         elif sourceType == self.SourceTypeTpkProject:
 335                                 if not os.path.exists(os.path.join(os.getcwd(), "sample-project-src/description.xml")):
 336                                         tizenXmlFilePath = os.path.join(os.getcwd(), "tizen-manifest.xml")
 337                                 else:
 338                                         descriptionXmlFilePath = os.path.join(os.getcwd(), "sample-project-src/description.xml")
 339                         profileType = "undefined"
 340                         if descriptionXmlFilePath is not None:
 341                                 if not os.path.exists(descriptionXmlFilePath):
 342                                         raise ValueError("Can't find description.xml file in %s. Can't determine sample template type." % descriptionXmlFilePath)
 343                                 xmlTree = ET.parse(descriptionXmlFilePath)
 344                                 xmlRoot = xmlTree.getroot()
 345                                 xmlnsPrefix = xmlRoot.tag.replace("Overview", "")
 346                                 profileType = xmlRoot.iter(xmlnsPrefix + 'ProfileName').next().text.lower()
 347                         if tizenXmlFilePath is not None:
 348                                 if not os.path.exists(tizenXmlFilePath):
 349                                         raise ValueError("Can't find tizen-manifest.xml file in %s. Can't determine sample template type." % tizenXmlFilePath)
 350                                 xmlTree = ET.parse(tizenXmlFilePath)
 351                                 xmlRoot = xmlTree.getroot()
 352                                 xmlnsPrefix = xmlRoot.tag.replace("manifest", "")
 353                                 profileType = xmlRoot.find(xmlnsPrefix + 'profile').attrib["name"]
 354
 355                         if profileType == "wearable":
 356                                 ret = self.SampleTemplateTypeWearable
 357                         elif profileType == "mobile":
 358                                 ret = self.SampleTemplateTypeMobile
 359                 if ret is None:
 360                         raise ValueError("Can't determine sample template type")
 361                 return ret
 362
 363         def buildTpk(self):
 364                 subprocess.check_call(["rm", "-rf", "Debug"], stderr=subprocess.STDOUT)
 365                 try:
 366                         output = subprocess.check_output([self.config.tizenBinary, "build-native", "-a", "arm"], stderr=subprocess.STDOUT)
 367                         return True, output
 368                 except subprocess.CalledProcessError as error:
 369                         return False, traceback.format_exc() + "\n" + error.output
 370         def buildSampleFromTpkBranch(self, nativeSampleChange, changeInfo = None):
 371                 print "========> TPK_BUILD for ", nativeSampleChange.nativeSample.projectName, " and changeId:", nativeSampleChange.changeId
 372                 curDir = os.getcwd()
 373                 try:
 374                         if changeInfo is None:
 375                                 changeInfo = self.gerrit.getChangeInfo(nativeSampleChange)
 376                         os.chdir(nativeSampleChange.nativeSample.projectPath)
 377                         if changeInfo['status'] != "MERGED" and changeInfo['status'] != "ABANDONED":
 378                                 if self._currentSourceType() == self.SourceTypeTpkProject:
 379                                         print 'building tpk branch'
 380
 381                                         if os.path.exists(os.path.join(nativeSampleChange.nativeSample.projectPath, "Build")):
 382                                                 result, output = self.buildTpk()
 383                                                 if result:
 384                                                         print output
 385                                                         print "SUCCESS: built change " + nativeSampleChange.changeId + " from project " + nativeSampleChange.nativeSample.projectName
 386                                                 else:
 387                                                         print "ERROR:", output
 388                                                         print "FAIL: built change " + nativeSampleChange.changeId + " from project " + nativeSampleChange.nativeSample.projectName
 389                                                         return False, output
 390                                         else:
 391                                                 print 'No Tizen CLI configuration in ' + nativeSampleChange.nativeSample.projectPath
 392                                 else:
 393                                         print "Can't built for source type other than tpk"
 394                                         return False, 'Building non-tpk projects are not yet handled by buildbot'
 395                         else:
 396                                 print "change already " + changeInfo['status']
 397                 finally:
 398                         os.chdir(curDir)
 399                 return True, ""
 400         def invokeConvert(self, revision, outputDirectory = None):
 401                 if outputDirectory is None:
 402                         outputDirectory = os.getcwd()
 403                 try:
 404                         subprocess.check_call([self.config.convertToSampleProjectBinary, "-v", "-r", revision, "-o", outputDirectory], stderr=subprocess.STDOUT)
 405                 except subprocess.CalledProcessError as error:
 406                         return False, traceback.format_exc() + "\n" + error.output
 407
 408         def invokeTcc(self):
 409                 templateType = self._sampleTemplateType()
 410                 templatePostfix = "undefined"
 411                 if templateType == self.SampleTemplateTypeMobile:
 412                         templatePostfix = "mobile"
 413                 elif templateType == self.SampleTemplateTypeWatchface:
 414                         templatePostfix = "watchface"
 415                 elif templateType == self.SampleTemplateTypeWearable:
 416                         templatePostfix = "wearable"
 417                 elif templateType == self.SampleTemplateTypeService:
 418                         templatePostfix = "service"
 419                 templatePath = os.path.join(self.config.sampleCoreComponentsPath, "rule/" + templatePostfix)
 420                 try:
 421                         tccOutput = subprocess.check_output([self.config.tccBinary, "-c", templatePath, "."], stderr=subprocess.STDOUT)
 422                         result = re.search("[0-9]+(?= code sections are modified)", tccOutput)
 423                         if int(result.group(0)) > 0:
 424                                 return False, tccOutput
 425                         result = re.search("[0-9]+(?= template files are removed)", tccOutput)
 426                         if int(result.group(0)) > 0:
 427                                 return False, tccOutput
 428                 except subprocess.CalledProcessError as error:
 429                         return False, traceback.format_exc() + "\n" + error.output
 430
 431                 return True, tccOutput
 432
 433
 434         def checkSampleUsingTcc(self, nativeSampleChange, changeInfo = None):
 435                 print "========> TCC for ", nativeSampleChange.nativeSample.projectName, " and changeId:", nativeSampleChange.changeId
 436                 curDir = os.getcwd()
 437                 try:
 438                         if changeInfo is None:
 439                                 changeInfo = self.gerrit.getChangeInfo(nativeSampleChange)
 440
 441                         if changeInfo['status'] != "MERGED" and changeInfo['status'] != "ABANDONED":
 442                                         os.chdir(nativeSampleChange.nativeSample.projectPath)
 443                                         self._cleanRepoAndCheckoutToRevision(sampleChange = nativeSampleChange)
 444                                         sourceType = self._currentSourceType()
 445                                         if sourceType == self.SourceTypeTpkProject:
 446                                                 tempTccOutputDir = tempfile.mkdtemp()
 447                                                 self.invokeConvert(revision = nativeSampleChange.revisionId, outputDirectory = tempTccOutputDir)
 448                                                 os.chdir(tempTccOutputDir)
 449                                         elif sourceType != self.SourceTypeSampleProject:
 450                                                 raise ValueError("Wrong source type to use tcc tool")
 451                                         result, tccOutput = self.invokeTcc()
 452                                         print tccOutput
 453                                         if not result:
 454                                                 print 'FAIL: tcc for ', nativeSampleChange.nativeSample.projectName, ' and changeId:', nativeSampleChange.changeId
 455                                                 return False, tccOutput
 456                         else:
 457                                 print "change already " + changeInfo['status']
 458                 finally:
 459                         self._cleanRepoAndCheckoutToRevision(sampleChange = nativeSampleChange)
 460                         os.chdir(curDir)
 461                 print 'SUCCCESS: tcc for ', nativeSampleChange.nativeSample.projectName, ' and changeId:', nativeSampleChange.changeId
 462                 return True, ""
 463         def invokeCheckpatchTizen(self):
 464                 try:
 465                         filesToCheck = []
 466                         for root, dirs, files in os.walk(os.getcwd()):
 467                                 for fileName in files:
 468                                         if fileName.endswith(".c") or fileName.endswith(".h"):
 469                                                 filesToCheck.append(os.path.join(root, fileName))
 470                         checkPatchTizenOutput = subprocess.check_output(["perl", self.config.checkpatchTizenBinary] + filesToCheck, stderr=subprocess.STDOUT)
 471                         return True, checkPatchTizenOutput
 472                 except subprocess.CalledProcessError as err:
 473                         return False, traceback.format_exc() + "\noutput:" + err.output
 474
 475         def checkSampleUsingCheckpatchTizen(self, nativeSampleChange, changeInfo):
 476                 print "========> CHECKPATCH_TIZEN for ", nativeSampleChange.nativeSample.projectName, " and changeId:", nativeSampleChange.changeId
 477                 curDir = os.getcwd()
 478                 try:
 479                         if changeInfo is None:
 480                                 changeInfo = self.gerrit.getChangeInfo(nativeSampleChange)
 481
 482                         if changeInfo['status'] != "MERGED" and changeInfo['status'] != "ABANDONED":
 483                                 os.chdir(nativeSampleChange.nativeSample.projectPath)
 484                                 self._cleanRepoAndCheckoutToRevision(sampleChange = nativeSampleChange)
 485                                 res, output = self.invokeCheckpatchTizen()
 486                                 if res:
 487                                         print output
 488                                         print "SUCCESS: sources checked with checkpatch_tizen.pl for " + nativeSampleChange.changeId + " from project " + nativeSampleChange.nativeSample.projectName
 489                                 else:
 490                                         print "ERROR:", output
 491                                         print "FAIL: sources checked failed with checkpatch_tizen.pl for " + nativeSampleChange.changeId + " from project " + nativeSampleChange.nativeSample.projectName
 492                                         return False, output
 493                         else:
 494                                 print "change already " + changeInfo['status']
 495                 finally:
 496                         self._cleanRepoAndCheckoutToRevision(sampleChange = nativeSampleChange)
 497                         os.chdir(curDir)
 498                 return True, ""
 499         def evaluateChange(self, nativeSampleChange):
 500                 print 'evaluating change:', nativeSampleChange
 501                 curDir = os.getcwd()
 502                 os.chdir(nativeSampleChange.nativeSample.projectPath)
 503
 504                 self._cleanRepoAndCheckoutToRevision(sampleChange = nativeSampleChange)
 505                 changeInfo = self.gerrit.getChangeInfo(nativeSampleChange)
 506
 507                 if changeInfo is not None:
 508                         if changeInfo['status'] != "MERGED" and changeInfo['status'] != "ABANDONED":
 509                                 nativeSampleChange.changeId = changeInfo['id']
 510
 511                                 result, message = self.buildSampleFromTpkBranch(nativeSampleChange, changeInfo)
 512                                 if result:
 513                                         self.gerrit.addCommentToChange(nativeSampleChange, "BuildBot: Compilation successful")
 514                                 else:
 515                                         self.gerrit.addCommentToChange(nativeSampleChange, "BuildBot: Compilation Failed:"+message)
 516
 517                                 result, message = self.checkSampleUsingTcc(nativeSampleChange, changeInfo)
 518                                 if result:
 519                                         self.gerrit.addCommentToChange(nativeSampleChange, "BuildBot: TCC Check successful")
 520                                 else:
 521                                         self.gerrit.addCommentToChange(nativeSampleChange, "BuildBot: TCC Check Failed:"+message)
 522
 523                                 result, message = self.checkSampleUsingCheckpatchTizen(nativeSampleChange, changeInfo)
 524                                 if result:
 525                                         self.gerrit.addCommentToChange(nativeSampleChange, "BuildBot: checkpatch_tizen successful")
 526                                 else:
 527                                         self.gerrit.addCommentToChange(nativeSampleChange, "BuildBot: checkpatch_tizen Failed:"+message)
 528                         else:
 529                                 print "Change already " + changeInfo['status']
 530                 else:
 531                         subject = "Can't find change id for change with revision : " + nativeSampleChange.revisionId + " of project:" + nativeSampleChange.nativeSample.projectName
 532                         print subject
 533                         self.emailSender.send(mailSubject = subject)
 534                 os.chdir(curDir)
 535
 536         def evaluateProject(self, nativeSample):
 537                 print "evaluating project", nativeSample.projectName
 538                 curDir = os.getcwd()
 539                 def convertResult(resultList):
 540                         ret = {'result': False, 'comment': None}
 541                         ret['result'] = resultList[0]
 542                         print resultList[1]
 543                         if resultList[0]:
 544                                 ret['comment'] = "OK"
 545                         else:
 546                                 ret['comment'] = resultList[1]
 547                         return ret
 548                 try:
 549                         os.chdir(nativeSample.projectPath)
 550                         subprocess.call(["git", "fetch", "origin"], stderr=subprocess.STDOUT)
 551                         self._cleanRepoAndCheckoutToRevision(repoPath = nativeSample.projectPath, revision="origin/tpk")
 552
 553                         #now we are on tpk branch - let's get first change
 554                         ret = {
 555                                 'TPK_BUILD': {'result': False, 'comment': None},
 556                                 'TCC_CHECK': {'result': False, 'comment': None},
 557                                 'CHECKPATCH_TIZEN': {'result': False, 'comment': None}
 558                                 }
 559
 560                         print '=======> TPK_BUILD for ', nativeSample.projectName
 561
 562                         ret["TPK_BUILD"] = convertResult(self.buildTpk())
 563
 564                         print '=======> CHECKPATCH_TIZEN ', nativeSample.projectName
 565                         ret["CHECKPATCH_TIZEN"] = convertResult(self.invokeCheckpatchTizen())
 566
 567                         self._cleanCurrentGitRepo()
 568                         tempTccOutputDir = tempfile.mkdtemp()
 569                         self.invokeConvert(revision = "origin/tpk", outputDirectory = tempTccOutputDir)
 570                         os.chdir(tempTccOutputDir)
 571                         print '=======> invoking TCC_CHECK ', nativeSample.projectName
 572                         ret["TCC_CHECK"] = convertResult(self.invokeTcc())
 573
 574                 finally:
 575                         os.chdir(curDir)
 576                 return ret
 577         def evaluatePendingChanges(self):
 578                 changesList = self.databaseModel.getPendingNativeSampleChanges(self)
 579
 580                 for i in range(len(changesList)):
 581                         try:
 582                                 changesList[i].nativeSample = self._getNativeSample(changesList[i].nativeSample.projectName)
 583                                 self.evaluateChange(changesList[i])
 584                                 self.databaseModel.deleteNativeSampleChange(changesList[i])
 585                         except KeyboardInterrupt:
 586                                 raise
 587                         except:
 588                                 subject = 'Tizen SAMPLE BUILD SYSTEM error: Something unexpected happened during build process'
 589                                 stacktrace = "Exception Info:\n\n" + traceback.format_exc()
 590                                 traceback.print_exc()
 591                                 self.emailSender.send(mailSubject = subject, mailText = stacktrace)
 592                                 print "Evaluating next change"
 593         def dailyRegressionCheck(self):
 594                 htmlText = "<HTML><TABLE border=\"1\" style=\"width:80%\"><TR><TH>Project Name</TH><TH>Check step</TH><TH>Result</TH><TH>Comment</TH></TR>"
 595                 for sampleProjectName in self.samplesList:
 596                         try:
 597                                 nativeSample = self._getNativeSample(sampleProjectName)
 598                                 if not os.path.exists(nativeSample.projectPath):
 599                                         self._cloneSampleFromGerrit(nativeSample)
 600                                 res = self.evaluateProject(nativeSample)
 601                                 spanText = "<TD rowspan=\"3\">%s</TD>" % sampleProjectName
 602                                 res.keys().sort()
 603                                 for i, checkStep in enumerate(res.keys()):
 604                                         htmlText += "<TR>"
 605                                         if i == 0:
 606                                                 htmlText += spanText
 607                                         htmlText += ("<TD>%s</TD><TD>%i</TD><TD>%s</TD></TR>") % (checkStep, res[checkStep]['result'], res[checkStep]['comment'].replace("\n", "<br>"))
 608                         except KeyboardInterrupt:
 609                                 raise
 610                         except:
 611                                 subject = 'Tizen DAILY REGRESSION BUILD SYSTEM error: Something unexpected happened during daily build process'
 612                                 stacktrace = "Exception Info:\n\n" + traceback.format_exc()
 613                                 traceback.print_exc()
 614                                 self.emailSender.send(mailSubject = subject, mailText = stacktrace)
 615                                 print "Evaluating next project"
 616                 self.emailSender.send(mailSubject = 'DAILY REGRESSION TESTS SUMMARY (' + str(datetime.date.today())+")", mailText = htmlText, mimeType='html')
 617                 htmlText += "</TABLE></HTML>"