tool/development/buildbot/nativesamples.py

   1 # -*- coding: utf-8 -*-
   2 import sys, os
   3 import sqlite3
   4 import ConfigParser
   5 import subprocess
   6 import re
   7 import json
   8 from pygerrit.ssh import GerritSSHClient
   9 import smtplib
  10 import base64
  11 import traceback
  12 import datetime
  13 import tempfile
  14 import xml.etree.ElementTree as ET
  15 import shutil
  16
  17 # Import the email modules we'll need
  18 from email.mime.text import MIMEText
  19 from email.mime.multipart import MIMEMultipart
  20
  21 reload(sys)
  22 sys.setdefaultencoding('utf-8')
  23
  24 def numberOfProcesses(pgrepArg):
  25         numberOfPollingProcesses = "0"
  26         try:
  27                 numberOfPollingProcesses = subprocess.check_output(["pgrep", "-c", "-f", pgrepArg])
  28         except subprocess.CalledProcessError as err:
  29                 numberOfPollingProcesses = err.output
  30         return int(numberOfPollingProcesses)
  31
  32
  33 class NativeSamplesConfig:
  34         configFilePath = None
  35         gerritServerName = None
  36         gerritUser = None
  37         gerritPort = None
  38         gerritSshPrefixUrl = None
  39         tizenSdkPath = None
  40         sampleCoreComponentsPath = None
  41         tizenBinary = None
  42         tccBinary = None
  43         checkpatchTizenBinary = None
  44         convertToSampleProjectBinary = None
  45         nativeSampleProjectList = None
  46
  47         emailNotificationsSmtpServerName = None
  48         emailNotificationsSmtpPort = None
  49         emailNotificationsSmtpUser = None
  50         emailNotificationsSmtpB64EncodedPassword = None
  51         emailNotificationsMailFrom = None
  52         emailNotificationsMailTo = None
  53         emailNotificationsMailCc = None
  54         emailNotificationsMailBcc = None
  55
  56         def __init__(self, configFilePath = None):
  57                 if configFilePath is not None:
  58                         self.readConfigFile(configFilePath)
  59         def readConfigFile(self, configFilePath):
  60                 if os.path.exists(configFilePath):
  61                         self.configFilePath = configFilePath
  62                         config = ConfigParser.SafeConfigParser({'GerritServerName': None,
  63                                                                                                         'GerritUser': None,
  64                                                                                                         'GerritPort': None,
  65                                                                                                         'TizenSdkPath': None,
  66                                                                                                         'SampleCoreComponentsPath': None,
  67                                                                                                         'SmtpServerName': None,
  68                                                                                                         'SmtpPort': None,
  69                                                                                                         'SmtpUser': None,
  70                                                                                                         'SmtpB64EncodedPassword': None,
  71                                                                                                         'MailFrom': None,
  72                                                                                                         'MailTo': None,
  73                                                                                                         'MailCc': None,
  74                                                                                                         'MailBcc': None})
  75                         config.read(configFilePath)
  76                         self.gerritServerName =  config.get('Gerrit', 'GerritServerName')
  77                         self.gerritUser = config.get('Gerrit', 'GerritUser')
  78                         self.gerritPort = config.get('Gerrit', 'GerritPort')
  79                         self.tizenSdkPath = config.get('BuildTools', 'TizenSdkPath')
  80                         self.sampleCoreComponentsPath =  config.get('BuildTools', 'SampleCoreComponentsPath')
  81                         #ssh://[username@]servername[:port]/
  82                         self.gerritSshPrefixUrl = "ssh://"
  83                         if self.gerritUser is not None:
  84                                 self.gerritSshPrefixUrl += self.gerritUser + "@"
  85                         self.gerritSshPrefixUrl += self.gerritServerName
  86                         if self.gerritPort is not None:
  87                                 self.gerritSshPrefixUrl += ":" + self.gerritPort
  88                         self.gerritSshPrefixUrl += "/"
  89                         self.nativeSampleProjectList = config.get('Gerrit', 'RepositoryList').split(",")
  90                         for i, prj in enumerate(self.nativeSampleProjectList):
  91                                 self.nativeSampleProjectList[i] = prj.strip("\n\t")
  92                         if os.path.exists(self.tizenSdkPath):
  93                                 self.tizenBinary = os.path.join(self.tizenSdkPath, "tools/ide/bin/tizen")
  94                         if os.path.exists(self.sampleCoreComponentsPath):
  95                                 self.tccBinary = os.path.join(self.sampleCoreComponentsPath, "tool/tcc.py")
  96                                 self.convertToSampleProjectBinary = os.path.join(self.sampleCoreComponentsPath, "tool/development/convert-tpk-to-sample-project.sh")
  97                                 self.checkpatchTizenBinary = os.path.join(self.sampleCoreComponentsPath, "tool/checkpatch_tizen.pl")
  98                         self.emailNotificationsSmtpServerName = config.get('EmailNotifications', 'SmtpServerName')
  99                         self.emailNotificationsSmtpPort = int(config.get('EmailNotifications', 'SmtpPort'))
 100                         self.emailNotificationsSmtpUser = config.get('EmailNotifications', 'SmtpUser')
 101                         self.emailNotificationsSmtpB64EncodedPassword = config.get('EmailNotifications', 'SmtpB64EncodedPassword')
 102                         self.emailNotificationsMailFrom = config.get('EmailNotifications', 'MailFrom')
 103                         self.emailNotificationsMailTo = config.get('EmailNotifications', 'MailTo').split(",")
 104                         self.emailNotificationsMailCc = config.get('EmailNotifications', 'MailCc').split(",")
 105                         self.emailNotificationsMailBcc = config.get('EmailNotifications', 'MailBcc').split(",")
 106                 else:
 107                         raise ValueError('Config file name:' + configFilePath + " does not exist")
 108
 109 class NativeSample:
 110         projectName = None
 111         projectPath = None
 112         repositoryUrl = None
 113         def __init__(self, projectName, projectPath, repositoryUrl):
 114                 self.projectName = projectName
 115                 self.projectPath = projectPath
 116                 self.repositoryUrl = repositoryUrl
 117         def __repr__(self):
 118                 return  ('NativeSample(projectName: %s, ' \
 119                                  'projectPath: %s, '\
 120                                  'repositoryUrl = %s)') % (
 121                                 self.projectName,
 122                                 self.projectPath,
 123                                 self.repositoryUrl)
 124
 125 class NativeSampleChange:
 126         revisionId = None
 127         changeId = None
 128         nativeSample = None
 129         def __init__(self, nativeSample, revId, changeId = None):
 130                 self.nativeSample = nativeSample
 131                 self.revisionId = revId
 132                 self.changeId = changeId
 133         def __repr__(self):
 134                 return  ('NativeSampleChange(nativeSample: %s, ' \
 135                                  'revisionId: %s, '\
 136                                  'changeId = %s)') % (
 137                                 self.nativeSample,
 138                                 self.revisionId,
 139                                 self.changeId)
 140
 141 class NativeSamplesDatabase:
 142         createDatabaseScript = """
 143         CREATE TABLE IF NOT EXISTS changes_to_evaluate (
 144                 id INTEGER NOT NULL PRIMARY KEY,
 145                 project_name TEXT NOT NULL,
 146                 revision_id TEXT NOT NULL
 147         );
 148         """
 149         insertChangeStatement = """
 150         INSERT INTO changes_to_evaluate(project_name, revision_id) VALUES(:project_name, :revision_id)
 151         """
 152         conn = None
 153         databaseFile = None
 154         def __init__(self, dbfile):
 155                 self.databaseFile = dbfile
 156                 self.conn = sqlite3.connect(dbfile, isolation_level = None)
 157                 self.conn.execute("PRAGMA busy_timeout = 30000")
 158                 self.cur = self.conn.cursor()
 159                 self.cur.executescript(self.createDatabaseScript)
 160                 self.conn.commit()
 161         def saveNativeSampleChange(self, nativeSampleChange):
 162                 self.cur.execute(self.insertChangeStatement, {'project_name':nativeSampleChange.nativeSample.projectName, 'revision_id': nativeSampleChange.revisionId})
 163         def saveNativeSampleChange(self, nativeSampleChange):
 164                 self.cur.execute(self.insertChangeStatement, {'project_name':nativeSampleChange.nativeSample.projectName, 'revision_id': nativeSampleChange.revisionId})
 165         def deleteNativeSampleChange(self, nativeSampleChange):
 166                 self.cur.execute("DELETE FROM changes_to_evaluate WHERE project_name = :project_name AND revision_id = :revision_id", {'project_name':nativeSampleChange.nativeSample.projectName, 'revision_id': nativeSampleChange.revisionId})
 167         def getPendingNativeSampleChanges(self, nativeSamples):
 168                 ret = []
 169                 for row in self.cur.execute("SELECT id, project_name, revision_id FROM changes_to_evaluate ORDER BY id ASC"):
 170                         ret.append(NativeSampleChange(NativeSample(row[1], None, None), row[2]))
 171                 return ret
 172
 173 class NativeSampleGerritManager:
 174         client = None
 175         version = None
 176         def __init__(self, config):
 177                 self.client = GerritSSHClient(config.gerritServerName, username=config.gerritUser, port = int(config.gerritPort), keepalive=True)
 178         def getVersion(self):
 179                 return self.client.run_gerrit_command("version").stdout.read()
 180         def getChangeInfo(self, nativeSampleChange):
 181                 if nativeSampleChange.revisionId is None:
 182                         raise ValueError('Native sample revision Id cannot be None for: ' + str(nativeSampleChange))
 183                 jsonChangeIdStr = self.client.run_gerrit_command("query --format=JSON --current-patch-set=" + nativeSampleChange.revisionId +" project:" + nativeSampleChange.nativeSample.projectName + " limit:1").stdout.read()
 184                 changeInfo = json.loads(jsonChangeIdStr.split("\n")[0])
 185                 if 'id' in changeInfo.keys() and 'currentPatchSet' in changeInfo.keys():
 186                         return changeInfo
 187                 else:
 188                         return None
 189         def addCommentToChange(self, nativeSampleChange, commentText):
 190                 command = subprocess.list2cmdline(["review", "--message="+ commentText, nativeSampleChange.revisionId])
 191                 result = self.client.run_gerrit_command(command)
 192                 print result.stdout.read(), result.stderr.read()
 193
 194 class EmailSender:
 195         smtpServer = None
 196         smtpPort = None
 197         smtpUser = None
 198         smtpPassword = None
 199         mailFrom = None
 200         mailTo = None
 201         mailCc = None
 202         mailBcc = None
 203         def __init__(self, config):
 204                 self.smtpServer = config.emailNotificationsSmtpServerName
 205                 self.smtpPort = config.emailNotificationsSmtpPort
 206                 self.smtpUser = config.emailNotificationsSmtpUser
 207                 self.smtpPassword = config.emailNotificationsSmtpB64EncodedPassword
 208                 self.mailFrom = config.emailNotificationsMailFrom
 209                 self.mailTo = config.emailNotificationsMailTo
 210                 self.mailCc = config.emailNotificationsMailCc
 211                 self.mailBcc = config.emailNotificationsMailBcc
 212         def send(self, mailSubject, mailText = None, mailTo = None, mailFrom = None, mailCc = None, mailBcc = None, mimeType = None):
 213                 msg = MIMEText(mailText, mimeType or 'plain', 'utf-8')
 214                 msg['Subject'] = mailSubject
 215                 msg['From'] = self.mailFrom
 216                 msg['To'] = ",".join(mailTo or self.mailTo)
 217                 msg['CC'] = ",".join(mailCc or self.mailCc)
 218                 msg['BCC'] = ",".join(mailBcc or self.mailBcc)
 219
 220                 s = smtplib.SMTP(host = self.smtpServer, port = self.smtpPort)
 221
 222                 s.login(self.smtpUser, base64.b64decode(self.smtpPassword))
 223                 s.sendmail(msg['From'], msg['To'].split(","), msg.as_string())
 224                 s.quit()
 225
 226 class NativeSamples:
 227         #enum like source type
 228         SourceTypeSampleProject = 0
 229         SourceTypeTpkProject = 1
 230         SampleTemplateTypeMobile = 0
 231         SampleTemplateTypeWearable = 1
 232         SampleTemplateTypeWatchface = 2
 233         SampleTemplateTypeService = 3
 234         samplesList = []
 235         config = None
 236         nativeSamplesRootPath = None
 237         databaseModel = None
 238         gerrit = None
 239         emailSender = None
 240         def __init__(self, rootBuildPath):
 241                 self.nativeSamplesRootPath = rootBuildPath
 242                 self.databaseModel = NativeSamplesDatabase(os.path.join(rootBuildPath, ".native-samples.db"))
 243                 self.config = NativeSamplesConfig(os.path.join(rootBuildPath, ".native-samples.cfg"))
 244                 self.samplesList = self.config.nativeSampleProjectList
 245                 self.gerrit = NativeSampleGerritManager(self.config)
 246                 self.emailSender = EmailSender(self.config)
 247         def _cloneSampleFromGerrit(self, nativeSample):
 248                 if os.path.exists(nativeSample.projectPath):
 249                         raise ValueError('Path ' + projectPath + ' of project ' + projectName + ' clone already exist')
 250                 print "Cloning repository of ", nativeSample
 251                 subprocess.check_call(["git", "clone", nativeSample.repositoryUrl, nativeSample.projectPath])
 252         def _getNativeSample(self, projectName):
 253                 return NativeSample(projectName, os.path.join(self.nativeSamplesRootPath, projectName), self.config.gerritSshPrefixUrl + projectName)
 254
 255         def pollForChanges(self, projectList = None):
 256                 if projectList is None:
 257                         projectList = self.samplesList
 258                 for nativeSampleProjectName in projectList:
 259                         nativeSample = self._getNativeSample(nativeSampleProjectName)
 260                         if not os.path.exists(nativeSample.projectPath):
 261                                 self._cloneSampleFromGerrit(nativeSample)
 262                         gitCommandList = ["git", "--git-dir=" + os.path.join(nativeSample.projectPath, ".git")]
 263                         fetchOutput = ""
 264                         try:
 265                                 fetchOutput = subprocess.check_output(gitCommandList + ["fetch", "origin", "refs/changes/*:refs/remotes/origin/gerrit/*"], stderr=subprocess.STDOUT)
 266                         except subprocess.CalledProcessError as error:
 267                                 if error.returncode == 128:
 268                                         print 'network or server error - trying to fetch next project'
 269                                 else:
 270                                         raise
 271                         if len(fetchOutput) > 0:
 272                                 print fetchOutput
 273                         changeIds = re.findall("(?<=-> ).*", fetchOutput)
 274                         if changeIds is None:
 275                                 continue
 276                         for changeId in changeIds:
 277                                 revisionId = subprocess.check_output(gitCommandList + [ "log", "--pretty=format:%H", "-1", changeId])
 278                                 self.databaseModel.saveNativeSampleChange(NativeSampleChange(nativeSample, revisionId))
 279         def _cleanGitRepo(self, repoDir):
 280                 curDir = os.getcwd()
 281                 try:
 282                         os.chdir(repoDir)
 283                         subprocess.call(["git", "checkout", "-q", "."], stderr=subprocess.STDOUT)
 284                         subprocess.check_call(["git", "clean", "-fdxq", "."], stderr=subprocess.STDOUT)
 285                 finally:
 286                         os.chdir(curDir)
 287         def _cleanCurrentGitRepo(self):
 288                 self._cleanGitRepo(os.getcwd())
 289         def _cleanRepoAndCheckoutToRevision(self, sampleChange = None, repoPath = None, revision = None):
 290                 curDir = os.getcwd()
 291                 try:
 292                         os.chdir(repoPath or sampleChange.nativeSample.projectPath or curDir)
 293                         self._cleanCurrentGitRepo()
 294                         subprocess.check_call(["git", "checkout", "-qf", revision or sampleChange.revisionId])
 295                 finally:
 296                         os.chdir(curDir)
 297         def _sourceType(self, sampleDirectoryPath = None):
 298                 curDir = os.getcwd()
 299                 try:
 300                         if sampleDirectoryPath is not None:
 301                                 os.chdir(sampleDirectoryPath)
 302
 303                         #if sample contains sample.xml then we assume that it is sample project type
 304                         if os.path.exists("sample.xml"):
 305                                 return self.SourceTypeSampleProject
 306                         if os.path.exists("tizen-manifest.xml"):
 307                                 return self.SourceTypeTpkProject
 308                         raise ValueError("Can't determine source project type in path:" + os.getcwd())
 309                 finally:
 310                         os.chdir(curDir)
 311         def _currentSourceType(self):
 312                 return self._sourceType(os.getcwd())
 313         def _directoryContainsFileWithString(self, directory, stringToFind):
 314                 try:
 315                         subprocess.check_call(["grep", "-qr", stringToFind, directory])
 316                         return True
 317                 except subprocess.CalledProcessError:
 318                         return False
 319
 320         def _sampleTemplateType(self, sampleDirectoryPath = None):
 321                 if sampleDirectoryPath is None:
 322                         sampleDirectoryPath = os.getcwd()
 323                 ret = None
 324                 #if it is service type then profile doesn't matter
 325                 if self._directoryContainsFileWithString(sampleDirectoryPath, "service_app_lifecycle_callback_s"):
 326                         ret = self.SampleTemplateTypeService
 327                 elif self._directoryContainsFileWithString(sampleDirectoryPath, "watch_app_lifecycle_callback_s"):
 328                         ret = self.SampleTemplateTypeWatchface
 329                 if ret is None:
 330                         sourceType = self._sourceType()
 331                         tizenXmlFilePath = None
 332                         descriptionXmlFilePath = None
 333                         if sourceType == self.SourceTypeSampleProject:
 334                                 descriptionXmlFilePath = os.path.join(os.getcwd(), "description.xml")
 335                         elif sourceType == self.SourceTypeTpkProject:
 336                                 if not os.path.exists(os.path.join(os.getcwd(), "sample-project-src/description.xml")):
 337                                         tizenXmlFilePath = os.path.join(os.getcwd(), "tizen-manifest.xml")
 338                                 else:
 339                                         descriptionXmlFilePath = os.path.join(os.getcwd(), "sample-project-src/description.xml")
 340                         profileType = "undefined"
 341                         if descriptionXmlFilePath is not None:
 342                                 if not os.path.exists(descriptionXmlFilePath):
 343                                         raise ValueError("Can't find description.xml file in %s. Can't determine sample template type." % descriptionXmlFilePath)
 344                                 xmlTree = ET.parse(descriptionXmlFilePath)
 345                                 xmlRoot = xmlTree.getroot()
 346                                 xmlnsPrefix = xmlRoot.tag.replace("Overview", "")
 347                                 profileType = xmlRoot.iter(xmlnsPrefix + 'ProfileName').next().text.lower()
 348                         if tizenXmlFilePath is not None:
 349                                 if not os.path.exists(tizenXmlFilePath):
 350                                         raise ValueError("Can't find tizen-manifest.xml file in %s. Can't determine sample template type." % tizenXmlFilePath)
 351                                 xmlTree = ET.parse(tizenXmlFilePath)
 352                                 xmlRoot = xmlTree.getroot()
 353                                 xmlnsPrefix = xmlRoot.tag.replace("manifest", "")
 354                                 profileType = xmlRoot.find(xmlnsPrefix + 'profile').attrib["name"]
 355
 356                         if profileType == "wearable":
 357                                 ret = self.SampleTemplateTypeWearable
 358                         elif profileType == "mobile":
 359                                 ret = self.SampleTemplateTypeMobile
 360                 if ret is None:
 361                         raise ValueError("Can't determine sample template type")
 362                 return ret
 363
 364         def buildTpk(self):
 365                 subprocess.check_call(["rm", "-rf", "Debug"], stderr=subprocess.STDOUT)
 366                 try:
 367                         output = subprocess.check_output([self.config.tizenBinary, "build-native", "-a", "arm"], stderr=subprocess.STDOUT)
 368                         #sometimes build-native returns 0 exit status if linker error occured
 369                         return not "clang++: error" in output, output
 370                 except subprocess.CalledProcessError as error:
 371                         return False, traceback.format_exc() + "\n" + error.output
 372
 373         def buildSampleFromTpkBranch(self, nativeSampleChange, changeInfo = None):
 374                 print "========> TPK_BUILD for ", nativeSampleChange.nativeSample.projectName, " and changeId:", nativeSampleChange.changeId
 375                 curDir = os.getcwd()
 376                 try:
 377                         if changeInfo is None:
 378                                 changeInfo = self.gerrit.getChangeInfo(nativeSampleChange)
 379                         os.chdir(nativeSampleChange.nativeSample.projectPath)
 380                         if os.path.exists(os.path.join(nativeSampleChange.nativeSample.projectPath, "Build")):
 381                                 result, output = self.buildTpk()
 382                                 if result:
 383                                         print output
 384                                         print "SUCCESS: built change " + nativeSampleChange.changeId + " from project " + nativeSampleChange.nativeSample.projectName
 385                                 else:
 386                                         print "ERROR:", output
 387                                         print "FAIL: built change " + nativeSampleChange.changeId + " from project " + nativeSampleChange.nativeSample.projectName
 388                                         return False, output
 389                         else:
 390                                 print 'No Tizen CLI configuration in ' + nativeSampleChange.nativeSample.projectPath
 391                 finally:
 392                         os.chdir(curDir)
 393                 return True, ""
 394         def buildTpkFromProject(self, nativeSample, uniquePostfix):
 395                 curDir = os.getcwd()
 396                 tmpProjectPath = None
 397                 destSamplesPathDir = None
 398                 try:
 399                         os.chdir(nativeSample.projectPath)
 400                         projectProfileWithVersion = None
 401                         projectDefFile = os.path.join(nativeSample.projectPath,'project/project_def.prop')
 402                         if not os.path.exists(projectDefFile):
 403                                 return False, "No Tizen CLI project file:" + projectDefFile
 404                         f = open(projectDefFile)
 405                         for line in f:
 406                                 res = re.search("\s*profile\s*=\s*", line)
 407                                 if res is not None:
 408                                         projectProfileWithVersion = re.sub("\s*profile\s*=\s*", "", line).strip("\n\t ")
 409                                         break
 410                         tizenVersion = "tizen" + re.search("-[0-9].[0-9](.[0-9])?", projectProfileWithVersion).group(0)
 411                         profileName = re.search(".*(?=-)", projectProfileWithVersion).group(0)
 412                         tmpProjectPath = tempfile.mkdtemp()
 413                         tmpProjectName = "SampleProjectToBuild"
 414                         projectBasename = os.path.basename(nativeSample.projectPath)
 415                         samplesUniqueProjectName = projectBasename + uniquePostfix
 416                         projectPrefixInSamplesDir = "buildbot"
 417                         destSamplesPathDir = os.path.join(self.config.tizenSdkPath, "platforms", tizenVersion, profileName, "samples/Template/Native", projectPrefixInSamplesDir)
 418                         if not os.path.exists(destSamplesPathDir):
 419                                 os.makedirs(destSamplesPathDir)
 420                         destSamplesPathDir = os.path.join(destSamplesPathDir, samplesUniqueProjectName)
 421                         print "Copying sample dir to " + destSamplesPathDir
 422                         subprocess.check_call(["cp", "-r", nativeSample.projectPath, destSamplesPathDir])
 423                         print "removing " + destSamplesPathDir + "/.git"
 424                         shutil.rmtree(os.path.join(destSamplesPathDir, ".git"), ignore_errors=True)
 425                         createProjectArgs = [self.config.tizenBinary, "create", "native-project", "-n", tmpProjectName, "-p", projectProfileWithVersion, "-t", samplesUniqueProjectName, "--", tmpProjectPath]
 426                         print "Creating project from sample template: " + " ".join(createProjectArgs)
 427                         subprocess.check_call(createProjectArgs)
 428                         os.chdir(os.path.join(tmpProjectPath, tmpProjectName))
 429                         print "building project using command line"
 430                         return self.buildTpk()
 431                 except subprocess.CalledProcessError as error:
 432                         return False, traceback.format_exc() + "\n" + error.output
 433                 finally:
 434                         if tmpProjectPath is not None:
 435                                 shutil.rmtree(tmpProjectPath, ignore_errors=True)
 436                         if destSamplesPathDir is not None:
 437                                 shutil.rmtree(destSamplesPathDir, ignore_errors=True)
 438                         os.chdir(curDir)
 439         def buildSampleFromProjectBranch(self, nativeSampleChange, changeInfo = None):
 440                 print "========> SAMPLE_PROJECT_BUILD for ", nativeSampleChange.nativeSample.projectName, " and changeId:", nativeSampleChange.changeId
 441                 curDir = os.getcwd()
 442                 try:
 443                         tmpProjectPath = None
 444                         destSamplesPathDir = None
 445                         if changeInfo is None:
 446                                 changeInfo = self.gerrit.getChangeInfo(nativeSampleChange)
 447                         if os.path.exists(os.path.join(nativeSampleChange.nativeSample.projectPath, "project/Build")):
 448                                 print "building project using command line"
 449                                 result, output = self.buildTpkFromProject(nativeSampleChange.nativeSample, nativeSampleChange.changeId)
 450                                 if result:
 451                                         print output
 452                                         print "SUCCESS: built change " + nativeSampleChange.changeId + " from project " + nativeSampleChange.nativeSample.projectName
 453                                 else:
 454                                         print "ERROR:", output
 455                                         print "FAIL: built change " + nativeSampleChange.changeId + " from project " + nativeSampleChange.nativeSample.projectName
 456                                         return False, output
 457                         else:
 458                                 print 'No Tizen CLI configuration in ' + nativeSampleChange.nativeSample.projectPath
 459                 finally:
 460                         if tmpProjectPath is not None:
 461                                 shutil.rmtree(tmpProjectPath, ignore_errors=True)
 462                         if destSamplesPathDir is not None:
 463                                 shutil.rmtree(destSamplesPathDir, ignore_errors=True)
 464                         os.chdir(curDir)
 465                 return True, ""
 466         def invokeConvert(self, revision, outputDirectory = None):
 467                 if outputDirectory is None:
 468                         outputDirectory = os.getcwd()
 469                 try:
 470                         subprocess.check_call([self.config.convertToSampleProjectBinary, "-v", "-r", revision, "-o", outputDirectory], stderr=subprocess.STDOUT)
 471                 except subprocess.CalledProcessError as error:
 472                         return False, traceback.format_exc() + "\n" + error.output
 473
 474         def invokeTcc(self):
 475                 templateType = self._sampleTemplateType()
 476                 templatePostfix = "undefined"
 477                 if templateType == self.SampleTemplateTypeMobile:
 478                         templatePostfix = "mobile"
 479                 elif templateType == self.SampleTemplateTypeWatchface:
 480                         templatePostfix = "watchface"
 481                 elif templateType == self.SampleTemplateTypeWearable:
 482                         templatePostfix = "wearable"
 483                 elif templateType == self.SampleTemplateTypeService:
 484                         templatePostfix = "service"
 485                 templatePath = os.path.join(self.config.sampleCoreComponentsPath, "rule/" + templatePostfix)
 486                 try:
 487                         tccOutput = subprocess.check_output([self.config.tccBinary, "-c", templatePath, "."], stderr=subprocess.STDOUT)
 488                         result = re.search("[0-9]+(?= code sections are modified)", tccOutput)
 489                         if int(result.group(0)) > 0:
 490                                 return False, tccOutput
 491                         result = re.search("[0-9]+(?= template files are removed)", tccOutput)
 492                         if int(result.group(0)) > 0:
 493                                 return False, tccOutput
 494                 except subprocess.CalledProcessError as error:
 495                         return False, traceback.format_exc() + "\n" + error.output
 496
 497                 return True, tccOutput
 498
 499
 500         def checkSampleUsingTcc(self, nativeSampleChange, changeInfo = None):
 501                 print "========> TCC for ", nativeSampleChange.nativeSample.projectName, " and changeId:", nativeSampleChange.changeId
 502                 curDir = os.getcwd()
 503                 try:
 504                         if changeInfo is None:
 505                                 changeInfo = self.gerrit.getChangeInfo(nativeSampleChange)
 506
 507                         if changeInfo['status'] != "MERGED" and changeInfo['status'] != "ABANDONED":
 508                                         os.chdir(nativeSampleChange.nativeSample.projectPath)
 509                                         self._cleanRepoAndCheckoutToRevision(sampleChange = nativeSampleChange)
 510                                         sourceType = self._currentSourceType()
 511                                         if sourceType == self.SourceTypeTpkProject:
 512                                                 tempTccOutputDir = tempfile.mkdtemp()
 513                                                 self.invokeConvert(revision = nativeSampleChange.revisionId, outputDirectory = tempTccOutputDir)
 514                                                 os.chdir(tempTccOutputDir)
 515                                         elif sourceType != self.SourceTypeSampleProject:
 516                                                 raise ValueError("Wrong source type to use tcc tool")
 517                                         result, tccOutput = self.invokeTcc()
 518                                         print tccOutput
 519                                         if not result:
 520                                                 print 'FAIL: tcc for ', nativeSampleChange.nativeSample.projectName, ' and changeId:', nativeSampleChange.changeId
 521                                                 return False, tccOutput
 522                         else:
 523                                 print "change already " + changeInfo['status']
 524                 finally:
 525                         self._cleanRepoAndCheckoutToRevision(sampleChange = nativeSampleChange)
 526                         os.chdir(curDir)
 527                 print 'SUCCCESS: tcc for ', nativeSampleChange.nativeSample.projectName, ' and changeId:', nativeSampleChange.changeId
 528                 return True, ""
 529         def invokeCheckpatchTizen(self):
 530                 try:
 531                         filesToCheck = []
 532                         for root, dirs, files in os.walk(os.getcwd()):
 533                                 for fileName in files:
 534                                         if fileName.endswith(".c") or fileName.endswith(".h"):
 535                                                 filesToCheck.append(os.path.join(root, fileName))
 536                         checkPatchTizenOutput = subprocess.check_output(["perl", self.config.checkpatchTizenBinary] + filesToCheck, stderr=subprocess.STDOUT)
 537                         return True, checkPatchTizenOutput
 538                 except subprocess.CalledProcessError as err:
 539                         return False, traceback.format_exc() + "\noutput:" + err.output
 540         def invokeAspell(self):
 541                 try:
 542                         aspellSummary = ""
 543                         aspellIgnoreDictPath = ""
 544                         aspellIgnoreDictPathArg = ""
 545                         sourceType = self._currentSourceType()
 546                         if sourceType == self.SourceTypeTpkProject:
 547                                 aspellIgnoreDictPath = os.path.join(os.getcwd(), "sample-project-src/aspell.ignore.txt")
 548                         elif sourceType == self.SourceTypeSampleProject:
 549                                 aspellIgnoreDictPath = os.path.join(os.getcwd(), "aspell.ignore.txt")
 550                         if os.path.exists(aspellIgnoreDictPath):
 551                                 aspellIgnoreDictPathArg = " --personal=" + aspellIgnoreDictPath + " "
 552                         for root, dirs, files in os.walk(os.getcwd()):
 553                                 for fileName in files:
 554                                         if fileName.endswith(".c") or fileName.endswith(".h"):
 555
 556                                                 fullFilePath = os.path.join(root, fileName)
 557                                                 relativeFilePath = os.path.relpath(fullFilePath)
 558                                                 file = open(fullFilePath)
 559                                                 lines = file.readlines()
 560                                                 file.close()
 561                                                 aspell = subprocess.Popen("bash -c 'set -o pipefail; aspell pipe list --mode=ccpp --run-together --lang=en_US " + aspellIgnoreDictPathArg + "< " +
 562                                                                  fullFilePath + " | grep \"\w\+ [0-9]\+ [0-9]\+:\" || true'", shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 563                                                 stdoutOutput,stderrOutput = aspell.communicate()
 564                                                 if aspell.returncode != 0 or stderrOutput:
 565                                                         return False, "Error from aspell (error code:" + str(aspell.returncode) + "):\n" + stderrOutput
 566                                                 #remove empty lines
 567                                                 outputLines = filter(None, stdoutOutput.split("\n"))
 568
 569                                                 if outputLines is not None and len(outputLines) > 0:
 570                                                         words = []
 571                                                         hints = []
 572                                                         for outputLine in outputLines:
 573                                                                 tmpWord = re.search("[\w']+(?= )", outputLine).group(0)
 574                                                                 if tmpWord not in words:
 575                                                                         words.append(tmpWord)
 576                                                                         hints.append(outputLine.split(":")[1])
 577                                                         longestOutputLine = len(max(words, key=len))
 578                                                         checkResult = []
 579                                                         for i,word in enumerate(words):
 580                                                                 for lineNum, inputFileLine in enumerate(lines):
 581                                                                         #standard regex \b doesn't include _ but aspell treats it as beginning of a word
 582                                                                         reRes = re.search("(\\b{0}\\b)|(_+{0}\\b)|(_+{0}_+)|(\\b{0}_+)".format(word), inputFileLine)
 583                                                                         if reRes is not None:
 584                                                                                 checkResult.append(str(lineNum + 1).rjust(5, " ") + ": " + word.ljust(longestOutputLine + 1, " ") + " -> " + re.sub("\\b"+word+"\\b", word.upper(),inputFileLine.strip()) +
 585                                                                                                                 "\n" + "(".rjust(5 + 2 + longestOutputLine + 1 + 4, " ") + hints[i] + ")")
 586                                                         if len(words) > len(checkResult):
 587                                                                 return False, "There is some problem in regular expression (words set is bigger than found set)"
 588
 589                                                         if len(checkResult) > 0:
 590                                                                 #sort by line number - the output is like LINU_NUM:WORD
 591                                                                 #   12: someunknownword
 592                                                                 checkResult.sort(key=lambda line: int(line.split(":")[0].strip()))
 593
 594                                                                 aspellSummary += "\nASPELL CHECK FILE: " + relativeFilePath + "\n"
 595                                                                 aspellSummary += "\n".join(checkResult)
 596                         return len(aspellSummary) == 0, aspellSummary
 597                 except subprocess.CalledProcessError as err:
 598                         return False, traceback.format_exc() + "\noutput:" + err.output
 599
 600         def checkSampleUsingCheckpatchTizen(self, nativeSampleChange, changeInfo):
 601                 print "========> CHECKPATCH_TIZEN for ", nativeSampleChange.nativeSample.projectName, " and changeId:", nativeSampleChange.changeId
 602                 curDir = os.getcwd()
 603                 try:
 604                         if changeInfo is None:
 605                                 changeInfo = self.gerrit.getChangeInfo(nativeSampleChange)
 606
 607                         if changeInfo['status'] != "MERGED" and changeInfo['status'] != "ABANDONED":
 608                                 os.chdir(nativeSampleChange.nativeSample.projectPath)
 609                                 self._cleanRepoAndCheckoutToRevision(sampleChange = nativeSampleChange)
 610                                 res, output = self.invokeCheckpatchTizen()
 611                                 if res:
 612                                         print output
 613                                         print "SUCCESS: sources checked with checkpatch_tizen.pl for " + nativeSampleChange.changeId + " from project " + nativeSampleChange.nativeSample.projectName
 614                                 else:
 615                                         print "ERROR:", output
 616                                         print "FAIL: sources checked failed with checkpatch_tizen.pl for " + nativeSampleChange.changeId + " from project " + nativeSampleChange.nativeSample.projectName
 617                                         return False, output
 618                         else:
 619                                 print "change already " + changeInfo['status']
 620                 finally:
 621                         os.chdir(curDir)
 622                 return True, ""
 623         def checkSampleUsingAspell(self, nativeSampleChange, changeInfo):
 624                 print "========> ASPELL_CHECK for ", nativeSampleChange.nativeSample.projectName, " and changeId:", nativeSampleChange.changeId
 625                 curDir = os.getcwd()
 626                 try:
 627                         if changeInfo is None:
 628                                 changeInfo = self.gerrit.getChangeInfo(nativeSampleChange)
 629
 630                         os.chdir(nativeSampleChange.nativeSample.projectPath)
 631                         self._cleanRepoAndCheckoutToRevision(sampleChange = nativeSampleChange)
 632                         res, output = self.invokeAspell()
 633                         if res:
 634                                 print output
 635                                 print "SUCCESS: sources checked with aspell for " + nativeSampleChange.changeId + " from project " + nativeSampleChange.nativeSample.projectName
 636                         else:
 637                                 print "ERROR:", output
 638                                 print "FAIL: sources checked failed with aspell for " + nativeSampleChange.changeId + " from project " + nativeSampleChange.nativeSample.projectName
 639                                 return False, output
 640                 finally:
 641                         os.chdir(curDir)
 642                 return True, ""
 643         def evaluateChange(self, nativeSampleChange):
 644                 print 'evaluating change:', nativeSampleChange
 645                 curDir = os.getcwd()
 646                 os.chdir(nativeSampleChange.nativeSample.projectPath)
 647
 648                 self._cleanRepoAndCheckoutToRevision(sampleChange = nativeSampleChange)
 649                 changeInfo = self.gerrit.getChangeInfo(nativeSampleChange)
 650
 651                 if changeInfo is not None:
 652                         if changeInfo['status'] != "MERGED" and changeInfo['status'] != "ABANDONED":
 653                                 nativeSampleChange.changeId = changeInfo['id']
 654
 655                                 if self._currentSourceType() == self.SourceTypeTpkProject:
 656                                         result, message = self.buildSampleFromTpkBranch(nativeSampleChange, changeInfo)
 657                                         if result:
 658                                                 self.gerrit.addCommentToChange(nativeSampleChange, "BuildBot: Compilation successful")
 659                                         else:
 660                                                 self.gerrit.addCommentToChange(nativeSampleChange, "BuildBot: Compilation failed:\n"+message)
 661                                 elif self._currentSourceType() == self.SourceTypeSampleProject:
 662                                         result, message = self.buildSampleFromProjectBranch(nativeSampleChange, changeInfo)
 663                                         if result:
 664                                                 self.gerrit.addCommentToChange(nativeSampleChange, "BuildBot: Project creation and compilation successful")
 665                                         else:
 666                                                 self.gerrit.addCommentToChange(nativeSampleChange, "BuildBot: Project creation and/or compilation failed:\n"+message)
 667
 668                                 result, message = self.checkSampleUsingTcc(nativeSampleChange, changeInfo)
 669                                 if result:
 670                                         self.gerrit.addCommentToChange(nativeSampleChange, "BuildBot: TCC Check successful")
 671                                 else:
 672                                         self.gerrit.addCommentToChange(nativeSampleChange, "BuildBot: TCC Check failed:\n"+message)
 673
 674                                 result, message = self.checkSampleUsingCheckpatchTizen(nativeSampleChange, changeInfo)
 675                                 if result:
 676                                         self.gerrit.addCommentToChange(nativeSampleChange, "BuildBot: checkpatch_tizen successful")
 677                                 else:
 678                                         self.gerrit.addCommentToChange(nativeSampleChange, "BuildBot: checkpatch_tizen failed:\n"+message)
 679
 680                                 result, message = self.checkSampleUsingAspell(nativeSampleChange, changeInfo)
 681                                 if result:
 682                                         self.gerrit.addCommentToChange(nativeSampleChange, "BuildBot: spelling check successful")
 683                                 else:
 684                                         self.gerrit.addCommentToChange(nativeSampleChange, "BuildBot: fix spelling check errors or add words to ignore list:\n"+message)
 685
 686                         else:
 687                                 print "Change already " + changeInfo['status']
 688                 else:
 689                         subject = "Can't find change id for change with revision : " + nativeSampleChange.revisionId + " of project:" + nativeSampleChange.nativeSample.projectName
 690                         print subject
 691                         self.emailSender.send(mailSubject = subject)
 692                 os.chdir(curDir)
 693
 694         def evaluateProject(self, nativeSample):
 695                 print "evaluating project", nativeSample.projectName
 696                 curDir = os.getcwd()
 697                 def convertResult(resultList):
 698                         ret = {'result': False, 'comment': None}
 699                         ret['result'] = resultList[0]
 700                         print resultList[1]
 701                         if resultList[0]:
 702                                 ret['comment'] = "OK"
 703                         else:
 704                                 ret['comment'] = resultList[1]
 705                         return ret
 706                 try:
 707                         os.chdir(nativeSample.projectPath)
 708                         subprocess.call(["git", "fetch", "origin"], stderr=subprocess.STDOUT)
 709
 710
 711                         ret = {
 712                                 'SAMPLE_PROJECT_BUILD': {'result': False, 'comment': ""},
 713                                 'TPK_BUILD': {'result': False, 'comment': ""},
 714                                 'TCC_CHECK': {'result': False, 'comment': ""},
 715                                 'CHECKPATCH_TIZEN': {'result': False, 'comment': ""},
 716                                 'ASPELL_CHECK': {'result': False, 'comment': ""}
 717                                 }
 718
 719                         self._cleanRepoAndCheckoutToRevision(repoPath = nativeSample.projectPath, revision="origin/tizen_2.4")
 720                         #now we are on tizen_2.4 branch
 721                         print '=======> SAMPLE_PROJECT_BUILD for ', nativeSample.projectName
 722                         ret["SAMPLE_PROJECT_BUILD"] = convertResult(self.buildTpkFromProject(nativeSample, "daily_build_origin_tizen_2.4"))
 723
 724                         print '=======> invoking TCC_CHECK ', nativeSample.projectName
 725                         ret["TCC_CHECK"] = convertResult(self.invokeTcc())
 726
 727                         print '=======> CHECKPATCH_TIZEN ', nativeSample.projectName
 728                         ret["CHECKPATCH_TIZEN"] = convertResult(self.invokeCheckpatchTizen())
 729
 730                         print '=======> ASPELL_CHECK ', nativeSample.projectName
 731                         ret["ASPELL_CHECK"] = convertResult(self.invokeAspell())
 732
 733                         self._cleanRepoAndCheckoutToRevision(repoPath = nativeSample.projectPath, revision="origin/tpk")
 734                         #now we are on tpk branch - let's get first change
 735                         print '=======> TPK_BUILD for ', nativeSample.projectName
 736                         ret["TPK_BUILD"] = convertResult(self.buildTpk())
 737
 738                 finally:
 739                         os.chdir(curDir)
 740                 return ret
 741         def evaluatePendingChanges(self):
 742                 changesList = self.databaseModel.getPendingNativeSampleChanges(self)
 743
 744                 for i in range(len(changesList)):
 745                         try:
 746                                 changesList[i].nativeSample = self._getNativeSample(changesList[i].nativeSample.projectName)
 747                                 self.evaluateChange(changesList[i])
 748                                 self.databaseModel.deleteNativeSampleChange(changesList[i])
 749                         except KeyboardInterrupt:
 750                                 raise
 751                         except:
 752                                 subject = 'Tizen SAMPLE BUILD SYSTEM error: Something unexpected happened during build process'
 753                                 stacktrace = "Exception Info:\n\n" + traceback.format_exc()
 754                                 traceback.print_exc()
 755                                 self.emailSender.send(mailSubject = subject, mailText = stacktrace)
 756                                 print "Evaluating next change"
 757         def dailyRegressionCheck(self):
 758                 htmlText = "<HTML><TABLE border=\"1\" style=\"width:80%\"><TR><TH>Project Name</TH><TH>Check step</TH><TH>Result</TH><TH>Comment</TH></TR>"
 759                 def escapeHtml(s):
 760                         return  s.replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;")
 761                 for sampleProjectName in self.samplesList:
 762                         try:
 763                                 nativeSample = self._getNativeSample(sampleProjectName)
 764                                 if not os.path.exists(nativeSample.projectPath):
 765                                         self._cloneSampleFromGerrit(nativeSample)
 766                                 res = self.evaluateProject(nativeSample)
 767                                 spanText = "<TD rowspan=\"" + str(len(res.keys())) + "\">%s</TD>" % sampleProjectName
 768                                 res.keys().sort()
 769                                 for i, checkStep in enumerate(res.keys()):
 770                                         htmlText += "<TR>"
 771                                         if i == 0:
 772                                                 htmlText += spanText
 773                                         htmlText += ("<TD>%s</TD><TD>%i</TD><TD><PRE>%s</PRE></TD></TR>") % (checkStep, res[checkStep]['result'], escapeHtml(res[checkStep]['comment']))
 774                         except KeyboardInterrupt:
 775                                 raise
 776                         except:
 777                                 subject = 'Tizen DAILY REGRESSION BUILD SYSTEM error: Something unexpected happened during daily build process'
 778                                 stacktrace = "Exception Info:\n\n" + traceback.format_exc()
 779                                 traceback.print_exc()
 780                                 self.emailSender.send(mailSubject = subject, mailText = stacktrace)
 781                                 print "Evaluating next project"
 782                 htmlText += "</TABLE></HTML>"
 783                 self.emailSender.send(mailSubject = 'DAILY REGRESSION TESTS SUMMARY (' + str(datetime.date.today())+")", mailText = htmlText, mimeType='html')