From a505b88addadef2bc74a48c10193ca1ce0490751 Mon Sep 17 00:00:00 2001 From: hyokeun Date: Tue, 1 Aug 2017 18:04:49 +0900 Subject: [PATCH] ABS: change code structure Change-Id: I4b1508d2376200666973f31705292bf723e7e2bf --- abs/job_abs_batch_all.py | 798 -------------------- abs/job_abs_build.py | 770 ++++++++++++++++++++ abs/job_abs_main.py | 1411 ------------------------------------ abs/job_abs_update.py | 134 ++++ abs/job_abs_update_vm.py | 435 ----------- common/tizen_studio.py | 241 ++++++ debian/jenkins-scripts-abs.install | 1 + packaging/jenkins-scripts.spec | 7 +- 8 files changed, 1150 insertions(+), 2647 deletions(-) delete mode 100644 abs/job_abs_batch_all.py create mode 100644 abs/job_abs_build.py delete mode 100644 abs/job_abs_main.py create mode 100644 abs/job_abs_update.py delete mode 100644 abs/job_abs_update_vm.py create mode 100644 common/tizen_studio.py diff --git a/abs/job_abs_batch_all.py b/abs/job_abs_batch_all.py deleted file mode 100644 index b9a912a..0000000 --- a/abs/job_abs_batch_all.py +++ /dev/null @@ -1,798 +0,0 @@ -#!/usr/bin/env python -# vim: ai ts=4 sts=4 et sw=4 -# - -import os -import re -import sys -import subprocess -import urllib, urllib2 -import requests -from bs4 import BeautifulSoup -from urllib2 import urlopen, ProxyHandler, build_opener, install_opener, URLError, HTTPError -import time -from datetime import datetime, timedelta -import json -import base64 -import ast -import xml.etree.cElementTree as ET -import inspect -from random import randint - -sys.path.insert(1, os.path.join(sys.path[0], '..')) - -from common.buildtrigger import trigger_info, trigger_next, get_jenkins_build_data -from common.mapping import git_obs_map_full_list -from common.utils import list_files_in_url, unicode_to_str -from common.git import Git, clone_gitproject -from common.send_mail import prepare_mail, makemail - -mail_title_prefix = '[ABS]' - -EMAIL_HEADER = 'App Build System(ABS) reports entire project build.\n' \ - 'It is intended for build check with new private rootstrap.\n' \ - '(Base souce code from the most recent accepted SR)\n\n' -EMAIL_FOOTER = '\n\n--------------------------------------------------------\n'\ - 'Automatically generated by backend service.\n'\ - 'Please DO NOT Reply!' - -def grap_text_from_url(url): - - html = urlopen(url).read() - soup = BeautifulSoup(html) - - # rip all script and style elements out - for script in soup(["script", "style"]): - script.extract() - - text = soup.get_text() - lines = (line.strip() for line in text.splitlines()) - chunks = (phrase.strip() for line in lines for phrase in line.split(" ")) - text = '\n'.join(chunk for chunk in chunks if chunk) - - return text - -def send_mail(title, msg_text): - - if ': 0 error' in title: - return - - title = "%s %s " % (mail_title_prefix, title) - - # Record the MIME types of both parts - text/plain and text/html. - msg = '\n' + EMAIL_HEADER + msg_text + '\n\n' + EMAIL_FOOTER - print '\n[TITLE]\n%s\n[MSG]\n%s' % (title, msg) - - email_to = [] - email_to.extend(os.getenv('ABS_MAILINGLIST').split(',')) - - prepare_mail("%s.env" % os.getenv('BUILD_TAG'), \ - title, \ - msg, \ - os.getenv('NOREPLY_EMAIL_SENDER'), \ - email_to) - -class Git_gerrit_if(object): - - def __init__(self, \ - gitcache=os.getenv('ABS_GIT_CACHE_DIR'), \ - hostname=os.getenv('ABS_GERRIT_HOSTNAME'), \ - username=os.getenv('ABS_GERRIT_USERNAME'), \ - sshport=os.getenv('ABS_GERRIT_SSHPORT')): - self.hostname = hostname - self.username = username - self.sshport = sshport - self.gitcache = gitcache - self.remote = 'ssh://%s@%s:%d/' \ - % (self.username, \ - self.hostname, \ - int(self.sshport)) - -class Trigger_for_abs_update(object): - - profiles = os.getenv('ROOTSTRAP_PROFILES').split(',') - trigger_arg = '' - - kvm_root = '/'.join(os.getenv('ABS_VM_IMAGE').split('/')[:-1]) - pattern = r'tizen[a-zA-Z_-]*[0-9]{8}.[0-9]{1,2}' - - def __init__(self, new_rs=None, silent=False): - - self.silent = silent - self.new_rootstrap_url = self.trigger_arg = new_rs - if new_rs != None and new_rs.startswith('http'): - self.new_rootstrap_url = new_rs - self.trigger_arg = re.match(r'.*\/(.*-.*_[0-9.]{1,})\/.*', new_rs).group(1) - for x in os.getenv('ROOTSTRAP_PROFILES').split(','): - expr = '(%s)_(\d{8}).([0-9]+)' % (x.lower().replace(':', '-')) - if re.search(expr, new_rs): - self.profiles = [x] - break - elif new_rs != None: - for x in os.getenv('ROOTSTRAP_PROFILES').split(','): - expr = '(%s)_(\d{8}).([0-9]+)' % (x.lower().replace(':', '-')) - if re.search(expr, new_rs): - self.profiles = [x] - break - if len(self.profiles) > 1: - raise Exception('Sorry. Your selection is not in my options:%s' % self.profiles) - print ' Version:%s Profile:%s' \ - % (self.trigger_arg, self.profiles) - - def get_rootstrap_url(self, profile, version): - return self.new_rootstrap_url - - def run_update_trigger(self, profile): - - if self.trigger_arg != '': - rs_version = self.trigger_arg - print 'Rootstrap from pre-defined variable: %s' % rs_version - else: - raise Exception('Sorry. No rootstrap url argument given!') - - print('-----[trigger update_vm job for {} ]-----'.format(rs_version)) - trigger_data = {"contents": - {"version": rs_version, - "project": profile, - "rs_url": self.get_rootstrap_url(profile, rs_version)}, - "title": "sdk_rootstrap_updated" } - trigger_next("update_vm_%s" % (rs_version), trigger_data) - if not self.silent: - print "\"TitleBatch\": \"update_vm(%s)\"" % (rs_version) - - def main(self): - - print('-----[JOB STARTED: Trigger_for_abs_update]-----') - for profile in self.profiles: - self.run_update_trigger(profile) - -class Initiate_build_all(object): - - package_commit_list = {} - project = '' # Tizen:Mobile, Tizen:Wearable - version = '' # tizen-mobile_20160405.3, tizen-wearable_20160405.3 - b_prerelease = False - - def __init__(self): - - fields = trigger_info(os.getenv('TRIGGER_INFO')) - self.project = fields['project'] - if 'version' in fields: - self.version = fields['version'] - else: - self.b_prerelease = True - self.commit_message = fields['commit_message'] - self.tag_name = fields['tag_name'] - self.staging = 'abs' - print '\nBuild all for %s - %s\n' % (self.project, self.version) - - self.workspace = os.getenv('WORKSPACE') - self.gerrit_if = Git_gerrit_if() - - def get_mapping_list(self): - - print '****' - print self.project - print self.staging - print self.gerrit_if.gitcache - print self.gerrit_if.hostname - print self.gerrit_if.username - print self.gerrit_if.sshport - mapping_list = git_obs_map_full_list(obs_project=self.project, staging_project=self.staging, \ - gitcache=self.gerrit_if.gitcache, \ - gerrit_hostname=self.gerrit_if.hostname, \ - gerrit_username=self.gerrit_if.username, \ - gerrit_sshport=self.gerrit_if.sshport) - print mapping_list - - for item in mapping_list: - if not all(name in ['Project_name', 'Branch_name', \ - 'OBS_project', 'OBS_staging_project', 'OBS_package'] for name in item): - continue - prj_name = item['Project_name'] - _branch = item['Branch_name'] - _obs_prj = item['OBS_project'] - _stg = item['OBS_staging_project'] - _pkg = item['OBS_package'] - _brch = '' - for _br in _branch.split(','): - if _brch == '' or len(_br) < len(_brch): - _brch = _br - single_list = {'git_path': prj_name, \ - 'branch_name': _brch, \ - 'obs_project': _obs_prj, \ - 'staging_project': _stg, \ - 'package': _pkg} - self.package_commit_list[_pkg] = single_list - continue - - def set_commit_message(self, git_path, msg): - - for x in self.package_commit_list: - single_list = self.package_commit_list.get(x) - if single_list['git_path'] == git_path: - single_list['commit_message'] = msg - self.package_commit_list[x] = single_list - return - - def set_commit_ids(self, full_commit_list): - - print '\nLIST FOR BUILD WITH COMMIT ID:' - for x in self.package_commit_list: - single_list = self.package_commit_list.get(x) - single_list['commitid'] = full_commit_list.get(single_list['git_path']) - self.package_commit_list[x] = single_list - print self.package_commit_list.get(x) - print '\n' - - def list_todo_packages(self): - - self.get_mapping_list() - full_commit_list = {} - #ABS will find accepted sr tags. Do not need to parse commit ids here! - self.set_commit_ids(full_commit_list) - - def retrieve_last_accepted_tag(self, mygit=None, gitpath=None, profile=None, obs_project=None): - # full build will use the most recent accepted tag. - t_outs, t_err, t_code = mygit._git_inout('for-each-ref', \ - ['--sort=-taggerdate', '--format=%(refname)%0a%(*subject)%0a%(*body)', \ - 'refs/tags/accepted/%s/' % ('/'.join(obs_project.split(':')).lower()), '--count=1']) - if len(t_outs) == 0 or '/%s/' % profile not in t_outs.split('\n')[0]: - print 'Sorry. Most recent accepted tag %s is not desired one.' % t_outs.split('\n')[0] - return None - accepted_tag = t_outs.split('\n')[0].replace('refs/tags/','').rstrip() - orig_tag = t_outs[t_outs.rfind('Reference: ')+11:].split('\n')[0].split(' ')[0].strip().rstrip() - if orig_tag.startswith('submit/tizen'): - print 'FULL BUILD! Accepted tag from _tpk branch %s -> %s' % (accepted_tag, orig_tag) - return orig_tag - - t_outs, t_err, t_code = mygit._git_inout('show', \ - [accepted_tag]) - if len(t_outs) == 0 or '- Git project: %s' % gitpath not in t_outs: - print 'Sorry(1). Fail to retrieve original tag from %s' % accepted_tag - return None - orig_tag = t_outs[t_outs.rfind('- Tag:')+6:].split('\n')[0].strip().rstrip() - if orig_tag.startswith('submit/tizen'): - print 'FULL BUILD! Accepted tag from source branch %s -> %s' % (accepted_tag, orig_tag) - return orig_tag - - print 'Sorry(2). Fail to retrieve original tag from %s' % accepted_tag - return None - - def initiate_submit_request(self): - - print 'initiate_submit_request' - tag_name = re.search(r'(submit/tizen.*/[0-9]+.[0-9]+)', self.tag_name) - if tag_name is not None: - self.tag_name = tag_name.groups()[0] - else: - date_str = str(datetime.now()) - tag_name_t = 'submit/' + self.project.replace(':', '_').lower() + '/' \ - + date_str.split(' ')[0].replace('-', '') \ - + '.' + date_str.split(' ')[1].split('.')[0].replace(':', '') - self.tag_name = tag_name_t - print '\n==============\nTag to push : %s\n==============' % self.tag_name - - # Fetch first - for x in self.package_commit_list: - pkg_info = self.package_commit_list.get(x) - gerrit_project = pkg_info['git_path'] - print '\n\n********* %s ********' % os.path.basename(gerrit_project).upper() - prjdir = os.path.join(self.workspace, os.path.basename(gerrit_project)) - if not clone_gitproject(gerrit_project, prjdir, \ - git_cache_dir=self.gerrit_if.gitcache, \ - gerrit_hostname = self.gerrit_if.hostname, \ - gerrit_username = self.gerrit_if.username, \ - gerrit_sshport = self.gerrit_if.sshport): - raise LocalError('Error cloning project %s' % (gerrit_project)) - mygit = Git(prjdir) - #TODO: Find accepted tag. - sr_tag = self.retrieve_last_accepted_tag(mygit, gerrit_project, \ - pkg_info['obs_project'].split(':')[-1].lower(), \ - pkg_info['obs_project']) - pkg_info['commitid'] = sr_tag - mygit.checkout(pkg_info['commitid']) - t_outs, t_err, t_code = mygit._git_inout('log', ['--format=%B', '-n 1']) - #commit_message = t_outs[:t_outs.rfind("\nChange-Id: ")] - #commit_message = '[ABS] %s\nOriginal Commit:%s' % (commit_message, pkg_info['commitid']) - self.set_commit_message(gerrit_project, self.commit_message) - sys.stdout.flush() - - print '\n\n%s' % self.package_commit_list - return - - # Push tag - for x in self.package_commit_list: - gerrit_project = self.package_commit_list.get(x)['git_path'] - print '\nPushing tag for %s started at %s' % (gerrit_project, str(datetime.now())) - prjdir = os.path.join(self.workspace, os.path.basename(gerrit_project)) - mygit = Git(prjdir) - mygit.create_tag(self.tag_name, \ - msg=self.package_commit_list.get(x)['commit_message'], - commit=self.package_commit_list.get(x)['commitid']) - #mygit.push_tag(self.gerrit_if.remote + gerrit_project, sr_tag) - print 'Pushing tag for %s finished at %s' % (gerrit_project, str(datetime.now())) - - print "\"TitleBatch\": \"full_sr(%s)\"" % (self.version) - - def build_all_together(self): - - print 'build_all_together()' - - index = 1 - for x in self.package_commit_list: - single_dict = self.package_commit_list.get(x) - trigger_data = { -# "rs_url": None, - "obs_project": single_dict['obs_project'], - "obs_package": single_dict['package'], - "source": - {"package": single_dict['git_path'].split('/')[-1], - "branch": single_dict['branch_name'], - "git_path": single_dict['git_path'], - "tag": single_dict['commitid']}, - "full_build": self.version, - "event": { -# "GERRIT_REFNAME": None, - "GERRIT_EVENT_ACCOUNT_EMAIL": "abs.robot@tizen.do.not.reply", -# "GERRIT_CHANGE_NUMBER": None, - "GERRIT_EVENT_ACCOUNT_NAME": "abs-robot", -# "GERRIT_REFSPEC": None, - "GERRIT_PROJECT": single_dict['git_path'], -# "GERRIT_PATCHSET_REVISION": None, -# "GERRIT_BRANCH": None, - "GERRIT_NEWREV": single_dict['commitid']}, - "index": str(index)} - - if trigger_data['source']['tag'] is None: - print 'Cannot find mapping for %s - %s' % (trigger_data['source']['git_path'], trigger_data['obs_project']) - print 'ABS build for %s initiated.' % (trigger_data['source']['package']) - trigger_next('build_class_all_together_%d_%d_%s' \ - % (int(os.getenv('BUILD_NUMBER')), index, trigger_data['source']['package']), \ - trigger_data) - index += 1 - - print "\"TitleBatch\": \"full_build(%s)\"" % (self.version) - - def main(self): - - print('-----[JOB STARTED: Initiate_build_all]-----') - self.list_todo_packages() - #TODO: Enable initiate_submit_request() for pre-release process - if self.b_prerelease: - self.initiate_submit_request() - else: - self.build_all_together() - -class Jenkins(object): - - jenkinsUrl = '' - jobName = '' - cred = None - - def __init__(self, jobName=None): - - jenkinsUrl = os.getenv('JOB_URL').replace('/'+os.getenv('JOB_NAME'), '') - auth = {'username': os.getenv('ABS_JENKINS_USER'), 'password': os.getenv('ABS_JENKINS_PASS')} - self.jenkinsUrl = jenkinsUrl.replace('http://', 'http://%s:%s@' \ - % (auth['username'], auth['password'])) - self.cred = {'url': os.getenv('JENKINS_URL'), \ - 'username': auth['username'], \ - 'password': auth['password']} - if jobName != None: - self.jobName = jobName - else: - self.jobName = os.getenv('ABS_BUILDER_JOB_NAME') - -class GatherBuildResult(Jenkins): - - buildJob = None - buildNumber = None - result = '' - sr_tag = 'sr_tag' - - def __init__(self, buildJob=None, buildNumber=None, raw_data=None): - - super(GatherBuildResult, self).__init__() - - self.buildJob = buildJob - self.buildNumber = buildNumber - if self.buildNumber == None: - self.buildNumber = 'lastBuild' - if self.buildJob == None: - self.buildJob = self.jobName - self.get_build_data(raw_data) - - def get_build_data(self, raw_data=None): - - if raw_data is None: - build_data = get_jenkins_build_data(job=self.buildJob, \ - build_num=self.buildNumber, cred=self.cred) - else: - build_data = raw_data - - self.result = build_data['result'] - self.duration = time.strftime('%Mmin %Ssec', time.gmtime(float(build_data['duration']) / 1000.0)) - self.timestamp = time.strftime('%Y%m%d.%H%M%S', time.gmtime(float(build_data['timestamp']) / 1000.0)) - self.number = build_data['number'] - self.git_size_from = '0' - self.git_size_to = '0' - for action in build_data['actions']: - if 'parameters' in action: - for param in action['parameters']: - if 'name' in param and 'value' in param and param['name'] == 'TRIGGER_INFO': - myvalue = param['value'] - self.trigger_info = ast.literal_eval( \ - base64.b64decode(myvalue).replace('null', '\"none\"')) - if 'text' in action: - if 'submit' in action['text']: - self.sr_tag = '/'.join(action['text'].split('/')[2:]) - else: - self.git_size_from = action['text'].split('/')[0] - self.git_size_to = action['text'].split('/')[1] - - def get_sr_tag(self): - return self.sr_tag - - def get_build_number(self): - return str(self.number) - - def get_build_result(self): - return self.result - - def get_build_duration(self): - return self.duration.replace('00min ', '') - - def get_build_timestamp(self): - return self.timestamp - - def get_obs_project(self): - return self.trigger_info['obs_project'] - - def get_profile(self): - return self.get_obs_project().split(':')[-1].lower() - - def get_branch_name(self): - return self.trigger_info['source']['branch'] - - def get_parameter_with(self, parm1, parm2=None): - if parm1 not in self.trigger_info or \ - parm2 is not None and parm2 not in self.trigger_info[parm1]: - return '-' - if parm2 is None: - return self.trigger_info[parm1] - else: - return self.trigger_info[parm1][parm2] - - def get_full_build(self): - return self.get_parameter_with('full_build') - - def get_package_name(self): - return self.get_parameter_with('source', 'package') - - def get_git_path(self): - return self.get_parameter_with('event', 'GERRIT_PROJECT') - - def get_account_name(self): - return self.get_parameter_with('event', 'GERRIT_EVENT_ACCOUNT_NAME') - - def get_account_email(self): - return self.get_parameter_with('event', 'GERRIT_EVENT_ACCOUNT_EMAIL') - - def get_revision(self): - return self.get_parameter_with('event', 'GERRIT_NEWREV') - - def get_patchset_revision(self): - return self.get_parameter_with('event', 'GERRIT_PATCHSET_REVISION') - - def get_refname(self): - return self.get_parameter_with('event', 'GERRIT_REFNAME') - - def get_refspec(self): - return self.get_parameter_with('event', 'GERRIT_REFSPEC') - - def get_build_cause(self): - if self.get_refname() != '-' and self.get_refname().startswith('refs/tags/submit/'): - return 'submit' - elif self.get_refname() != '-' and self.get_refname().startswith('refs/tags/'): - return 'tag' - elif self.get_refspec() != '-' and self.get_refspec().startswith('refs/changes/'): - return 'review' - else: - return 'commit' - - def get_reference_name(self): - build_cause = self.get_build_cause() - if build_cause == 'submit' or build_cause == 'tag': - return self.get_refname(), self.get_revision() - elif build_cause == 'review': - return self.get_refspec(), self.get_patchset_revision() - elif build_cause == 'commit': - return self.get_refname(), self.get_revision() - else: - return None, None - - def get_git_size(self): - return self.git_size_from + ',' + self.git_size_to - -class ReportFullBuildResult(object): - - list_builds = [] - email_body = [] - profile = '' #TODO: Not a entire value but one of them - full_build_cause = None - - def main(self): - - marker = {'SUCCESS': 'O', 'FAILURE': 'X', 'ABORTED': 'X'} - - print '\nTotal number = %s' % (os.getenv('RESULT_PARSE')) - for i in range(0, int(os.getenv('RESULT_PARSE'))): - my_build_job = os.getenv('ABS_BUILD_NUMBER_%d' % (i)).split('#')[0].strip() - my_build_id = os.getenv('ABS_BUILD_NUMBER_%d' % (i)).split('#')[-1].rstrip() - x = GatherBuildResult(buildJob=my_build_job, buildNumber=my_build_id) - self.list_builds.append(x) - self.profile = x.get_profile() - self.branch_name = x.get_branch_name() - if self.full_build_cause is None: - self.full_build_cause = x.get_full_build() - print 'Full build cause: %s\n' % self.full_build_cause - self.email_body.append('Full build cause: %s\n' % self.full_build_cause) - if x.get_build_result() != 'SUCCESS': - self.email_body.append('%s %s (%s)' \ - % (marker[x.get_build_result()], x.get_package_name(), x.get_sr_tag())) - print ' %s >> %s' % (os.getenv('ABS_BUILD_NUMBER_%d' %(i)), - self.email_body[-1].rstrip()) - - fail_count = 0 - for x in self.list_builds: - if x.get_build_result() != 'SUCCESS': - fail_count += 1 - path = os.path.join(x.jenkinsUrl, x.buildJob, x.buildNumber, 'consoleText') - log_title = '\n[[ %s ]]\n' \ - % x.get_package_name().upper() - print log_title - full_text = unicode_to_str(requests.get(path).text) - - exp = r'/home/build/tizen-sdk-cli/tools/smart-build-interface/../../platforms/[a-zA-Z-0-9./]{1,}/rootstraps/[a-zA-Z]{1,}-[0-9.]{1,}-(emulator|device).core.private.[0-9_]{1,}' - full_text = re.sub(exp, "{PRIVATE_ROOTSTRAP} ", full_text) - - # No accepted tags so far - idx = full_text.rfind('Full build cannot be proceed. No matching SR tag') - if idx != -1: - full_text = 'No accepted tpk tags for this package' - fail_count -= 1 - - # No rootstrap found - idx = full_text.rfind("+ rs_list='") - if idx == -1: - full_text = 'System admin need to check rootstrap generation failure.' - - #If Exception string found - idx = full_text.rfind('Exception ') - idx2 = -1 - if idx != -1: - idx2 = full_text[idx:].find('\n') - if idx != -1 and idx2 != -1: - full_text = full_text[idx:idx+idx2] - else: - for _str in ['PLATFORM_VER\t', 'Finished build-native', 'Automatically generated by backend service.', \ - 'Build step \'Execute shell\' marked build as', 'compilation terminated.']: - idx = full_text.rfind(_str) - if idx != -1: - full_text = full_text[:idx] - - # Trim the log - full_text = full_text[len(full_text)-384:] - print '%s' % full_text - self.email_body.append(log_title + '\n' + full_text) - - send_mail(' FULL BUILD (%s) : %d error' % (self.full_build_cause, fail_count), '\n'.join(self.email_body)) - - #if fail_count != 0: - # self.full_build_cause = self.full_build_cause + '-FAIL' - Trigger_for_abs_update(self.full_build_cause, silent=True).main() - - print "\n\n\"TitleBatch\": \"Result\"" - -class SummaryReport(Jenkins): - - span_start = 0 - span_emd = 0 - - def __init__(self, span=None): - - super(SummaryReport, self).__init__() - if span is None: span = os.getenv('REPORT_SPAN') - self._parse_date(span) - self.report_file = os.path.join('/'.join(os.getenv('ABS_VM_IMAGE').split('/')[:-1]), 'abs_history.report') - print "\n\n================\n==== BUILD HISTORY ====\n================\n" - - def _parse_date(self, date=None): - - if date is None: - date = '2D' - - today = time.strftime('%Y%m%d', time.gmtime()) - print 'Today: %s' % today - print 'Now: %s' % str(datetime.now()) - - self.current_date = datetime.now() -# self.span_end = self.span_start = current_date.strftime('%Y%m%d') - span_end = span_start = self.current_date - - if 'all' in date.lower(): - span_end = span_start = None - elif re.search(r'([0-9]+)[D]', date): - delta = re.search(r'([0-9]+)[D]', date).groups()[0] - span_start = self.current_date - timedelta(days=int(delta)-1) - elif re.search(r'([0-9]+)[W]', date): - delta = re.search(r'([0-9]+)[W]', date).groups()[0] - span_start = self.current_date - timedelta(days=int(delta)*7-1) - elif re.search(r'([0-9]+)[M]', date): - delta = re.search(r'([0-9]+)[M]', date).groups()[0] - span_start = self.current_date - timedelta(days=int(delta)*30-1) - elif re.search(r'([0-9]+)[Y]', date): - delta = re.search(r'([0-9]+)[Y]', date).groups()[0] - span_start = self.current_date - timedelta(days=int(delta)*365-1) - elif re.search(r'(\d{8})-(\d{8})', date): - span_start, span_end = re.search(r'(\d{8})-(\d{8})', date).groups() - span_start = datetime.strptime(span_start, '%Y%m%d') - span_end = datetime.strptime(span_end, '%Y%m%d') - - if span_end is None or span_start is None: - self.span_start = self.span_end = None - else: - self.span_start = span_start.strftime('%Y%m%d') - self.span_end = span_end.strftime('%Y%m%d') - - print 'start:%s, end:%s' % (self.span_start, self.span_end) - - def generate_log(self): - - # No Date Result Git Name Email Tag/Branch Revision Elapsed - old_history = [] - self.new_history = [] - - if os.path.isfile(self.report_file): - with open(self.report_file, 'r') as rf: - for x in rf.readlines(): - old_history.append(x.rstrip().split(',')) - - if len(old_history) > 0: - last_history = old_history[0][0] - else: - last_history = '0' - - # No Date Result Git Name Email Tag/Branch Revision Elapsed - - build_data = get_jenkins_build_data(job=self.jobName, \ - build_num=None, cred=self.cred) - sys.stdout.flush() - - for build in build_data: - number = build['number'] - duration = build['duration'] - timestamp = build['timestamp'] - this_date = time.strftime('%Y%m%d', time.gmtime(float(timestamp) / 1000.0)) - - if (self.span_end == None and self.span_start == None) or \ - (self.span_end >= this_date and self.span_start <= this_date): - - if int(last_history) >= int(number): - break - - x = GatherBuildResult(buildJob=None, buildNumber=None, raw_data=build) - if x.get_build_result() is None: - continue - if 'tizen' in x.get_full_build(): - continue - if 'jenkins' in x.get_account_name().lower(): - continue - - myhistory = [] - myhistory.append(x.get_build_number()) - myhistory.append(x.get_build_timestamp()) - myhistory.append(x.get_build_result()) - myhistory.append(x.get_git_path()) - myhistory.append(x.get_account_name()) - myhistory.append(x.get_account_email()) - myhistory.append(x.get_reference_name()[0]) - myhistory.append(x.get_reference_name()[1]) - myhistory.append(x.get_build_duration()) - myhistory.append(x.get_git_size()) - self.new_history.append(myhistory) - print '**********************************' - print myhistory - - with open(self.report_file, 'w') as newf: - for x in old_history: - self.new_history.append(x) - for x in self.new_history: - newf.write(','.join(x) + '\n') - - def report_log(self): - - self._history = [] - - if os.path.isfile(self.report_file): - with open(self.report_file, 'r') as rf: - for x in rf.readlines(): - self._history.append(x.rstrip().split(',')) - - for build in self._history: - # No Date Result Git Name Email Tag/Branch Revision Elapsed - _no, _da, _rt, _gt, _nm, _ml, _tg, _rv, _tm, _sz1, _sz2 = build - this_date = _da.split('.')[0] - if (self.span_end == None and self.span_start == None) or \ - (self.span_end >= this_date and self.span_start <= this_date): - print ' ' - print '[%s] >> %s' % (_gt, _rt) - print ' %s(%s)' % (_nm, _ml) - print ' %s @ %s' % (_tg, _rv) - print ' %s #%s (%s)' % (_da, _no, _tm) - - def generate_html_report(self): - - with open(os.path.join(os.getenv('WORKSPACE'), 'jenkins-scripts', \ - 'abs', 'report_template'), 'r') as tf: - js_template = tf.read().replace('%', '%%').replace('RAW_DATA_FROM_BACKEND', '%s'); - - raw_data = [] - with open(self.report_file, 'r') as rf: - for x in rf.readlines(): - _no, _dt, _rs, _pk, _nm, _em, _tg, _rv, _ep, _si, _so = x.split(','); - raw_data.append( - {'num': _no, - 'date': _dt, - 'result': _rs, - 'package': _pk, - 'name': _nm, - 'email': _em, - 'tag': _tg, - 'rev': _rv, - 'elapsed': _ep, - 'sizein': _si, - 'sizeout': _so}); - - html_path = os.path.join(os.getenv('WORKSPACE'), 'html') - html_file = 'index.html' - os.makedirs(html_path) - - html_contents = js_template % (raw_data) - - with open(os.path.join(html_path, html_file), "w") as w_file: - w_file.write(html_contents) - - def main(self): - - self.generate_log() - #self.report_log() - self.generate_html_report() - print "\"TitleBatch\": \"Report\"" - -if __name__ == '__main__': - - if True: - if os.getenv('REPORT_SPAN'): - trigger = SummaryReport() - sys.exit(trigger.main()) - - # Full build result parsing - elif os.getenv('RESULT_PARSE'): - trigger = ReportFullBuildResult() - ret = trigger.main() - trigger = SummaryReport(span='All') - trigger.main() - sys.exit(ret) - - elif os.getenv('TRIGGER_INFO'): # Full build - trigger = Initiate_build_all() - sys.exit(trigger.main()) - elif os.getenv('NEW_ROOTSTRAP'): # ABS VM update - trigger = Trigger_for_abs_update(new_rs=os.getenv('NEW_ROOTSTRAP')) - sys.exit(trigger.main()) - else: #Default - print 'INVALID OPTION...' - sys.exit(-1) - diff --git a/abs/job_abs_build.py b/abs/job_abs_build.py new file mode 100644 index 0000000..530f82c --- /dev/null +++ b/abs/job_abs_build.py @@ -0,0 +1,770 @@ +#!/usr/bin/env python +# +# Copyright (c) 2014, 2015, 2016 Samsung Electronics.Co.Ltd. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation; version 2 of the License +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# for more details. +# +"""This job is triggered by gerrit submit ref update event for preloaded app. + It will generate tizen app package(.tpk) and requests prerelease routine. +""" + +import os +import sys +import shutil +import re +import glob +import zipfile +import ast + +from datetime import datetime +from random import randint +from time import sleep +import xml.etree.cElementTree as ElementTree + +sys.path.insert(1, os.path.join(sys.path[0], '..')) + +from common.utils import parse_link +from common.mapping import git_obs_map +from common.git import Git, clone_gitproject +from common.gerrit import Gerrit, is_ref_deleted, GerritEnv +from common.buildservice import BuildService +from common.prerelease import get_prerelease_project_name +from common.buildtrigger import trigger_info, trigger_next +from common.send_mail import prepare_mail +from common.workflow import find_specfile, parse_specfile +from common.tizen_studio import TizenStudio + +# set default char-set endcoding to utf-8 +reload(sys) +sys.setdefaultencoding('utf-8') # pylint: disable-msg=E1101 + +class LocalError(Exception): + """Local error exception.""" + pass + +class MailSender(object): + + email_title = '[ABS]' + email_head = 'App Build System(ABS) inform you the SDK build result.\n' \ + '(ABS will trigger build if you issues an SR on preloaded app branch)\n\n' + email_body = '' + email_footer = '\n\n--------\n'\ + 'Automatically generated by backend service.\n' + + email_to = [] + + def __init__(self, receiver=None, title=None, body=None): + if receiver is not None: self.email_to = receiver + if title is not None: self.email_title = title + if body is not None: self.email_body = body + + def add_receiver(self, add_new): + if type(add_new) == list: + self.email_to.extend(x for x in add_new) + elif type(add_new) == str: + self.email_to.append(add_new) + else: + print 'TYPE(%s) == %s' % (add_new, type(add_new)) + + def add_title(self, add_new): + self.email_title = self.email_title + ' ' + add_new + + def add_message(self, add_new, top=None): + if top is not None: + self.email_body = add_new + '\n' + self.email_body + else: + self.email_body = self.email_body + '\n' + add_new + + def add_maintainers(self, project): + mygerrit = Gerrit(GerritEnv().hostname, GerritEnv().username,\ + port=int(GerritEnv().sshport)) + grps = mygerrit.ls_groups(['--project %s' % project]) + dest_grp = [s for s in grps if " - Maintainers" in s] + for dg in dest_grp: + mbrs = mygerrit.ls_members(['\'\"%s\"\'' % dg, '--recursive']) + for line in mbrs: + self.add_receiver(line.split('\t')[3]) + + def send_mail(self): + + if 'SUCCESS' not in self.email_title: + self.add_title('FAILURE') + + self.email_body = self.email_head + self.email_body + self.email_footer + + self.email_to = [x for x in self.email_to if x != 'n/a'] + print '\n\n' + print self.email_title + m_body = '' + for m in self.email_body.splitlines(): + m_body += '\n'.join(m[i:i+128] for i in xrange(0, len(m), 128)) + '\n' + self.email_body = m_body + print self.email_body + print self.email_to + + prepare_mail("%s.env" % os.getenv('BUILD_TAG'), \ + self.email_title, \ + self.email_body, \ + os.getenv('NOREPLY_EMAIL_SENDER'), \ + self.email_to) + + if 'SUCCESS' not in self.email_title: + return 4 + +def set_gerrit_event(fields): + """ Merge all the required input arguments """ + + event = fields.get('event') + source = fields.get('source', None) + + # FROM GERRIT EVENT + _data = {'obs_project': fields.get('obs_project'), \ + 'obs_package': fields.get('obs_package'), \ + 'project': event.get('GERRIT_PROJECT'), \ + 'package': os.path.basename(event.get('GERRIT_PROJECT')), \ + 'email': event.get('GERRIT_EVENT_ACCOUNT_EMAIL'), \ + 'name': event.get('GERRIT_EVENT_ACCOUNT_NAME'), \ + 'newrev': event.get('GERRIT_NEWREV', None), \ + 'refname': event.get('GERRIT_REFNAME', None), \ + 'changenumber': event.get('GERRIT_CHANGE_NUMBER', None), \ + 'refspec': event.get('GERRIT_REFSPEC', None), \ + 'patchset': event.get('GERRIT_PATCHSET_REVISION', None), \ + 'type': fields.get('type'), \ + 'tag': fields.get('tag'), \ + 'branch': fields.get('branch'), \ + 'revision': fields.get('revision'), \ + 'buildcheck': fields.get('full_build', None)} + _data['profile'] = _data.get('obs_project').split(':')[-1].lower() + + # Set new branch and tag + if _data.get('branch').endswith('_' + _data.get('profile')): + _data['new_branch'] = _data.get('branch') + '_tpk' + else: + _data['new_branch'] = _data.get('branch') + '_' + _data.get('profile') + '_tpk' + _data['new_tag'] = _data.get('tag').replace(_data.get('branch'), _data.get('new_branch')) + + print 'TOUCHED_DATA:' + for f in _data: + print ' [%s]: %s' % (f, _data[f]) + return _data + +def retrieve_last_accepted_tag(working_git, project, profile=None, obs_project=None): + """ Find most recent accepted tag in case of build check mode""" + + # full build will use the most recent accepted tag. + if obs_project == 'Tizen:Unified': + check_branch = 'tizen' + else: + check_branch = ('/'.join(obs_project.split(':')).lower()) + t_outs, t_err, t_code = working_git._git_inout('for-each-ref', \ + ['--sort=-taggerdate', '--format=%(refname)%0a%(*subject)%0a%(*body)', \ + 'refs/tags/accepted/%s/' % check_branch, '--count=1']) + #if len(t_outs) == 0 or '/%s/' % profile not in t_outs.split('\n')[0]: + if len(t_outs) == 0: + print 'Sorry. Most recent accepted tag %s is not desired one.' % t_outs.split('\n')[0] + return None + accepted_tag = t_outs.split('\n')[0].replace('refs/tags/','').rstrip() + orig_tag = t_outs[t_outs.rfind('Reference: ')+11:].split('\n')[0].split(' ')[0].strip().rstrip() + if orig_tag.startswith('submit/tizen'): + print 'FULL BUILD! Accepted tag from _tpk branch %s -> %s' % (accepted_tag, orig_tag) + return orig_tag + + t_outs, t_err, t_code = working_git._git_inout('show', \ + [accepted_tag]) + if len(t_outs) == 0 or '- Git project: %s' % project not in t_outs: + print 'Sorry(1). Fail to retrieve original tag from %s' % accepted_tag + return None + orig_tag = t_outs[t_outs.rfind('- Tag:')+6:].split('\n')[0].strip().rstrip() + if orig_tag.startswith('submit/tizen'): + print 'FULL BUILD! Accepted tag from source branch %s -> %s' % (accepted_tag, orig_tag) + return orig_tag + + print 'Sorry(2). Fail to retrieve original tag from %s' % accepted_tag + return None + +def retrieve_project_property(src_git, tag): + """ Search APP properties """ + + ret_prop = {} + + def retrieve_license(src_root): + sdk_license = '' + license_file = os.path.join(src_root, 'LICENSE') + if os.path.isfile(license_file): + shutil.copy(license_file, os.path.dirname(src_root)) + with open(license_file, 'r') as r_file: + for x in r_file.readlines(): + matchline = re.search(r'.*Licensed under the (.*) License, Version (.*) \(the \"License\"\).*;', x) + if matchline: + sdk_license = '%s-%s' % (matchline.groups()[0], matchline.groups()[1]) + print 'SDK_LICENSE: [%s]' % sdk_license + return sdk_license + + def retrieve_properties_from_project_def(src_root): + + # Single project + def_file = os.path.join(src_root, 'tizen-manifest.xml') + # Multi project + multiproject_list_f = os.path.join(src_root, 'WORKSPACE') + if os.path.isfile(multiproject_list_f): + with open(multiproject_list_f, 'r') as r_file: + for x in r_file.readlines(): + if os.path.isdir(os.path.join(src_root, x.rstrip())) and \ + os.path.isfile(os.path.join(src_root, x.rstrip(), 'tizen-manifest.xml')): + def_file = os.path.join(src_root, x.rstrip(), 'tizen-manifest.xml') + + print 'Property file: %s' % def_file + if os.path.isfile(def_file): + root = ElementTree.ElementTree(file=def_file).getroot() + sdk_package = root.attrib['package'] + sdk_version = root.attrib['version'] + else: + raise LocalError('Property file [%s] not found!' % def_file) + print 'TIZEN-MANIFEST.XML (%s) (%s)' % (sdk_package, sdk_version) + return {'sdk_package': sdk_package, 'sdk_version': sdk_version} + + def retrieve_commit_id(src_git): + commit_id = src_git.rev_parse('HEAD') + print 'commit from (HEAD) = %s' % (commit_id) + return commit_id + + def retrieve_revision_number(src_git, tag): + _ret = {} + try: + _ret['revision_number'] = src_git.rev_parse(tag) + except Exception as err: + print repr(err) + print 'revision from (%s) = %s' % (tag, _ret) + return _ret + + def retrieve_commit_message(src_git): + t_outs, t_err, t_code = src_git._git_inout('log', ['--format=%B', '-n 1']) + commit_message = t_outs[:t_outs.rfind("\nChange-Id: ")].rstrip() + '\n' + print "author comment : %s" % commit_message + return commit_message + + def retrieve_sr_message(src_git, tag): + t_outs, t_err, t_code = src_git._git_inout('show', [tag, '--format=##chkstr:%b']) + sr_message = '\n'.join(t_outs[:t_outs.rfind("##chkstr:")].split('\n')[3:]) + print "sr comment from(%s) : [%s]" % (tag, sr_message.rstrip()) + return sr_message + + ret_prop.update(retrieve_properties_from_project_def(src_git.path)) + ret_prop['commit_id'] = retrieve_commit_id(src_git) + ret_prop['revision_number'] = retrieve_revision_number(src_git, tag).get('revision_number', \ + ret_prop.get('commit_id')) + ret_prop['commit_message'] = retrieve_commit_message(src_git) + ret_prop['sr_message'] = retrieve_sr_message(src_git, tag) + ret_prop['sdk_license'] = retrieve_license(src_git.path) + + for r in ret_prop: + print ' [%s]: %s' % (r, ret_prop[r].rstrip()) + return ret_prop + +def fetch_source(gerrit_env, src_root, project, branch=None, tag=None, patchset=False, \ + build_check=None, profile=None, obs_project=None): + """ Cloning APP source code """ + + print 'Cloning into bare repo %s' % os.path.join(gerrit_env.gitcache, project) + clone_gitproject(project, \ + '%s.git' % os.path.join(gerrit_env.gitcache, project), \ + gerrit_hostname=gerrit_env.hostname, \ + gerrit_username=gerrit_env.username, \ + gerrit_sshport=gerrit_env.sshport, \ + bare=True) + print 'Cloning clone %s' % src_root + if not clone_gitproject(project, \ + src_root, \ + gerrit_hostname=gerrit_env.hostname, \ + gerrit_username=gerrit_env.username, \ + gerrit_sshport=gerrit_env.sshport): + raise LocalError('Error cloning project %s' % project) + + working_git = Git(src_root) + + if patchset: + if tag and tag.startswith('refs/changes/'): + working_git._git_inout('fetch', ['origin', '%s' % tag]) + working_git._git_inout('reset', ['--hard', 'FETCH_HEAD']) + return working_git, tag + else: + raise LocalError('Patchset but no tag') + + if build_check is not None: + tag = retrieve_last_accepted_tag(working_git, project, profile=profile, obs_project=obs_project) + print 'Force to checkout build check for [%s]' % tag + + if tag is None: + working_git.checkout(branch) + else: + working_git.checkout(tag) + + assert tag and tag.startswith('submit/tizen') + return working_git, tag + +def zipping_workspace(src_root): + """ Create tarball to share source code easily between VM machine """ + + def ZipDir(inputDir, outputZip): + '''Zip up a directory and preserve symlinks and empty directories''' + zipOut = zipfile.ZipFile(outputZip, 'w', compression=zipfile.ZIP_DEFLATED) + + rootLen = len(os.path.dirname(inputDir)) + def _ArchiveDirectory(parentDirectory): + contents = os.listdir(parentDirectory) + #store empty directories + if not contents: + archiveRoot = parentDirectory[rootLen:].replace('\\', '/').lstrip('/') + zipInfo = zipfile.ZipInfo(archiveRoot+'/') + zipOut.writestr(zipInfo, '') + for item in contents: + fullPath = os.path.join(parentDirectory, item) + if fullPath.endswith('/.git'): continue + if os.path.isdir(fullPath) and not os.path.islink(fullPath): + _ArchiveDirectory(fullPath) + else: + archiveRoot = fullPath[rootLen:].replace('\\', '/').lstrip('/') + if os.path.islink(fullPath): + zipInfo = zipfile.ZipInfo(archiveRoot) + zipInfo.create_system = 3 + # long type of hex val of '0xA1ED0000L', + # say, symlink attr magic... + zipInfo.external_attr = 2716663808L + zipOut.writestr(zipInfo, os.readlink(fullPath)) + else: + zipOut.write(fullPath, archiveRoot, zipfile.ZIP_DEFLATED) + _ArchiveDirectory(inputDir) + zipOut.close() + + current_dir = os.getcwd() + os.chdir(os.path.dirname(src_root)) + src_d = os.path.basename(src_root) + out_f = src_d + '.zip' + ZipDir(src_d, out_f) + os.chdir(current_dir) + +def generate_spec_file(tizen_studio, src_git, prop, data): + """ Use spec file template """ + + found_archs = [] + archs_lookup = [['i386', 'i486', 'i586', 'i686', 'x86'], \ + ['arm', 'armv7l'], \ + ['x86_64'], \ + ['aarch64']] + + for fn in os.listdir(tizen_studio.builddir): + mtch = re.search(r'(.*)-([0-9.]+)-(.*)(? 0: + try: + src_git.push(repo = 'origin', src = '%s' % new_branch, force=True) + break + except Exception as err: + print 'git push exception: %s' % repr(err) + push_err_msg = push_err_msg + '\n' + str(err) + push_retry -= 1 + sleep(5) + if not push_retry: + mailer.add_title('Git push tpk branch failed') + mailer.add_message('Result: FAIL') + mailer.add_message('\n\nGit push branch %s failed %s' % (new_branch, push_err_msg)) + return + + if data.get('gerrit_infra', '').startswith('public_gerrit'): + gerrit_env = GerritEnv('PUBLIC_') + else: + gerrit_env = GerritEnv('') + remote = 'ssh://%s@%s:%d/%s' % (gerrit_env.username, \ + gerrit_env.hostname, \ + int(gerrit_env.sshport), \ + data.get('project')) + + message = '%s\n' \ + '[ABS] Ready.\n' \ + '- Original Tag: %s (%s)\n' \ + '- Original Commit: %s\n' \ + '- Requested By: %s\n' \ + % (prop.get('sr_message'), data.get('tag'), \ + prop.get('revision_number'), prop.get('commit_id'), data.get('email')) + + try: + src_git.create_tag(name=new_tag, msg=message, commit=commit_id_new) + except Exception as err: + mailer.add_title('Creating tag failed') + mailer.add_message('Result: FAIL') + mailer.add_message('\n\nCreating tag %s failed: %s' % (new_tag, str(err))) + return + + try: + revision_number_new = src_git.self.rev_parse(new_tag) + except: + revision_number_new = prop.get('commit_id') + + push_err_msg = '' + push_retry = 5 + while push_retry >0: + try: + src_git.push_tag(remote, new_tag) + break + except Exception as err: + print 'Cannot push tag %s (%s)' % (new_tag, str(err)) + push_err_msg = push_err_msg + '\n' + str(err) + push_retry -= 1 + sleep(5) + if not push_retry: + mailer.add_title('Git push tag failed') + mailer.add_message('Result: FAIL') + mailer.add_message('\n\nGit push tag %s failed: %s' % (new_tag, str(err))) + return + + mailer.add_title('SUCCESS') + mailer.add_message('To view, visit %s/project/show/%s' \ + % (build.apiurl, \ + get_prerelease_project_name(data.get('obs_project'), data.get('new_tag')))) + +def start_build(fields): + """ Running build using Tizen Studio """ + + # Adjust arguments + data = set_gerrit_event(fields) + + if fields.get('gerrit_infra').startswith('public_gerrit'): + gerrit_env = GerritEnv('PUBLIC_') + build = BuildService(os.getenv('PUBLIC_OBS_API_URL'), \ + os.getenv('PUBLIC_OBS_API_USERNAME'), \ + os.getenv('PUBLIC_OBS_API_PASSWD')) + else: + gerrit_env = GerritEnv('') + build = BuildService(os.getenv('OBS_API_URL'), \ + os.getenv('OBS_API_USERNAME'), \ + os.getenv('OBS_API_PASSWD')) + + mailer = MailSender() + #TODO: + #mailer.add_receiver(data.get('email')) + mailer.add_message('Package: %s' % data.get('project')) + #TODO: + #mailer.add_receiver(os.getenv('ABS_MAILINGLIST').split(',')) + mailer.add_receiver('hyokeun.jeon@samsung.com') + + non_exist_prjs = [ prj for prj in set([data.get('obs_project')]) if not build.exists(prj) ] + if non_exist_prjs: + raise LocalError('Target OBS project %s does not exist' % non_exist_prjs) + + # Working directory + basedir = os.path.join(os.getenv('WORKSPACE'), os.getenv('JOB_NAME')) + if os.path.exists(basedir): + shutil.rmtree(basedir) + builddir = os.path.join(basedir, 'build') + os.makedirs(builddir) + + # Fetch Source Code + src_root = os.path.join(builddir, os.path.basename(data.get('project'))) + src_git, int_tag = fetch_source(gerrit_env, \ + src_root, \ + data.get('project'), branch=data.get('branch'), \ + tag=data.get('tag'), \ + patchset=data.get('type')=='REVIEW', \ + build_check=data.get('buildcheck'), \ + profile=data.get('profile'), \ + obs_project=data.get('obs_project')) + print 'Got tag: [%s]' % int_tag + + # Find package name from spec file + if fields.get('use_specname', None): + # search specfile under packaging directory + resp = find_specfile(src_root, \ + parse_link('%s/%s' % (src_root, 'packaging')), \ + data.get('tag'), {'project': data.get('project')}, None) + if resp['spec']: + # parse specfile + resp = parse_specfile(resp['spec'], data.get('project'), None, None) + if resp['spec']: + print 'spec name = %s' %(resp['spec'].name) + data['obs_package'] = resp['spec'].name + + project_properties = retrieve_project_property(src_git, int_tag) + + mailer.add_title(data.get('obs_package')) + mailer.add_title('(' + data.get('branch') + ')') + if data.get('buildcheck', None): + mailer.add_message('Build cause: New Rootstrap %s' % data.get('buildcheck')) + mailer.add_message('Tag/Branch: %s' % data.get('tag')) + mailer.add_message('Target OBS project: %s' % data.get('obs_project')) + mailer.add_message('SR subject: %s' % project_properties.get('sr_message'), top=True) + + # Make tarball to share source codes easily with vm machine + zipping_workspace(src_root) + + parallel_jobs = '' + if data.get('buildcheck'): + parallel_jobs = '"-j 4"' + #### Running QEMU to launch Tizen Studio #### + print '[ TizenStudio START ] %s' % (str(datetime.now())) + sys.stdout.flush() + vm_image = ast.literal_eval(os.getenv('ABS_PROJECT_VM_MAP')).get(data.get('obs_project')) + my_tizen_studio = TizenStudio(vm_image) + ret = my_tizen_studio.build_app_source(package=data.get('package'), \ + profile=data.get('profile'), \ + gitpath=data.get('project'), \ + build_mode=os.getenv('BUILD_MODE'), \ + parallel_jobs=parallel_jobs) + print '[ TizenStudio END ] %s' % (str(datetime.now())) + + if data.get('buildcheck', None) == None and data.get('type', None) == 'SUBMIT': + post_processing(my_tizen_studio, src_git, project_properties, data, builddir, mailer, build) + mailer.send_mail() + + if 'FAIL' in mailer.email_title: + return 1 + +def main(argv): + """ + Script entry point. + """ + + print '---[SCRIPT START at %s]---' % str(datetime.now()) + + args = os.getenv('TRIGGER_INFO', None) + + # Manual trigger (Actual build step) + if args: + trigger_data = trigger_info(args) + return start_build(trigger_data) + + # Regular gerrit event (Dispatcher) + elif os.getenv('GERRIT_PROJECT', None): + + gerrit_infra = '' + if os.getenv('GERRIT_NAME').startswith('public_gerrit'): + gerrit_infra = 'PUBLIC_' + gerrit_env = GerritEnv(gerrit_infra) + + project = os.getenv('GERRIT_PROJECT') + + # Distingush event type + if os.getenv('GERRIT_CHANGE_NUMBER', None): + derived_data = {'type': 'REVIEW', \ + 'tag': os.getenv('GERRIT_REFSPEC'), \ + 'branch': os.getenv('GERRIT_BRANCH'), \ + 'revision': os.getenv('GERRIT_PATCHSET')} + elif os.getenv('GERRIT_REFNAME', None) and os.getenv('GERRIT_REFNAME').startswith('refs/tags/submit/'): + tag = os.getenv('GERRIT_REFNAME').split('refs/tags/')[1] + derived_data = {'type': 'SUBMIT', \ + 'tag': tag, \ + 'branch': tag[len('submit/'):tag.rfind('/', len('submit/'))], \ + 'revision': os.getenv('GERRIT_NEWREV')} + else: + derived_data = {'type': 'COMMIT', \ + 'tag': os.getenv('GERRIT_NEWREV'), \ + 'branch': os.getenv('GERRIT_REFNAME'), \ + 'revision': os.getenv('GERRIT_NEWREV')} + print derived_data + + obs_target_prjs = git_obs_map(project, derived_data.get('branch'), \ + gitcache=gerrit_env.gitcache, \ + gerrit_hostname=gerrit_env.hostname, \ + gerrit_username=gerrit_env.username, \ + gerrit_sshport=gerrit_env.sshport) + + index = 0 + for item in obs_target_prjs: + obs_target_prj = item['OBS_project'] + obs_stg_prj = item['OBS_staging_project'] + obs_pkg = item['OBS_package'] + obs_use_specname = item['OBS_use_specname'] + + if obs_stg_prj != 'abs': + continue + print 'Trying to call builder for: Branch(%s), Package(%s), OBS(%s), Staging(%s)' \ + % (derived_data.get('branch'), obs_pkg, obs_target_prj, obs_stg_prj) + + if is_ref_deleted(os.getenv('GERRIT_OLDREV'), os.getenv('GERRIT_NEWREV')): + print 'REF DELETED(%s)!' % os.getenv('GERRIT_OLDREV') + continue + + if obs_pkg is not None: + obs_package = obs_pkg + else: + obs_package = os.path.basename(project) + + derived_data['obs_project'] = obs_target_prj + derived_data['obs_package'] = obs_package + derived_data['use_specname'] = obs_use_specname == 'yes' + + trigger_data = {'source': {'package': derived_data.get('obs_package'), \ + 'branch': derived_data.get('branch'), \ + 'tag': derived_data.get('tag'), \ + }, \ + 'event': {'GERRIT_PROJECT': project, \ + 'GERRIT_EVENT_ACCOUNT_EMAIL': os.getenv('GERRIT_EVENT_ACCOUNT_EMAIL'), \ + 'GERRIT_EVENT_ACCOUNT_NAME': os.getenv('GERRIT_EVENT_ACCOUNT_NAME'), \ + 'GERRIT_NEWREV': os.getenv('GERRIT_NEWREV'), \ + 'GERRIT_REFNAME': os.getenv('GERRIT_REFNAME', None), \ + 'GERRIT_CHANGENUMBER': os.getenv('GERRIT_CHANGENUMBER', None), \ + 'GERRIT_BRANCH': os.getenv('GERRIT_BRANCH', None), \ + 'GERRIT_REFSPEC': os.getenv('GERRIT_REFSPEC', None), \ + 'GERRIT_PATCHSET_REVISION': os.getenv('GERRIT_PATCHSET_REVISION', None), \ + } , \ + 'gerrit_infra': gerrit_infra + } + trigger_data.update(derived_data) + + index += 1 + # Forward it to build job + trigger_next('ABS_BUILD_%d_%d_%s' \ + % (int(os.getenv('BUILD_NUMBER')), index, derived_data.get('obs_package')), \ + trigger_data, extra_params={'BACKEND_SELECTION': 'ABS'}) + +if __name__ == '__main__': + try: + sys.exit(main(sys.argv[1:])) + except LocalError, err: + print err + sys.exit(1) + diff --git a/abs/job_abs_main.py b/abs/job_abs_main.py deleted file mode 100644 index d1beb0e..0000000 --- a/abs/job_abs_main.py +++ /dev/null @@ -1,1411 +0,0 @@ -#!/usr/bin/env python -# -# Copyright (c) 2014, 2015, 2016 Samsung Electronics.Co.Ltd. -# -# This program is free software; you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the Free -# Software Foundation; version 2 of the License -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY -# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License -# for more details. -# -"""This job is triggered by gerrit submit ref update event for preloaded app. - It will generate tizen app package(.tpk) and requests prerelease routine. -""" - -import os -import sys -import shutil -import subprocess -import re -import glob -import zipfile -import tarfile -import json - -from subprocess import Popen, PIPE -from datetime import datetime -import time -from random import randint -import xml.etree.cElementTree as ElementTree -from xml.sax.saxutils import escape -from time import sleep - -from gitbuildsys.errors import ObsError -from gbp.git.repository import GitRepositoryError - -sys.path.insert(1, os.path.join(sys.path[0], '..')) - -from common.upload_service import upload_obs_service, UploadError, upload_obs_files - -from common import utils, mapping -from common.utils import set_permissions, tail -from common.mapping import git_obs_map - -from common.git import Git, clone_gitproject -from common.gerrit import Gerrit, is_ref_deleted - -from common.buildservice import BuildService -from common.prerelease import get_prerelease_project_name - -from common.buildtrigger import trigger_info, trigger_next -from common.send_mail import prepare_mail -from common import runner - -# set default char-set endcoding to utf-8 -reload(sys) -sys.setdefaultencoding('utf-8') # pylint: disable-msg=E1101 - -class LocalError(Exception): - """Local error exception.""" - pass - -class GerritEnv(object): - - def __init__(self, event=None): - - self.hostname = os.getenv('ABS_GERRIT_HOSTNAME') - self.username = os.getenv('ABS_GERRIT_USERNAME') - self.sshport = os.getenv('ABS_GERRIT_SSHPORT') - self.gitcache = os.getenv('ABS_GIT_CACHE_DIR') - -class ObsBuildService(object): - - def __init__(self): - self.build = BuildService(os.getenv('PUBLIC_OBS_API_URL'), \ - os.getenv('PUBLIC_OBS_API_USERNAME'), \ - os.getenv('PUBLIC_OBS_API_PASSWD')) - self.obs_url = os.getenv('PUBLIC_OBS_URL') - - def staging_project_name(self, prj_name, tag_name): - return get_prerelease_project_name(prj_name, tag_name) - -#### SOURCE COPY FROM JOB_SUBMIT.PY #### -def change_release_name(build, project, git_tag): - """ - Change release name from project config in OBS - Add the datetime into release name. - Eg: 'Release: .' ----> 'Release: 20141010..' - """ - # get project config - config = build.get_project_config(project) - release_name = 'Release: %s' % (git_tag.split('/')[-1]) - res = re.findall(r'^Release: ?\S+$', config, flags=re.MULTILINE) - if res: - if git_tag.split('/')[-1] not in res[0]: - note = '#Insert time from submission into release name\n' - release_name = '%s.%s' % (release_name, - res[0].split('Release:')[1].strip()) - config = config.replace(res[0], '%s%s' % (note, release_name), 1) - else: - note = '#Add release name into prjconf\n' - config = note + '%s\n' % release_name + config - # set project config - build.set_project_config(project, config) - -def copy_person_project_meta(build, obs_target_prj, obs_project): - """copy the maintainer list from obs_target_prj meta to corresponding - prerelease project - """ - src_meta_xml = build.get_meta(obs_target_prj) - src_xml_root = ElementTree.fromstringlist(src_meta_xml) - # get peron list from obs_target_prj meta - person_dict = {} - for person in src_xml_root.findall('person'): - if person.get('userid') in person_dict: - person_dict[person.get('userid')].append(person.get('role')) - else: - person_dict[person.get('userid')] = [person.get('role')] - # add person to prerelease project - if person_dict: - build.addPerson(obs_project, person_dict) - -def create_related_packages(build, obs_target_prj, obs_pre_prj, pre_package): - """create the 'link' package that relate the original package - obs_target_prj is the base project - obs_pre_prj is the prelease project - pre_package is the original package - """ - sourceinfo = build.get_sourceinfo_list(obs_target_prj) - for package in sourceinfo: - if sourceinfo[package]: - link_prj, link_pkg = sourceinfo[package][-1].split('/') - if link_prj == obs_target_prj and link_pkg == pre_package: - build.create_link_pac(obs_pre_prj, pre_package, \ - obs_pre_prj, package) - #TODO: When to enable below feature??? - return - if re.search("_aggregate", package): - print "Copypac aggregate package: %s/%s" %(obs_pre_prj, package) - build.create_copy_pac(obs_target_prj, package, obs_pre_prj,\ - package) - aggregate_file_name="_aggregate" - build.get_source_file(obs_target_prj, package, aggregate_file_name) - content = "" - with open(aggregate_file_name, 'r') as f: - content = f.read() - content_xml_root = ElementTree.fromstringlist(content) - for element in content_xml_root.findall('aggregate'): - element.set('project',obs_pre_prj) - content = ElementTree.tostring(content_xml_root) - with open(aggregate_file_name, 'w') as f: - f.write(content) - commit_msg="uploaded to copy pac %s/%s from %s" % (obs_pre_prj, package, obs_target_prj) - try: - build.commit_files(obs_pre_prj, package, - [(aggregate_file_name, True)], commit_msg) - except ObsError, error: - raise UploadError("Unable to upload _aggregate to %s: %s" % \ - (obs_pre_prj, error)) - print "Copypac done." - -def create_project(git_url, git_project, git_tag, git_revision, build, - obs_target_prj, obs_project, submitter, package, files=None): - """Create prerelease OBS project and upload sources for the build.""" - - # Create review project if it doesn't exist - print "Creating temporary review OBS project %s" % obs_project - info = {'projects': [git_project], - 'obs_target_prj': obs_target_prj, - 'git_tag': git_tag, - 'git_commit': git_revision, - 'obs_url': os.path.join(os.getenv('PUBLIC_OBS_URL'), \ - 'project/show?project=%s' % obs_project), - 'images': []} - if submitter: - info['submitter'] = escape(submitter) - - if build.exists(obs_project): - # update project info - build.update_info(info, obs_project) - # unlink the project to upload packages - try: - build.unlink_project(obs_project) - except ObsError, error: - print 'Modify the meta conf to unlink failed: %s' % error - else: - if not build.exists(obs_target_prj): - raise LocalError("Target project %s doesn't exist" % obs_target_prj) - try: - build.create_project(obs_project, obs_target_prj, - description=json.dumps(info)) - except ObsError, error: - LocalError("Unable to create project %s: %s" % (obs_project, error)) - - # change release name of project config in OBS - change_release_name(build, obs_project, git_tag) - - #disable publish flag - build.disable_build_flag(obs_project, repo = None, flag="publish", status="disable") - - #disable build flag - build.disable_build_flag(obs_project, repo = None, flag="build", status="disable") - - try: - if files is None: - upload_obs_service(git_url, git_project, git_tag, - git_revision, obs_project, build, package) - else: - upload_obs_files(git_project, git_tag, git_revision, \ - obs_project, build, package, files) - except UploadError, err: - raise LocalError(err) - - build.link_project(obs_project, src=obs_target_prj, linktype="localdep") - - # copy the maintainer list from obs_target_prj meta to corresponding - # prerelease project - copy_person_project_meta(build, obs_target_prj, obs_project) - - #create the 'link' package that relate the original package - create_related_packages(build, obs_target_prj, obs_project, package) - - #Wait 10 seconds to upload the package to the OBS - sleep(10) - - #default build flag - build.default_build_flag(obs_project, repo = None, flag="build") - - #default publish flag - build.default_build_flag(obs_project, repo = None, flag="publish") - -def create_tarball(tpk_dir, src_dir): - os.makedirs(src_dir) - _tar_file = src_dir + '.tar.gz' - _tar = tarfile.open(_tar_file, 'w:gz') - for filename in os.listdir(tpk_dir): - if re.match('.*\.tpk', filename): - shutil.copy(os.path.join(tpk_dir, filename), os.path.join(src_dir, filename)) - current_dir = os.getcwd() - os.chdir(os.path.dirname(src_dir)) - _tar.add(os.path.basename(src_dir)) - os.chdir(current_dir) - _tar.close() - return _tar_file - -class MailSender(object): - - email_title = '[ABS]' - email_head = 'App Build System(ABS) inform you the SDK build result.\n' \ - '(ABS will trigger build if you issues an SR on preloaded app branch)\n\n' - email_body = '' - email_footer = '\n\n--------\n'\ - 'Automatically generated by backend service.\n' - - email_to = [] - - def __init__(self, receiver=None, title=None, body=None): - if receiver is not None: self.email_to = receiver - if title is not None: self.email_title = title - if body is not None: self.email_body = body - - def add_receiver(self, add_new): - if type(add_new) == list: - self.email_to.extend(x for x in add_new) - elif type(add_new) == str: - self.email_to.append(add_new) - else: - print 'TYPE(%s) == %s' % (add_new, type(add_new)) - - def add_title(self, add_new): - self.email_title = self.email_title + ' ' + add_new - - def add_message(self, add_new, top=None): - if top is not None: - self.email_body = add_new + '\n' + self.email_body - else: - self.email_body = self.email_body + '\n' + add_new - - def add_maintainers(self, project): - mygerrit = Gerrit(GerritEnv().hostname, GerritEnv().username,\ - port=int(GerritEnv().sshport)) - grps = mygerrit.ls_groups(['--project %s' % project]) - dest_grp = [s for s in grps if " - Maintainers" in s] - for dg in dest_grp: - mbrs = mygerrit.ls_members(['\'\"%s\"\'' % dg, '--recursive']) - for line in mbrs: - self.add_receiver(line.split('\t')[3]) - - def send_mail(self): - - if 'SUCCESS' not in self.email_title: - self.add_title('FAILURE') - - self.email_body = self.email_head + self.email_body + self.email_footer - - self.email_to = [x for x in self.email_to if x != 'n/a'] - print '\n\n' - print self.email_title - m_body = '' - for m in self.email_body.splitlines(): - m_body += '\n'.join(m[i:i+128] for i in xrange(0, len(m), 128)) + '\n' - self.email_body = m_body - print self.email_body - print self.email_to - - prepare_mail("%s.env" % os.getenv('BUILD_TAG'), \ - self.email_title, \ - self.email_body, \ - os.getenv('NOREPLY_EMAIL_SENDER'), \ - self.email_to) - - if 'SUCCESS' not in self.email_title: - return 4 - -class VirtualMachine(object): - - parallel_jobs_cfg = '' - - def __init__(self, basedir, version_with=None, identifier=None): - self.virt_root = basedir - self.builddir = os.path.join(self.virt_root, 'build') - self.vm_image = os.getenv('ABS_VM_IMAGE') - if identifier is not None: - #TODO: HYOKEUN - if identifier == 'Tizen:Unified': - print 'Set tizen-unified' - identifier = 'tizen-unified' - else: - identifier = '-'.join(identifier.lower().split(':')[:-1]) - self.vm_image = self.vm_image.replace('REL_VER', identifier) - if version_with is not None: - self.vm_image = self.vm_image + '.v-' + version_with - self.parallel_jobs_cfg = '-j 4' - self._generate_qemu_command() - - def _generate_qemu_command(self): - - ssh_connection = os.getenv('SSH_CONNECTION') - try: - lastip = int(ssh_connection.split(' ')[2].split('.')[3]) - except: - lastip = randint(0x00, 0xff) - print 'Use random(%s) instead of ssh_connection(%s)' % (lastip, ssh_connection) - exeid = int(os.getenv('EXECUTOR_NUMBER', '1')) % 256 - processid = os.getpid() % 256 - buildid = int(os.getenv('BUILD_NUMBER', randint(0x00, 0xff))) % 256 - mac_address = '52:%02x:%02x:%02x:%02x:%02x' % \ - (lastip, exeid, processid, buildid, randint(0x00, 0xff)) - self.cmd = 'qemu-system-x86_64 -machine accel=kvm:xen:tcg ' \ - '-name ubuntu -M pc -m %d -smp %d ' \ - '-drive file=%s,snapshot=on '\ - '-virtfs local,id=test_dev,path=%s,security_model=mapped,mount_tag=share ' \ - '-net nic,macaddr=%s -net user -nographic' \ - % (int(os.getenv("ABS_VM_MEMORY")), int(os.getenv("ABS_VM_CPUS")), \ - self.vm_image, self.virt_root, mac_address) - print "Running kvm machine...\n %s" % (self.cmd) - - def generate_build_cmd(self, package, profile=None, gitpath=None): - - BUILD_ROOT = "/home/build" - SDK_PATH = os.path.join(BUILD_ROOT, "tizen-sdk-cli") - SHARE_ROOT = "/share/build" - PROFILE = profile - - if gitpath: - print 'Force set profile from %s' % gitpath - if 'profile/mobile/apps/native/' in gitpath: - PROFILE = 'mobile' - elif 'profile/wearable/apps/native/' in gitpath: - PROFILE = 'wearable' - else: - raise LocalError('Not supported profile %s' % gitpath) - - if os.getenv('BUILD_MODE') == 'Debug': - build_mode = 'Debug' - else: - build_mode = 'Release' - - #TODO: 64 bit build support?? - buildcmd = '$!/bin/bash \n' \ - 'set -x\n' \ - 'ABS_CMD=/home/build/abs\n' \ - 'SHARE_ROOT=%s\nBUILD_ROOT=%s\nSDK_PATH=%s\nPROFILE=%s\n' \ - 'PACKAGE=%s\n' \ - 'TMP_DIR=${BUILD_ROOT}/${PACKAGE}/_abs_out_\n' \ - 'SDK_CMD=$SDK_PATH/tools/ide/bin/tizen; LIST="$SDK_CMD list rootstrap "\n' \ - 'chown -R build:build $SHARE_ROOT\n' \ - 'su - build -c "cp $SHARE_ROOT/abs $BUILD_ROOT/"\n' \ - 'su - build -c "unzip $SHARE_ROOT/${PACKAGE}.zip -d $BUILD_ROOT/ > /dev/null"\n' \ - 'function _clear { \n' \ - ' mv ${TMP_DIR}/*.log ${SHARE_ROOT}\n' \ - ' exit $1 \n' \ - '}\n' \ - 'UPDATER="$SDK_PATH/update-manager/update-manager-cli.bin"\n' \ - 'if [ ! -f $UPDATER ]; then\n' \ - ' UPDATER="$SDK_PATH/package-manager/package-manager-cli.bin"\n' \ - 'fi\n' \ - 'export DISPLAY=:0\n' \ - 'su - build -c "$UPDATER show-info"\n' \ - 'su - build -c "tizen version"\n' \ - 'su - build -c "df -hT"\n' \ - 'rs_list=`su - build -c "tizen list rootstrap | grep ${PROFILE}-.*.core.private.* | cut -d \' \' -f 1"`\n' \ - 'echo $rs_list >> $SHARE_ROOT/rsver\n' \ - 'for rs in $rs_list; do\n' \ - ' echo "BUILDING START TIME: `date`"\n\n' \ - ' #if [[ $rs == *"64.core"* ]]; then\n' \ - ' # echo "SKIP! 64-bit not supported!"; continue;\n' \ - ' #else\n' \ - ' su - build -c "${ABS_CMD} build -r $rs -w ${BUILD_ROOT}/${PACKAGE} -s ABS -c %s %s"\n' \ - ' #fi\n' \ - ' mv ${TMP_DIR}/*.tpk ${SHARE_ROOT}\n' \ - ' ret=$?; echo "BUILDING FINISH TIME: `date`"\n\n' \ - ' if [ $ret != 0 ]; then\n' \ - ' echo $rs build fail; _clear 8\n' \ - ' fi\n' \ - 'done\n' \ - '_clear 0\n' \ - 'su - build -c "df -hT"\n' \ - % (SHARE_ROOT, BUILD_ROOT, SDK_PATH, PROFILE, package, build_mode, self.parallel_jobs_cfg) - - self.run_script = buildcmd - print self.run_script - - with open(os.path.join(self.builddir, 'run'), 'w') as fcmdl: - fcmdl.write('%s' % self.run_script) - os.chmod(os.path.join(self.builddir, 'run'), 0777) - - def get_log(self): - - self.buildlog = '' - for filename in os.listdir(self.builddir): - if re.match('build.*\.log', filename): - onefile = tail(os.path.join(self.builddir, filename), c=2304) - self.buildlog = self.buildlog + onefile[:onefile.rfind("Finished build-native")] - - def check_vm_result(self): - - # Get installed rootstrap version - built_version = None - with open(os.path.join(self.builddir, 'rsver')) as rsverfile: - built_version = rsverfile.read().replace(' ', ', ').replace('\n', ' ').encode('utf8') - built_version = built_version.split('.')[-1] - print 'Installed RS version... %s' % built_version - if built_version is None: - print 'Not able detect installed Rootstrap version' - - self.built_images = [] - for filename in os.listdir(self.builddir): - if re.match('.*\.tpk', filename): - self.built_images.append(filename) - self.get_log() - if int(self.status) != 0: - return [built_version, 'FAIL::CLI BUILD', self.buildlog] - if not glob.glob(os.path.join(self.builddir, '*.tpk')): - for filename in os.listdir(self.builddir): - if re.match('pkg.*\.log', filename): - self.buildlog = self.buildlog + tail(os.path.join(self.builddir, filename)) - return [built_version, 'FAIL::CLI PKG', self.buildlog] - return [built_version, None, None] - - def run(self): - - subprocess.call(self.cmd, stdout=sys.stdout, - stderr=sys.stderr, shell=True) - try: - with open(os.path.join(self.builddir,'testresult')) as statusf: - self.status = statusf.read().strip() - print 'KVM returned [%s]' % (self.status) - except IOError, _err: - raise LocalError('KVM Failed') - -class AbsRepository(object): - - profile = '' - - def __init__(self, project, email=None, name=None, \ - newrev=None, refname=None, changenumber=None, \ - branch=None, refspec=None, patchset=None, full_build=None, \ - obs_project=None, obs_package=None): - self.gerrit_event = {'project' : project, \ - 'email' : email, \ - 'name' : name, \ - 'newrev' : newrev, \ - 'refname' : refname, \ - 'changenumber': changenumber, \ - 'branch' : branch, \ - 'refspec' : refspec, \ - 'patchset' : patchset} - - print 'AbsRepository:\n%s' % self.gerrit_event - - self.full_build = full_build - self.set_gerrit_event() - self.set_obs_info(obs_project, obs_package) - - def set_gerrit_event(self): - - def _set_basic_data(self, req_type=None, tag=None, branch=None, revision=None): - if req_type is not None: self.request_type = req_type - if tag is not None: self.tag = tag - if branch is not None: self.branch = branch - #if revision is not None: self.revision = revision - self.revision = revision - print 'ABS basic data: Type(%s), Branch(%s), Tag(%s), Rev(%s)' \ - % (self.request_type, self.branch, self.tag, self.revision) - - self.project = self.gerrit_event['project'] - self.package = os.path.basename(self.project) - - if self.full_build is not None: - _set_basic_data(self, 'FULLBUILD', \ - 'full-build-tag', \ - self.gerrit_event['branch'], \ - 'full-build-rev') - elif self.gerrit_event['changenumber'] is not None: - _set_basic_data(self, 'REVIEW', \ - self.gerrit_event['refspec'], \ - self.gerrit_event['branch'], \ - self.gerrit_event['patchset']) - elif self.gerrit_event['refname'] is not None: - if self.gerrit_event['refname'].startswith('refs/tags/submit/'): - _set_basic_data(self, 'SUBMIT', \ - self.gerrit_event['refname'].replace('refs/tags/', ''), \ - self.gerrit_event['refname'].replace('refs/tags/', '').split('/')[1], \ - self.gerrit_event['newrev']) - elif self.gerrit_event['refname'].startswith('refs/tags/'): - raise LocalError('ABS will not process %s' % self.gerrit_event['refname']) - else: - _set_basic_data(self, 'COMMIT', \ - self.gerrit_event['newrev'], \ - self.gerrit_event['refname'], \ - self.gerrit_event['newrev']) - else: - raise LocalError('ABS invalid request') - - def set_package(self, package=None): - if package is not None: - self.package = package - else: - self.pacakge = os.path.basename(self.gerrit_event['project']) - - def set_new_tag(self): - if self.branch.endswith('_' + self.profile): - self.new_tag = self.tag.replace(self.branch, self.branch + '_tpk') - else: - self.new_tag = self.tag.replace(self.branch, self.branch + '_' + self.profile + '_tpk') - print 'set new tag [%s]' % self.new_tag - - def set_new_branch(self): - if self.branch.endswith('_' + self.profile): - self.new_branch = self.branch + '_tpk' - else: - self.new_branch = self.branch + '_' + self.profile + '_tpk' - print 'set new branch [%s]' % self.new_branch - - def set_obs_info(self, obs_project, obs_package): - if obs_project is not None: - self.obs_project = obs_project - self.profile = self.obs_project.split(':')[-1].lower() - print 'Set obs project to [%s]' % self.obs_project - print 'Set profile to [%s]' % self.profile - self.set_new_tag() - self.set_new_branch() - if obs_package is not None: - self.obs_package = obs_package - - def set_sr_tag(self, sr_tag): - self.sr_tag = sr_tag - -class SourceWork(object): - - def __init__(self, builddir, project, branch=None, tag=None): - self.workspace = os.path.join(builddir, os.path.basename(project)) - self.project = project - self.working_git = self.clone_from_gerrit(project=self.project) - - def clone_from_gerrit(self, project=None): - - if project is not None: - self.workspace = os.path.join(os.path.dirname(self.workspace), os.path.basename(project)) - self.project = project - - gerrit_env = GerritEnv() - for retry in xrange(20): - try: - print "Cloning into bare repo" - print 'work=%s' % os.path.join(os.getenv('GIT_CACHE_DIR'), self.project) - clone_gitproject(self.project, '%s.git' % \ - os.path.join(os.getenv('GIT_CACHE_DIR'), self.project), \ - gerrit_hostname=gerrit_env.hostname, \ - gerrit_username=gerrit_env.username, \ - gerrit_sshport=gerrit_env.sshport, \ - bare=True) - - print "trying to clone at %s " % (str(datetime.now())) - if not clone_gitproject(self.project, self.workspace, \ - gerrit_hostname=gerrit_env.hostname, \ - gerrit_username=gerrit_env.username, \ - gerrit_sshport=gerrit_env.sshport): - raise LocalError('Error cloning project %s' % self.project) - print "finished clone at %s " % (str(datetime.now())) - break - except Exception, e: - if retry >= 20: - print 'Error git clone!!!' - return None - #raise LocalError('Error git clone for %s' % self.project) - print 'Retry cloning in 30 sec. %s' % str(e) - time.sleep(30) - - return Git(self.workspace) - - def checkout_git_project(self, git=None, branch=None, tag=None, patchset=False, full_build=None, profile=None, obs_project=None): - print 'Trying to checkout project(%s), Branch(%s), Tag(%s) at %s' \ - % (self.project, branch, tag, str(datetime.now())) - if git is None: - git = self.working_git - - if patchset == True: - if tag and tag.startswith('refs/changes/'): - git._git_inout('fetch', ['origin', '%s' % tag]) - git._git_inout('reset', ['--hard', 'FETCH_HEAD']) - return - else: - raise LocalError('Patchset detected but no tag') - - if full_build is not None: - tag = self.retrieve_last_accepted_tag(profile=profile, obs_project=obs_project) - print 'Force to checkout full build for [%s]' % tag - - try: - if tag is None: - git.checkout(branch) - else: - git.checkout(tag) - except Exception, e: - raise LocalError('Checking out failed. Project(%s), Branch(%s), Tag(%s) Reason(%s)' \ - % (self.project, branch, tag, str(e))) - return tag - - def zipping_workspace(self): - current_dir = os.getcwd() - os.chdir(os.path.dirname(self.workspace)) - zipf = zipfile.ZipFile(os.path.basename(self.workspace)+'.zip', 'w', zipfile.ZIP_DEFLATED) - for root, dirs, files in os.walk(os.path.basename(self.workspace)): - for file in files: - zipf.write(os.path.join(root, file)) - zipf.close() - os.chdir(current_dir) - - def zipping_workspace_low(self): - current_dir = os.getcwd() - os.chdir(os.path.dirname(self.workspace)) - src_d = os.path.basename(self.workspace) - out_f = src_d + '.zip' - out = runner.show('zip --symlinks -r %s %s' % (out_f, src_d)) - os.chdir(current_dir) - - def zipping_workspace_workaround(self): - current_dir = os.getcwd() - os.chdir(os.path.dirname(self.workspace)) - src_d = os.path.basename(self.workspace) - out_f = src_d + '.zip' - self.ZipDir(src_d, out_f) - os.chdir(current_dir) - - def ZipDir(self, inputDir, outputZip): - '''Zip up a directory and preserve symlinks and empty directories''' - zipOut = zipfile.ZipFile(outputZip, 'w', compression=zipfile.ZIP_DEFLATED) - - rootLen = len(os.path.dirname(inputDir)) - def _ArchiveDirectory(parentDirectory): - contents = os.listdir(parentDirectory) - #store empty directories - if not contents: - archiveRoot = parentDirectory[rootLen:].replace('\\', '/').lstrip('/') - zipInfo = zipfile.ZipInfo(archiveRoot+'/') - zipOut.writestr(zipInfo, '') - for item in contents: - fullPath = os.path.join(parentDirectory, item) - if fullPath.endswith('/.git'): continue - if os.path.isdir(fullPath) and not os.path.islink(fullPath): - _ArchiveDirectory(fullPath) - else: - archiveRoot = fullPath[rootLen:].replace('\\', '/').lstrip('/') - if os.path.islink(fullPath): - zipInfo = zipfile.ZipInfo(archiveRoot) - zipInfo.create_system = 3 - # long type of hex val of '0xA1ED0000L', - # say, symlink attr magic... - zipInfo.external_attr = 2716663808L - zipOut.writestr(zipInfo, os.readlink(fullPath)) - else: - zipOut.write(fullPath, archiveRoot, zipfile.ZIP_DEFLATED) - _ArchiveDirectory(inputDir) - zipOut.close() - - def register_user_account(self, name, email): - try: - with open(os.path.join(self.workspace, ".git", "config"), "a") as myfile: - myfile.write("[user]\n\tname = %s\n\temail = %s\n" \ - % (name, email)) - except: - raise LocalError('Setting up git user failure. Exit now') - print "Tagging for - user:%s, email:%s" \ - % (self.working_git.get_config('user.name'), self.working_git.get_config('user.email')) - - def checkout_tpk_branch(self, new_branch): - if self.working_git.has_branch('origin' + new_branch, remote = True): - print 'Branch (%s) already exist' % new_branch - self.working_git.checkout(new_branch) - else: - self.working_git._git_inout('checkout', ['--orphan', '%s' % new_branch]) - self.working_git._git_inout('rm', ['-rf', '.']) - - print self.working_git._git_inout('fetch', ['origin'])[0] - print self.working_git._git_inout('reset', ['origin/' + new_branch, '--hard'])[0] - - def retrieve_last_accepted_tag(self, profile=None, obs_project=None): - # full build will use the most recent accepted tag. - if obs_project == 'Tizen:Unified': - check_branch = 'tizen' - else: - check_branch = ('/'.join(obs_project.split(':')).lower()) - t_outs, t_err, t_code = self.working_git._git_inout('for-each-ref', \ - ['--sort=-taggerdate', '--format=%(refname)%0a%(*subject)%0a%(*body)', \ - 'refs/tags/accepted/%s/' % check_branch, '--count=1']) - #if len(t_outs) == 0 or '/%s/' % profile not in t_outs.split('\n')[0]: - if len(t_outs) == 0: - print 'Sorry. Most recent accepted tag %s is not desired one.' % t_outs.split('\n')[0] - return None - accepted_tag = t_outs.split('\n')[0].replace('refs/tags/','').rstrip() - orig_tag = t_outs[t_outs.rfind('Reference: ')+11:].split('\n')[0].split(' ')[0].strip().rstrip() - if orig_tag.startswith('submit/tizen'): - print 'FULL BUILD! Accepted tag from _tpk branch %s -> %s' % (accepted_tag, orig_tag) - return orig_tag - - t_outs, t_err, t_code = self.working_git._git_inout('show', \ - [accepted_tag]) - if len(t_outs) == 0 or '- Git project: %s' % self.project not in t_outs: - print 'Sorry(1). Fail to retrieve original tag from %s' % accepted_tag - return None - orig_tag = t_outs[t_outs.rfind('- Tag:')+6:].split('\n')[0].strip().rstrip() - if orig_tag.startswith('submit/tizen'): - print 'FULL BUILD! Accepted tag from source branch %s -> %s' % (accepted_tag, orig_tag) - return orig_tag - - print 'Sorry(2). Fail to retrieve original tag from %s' % accepted_tag - return None - - def retrieve_license(self): - self.sdk_license = '' - license_file = os.path.join(self.workspace, 'LICENSE') - if os.path.isfile(license_file): - shutil.copy(license_file, os.path.dirname(self.workspace)) - with open(license_file, 'r') as r_file: - for x in r_file.readlines(): - matchline = re.search(r'.*Licensed under the (.*) License, Version (.*) \(the \"License\"\).*;', x) - if matchline: - self.sdk_license = '%s-%s' % (matchline.groups()[0], matchline.groups()[1]) - print 'LICENSE => [%s]' % self.sdk_license - - def retrieve_properties_from_project_def(self): - - # Single project - def_file = os.path.join(self.workspace, 'tizen-manifest.xml') - # Multi project - multiproject_list_f = os.path.join(self.workspace, 'WORKSPACE') - if os.path.isfile(multiproject_list_f): - with open(multiproject_list_f, 'r') as r_file: - for x in r_file.readlines(): - if os.path.isdir(os.path.join(self.workspace, x.rstrip())) and \ - os.path.isfile(os.path.join(self.workspace, x.rstrip(), 'tizen-manifest.xml')): - def_file = os.path.join(self.workspace, x.rstrip(), 'tizen-manifest.xml') - - print 'Property file: %s' % def_file - if os.path.isfile(def_file): - root = ElementTree.ElementTree(file=def_file).getroot() - self.sdk_package = root.attrib['package'] - self.sdk_version = root.attrib['version'] - else: - raise LocalError('Property file [%s] not found!' % def_file) - print 'TIZEN-MANIFEST.XML (%s) (%s)' % (self.sdk_package, self.sdk_version) - - def retrieve_commit_id(self): - commit_id = self.working_git.rev_parse('HEAD') - print 'commit from (HEAD) = %s' % (commit_id) - return commit_id - - def retrieve_revision_number(self, tag): - try: - revision_number = self.working_git.rev_parse(tag) - except: - revision_number = self.commit_id - print 'revision from (%s) = %s' % (tag, revision_number) - return revision_number - - def retrieve_commit_message(self): - t_outs, t_err, t_code = self.working_git._git_inout('log', ['--format=%B', '-n 1']) - self.commit_message = t_outs[:t_outs.rfind("\nChange-Id: ")].rstrip() + '\n' - print "author comment : %s" % self.commit_message - - def retrieve_sr_message(self, tag): - t_outs, t_err, t_code = self.working_git._git_inout('show', [tag, '--format=##chkstr:%b']) - sr_message = '\n'.join(t_outs[:t_outs.rfind("##chkstr:")].split('\n')[3:]) - print "sr comment from(%s) : [%s]" % (tag, sr_message.rstrip()) - return sr_message - - def retrieve_project_property(self, tag): - self.retrieve_properties_from_project_def() - self.commit_id = self.retrieve_commit_id() - self.revision_number = self.retrieve_revision_number(tag) - self.retrieve_commit_message() - self.sr_message = self.retrieve_sr_message(tag) - self.retrieve_license() - - def push_tpk_into_new_branch(self, tpk_dir, branch, obs_pkg_name, tag, new_tag, email, rs_ver): - # Check existance of tag first! - try: - chk_tag = self.working_git.rev_parse(new_tag) - print 'Checking tag [%s]' % (chk_tag) - if chk_tag: - print 'TAG(%s) already exist.' % new_tag - #self.working_git.push(repo = 'origin', src = ':' + new_tag) - #self.working_git._git_inout('tag', ['-d', new_tag]) - #raise LocalError('Tag(%s) already exist' % new_tag) - return 'Tag(%s) already exist.' % new_tag - except: - pass - try: - self.working_git.remove_files('*') - except: - pass - try: - self.working_git.remove_files('packaging/*') - except: - pass - if not os.path.exists(os.path.join(self.workspace, 'packaging')): - os.makedirs(os.path.join(self.workspace, 'packaging')) - for filename in os.listdir(tpk_dir): - if re.match('.*\.tpk', filename): - print "Found [%s]." % (filename) - shutil.copy(os.path.join(tpk_dir, filename), os.path.join(self.workspace, filename)) - self.working_git.add_files('*.tpk', True) - - with open(os.path.join(self.workspace, "packaging", obs_pkg_name + ".spec"), "w") as text_file: - text_file.write("%s" % self.spec) - self.working_git.add_files('packaging/*.spec', True) - - #TODO: - if False: #self.sdk_license: - shutil.copy(os.path.join(os.path.dirname(self.workspace), 'LICENSE'), \ - os.path.join(self.workspace, 'LICENSE')) - self.working_git.add_files('LICENSE', True) - - commit_message = "[ABS] %s\nReference: %s\nCommit id: %s\nRequested by: %s\nSDK rootstrap version: %s" \ - % (self.commit_message, tag, self.commit_id, email, rs_ver) - self.working_git.commit_staged(commit_message) - self.commit_id_new = self.retrieve_commit_id() - - push_err_msg = '' - push_retry = 20 - while push_retry > 0: - try: - self.working_git.push(repo = 'origin', src = '%s' % (branch), force=True) - return None - except GitRepositoryError, gre: - print 'git push exception: %s' % str(gre) - push_err_msg = push_err_msg + '\n' + str(gre) - push_retry -= 1 - sleep(5) - if not push_retry: - print 'Push failed 20 times' - return push_err_msg - - def push_tag_new_branch(self, tag, new_tag, mail): - - gerrit_env = GerritEnv() - remote = 'ssh://%s@%s:%d/%s' % (gerrit_env.username, \ - gerrit_env.hostname, \ - int(gerrit_env.sshport), \ - self.project) - message = '%s\n' \ - '[ABS] Ready.\n' \ - '- Original Tag: %s (%s)\n' \ - '- Original Commit: %s\n' \ - '- Requested By: %s\n' \ - % (self.sr_message, tag, self.revision_number, self.commit_id, mail) - - try: - self.working_git.create_tag(name=new_tag, msg=message, commit=self.commit_id_new) - except GitRepositoryError, e: - #raise LocalError('Cannot create tag %s (%s)' % (new_tag, str(e))) - print 'Cannot create tag %s (%s)' % (new_tag, str(e)) - return str(e) - - self.revision_number_new = self.retrieve_revision_number(new_tag) - - push_err_msg = '' - push_retry = 20 - while push_retry >0: - try: - self.working_git.push_tag(remote, new_tag) - return None - except GitRepositoryError, e: - #raise LocalError('Cannot push tag %s (%s)' % (new_tag, str(e))) - print 'Cannot push tag %s (%s)' % (new_tag, str(e)) - push_err_msg = push_err_msg + '\n' + str(e) - push_retry -= 1 - sleep(5) - if not push_retry: - print 'Push tag failed 20 times' - return push_err_msg - - def generate_spec_file(self, obs_pkg_name, rsver, no_tpk_branch='false', include_archs=None): - - group = '' - license = self.sdk_license - summary = '' - if os.path.isdir(os.path.join(self.workspace, 'packaging')): - for filename in os.listdir(os.path.join(self.workspace, 'packaging')): - if re.match('.*\.spec', filename): - with open(os.path.join(self.workspace, 'packaging', filename)) as f_spec: - for x in f_spec.readlines(): - if group == '' and re.match('Group:.*', x): - group = x.split(':')[1].strip().rstrip() - if license == '' and re.match('License:.*', x): - license = x.split(':')[1].strip().rstrip() - if summary == '' and re.match('Summary:.*', x): - summary = x.split(':')[1].strip().rstrip() - if group == '': group = 'N/A' - if license == '': license = 'N/A' - if summary == '': summary = obs_pkg_name.replace('org.tizen.', '') - - vcs_desc = '#' - if no_tpk_branch != 'true': - vcs = '#VCS_FROM: %s' % (self.project + "#" + self.commit_id) - vcs_desc = vcs.split(' ')[-1] - else: - vcs = 'VCS: %s' % (self.project + "#" + self.commit_id) - - if os.getenv('EXCLUDE_ARCH'): - exclude_arch = 'ExcludeArch: ' - for exa in os.getenv('EXCLUDE_ARCH').split(','): - exclude_arch = exclude_arch + ' ' + exa - else: - exclude_arch = '' - - if include_archs is not None: - exclusive_arch = 'ExclusiveArch: ' - for exa in include_archs: - exclusive_arch = exclusive_arch + ' ' + exa - else: - exclusive_arch = '' - - if False: #if oss.getenv('BUILD_MODE') == 'Debug': - build_mode = '-debug' - else: - build_mode = '%{nil}' - - header = "Name: %s\n%s\n#RS_Ver: %s\n" \ - "Summary: %s\nVersion: %s\nRelease: 1\n" \ - "Group: %s\nLicense: %s\n" \ - "Source0: %%{name}-%%{version}.tar.gz\n" \ - "\n" \ - "%s\n" \ - "%s\n" \ - "BuildRequires: pkgconfig(libtzplatform-config)\n" \ - "Requires(post): /usr/bin/tpk-backend\n\n" \ - "%%define internal_name %s\n" \ - "%%define preload_tpk_path %%{TZ_SYS_RO_APP}/.preload-tpk \n" \ - "\n" \ - "%%define build_mode %s\n\n" \ - "%%ifarch arm armv7l\n%%define target arm\n%%endif\n" \ - "%%ifarch aarch64\n%%define target aarch64\n%%endif\n" \ - "%%ifarch x86_64\n%%define target x86_64\n%%endif\n" \ - "%%ifarch i386 i486 i586 i686\n%%define target x86\n%%endif\n" \ - "%%description\n" \ - "%s\nThis is a container package which have preload TPK files\n\n" \ - % (obs_pkg_name, vcs, rsver, summary, self.sdk_version, \ - group, license, exclusive_arch, exclude_arch, self.sdk_package, build_mode, vcs_desc) - - content = "%prep\n%setup -q\n\n%build\n\n" \ - "%install\n" \ - "rm -rf %{buildroot}\n" \ - "mkdir -p %{buildroot}/%{preload_tpk_path}\n" \ - "install %{internal_name}-%{version}-%{target}%{build_mode}.tpk %{buildroot}/%{preload_tpk_path}/\n" \ - "\n" \ - "%post\n" \ - "\n" \ - "%files\n%defattr(-,root,root,-)\n" \ - "%{preload_tpk_path}/*\n" - #TODO: App sources are open to the public. Installing LICENSE file is not required. - #if self.sdk_license: - # content = content + '%license LICENSE\n' - - self.spec = header + content - #print '\n\n==========\n%s\n===========\n\n' % self.spec - - def get_git_repo_size(self, quiet=True): - if quiet == False: - self.working_git._git_inout('gc', ['--quiet']) - pack_size, t_err, t_code = self.working_git._git_inout('count-objects', ['-v']) - pack_size = pack_size[pack_size.find('size-pack:')+11:].split('\n')[0] - if quiet == False: - print '\"UNPACK-SIZE\": \"%s/%s\"' % (self.pack_size, pack_size) - else: - self.pack_size = pack_size - -class Builder(object): - - full_build = None - create_obs_job = False - - def __init__(self): - fields = trigger_info(os.getenv('TRIGGER_INFO')) - if 'event' not in fields: - self.create_obs_job = True - return - - event = fields['event'] - - if 'full_build' in fields: - self.full_build = fields['full_build'] - print 'Full build requested for %s' % self.full_build - self._abs = AbsRepository(project= event['GERRIT_PROJECT'], \ - email= event['GERRIT_EVENT_ACCOUNT_EMAIL'], \ - name= event['GERRIT_EVENT_ACCOUNT_NAME'], \ - branch= fields['source']['branch'], \ - full_build= self.full_build, \ - obs_project= fields['obs_project'], \ - obs_package= fields['obs_package']) - else: - self._abs = AbsRepository(project= event['GERRIT_PROJECT'], \ - email= event['GERRIT_EVENT_ACCOUNT_EMAIL'], \ - name= event['GERRIT_EVENT_ACCOUNT_NAME'], \ - newrev= event['GERRIT_NEWREV'], \ - refname= event['GERRIT_REFNAME'], \ - changenumber=event['GERRIT_CHANGE_NUMBER'], \ - branch= event['GERRIT_BRANCH'], \ - refspec= event['GERRIT_REFSPEC'], \ - patchset= event['GERRIT_PATCHSET_REVISION'], \ - obs_project= fields['obs_project'], \ - obs_package= fields['obs_package']) - - #self._abs.set_obs_info(fields['obs_project'], fields['obs_package']) - - def prepare_working_directory(self): - - self.basedir = os.path.join(os.getenv('WORKSPACE'), os.getenv('JOB_NAME')) - if os.path.exists(self.basedir): - shutil.rmtree(self.basedir) - self.builddir = os.path.join(self.basedir, 'build') - print 'basedir=%s, builddir=%s' % (self.basedir, self.builddir) - os.makedirs(self.builddir) - - def create_obs_project(self): - - fields = trigger_info(os.getenv('TRIGGER_INFO')) - - my_build = ObsBuildService() - my_mail = MailSender(receiver = fields['mail_contents']['receiver'], \ - title = fields['mail_contents']['title'], \ - body = fields['mail_contents']['body']) - - if True: - retry_count = 3 - while retry_count > 0: - try: - create_project(fields['url'], \ - fields['project'], \ - fields['_obs_tag'], \ - fields['_obs_revision'], \ - my_build.build, \ - fields['obs_project'], \ - fields['new_obs_project'], \ - fields['submitter'], \ - fields['obs_package'], \ - fields['_obs_files']) - - my_mail.add_title('SUCCESS') - my_mail.add_message('To view, visit %s/package/show?package=%s&project=%s' \ - % (my_build.obs_url, fields['obs_package'], fields['new_obs_project'])) - break - except Exception, err: - print err - sleep(15) - retry_count -= 1 - if not retry_count: - print 'retrying failed' - fail_title = 'OBS creation fail' - my_mail.add_title(fail_title) - my_mail.add_message('Result: FAIL') - my_mail.add_message('\n\nFail to create OBS prerelease project') - - return my_mail.send_mail() - - def main(self): - - if self.create_obs_job: - return self.create_obs_project() - - my_build = ObsBuildService() - my_mail = MailSender() - - self.prepare_working_directory() - - my_mail.add_receiver(self._abs.gerrit_event['email']) - my_mail.add_message('Package: %s' % self._abs.project) - if self._abs.request_type == 'SUBMIT': - my_mail.add_receiver(os.getenv('ABS_MAILINGLIST').split(',')) - #my_mail.add_maintainers(self._abs.project) - else: - my_mail.add_title('(preview)') - my_mail.add_title(self._abs.package) - my_mail.add_title('(' + self._abs.branch + ')') - if self.full_build is not None: - my_mail.add_message('Build cause: New Rootstrap %s' % self.full_build) - - #### Fetch Source Code - my_source = SourceWork(self.builddir, \ - self._abs.project) - if my_source.working_git is None: - print '\"Title\": \"%s/%s\"' \ - % (self._abs.package, self._abs.tag.replace('refs/','')) - raise LocalError('Fail to clone %s' % self._abs.project) - - int_tag = my_source.checkout_git_project(branch = self._abs.branch, \ - tag = self._abs.tag, \ - patchset=self._abs.request_type=='REVIEW', \ - full_build=self.full_build, \ - profile=self._abs.profile, \ - obs_project=self._abs.obs_project) - - my_source.get_git_repo_size() - - if self.full_build is None: - print '\"Title\": \"%s/%s\"' \ - % (self._abs.package, self._abs.tag.replace('refs/','')) - else: - #TODO: - self._abs.tag = int_tag - print '\"Title\": \"%s/%s/%s\"' \ - % (self._abs.package, self.full_build, self._abs.tag) - if int_tag == None or not int_tag.startswith('submit/tizen'): - raise LocalError('Full build cannot be proceed. No matching SR tag') - - my_mail.add_message('Tag/branch: %s' % self._abs.tag) - my_mail.add_message('Target OBS project: %s' % self._abs.obs_project) - - # Retrieve project settings - my_source.retrieve_project_property(self._abs.tag) - my_mail.add_message('SR subject: %s' % my_source.sr_message.rstrip(), top=True) - - my_source.zipping_workspace_workaround() - - my_vm = VirtualMachine(self.basedir, version_with=self.full_build, identifier=self._abs.obs_project) - my_vm.generate_build_cmd(self._abs.package, self._abs.profile, gitpath=self._abs.project) - - #### Running QEMU to launch Tizen SDK CLI build - print "[ s d k - k v m s t a r t ] %s " % (str(datetime.now())) - sys.stdout.flush() - ret = my_vm.run() - print "[ s d k - k v m f i n i s h ] %s " % (str(datetime.now())) - set_permissions(self.builddir, (0644, 0755)) - sys.stdout.flush() - - rs_ver, fail_title, fail_log = my_vm.check_vm_result() - my_mail.add_message('Sdk Rootstrap Version: %s' % rs_ver) - print 'VM returned [%s] [%s] [%s]\n' % (rs_ver, fail_title, fail_log) - if fail_title != None: - my_mail.add_title(fail_title) - my_mail.add_message('Result: FAIL') - my_mail.add_message('\n\n%s' % fail_log) - - if self.full_build is None: - return my_mail.send_mail() - - found_archs = [] - archs_lookup = [['i386', 'i486', 'i586', 'i686', 'x86'], \ - ['arm', 'armv7l'], \ - ['x86_64'], \ - ['aarch64']] - - for fn in os.listdir(my_vm.builddir): - mtch = re.search(r'(.*)-([0-9.]+)-(.*)(? old_tag=%s\n revision=%s\n commit=%s" \ - % (self._abs.tag, my_source.revision_number, my_source.commit_id) - - # TODO: Pre-release Job - submitter = '%s <%s>' % (self._abs.gerrit_event['name'], self._abs.gerrit_event['email']) - url = 'ssh://%s:%d' % (GerritEnv().hostname, int(GerritEnv().sshport)) - - if os.getenv('NO_TPK_BRANCH') == 'true': - new_obs_project = my_build.staging_project_name(self._abs.obs_project, self._abs.tag) - print '\n Trying to create OBS prerelease project(%s)\n' % (new_obs_project) - spec_file = os.path.join(self.builddir, self._abs.obs_package + ".spec") - with open(spec_file, "w") as text_file: - text_file.write("%s" % my_source.spec) - tarball_file = create_tarball(my_vm.builddir, \ - os.path.join(self.builddir, self._abs.obs_package + \ - '-' + my_source.sdk_version)) - _obs_files = [spec_file, tarball_file] - _obs_tag = self._abs.tag - _obs_revision = my_source.revision_number - print 'Directly uploading files %s for (%s,%s)' % (_obs_files, _obs_tag, _obs_revision) - #TODO: How to maintain tpk history??? - else: - new_obs_project = my_build.staging_project_name(self._abs.obs_project, self._abs.new_tag) - print '\n Trying to create OBS prerelease project(%s)\n' % (new_obs_project) - _obs_files = None - _obs_tag = self._abs.new_tag - _obs_revision = my_source.revision_number_new - print "----> new_tag=%s\n revision=%s\n commit=%s" \ - % (_obs_tag, _obs_revision, my_source.commit_id_new) - - #TODO: SEND SUCCESS MAIL - my_mail.add_title('SUCCESS') - my_mail.add_message('To view, visit %s/package/show?package=%s&project=%s' \ - % (my_build.obs_url, self._abs.obs_package, new_obs_project)) - my_mail.send_mail() - - ######## Create obs project job in master node ######## - trigger_data = {'url': url, \ - 'project': self._abs.project, \ - '_obs_tag': _obs_tag, \ - '_obs_revision': _obs_revision, \ - 'obs_project': self._abs.obs_project, \ - 'new_obs_project': new_obs_project, \ - 'submitter': submitter, \ - 'obs_package': self._abs.obs_package, \ - '_obs_files': _obs_files, \ - 'mail_contents': {'receiver': my_mail.email_to, \ - 'title': my_mail.email_title, \ - 'body': my_mail.email_body}} - trigger_next("CREATE_OBS", trigger_data) - - print "[ JOB FINISHED AT %s ]" % (str(datetime.now())) - -class Dispatcher(object): - - def __init__(self): - self._abs = AbsRepository(project= os.getenv('GERRIT_PROJECT'), \ - email= os.getenv('GERRIT_EVENT_ACCOUNT_EMAIL'), \ - name= os.getenv('GERRIT_EVENT_ACCOUNT_NAME'), \ - newrev= os.getenv('GERRIT_NEWREV'), \ - refname= os.getenv('GERRIT_REFNAME'), \ - changenumber=os.getenv('GERRIT_CHANGE_NUMBER'), \ - branch= os.getenv('GERRIT_BRANCH'), \ - refspec= os.getenv('GERRIT_REFSPEC'), \ - patchset= os.getenv('GERRIT_PATCHSET_REVISION')) - - print 'Dispatcher project=[%s], package=[%s]' % (self._abs.project, self._abs.package) - - def query_git_obs_map(self): - - # check whether git-obs-mapping.xml exist in local - gerrit_env = GerritEnv() - obs_target_prjs = git_obs_map(self._abs.project, self._abs.branch, \ - gitcache=gerrit_env.gitcache, \ - gerrit_hostname=gerrit_env.hostname, \ - gerrit_username=gerrit_env.username, \ - gerrit_sshport=gerrit_env.sshport) - - if not obs_target_prjs: - print "PRJ %s(%s) does not map any OBS. Exit now." % (self._abs.project, self._abs.branch) - return - - obs_target_prjs = [ x for x in obs_target_prjs \ - if x['OBS_project'] in os.getenv('SUPPORTED_PROFILES').split(',') ] - print 'After filter out redundant mappings: %s\n' % obs_target_prjs - - return obs_target_prjs - - def main(self): - - my_build = ObsBuildService() - - obs_map_list = self.query_git_obs_map() - if obs_map_list is None: - return - - index = 1 - for _target in self.query_git_obs_map(): - _obs_prj = _target['OBS_project'] - _stg = _target['OBS_staging_project'] - _pkg = _target['OBS_package'] - if _stg != 'abs': - continue - print "Trying to call builder for: Branch(%s), Package(%s), OBS(%s), Staging(%s)" \ - % (self._abs.branch, _pkg, _obs_prj, _stg) - try: - # Check existance of target OBS project - non_exist_prjs = [prj for prj in set([_obs_prj]) if not my_build.build.exists(prj)] - if non_exist_prjs: - print "Target OBS projects %s does not exist" % (non_exist_prjs) - continue - - # ref deleted check - if is_ref_deleted(os.getenv("GERRIT_OLDREV"), os.getenv("GERRIT_NEWREV")): - print "REF DELETED(%s)!" % (os.getenv("GERRIT_OLDREV")) - #TODO: Delete prerelease obs project if it still exist - continue - - if _pkg is not None: - obs_package = _pkg - else: - obs_package = self._abs.package - - trigger_data = {"obs_project": _obs_prj, - "obs_package": obs_package, - "source": - {"package": self._abs.package, - "branch": self._abs.branch, - "tag": self._abs.tag}, - "event": - {"GERRIT_PROJECT": self._abs.gerrit_event['project'], - "GERRIT_EVENT_ACCOUNT_EMAIL": self._abs.gerrit_event['email'], - "GERRIT_EVENT_ACCOUNT_NAME": self._abs.gerrit_event['name'], - "GERRIT_NEWREV": self._abs.gerrit_event['newrev'], - "GERRIT_REFNAME": self._abs.gerrit_event['refname'], - "GERRIT_CHANGE_NUMBER": self._abs.gerrit_event['changenumber'], - "GERRIT_BRANCH": self._abs.gerrit_event['branch'], - "GERRIT_REFSPEC": self._abs.gerrit_event['refspec'], - "GERRIT_PATCHSET_REVISION": self._abs.gerrit_event['patchset']}, - "upstream": os.getenv('JOB_NAME'), - "index": index} - - trigger_next("abs_class_builder_%d_%d_%s" \ - % (int(os.getenv("BUILD_NUMBER")), index, self._abs.tag.split("/")[-1]), trigger_data) - index += 1 - except Exception, ex: - print str(ex) - pass - - if index > 1: - print "\"Title\": \"%s/%s\"" % (self._abs.package, self._abs.tag.replace('refs/','')) - print '---[ a b s d i s p a t c h e r ] %s---' % str(datetime.now()) - -def main(argv): - """ - Script entry point. - """ - - print '---[SCRIPT START at %s]---' % str(datetime.now()) - - if os.getenv("TRIGGER_INFO"): - builder = Builder() - return builder.main() - else: - dispatcher = Dispatcher() - return dispatcher.main() - -if __name__ == '__main__': - try: - sys.exit(main(sys.argv[1:])) - except LocalError, err: - print err - sys.exit(1) - - diff --git a/abs/job_abs_update.py b/abs/job_abs_update.py new file mode 100644 index 0000000..5d0c256 --- /dev/null +++ b/abs/job_abs_update.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python +# vim: ai ts=4 sts=4 et sw=4 +# + +import os +import re +import sys +import subprocess +import urllib, urllib2 +import requests +from bs4 import BeautifulSoup +from urllib2 import urlopen, ProxyHandler, build_opener, install_opener, URLError, HTTPError +import time +from datetime import datetime, timedelta +import json +import base64 +import ast +import xml.etree.cElementTree as ET +import inspect +from random import randint + +sys.path.insert(1, os.path.join(sys.path[0], '..')) + +from common.buildtrigger import trigger_info, trigger_next, get_jenkins_build_data +from common.mapping import git_obs_map_full_list +from common.utils import list_files_in_url, unicode_to_str +from common.git import Git, clone_gitproject +from common.send_mail import prepare_mail, makemail +from common.gerrit import GerritEnv +from common.tizen_studio import TizenStudio + +def list_all_packages(profile, gerrit_env): + + list_all = [] + + if True: + mapping_list = git_obs_map_full_list(obs_project=profile, staging_project='abs', \ + gitcache=gerrit_env.gitcache, \ + gerrit_hostname=gerrit_env.hostname, \ + gerrit_username='spinrobot', \ + gerrit_sshport=gerrit_env.sshport) + for item in mapping_list: + print item + if not all(name in ['Project_name', 'Branch_name', \ + 'OBS_project', 'OBS_staging_project', 'OBS_package'] for name in item): + continue + prj_name = item['Project_name'] + _branch = item['Branch_name'] + _obs_prj = item['OBS_project'] + _stg = item['OBS_staging_project'] + _pkg = item['OBS_package'] + _brch = '' + for _br in _branch.split(','): + if _brch == '' or len(_br) < len(_brch): + _brch = _br + print ' >> Adding %s(%s) %s %s' % (prj_name, _brch, _obs_prj, _pkg) + list_all.append({'git_path': prj_name, \ + 'branch_name': _brch, \ + 'obs_project': _obs_prj, \ + 'staging_project': _stg, \ + 'package': _pkg}) + return list_all + +def main(new_rs, gerrit_name): + + gerrit_infra = '' + if gerrit_name.startswith('public_gerrit'): + gerrit_infra = 'PUBLIC_' + gerrit_env = GerritEnv(gerrit_infra) + + profiles = os.getenv('ABS_SUPPORTED_PROFILES').split(',') + trigger_arg = '' + + kvm_root = '/'.join(os.getenv('ABS_VM_IMAGE').split('/')[:-1]) + pattern = r'tizen[a-zA-Z_-]*[0-9]{8}.[0-9]{1,2}' + + new_rootstrap_url = new_rs + trigger_arg = new_rs + + assert new_rs + assert new_rs.startswith('http') + + rs_version = re.match(r'.*\/(.*-.*_[0-9.]{1,})\/.*', new_rs).group(1) + print 'new_rs: %s' % new_rs + print 'rs_versin: %s' % rs_version + + profiles_to_update = [] + + for x in os.getenv('ABS_SUPPORTED_PROFILES').split(','): + expr = '(%s)_(\d{8}).([0-9]+)' % (x.lower().replace(':', '-')) + if re.search(expr, new_rs): + profiles_to_update = [x] + break + print 'profiles_to_update:%s' % profiles_to_update + + assert profiles_to_update + + index = 0 + for up in profiles_to_update: + print 'trigger %s' % up + vm_image = ast.literal_eval(os.getenv('ABS_PROJECT_VM_MAP')).get(up) + my_tizen_studio = TizenStudio(vm_image) + image_to_check = my_tizen_studio.update_rootstrap(new_rs, rs_version) + + todo_list = list_all_packages(up, gerrit_env) + for single_dict in todo_list: + index += 1 + print '\n%s' % single_dict + trigger_data = {'obs_project': single_dict.get('obs_project'), \ + 'obs_package': single_dict.get('package'), \ + 'source': {'package': single_dict.get('git_path').split('/')[-1], \ + 'branch': single_dict.get('branch_name'), \ + 'git_path': single_dict.get('git_path'), \ + 'tag': single_dict.get('commitid', None)}, \ + 'full_build': rs_version, \ + 'event': {'GERRIT_EVENT_ACCOUNT_EMAIL': 'abs.robot@tizen.org', \ + 'GERRIT_EVENT_ACCOUNT_NAME': 'abs-robot', \ + 'GERRIT_PROJECT': single_dict.get('git_path'), \ + 'GERRIT_NEWREV': single_dict.get('commitid', None)}, \ + 'type': 'BUILD_CHECK', \ + 'tag': 'build-check', \ + 'branch': single_dict.get('branch_name'), \ + 'revision': 'build-check', \ + 'gerrit_infra': gerrit_infra, \ + 'index': str(index)} + print 'ABS build for %s initiated.' % (trigger_data['source']['package']) + trigger_next('ABS_BUILD_CHECK_%d_%d_%s' \ + % (int(os.getenv('BUILD_NUMBER')), index, trigger_data['source']['package']), \ + trigger_data) + +if __name__ == '__main__': + + sys.exit(main(os.getenv('NEW_ROOTSTRAP'), os.getenv('GERRIT_INFRA'))) + diff --git a/abs/job_abs_update_vm.py b/abs/job_abs_update_vm.py deleted file mode 100644 index e4d15f1..0000000 --- a/abs/job_abs_update_vm.py +++ /dev/null @@ -1,435 +0,0 @@ -#!/usr/bin/env python -# -# Copyright (c) 2014, 2015, 2016 Samsung Electronics.Co.Ltd. -# -# This program is free software; you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the Free -# Software Foundation; version 2 of the License -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY -# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License -# for more details. -# -"""This job is triggered by job_create_sdkrootstrap.py - to update VM images for ABS and RBS. -""" - -import os -import sys -import shutil -import re -import stat -import ast -import subprocess -from datetime import datetime -from random import randint - -sys.path.insert(1, os.path.join(sys.path[0], '..')) - -from common.utils import wget_noproxy, list_files_in_url -from common.buildtrigger import trigger_info, trigger_next -from common.git import Git, clone_gitproject - -class LocalError(Exception): - """Local error exception.""" - pass - -def run_inside_vm(vm_image, virt_root, snapshot_on): - """ - Run build/run inside VM - """ - - ssh_connection = os.getenv('SSH_CONNECTION') - try: - lastip = int(ssh_connection.split(' ')[2].split('.')[3]) - except (IndexError, ValueError, AttributeError): - print 'ssh_connection is %s, it is a incorrect format ,' \ - 'random instead' % ssh_connection - lastip = randint(0x00, 0xff) - exeid = int(os.getenv('EXECUTOR_NUMBER', '1')) % 256 - processid = os.getpid() % 256 - buildid = int(os.getenv('BUILD_NUMBER', randint(0x00, 0xff))) % 256 - mac_address = '52:%02x:%02x:%02x:%02x:%02x' % \ - (lastip, exeid, processid, buildid, randint(0x00, 0xff)) - if snapshot_on: - opt_snapshot = ",snapshot=on " - else: - opt_snapshot = "" - cmd = 'qemu-system-x86_64 -machine accel=kvm:xen:tcg ' \ - '-name ubuntu -M pc -m %d -smp %d ' \ - '-drive file=%s%s '\ - '-virtfs local,id=test_dev,path=%s,security_model=mapped,mount_tag=share ' \ - '-net nic,macaddr=%s -net user -nographic' \ - % (int(os.getenv("ABS_VM_MEMORY")), int(os.getenv("ABS_VM_CPUS")), \ - vm_image, opt_snapshot, virt_root, mac_address) - print "Running kvm machine...\n %s" % (cmd) - - subprocess.call(cmd, stdout=sys.stdout, - stderr=sys.stderr, shell=True) - - #read testresult from file - try: - with open(os.path.join(virt_root,'build','testresult')) as statusf: - status = statusf.read().strip() - print 'KVM returned [%s]' % (status) - return int(status) - except IOError, _err: - raise LocalError('KVM Failed') - -def _checkout_gitproject(prjdir, gerrit_project, git_branch, git_tag): - if not clone_gitproject(gerrit_project, prjdir, \ - gerrit_hostname=os.getenv("ABS_GERRIT_HOSTNAME"), \ - gerrit_username=os.getenv("ABS_GERRIT_USERNAME"), \ - gerrit_sshport=os.getenv("ABS_GERRIT_SSHPORT")): - print 'Error cloning project %s' % (gerrit_project) - return None - mygit = Git(prjdir) - print "Trying to checkout... %s %s %s" % (gerrit_project, git_branch, git_tag) - if not git_tag: - mygit.checkout(git_branch) - else: - mygit.checkout(git_tag) - return mygit - -def prepare_sdk_tool(builddir, sdk_tool_gerrit): - _path = sdk_tool_gerrit.split(',')[0] - _branch = sdk_tool_gerrit.split(',')[1] - prjdir = os.path.join(builddir, os.path.basename(_path)) - _git = _checkout_gitproject(prjdir, _path, _branch, None) - if _git is not None: - return _git.path - return None - -def prepare_working_directory(work_dir): - basedir = os.path.join(os.getenv("WORKSPACE"), work_dir) - if os.path.exists(basedir): - shutil.rmtree(basedir) - builddir = os.path.join(basedir, 'build') - print 'basedir=%s, builddir=%s ' % (basedir, builddir) - os.makedirs(builddir) - return basedir, builddir - -def wget_sdk_rootstrap(rs_url, builddir): - rs_device = "" - rs_emulator = "" - file_list = list_files_in_url(rs_url) - print file_list - - for r_file in file_list: - print "Check %s" % r_file - if re.compile(r'[%s]-3.0-rs-device.core.*.zip' % os.getenv('LIST_PROFILES')).search(r_file) is not None: - rs_device = os.path.basename(r_file) - print "Downloading file %s" % (rs_device) - elif re.compile(r'[%s]-3.0-rs-emulator.core.*.zip' % os.getenv('LIST_PROFILES')).search(r_file) is not None: - rs_emulator = os.path.basename(r_file) - print "Downloading file %s" % (rs_emulator) - else: - continue - - wget_noproxy(r_file, os.path.join(builddir, os.path.basename(r_file))) - - return rs_device[:-4], rs_emulator[:-4] - -def generate_update_abs_cmd(rs_version, rs_url, package_server, tool_path, wrapper_path): - # build command line - SDK_PATH = "/home/build/tizen-sdk-cli" - SHARE_ROOT = "/share/build" - TOOL_PATH = os.path.join(SHARE_ROOT, os.path.basename(tool_path)) if tool_path else '' - WRAPPER_PATH = os.path.join(SHARE_ROOT, os.path.basename(wrapper_path)) if wrapper_path else '' - PROFILE = rs_version.split('_')[0].split('-')[-1] - - buildcmd = '$!/bin/bash\nset -x\n' \ - 'SDK_PATH=%s; SDK_CMD=$SDK_PATH/tools/ide/bin/tizen; SHARE_ROOT=%s; ' \ - 'TOOL_PATH=%s; WRAPPER_PATH=%s; PROFILE=%s; PACKAGE_SERVER=%s\n' \ - 'ABS_CMD=/home/build/abs; TOOL_CMD=/home/build/sdk-build/pkg-cli\n' \ - 'if [ ! -z $TOOL_PATH ]; then su - build -c "cp -rf $TOOL_PATH /home/build/"; fi\n' \ - 'if [ ! -z $WRAPPER_PATH ]; then su - build -c "cp -pf $WRAPPER_PATH/abs $ABS_CMD; chmod +x $ABS_CMD"; fi\n' \ - 'LIST="$SDK_CMD list rootstrap "\n' \ - 'UPDATER="$SDK_PATH/update-manager/update-manager-cli.bin"\n' \ - 'export DISPLAY=:0\n' \ - 'su - build -c "df -hT"\n' \ - 'if [ -f $UPDATER ]; then\n' \ - ' mylist=`su - build -c "$UPDATER show-repo-info -r $PACKAGE_SERVER -d tizen_3.0 ' \ - ' | grep TizenSDK | cut -f 2 -d \' \'"`\n' \ - ' IFS=\' \' read -r -a snapshot <<< ${mylist}\n' \ - ' if [ ! -z ${snapshot[0]} ]; then\n' \ - ' su - build -c "$UPDATER install WebCLI -r $PACKAGE_SERVER -d tizen_3.0 ' \ - ' -s ${snapshot[0]} --remove-installed-sdk"\n' \ - ' su - build -c "$UPDATER install `echo \"$PROFILE\" | awk \'{print toupper($0)}\'`-3.0-' \ - ' NativeAppDevelopment-CLI -s ${snapshot[0]} --accept-license --remove-installed-sdk"\n' \ - ' fi\n' \ - 'else\n ' \ - ' UPDATER="$SDK_PATH/package-manager/package-manager-cli.bin"\n' \ - ' #su - build -c "$UPDATER update"\n' \ - 'fi\n' \ - 'su - build -c "tizen list rootstrap"\n' \ - 'su - build -c "tizen version"\n' \ - 'if [ $PROFILE == "unified" ]; then\n' \ - ' rm -rf $SDK_PATH/tools/smart-build-interface/plugins/mobile-*.xml\n' \ - ' rm -rf $SDK_PATH/platforms/tizen-3.0/mobile/rootstraps/\n' \ - ' rm -rf $SDK_PATH/platforms/tizen-4.0/mobile/rootstraps/\n' \ - ' rm -rf $SDK_PATH/tools/smart-build-interface/plugins/wearable-*.xml\n' \ - ' rm -rf $SDK_PATH/platforms/tizen-3.0/wearable/rootstraps/\n' \ - ' rm -rf $SDK_PATH/platforms/tizen-4.0/wearable/rootstraps/\n' \ - 'else\n' \ - ' rm -rf $SDK_PATH/tools/smart-build-interface/plugins/${PROFILE}-*.xml\n' \ - ' rm -rf $SDK_PATH/platforms/tizen-3.0/$PROFILE/rootstraps/\n' \ - ' rm -rf $SDK_PATH/platforms/tizen-4.0/$PROFILE/rootstraps/\n' \ - 'fi\n' \ - 'wget --no-proxy -P $SHARE_ROOT/ -r -nd -np -R index.html* %s/ &> /dev/null\n' \ - 'rs_list=`ls $SHARE_ROOT | grep .*-rs.*.zip`; if [ $? != 0 ]; then exit 6; fi\n' \ - 'for tgt in $rs_list; do \n' \ - ' disp_v=`echo ${tgt/-rs/} | sed -rn \'s/(.*.private).*/\\1/p\'`;\n' \ - ' su - build -c "$TOOL_CMD install-file -P $SHARE_ROOT/${tgt} -l $SDK_PATH --force" \n' \ - ' echo ret_val = \"$?\" \n' \ - ' if [ "$disp_v" == `$LIST | grep ${disp_v} | sed -rn \'s/(.*.private).*/\\1/p\'` ];\n' \ - ' then echo "OK"\n' \ - ' else \n' \ - ' echo "Rootstrap update failure"; exit 9\n' \ - ' fi\n' \ - 'done \n' \ - 'su - build -c "$LIST"\n' \ - 'su - build -c "$UPDATER show-info"\n' \ - 'su - build -c "rm -rf /home/build/.update-manager/run/*"\n' \ - 'su - build -c "rm -rf /home/build/.package-manager/run/*"\n' \ - 'su - build -c "df -hT"\n' \ - % (SDK_PATH, SHARE_ROOT, TOOL_PATH, WRAPPER_PATH, PROFILE, package_server, rs_url) - - return buildcmd - -#TODO: 64-bit support for creating kvm image... -def generate_create_abs_cmd(installer, package_server, sign_data): - # build command line - BUILD_ROOT = "/home/build" - SDK_PATH = os.path.join(BUILD_ROOT, "tizen-sdk-cli") - SDK_KEY_PATH = os.path.join(BUILD_ROOT, "tizen-sdk-cli-data", "keystore") - SHARE_ROOT = "/share/build" - - buildcmd = '$!/bin/bash \n' \ - 'set -x\n' \ - 'egrep "vmx|svm" /proc/cpuinfo\n' \ - 'LIST_PROFILES=%s\n' \ - 'SHARE_ROOT=%s\nBUILD_ROOT=%s\nSDK_PATH=%s\nSDK_KEY_PATH=%s\n' \ - 'UPDATER="$SDK_PATH/update-manager/update-manager-cli.bin"\n' \ - 'INSTALLER=%s\nPACKAGE_SERVER=%s\n' \ - 'KEY_PATH=%s\nKEY_A=$BUILD_ROOT/$KEY_PATH/%s\nKEY_A_P=%s\n' \ - 'KEY_D=$BUILD_ROOT/$KEY_PATH/%s\nKEY_D_P=%s\n' \ - 'add-apt-repository -y ppa:webupd8team/java\n' \ - 'echo debconf shared/accepted-oracle-license-v1-1 select true | debconf-set-selections\n' \ - 'echo debconf shared/accepted-oracle-license-v1-1 seen true | debconf-set-selections\n' \ - 'apt-get update\n' \ - 'apt-get install -y oracle-java7-installer oracle-java7-set-default\n' \ - 'apt-get install -y zip build-essential gettext libwebkitgtk-1.0-0 libglib2.0-0 libcurl3-gnutls ' \ - ' libsdl1.2debian libglib2.0-0 acl zlib1g libpixman-1-0 ' \ - ' bridge-utils openvpn git ruby-full xdg-utils xmlstarlet rpm2cpio\n' \ - 'wget --no-proxy $INSTALLER -O $SHARE_ROOT/`basename $INSTALLER` \n' \ - 'export DISPLAY=:0\nchmod +x $SHARE_ROOT/`basename $INSTALLER`\n' \ - 'sed -i \'/10.112.1.184/d\' /etc/environment\n' \ - 'su - build -c "$SHARE_ROOT/`basename $INSTALLER` --accept-license $SDK_PATH"\n' \ - 'su - build -c "$UPDATER install WebCLI -r $PACKAGE_SERVER -d tizen_3.0 --remove-installed-sdk"\n' \ - 'su - build -c "$UPDATER install MOBILE-3.0-NativeAppDevelopment-CLI --accept-license"\n' \ - 'su - build -c "$UPDATER install WEARABLE-3.0-NativeAppDevelopment-CLI --accept-license"\n' \ - 'su - build -c "echo PATH=\$PATH:$SDK_PATH/tools/ide/bin >> ~/.profile"\n' \ - 'su - build -c "mkdir -p $SDK_KEY_PATH/; touch $SDK_KEY_PATH/profiles.xml"\n' \ - 'cp -rf $SHARE_ROOT/$KEY_PATH $BUILD_ROOT/$KEY_PATH; chown -R build:build $BUILD_ROOT/$KEY_PATH/\n' \ - 'su - build -c "tizen cli-config -g default.profiles.path=\"$SDK_KEY_PATH/profiles.xml\""\n' \ - 'su - build -c "tizen security-profiles add -n ABS -a $KEY_A -p $KEY_A_P -d $KEY_D -dp $KEY_D_P"\n' \ - 'rm -rf $SDK_PATH/tools/smart-build-interface/plugins/{${LIST_PROFILES}}-3.0*.xml\n' \ - 'rm -rf $SDK_PATH/platforms/tizen-3.0/{${LIST_PROFILES}}/rootstraps\n' \ - 'su - build -c "tizen list rootstrap"\n' \ - % (os.getenv('LIST_PROFILES'), \ - SHARE_ROOT, BUILD_ROOT, SDK_PATH, SDK_KEY_PATH, \ - installer, package_server, \ - sign_data['path'], sign_data['author_key'], sign_data['author_pwd'], \ - sign_data['distributor_key'], sign_data['distributor_pwd'] \ - ) - - return buildcmd - -def prepare_job(vm_image, vm_image_new, version): - basedir, builddir = prepare_working_directory(os.getenv('JOB_NAME')) - if not vm_image or not os.access(vm_image, os.R_OK): - raise LocalError('vm_image %s not found' % vm_image) - shutil.copyfile(vm_image, vm_image_new) - os.chmod(vm_image_new, os.stat(vm_image_new).st_mode | stat.S_IWRITE) - print "copied from " + vm_image + " to " + vm_image_new - - return basedir, builddir - -def main_job(vm_image_new, builddir, basedir, buildcmd): - with open(os.path.join(builddir, 'run'), 'w') as fcmdl: - fcmdl.write('%s' % buildcmd) - os.chmod(os.path.join(builddir, 'run'), 0777) - print buildcmd - - #### Running QEMU to launch Tizen SDK CLI build #### - print "[ s d k - k v m s t a r t ] %s " % (str(datetime.now())) - sys.stdout.flush() - ret = run_inside_vm(vm_image_new, basedir, False) - print "[ s d k - k v m f i n i s h ] %s " % (str(datetime.now())) - - if int(ret) != 0: - os.remove(vm_image_new) - raise LocalError('KVM return failure') - -def post_job(vm_image, raw_new_image): - - vm_image_new = raw_new_image.replace('-FAIL', '') - if raw_new_image.endswith('-FAIL'): - os.remove(vm_image_new) - raise LocalError('Build check failed for %s, DO NOT UPDATE IT!' % (vm_image_new)) - - if not os.path.exists(vm_image_new) or not os.path.isfile(vm_image_new): - return False - - #### Rename the VM image #### - try: - os.rename(vm_image_new, vm_image) - #os.chmod(vm_image, os.stat(vm_image).st_mode ^ stat.S_IWRITE) - except OSError as e: - print e.errno - print e.strerror - os.remove(vm_image_new) - raise LocalError('Fail to rename from %s to %s' % (vm_image_new, vm_image)) - - #### Distribute to workers #### - #TODO: list slaves here except current one... - return True - -def create_abs(contents): - """ - Create ABS VM entry point. - """ - - print '---[ CREATE ABS ] %s---' % str(datetime.now()) - - vm_image = os.getenv('BLANK_VM_IMAGE') - vm_image_new = vm_image + ".v-" + contents['version'] - basedir, builddir = prepare_job(vm_image, vm_image_new, contents['version']) - - #### Download SDK installer - sdk_file = contents['sdk_file'] - #wget_noproxy(os.path.join(contents['sdk_release_url'], sdk_file), \ - # os.path.join(builddir, sdk_file)) - - #### Fetch signer #### - prepare_sdk_tool(builddir, os.getenv('SDK_SIGN_GIT')) - sign_p = os.path.basename(os.getenv('SDK_SIGN_GIT').split(',')[0]) - sign_data = {'path': sign_p, \ - 'author_key': os.getenv('SDK_SIGN_AUTHOR').split(',')[0], \ - 'author_pwd': os.getenv('SDK_SIGN_AUTHOR').split(',')[1], \ - 'distributor_key': os.getenv('SDK_SIGN_DIST').split(',')[0], \ - 'distributor_pwd': os.getenv('SDK_SIGN_DIST').split(',')[1]} - - #### Generate script to run(install required packages and SDK) inside VM #### - buildcmd = generate_create_abs_cmd(sdk_file, os.getenv('ABS_SDK_PACKAGE_SERVER'), sign_data) - - # Main Routine - main_job(vm_image_new, builddir, basedir, buildcmd) - - # Post Routine - post_job(os.getenv('ABS_VM_IMAGE'), vm_image_new) - - print "\"Title\": \"ABS_Create/%s\"" % (contents['version']) - print '---[ CREATE ABS ] %s---' % str(datetime.now()) - -def update_abs(contents): - """ - Update ABS VM entry point. - """ - - print '---[ UPDATE ABS ] %s---' % str(datetime.now()) - - kvm_root = '/'.join(os.getenv('ABS_VM_IMAGE').split('/')[:-1]) - vf = os.path.join(kvm_root, 'abs-rootstrap-' + contents['version']) - - vm_image = os.getenv('ABS_VM_IMAGE') - identifier = '-'.join(contents['project'].lower().split(':')[:-1]) - #TODO: HYOKEUN - if contents['project'] == 'Tizen:Unified': - print 'Set tizen-unified' - identifier = 'tizen-unified' - vm_image = vm_image.replace('REL_VER', identifier) - vm_image_new = vm_image + ".v-" + contents['version'] - - # Stage 2 of 2 - if post_job(vm_image, vm_image_new) == True: - # Refresh version file - for _file in os.listdir(kvm_root): - if _file.startswith('abs-rootstrap-' + contents['version'].split('_')[0]): - os.remove(os.path.join(kvm_root, _file)) - subprocess.call('touch {}'.format(vf), shell=True) - print "\"Title\": \"ABS_Update/%s\"" % (contents['version']) - return - - # check pre-installed version - #if os.path.exists(vf): - # print 'You already installed %s' % contents['version'] - # return - - # Stage 1 of 2 - for _file in os.listdir(kvm_root): - if _file.startswith(os.path.basename(vm_image) + '.v-' + contents['version'].split('_')[0]): - print 'Another instance already running... %s' % _file - #os.remove(os.path.join(kvm_root, _file)) - basedir, builddir = prepare_job(vm_image, vm_image_new, contents['version']) - - #### Fetch sdk-rootstrap #### - #rs_device, rs_emulator = wget_sdk_rootstrap(contents['rs_url'], builddir) - - #### Fetch sdk-tool #### - sdk_tool_path = prepare_sdk_tool(builddir, os.getenv('SDK_TOOL_GIT')) - - #### Prepare local sdk cli wrapper script #### - cli_wrapper_path = prepare_sdk_tool(builddir, os.getenv('ABS_CLI_SCRIPT_GIT')) - - #### Generate script to run(building app and make .tpk) inside VM #### - buildcmd = generate_update_abs_cmd(contents['version'], \ - contents['rs_url'], \ - os.getenv('ABS_SDK_PACKAGE_SERVER'), \ - sdk_tool_path, cli_wrapper_path) - - # Main Routine - main_job(vm_image_new, builddir, basedir, buildcmd) - - #### Request build all packages with new rootstrap - trigger_data = {'project': contents['project'], - 'version': contents['version']} - trigger_next("full_build_request_%s_%s" \ - % (contents['project'], contents['version']), trigger_data) - - print '---[ UPDATE ABS ] %s---' % str(datetime.now()) - -def update_none(args): - raise LocalError('Invalid parameters') - -def main(argv): - """ - Script entry point. - """ - - print '---[JOB STARTED]-------------------------' - - options = {'sdk_rootstrap_updated': update_abs, - 'sdk_released': create_abs, - 'default': update_none} - - if os.getenv("TRIGGER_INFO"): - print "TRIGGER_INFO exist, trigger builder script" - fields = trigger_info(os.getenv("TRIGGER_INFO")) - return options.get(fields['title'] , update_none)(fields['contents']) - else: - print "****************TEST*****************" - test_data = ast.literal_eval(os.getenv("TRIGGER_TEST")) - for key in test_data.keys(): - print "%s='%s'" % (key, test_data[key]) - return options.get(test_data['title'], update_none)(test_data['contents']) - -if __name__ == '__main__': - try: - sys.exit(main(sys.argv[1:])) - except LocalError, err: - print err - sys.exit(1) - diff --git a/common/tizen_studio.py b/common/tizen_studio.py new file mode 100644 index 0000000..b3b82e7 --- /dev/null +++ b/common/tizen_studio.py @@ -0,0 +1,241 @@ +#!/usr/bin/env python +# vim: ai ts=4 sts=4 et sw=4 +# +# Copyright (C) 2017 Samsung Electronics. Co,. Ltd. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; version 2 +# of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +""" +Interface of Tizen Studio +""" + +import os +import sys +import shutil +import re +import stat +import ast +import subprocess +import glob +from datetime import datetime +from random import randint + +sys.path.insert(1, os.path.join(sys.path[0], '..')) + +from common.utils import wget_noproxy, list_files_in_url, tail +from common.buildtrigger import trigger_info, trigger_next +from common.git import Git, clone_gitproject +from common.gerrit import GerritEnv + +class LocalError(Exception): + """Local error exception.""" + pass + +class TizenStudio(object): + + def __init__(self, vm_image): + + self.vm_root = os.getenv('ABS_VM_IMAGE_ROOT') + self.basedir = os.path.join(os.getenv('WORKSPACE'), os.getenv('JOB_NAME')) + self.builddir = os.path.join(self.basedir, 'build') + self.vm_image = os.path.join(self.vm_root, vm_image) + assert os.path.isfile(self.vm_image) + #if os.path.exists(self.basedir): + # shutil.rmtree(self.basedir) + if not os.path.exists(self.basedir) or not os.path.exists(self.builddir): + os.makedirs(self.builddir) + + # Fetch ABS cli wrapper from gerrit + self.wrapper_path = self.fetch_resources(os.getenv('ABS_CLI_SCRIPT_GIT').split(',')) + print 'wrapper_path:%s' % self.wrapper_path + + self.build_root = '/home/build' + self.sdk_path = os.path.join(self.build_root, 'tizen-sdk-cli') + self.share_root = '/share/build' + + def get_template(self, mode=None): + + with open(os.path.join(self.wrapper_path, '%s.template' % mode), 'r') as rf: + cmd_template = rf.read() + return cmd_template + + def run_tizen_studio(self, cmd_to_run, vm_image=None, snapshot_on=True): + + with open(os.path.join(self.builddir, 'run'), 'w') as fcmdl: + fcmdl.write('%s' % cmd_to_run) + os.chmod(os.path.join(self.builddir, 'run'), 0777) + + if vm_image is None: + vm_image = self.vm_image + + ssh_connection = os.getenv('SSH_CONNECTION') + try: + lastip = int(ssh_connection.split(' ')[2].split('.')[3]) + except (IndexError, ValueError, AttributeError): + print 'ssh_connection is %s, it is a incorrect format ,' \ + 'random instead' % ssh_connection + lastip = randint(0x00, 0xff) + exeid = int(os.getenv('EXECUTOR_NUMBER', '1')) % 256 + processid = os.getpid() % 256 + buildid = int(os.getenv('BUILD_NUMBER', randint(0x00, 0xff))) % 256 + mac_address = '52:%02x:%02x:%02x:%02x:%02x' % \ + (lastip, exeid, processid, buildid, randint(0x00, 0xff)) + if snapshot_on: + opt_snapshot = ",snapshot=on " + else: + opt_snapshot = "" + cmd = 'qemu-system-x86_64 -machine accel=kvm:xen:tcg ' \ + '-name ubuntu -M pc -m %d -smp %d ' \ + '-drive file=%s%s '\ + '-virtfs local,id=test_dev,path=%s,security_model=mapped,mount_tag=share ' \ + '-net nic,macaddr=%s -net user -nographic' \ + % (int(os.getenv("ABS_VM_MEMORY")), int(os.getenv("ABS_VM_CPUS")), \ + vm_image, opt_snapshot, self.basedir, mac_address) + print "Running kvm machine...\n %s" % (cmd) + + subprocess.call(cmd, stdout=sys.stdout, + stderr=sys.stderr, shell=True) + + #read testresult from file + try: + with open(os.path.join(self.basedir,'build','testresult')) as statusf: + status = statusf.read().strip() + print 'KVM returned [%s]' % (status) + return int(status) + except IOError, _err: + raise LocalError('KVM Failed') + + def fetch_resources(self, rc_repo_branch=[]): + + gerrit_env = GerritEnv('PUBLIC_') + + rc_git_path, rc_git_branch = rc_repo_branch + rc_dir = os.path.join(self.builddir, os.path.basename(rc_git_path)) + if not clone_gitproject(rc_git_path, rc_dir, \ + gerrit_hostname=gerrit_env.hostname, \ + gerrit_username='spinrobot', \ + gerrit_sshport=gerrit_env.sshport): + print 'Error clone project %s' % rc_git_path + rc_git = Git(rc_dir) + rc_git.checkout(rc_git_branch) + return rc_git.path + + def update_rootstrap(self, rootstrap_src_url, rootstrap_version): + + temp_image = os.path.join(self.vm_root, '.tmp.%s.v-%s' \ + % (os.path.basename(self.vm_image), rootstrap_version)) + print 'Coping %s from %s...' % (temp_image, self.vm_image) + sys.stdout.flush() + shutil.copy(self.vm_image, temp_image) + assert os.path.isfile(temp_image) + os.chmod(temp_image, os.stat(temp_image).st_mode | stat.S_IWRITE) + + # Fetch sdk-tool from gerrit + sdk_tool_path = self.fetch_resources(os.getenv('SDK_TOOL_GIT').split(',')) + print 'sdk_tool_path:%s' % sdk_tool_path + + update_cmd = self.get_template('update') + update_cmd = update_cmd.replace( \ + '__SDK_PATH__', self.sdk_path).replace( \ + '__SHARE_ROOT__', self.share_root).replace( \ + '__TOOL_PATH__', os.path.join(self.build_root, os.path.basename(sdk_tool_path))).replace( \ + '__WRAPPER_PATH__', os.path.join(self.build_root, os.path.basename(self.wrapper_path))).replace( \ + '__PROFILE__', rootstrap_version.split('_')[0].split('-')[-1]).replace( \ + '__PACKAGE_SERVER__', os.getenv('ABS_SDK_PACKAGE_SERVER')).replace( \ + '__ROOTSTRAP_URL__', rootstrap_src_url) + print 'UPDATE_CMD:\n%s' % update_cmd + + #### Running QEMU to launch Tizen Studio #### + print '[ TizenStudio START ] %s' % (str(datetime.now())) + sys.stdout.flush() + ret = self.run_tizen_studio(update_cmd, vm_image=temp_image, snapshot_on=False) + print '[ TizenStudio END ] %s' % (str(datetime.now())) + + assert int(ret) == 0 + os.rename(temp_image, self.vm_image) + + return self.vm_image + + def gather_build_result(self): + + # Get installed rootstrap version + built_version = None + with open(os.path.join(self.builddir, 'rsver')) as rsverfile: + built_version = rsverfile.read().replace(' ', ', ').replace('\n', ' ').encode('utf8') + built_version = built_version.split('.')[-1] + print 'Installed RS version... %s' % built_version + if built_version is None: + print 'Not able detect installed Rootstrap version' + + self.built_version = built_version + + # Get building log + self.buildlog = '' + for filename in os.listdir(self.builddir): + if re.match('build.*\.log', filename): + onefile = tail(os.path.join(self.builddir, filename), c=2304) + self.buildlog = self.buildlog + onefile[:onefile.rfind("Finished build-native")] + + if self.build_result != 0: + return [built_version, 'FAIL::CLI BUILD', self.buildlog] + + # Get packaging log + if not glob.glob(os.path.join(self.builddir, '*.tpk')): + for filename in os.listdir(self.builddir): + if re.match('pkg.*\.log', filename): + self.buildlog = self.buildlog + tail(os.path.join(self.builddir, filename)) + return [built_version, 'FAIL::CLI PKG', self.buildlog] + + return [built_version, None, None] + + def build_app_source(self, package, profile=None, gitpath=None, build_mode='Release', parallel_jobs=''): + + PROFILE = profile + if gitpath: + print 'Force set profile from %s' % gitpath + if 'profile/mobile/apps/native/' in gitpath: + PROFILE = 'mobile' + elif 'profile/wearable/apps/native/' in gitpath: + PROFILE = 'wearable' + #TODO: + elif 'scm/test' in gitpath: + PROFILE = 'mobile' + else: + raise LocalError('Not supported profile %s' % gitpath) + + build_cmd = self.get_template('build') + build_cmd = build_cmd.replace( \ + '__SDK_PATH__', self.sdk_path).replace( \ + '__SHARE_ROOT__', self.share_root).replace( \ + '__BUILD_ROOT__', self.build_root).replace( \ + '__PROFILE__', PROFILE).replace( \ + '__PACKAGE__', package).replace( \ + '__PARALLEL_JOBS__', parallel_jobs).replace( \ + '__BUILD_MODE__', build_mode) + print 'BUILD_CMD:\n%s' % build_cmd + + #### Running QEMU to launch Tizen Studio #### + print '[ TizenStudio START ] %s' % (str(datetime.now())) + sys.stdout.flush() + ret = self.run_tizen_studio(build_cmd, snapshot_on=True) + print '[ TizenStudio END ] %s' % (str(datetime.now())) + + self.build_result = int(ret) + return self.build_result + +if __name__ == '__main__': + print 'This is not callable module' + sys.exit(-1) + diff --git a/debian/jenkins-scripts-abs.install b/debian/jenkins-scripts-abs.install index 423feb1..8e95251 100644 --- a/debian/jenkins-scripts-abs.install +++ b/debian/jenkins-scripts-abs.install @@ -1 +1,2 @@ debian/tmp/abs/* /var/lib/jenkins/jenkins-scripts/abs/ +debian/tmp/common/tizen_studio.py /var/lib/jenkins/jenkins-scripts/common/ diff --git a/packaging/jenkins-scripts.spec b/packaging/jenkins-scripts.spec index 1a3a111..ac1e8f8 100644 --- a/packaging/jenkins-scripts.spec +++ b/packaging/jenkins-scripts.spec @@ -284,9 +284,10 @@ fi %files abs %defattr(-,jenkins,jenkins) %dir %{destdir}/abs -%{destdir}/abs/job_abs_batch_all.py -%{destdir}/abs/job_abs_main.py -%{destdir}/abs/job_abs_update_vm.py +%{destdir}/abs/job_abs_build.py +%{destdir}/abs/job_abs_update.py +%dir %{destdir}/common +%{destdir}/common/tizen_studio.py %{destdir}/abs/report_template %files tzs -- 2.7.4