From 17ecdc7c59f9ade3b07efbf04a331b54b7348e53 Mon Sep 17 00:00:00 2001 From: hyokeun Date: Tue, 13 Nov 2018 20:29:55 +0900 Subject: [PATCH] [ABS] Docker build with configurable meta Change-Id: Ie912cb0c7674a1c1825e37895aaca10e54b37a11 --- abs/job_abs_backend.py | 829 +++++++++++++++++++++++++++++++++++++ abs/job_abs_frontend.py | 179 ++++++++ abs/job_abs_frontend.sh | 41 ++ common/tizen_studio_docker.py | 240 +++++++++++ debian/jenkins-scripts-abs.install | 1 + packaging/jenkins-scripts.spec | 4 + 6 files changed, 1294 insertions(+) create mode 100755 abs/job_abs_backend.py create mode 100755 abs/job_abs_frontend.py create mode 100755 abs/job_abs_frontend.sh create mode 100644 common/tizen_studio_docker.py diff --git a/abs/job_abs_backend.py b/abs/job_abs_backend.py new file mode 100755 index 0000000..a10256e --- /dev/null +++ b/abs/job_abs_backend.py @@ -0,0 +1,829 @@ +#!/usr/bin/env python +# +# Copyright (c) 2014, 2015, 2016 Samsung Electronics.Co.Ltd. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation; version 2 of the License +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# for more details. +# +"""This job is triggered by gerrit submit ref update event for preloaded app. + It will generate tizen app package(.tpk) and requests prerelease routine. +""" + +import os +import sys +import shutil +import re +import glob +import zipfile +import requests +from pprint import pprint +import base64 +import json + +from datetime import datetime +from time import sleep +import xml.etree.cElementTree as ElementTree + +sys.path.insert(1, os.path.join(sys.path[0], '..')) + +from common.git import Git, clone_gitproject +from common.tizen_studio_docker import TizenStudio +from common.send_mail import prepare_mail +from common.buildtrigger import trigger_next + +# set default char-set endcoding to utf-8 +reload(sys) +sys.setdefaultencoding('utf-8') # pylint: disable-msg=E1101 + +def retrieve_project_property(src_git, tag): + """ Search APP properties """ + + ret_prop = {} + + def get_git_repo_size(quiet=True): + if quiet == False: + src_git._git_inout('gc', ['--quiet']) + pack_size, t_err, t_code = src_git._git_inout('count-objects', ['-v']) + pack_size = pack_size[pack_size.find('size-pack:')+11:].split('\n')[0] + print '"UNPACK-SIZE": \"%d\"' % int(pack_size) + try: + get_git_repo_size() + except Exception as err: + print repr(err) + + def retrieve_license(src_root): + LICENSE_FORM = r'.*Licensed under the (.*) License, Version (.*) \(the \"License\"\).*;' + sdk_license = None + for f in os.listdir(src_root): + if os.path.isfile(os.path.join(src_root, f)) and f.startswith('LICENSE'): + with open(os.path.join(src_root, f), 'r') as r_file: + for x in r_file.readlines(): + matchline = re.search(LICENSE_FORM, x) + if matchline: + sdk_license = '%s-%s' % \ + (matchline.groups()[0], matchline.groups()[1]) + print 'LICENSE %s from %s' % (sdk_license, f) + if sdk_license is None: + sdk_license = 'Samsung' + return sdk_license + + + def retrieve_properties_from_project_def(src_root): + + sdk_package = '' + sdk_version = '' + sdk_profile = '' + sdk_web_name = '' + ret_error = '' + app_type = None + + MANIFEST = 'tizen-manifest.xml' + + # As guided, the last is the main app from multi project listup. + def_file_list = [] + # Single project + def_file_list.append(os.path.join(src_root, MANIFEST)) + # Multi project + multiproject_list_f = os.path.join(src_root, 'WORKSPACE') + if os.path.isfile(multiproject_list_f): + with open(multiproject_list_f, 'r') as r_file: + for x in r_file.readlines(): + if os.path.isdir(os.path.join(src_root, x.rstrip())): + if os.path.isfile(os.path.join(src_root, x.rstrip(), MANIFEST)): + def_file_list.append(os.path.join(src_root, x.rstrip(), MANIFEST)) + elif os.path.isfile(os.path.join(src_root, x.rstrip(), 'config.xml')): + def_file_list.append(os.path.join(src_root, x.rstrip(), 'config.xml')) + + #for def_file in def_file_list: + #TODO: Should be the last item + def_file = def_file_list[-1] + if os.path.isfile(def_file): + print 'Property file: %s' % def_file + try: + with open(def_file, 'r') as df: + manifest_dump = df.read() + if os.path.basename(def_file) == 'tizen-manifest.xml': + xmlstring = re.sub('\\sxmlns="[^"]+"', '', manifest_dump, count=1) + _root = ElementTree.fromstring(xmlstring) + for child in _root: + if child.tag == 'profile': + sdk_profile = child.attrib.get('name', sdk_profile) + print 'Set SDK_PROFILE:[%s]' % sdk_profile + sdk_package = _root.attrib.get('package', sdk_package) + sdk_version = _root.attrib.get('version', sdk_version) + app_type = 'tpk' + elif os.path.basename(def_file) == 'config.xml': + xmlstring = re.sub('\\sxmlns:tizen="[^"]+"', '', manifest_dump, count=1) + xmlstring = re.sub('\\sxmlns="[^"]+"', '', manifest_dump, count=1) + xmlstring = xmlstring.replace('tizen:', '') + _root = ElementTree.fromstring(xmlstring) + for child in _root: + if child.tag == 'profile': + sdk_profile = child.attrib.get('name', sdk_profile) + print 'Set SDK_PROFILE:[%s]' % sdk_profile + elif child.tag == 'application': + sdk_package = child.attrib.get('package', sdk_package) + print 'Set SDK_PACKAGE:[%s]' % sdk_package + sdk_web_name = child.attrib.get('id', sdk_package) + if "%s." % sdk_package in sdk_web_name: + sdk_web_name = sdk_web_name.replace("%s." % sdk_package, "") + print 'Set SDK_WEB_NAME:[%s]' % sdk_web_name + #elif child.tag == 'name' and child.attrib == {}: + # sdk_web_name = child.text + # print 'Set SDK_WEB_NAME:[%s]' % sdk_web_name + sdk_version = _root.attrib.get('version', sdk_version) + app_type = 'wgt' + except Exception as err: + ret_error = err + if not sdk_profile: + sdk_profile = '' + # ret_error += 'ERROR: No "profile" tag in property file(tizen-manifest.xml)' + print 'TIZEN-MANIFEST.XML (%s) (%s)' % (sdk_package, sdk_version) + return {'sdk_package': sdk_package, 'sdk_version': sdk_version, \ + 'sdk_profile': sdk_profile, 'ret_error': ret_error, \ + 'app_type': app_type, 'sdk_web_name': sdk_web_name} + + def retrieve_commit_id(src_git): + commit_id = src_git.rev_parse('HEAD') + print 'commit from (HEAD) = %s' % (commit_id) + return commit_id + + def retrieve_revision_number(src_git, tag): + _ret = {} + try: + _ret['revision_number'] = src_git.rev_parse(tag) + except Exception as err: + print repr(err) + print 'revision from (%s) = %s' % (tag, _ret) + return _ret + + def retrieve_commit_message(src_git): + t_outs, t_err, t_code = src_git._git_inout('log', ['--format=%B', '-n 1']) + commit_message = t_outs[:t_outs.rfind("\nChange-Id: ")].rstrip() + '\n' + print "author comment : %s" % commit_message + return commit_message + + def retrieve_sr_message(src_git, tag): + t_outs, t_err, t_code = src_git._git_inout('show', [tag, '--format=##chkstr:%b']) + sr_message = '\n'.join(t_outs[:t_outs.rfind("##chkstr:")].split('\n')[3:]) + print "sr comment from(%s) : [%s]" % (tag, sr_message.rstrip()) + return sr_message + + def retrieve_committer(src_git): + t_outs, t_err, t_code = src_git._git_inout('log', ['--format=%cE', '-n 1']) + committer = t_outs.strip().rstrip().replace('<','').replace('>','') + print "committer : %s" % committer + return committer + + def retrieve_submitter(src_git, tag): + t_outs, st_err, st_ret = src_git._git_inout('for-each-ref', \ + ['--count=1', '--sort=-taggerdate', \ + '--format=%(taggeremail)', 'refs/tags/%s' % tag]) + submitter = t_outs.strip().rstrip().replace('<','').replace('>','') + print "submitter : %s" % submitter + return submitter + + def retrieve_commit_date(src_git): + t_outs, t_err, t_code = src_git._git_inout('log', ['--format=%ci', '-n 1']) + commit_date = t_outs + print "commit_date : %s" % commit_date + return commit_date + + def retrieve_submit_date(src_git, tag): + submit_date, sd_err, sd_ret = src_git._git_inout('for-each-ref', \ + ['--count=1', '--sort=-taggerdate', \ + '--format=%(taggerdate:iso)', 'refs/tags/%s' % tag]) + print "submit_date : %s" % submit_date + return submit_date + + ret_prop.update(retrieve_properties_from_project_def(src_git.path)) + ret_prop['commit_id'] = retrieve_commit_id(src_git) + ret_prop['revision_number'] = retrieve_revision_number(src_git, tag).get('revision_number', \ + ret_prop.get('commit_id')) + ret_prop['commit_message'] = retrieve_commit_message(src_git) + ret_prop['sr_message'] = retrieve_sr_message(src_git, tag) + ret_prop['sdk_license'] = retrieve_license(src_git.path) + + ret_prop['committer'] = retrieve_committer(src_git) + ret_prop['submitter'] = retrieve_submitter(src_git, tag) + ret_prop['commit_date'] = retrieve_commit_date(src_git) + ret_prop['submit_date'] = retrieve_submit_date(src_git, tag) + + for r in ret_prop: + if ret_prop[r]: + print ' [%s]: %s' % (r, ret_prop[r].rstrip()) + else: + print ' [%s]: %s' % (r, ret_prop[r]) + return ret_prop + +def fetch_source(src_root, project, branch=None, tag=None): + """ Cloning APP source code """ + + print 'Cloning source code %s' % src_root + if True: + if not clone_gitproject(project, src_root, \ + git_cache_dir='nonexist'): + raise LocalError('Error cloning project %s' % project) + working_git = Git(src_root) + if tag is not None: + working_git.checkout(tag) + return working_git, tag + elif branch is not None: + working_git.checkout(branch) + return working_git, branch + +def zipping_workspace(src_root): + """ Create tarball to share source code easily between VM machine """ + + def ZipDir(inputDir, outputZip): + '''Zip up a directory and preserve symlinks and empty directories''' + zipOut = zipfile.ZipFile(outputZip, 'w', compression=zipfile.ZIP_DEFLATED) + + rootLen = len(os.path.dirname(inputDir)) + def _ArchiveDirectory(parentDirectory): + contents = os.listdir(parentDirectory) + #store empty directories + if not contents: + archiveRoot = parentDirectory[rootLen:].replace('\\', '/').lstrip('/') + zipInfo = zipfile.ZipInfo(archiveRoot+'/') + zipOut.writestr(zipInfo, '') + for item in contents: + fullPath = os.path.join(parentDirectory, item) + if fullPath.endswith('/.git'): continue + if os.path.isdir(fullPath) and not os.path.islink(fullPath): + _ArchiveDirectory(fullPath) + else: + archiveRoot = fullPath[rootLen:].replace('\\', '/').lstrip('/') + if os.path.islink(fullPath): + zipInfo = zipfile.ZipInfo(archiveRoot) + zipInfo.create_system = 3 + # long type of hex val of '0xA1ED0000L', + # say, symlink attr magic... + zipInfo.external_attr = 2716663808L + zipOut.writestr(zipInfo, os.readlink(fullPath)) + else: + zipOut.write(fullPath, archiveRoot, zipfile.ZIP_DEFLATED) + _ArchiveDirectory(inputDir) + zipOut.close() + + current_dir = os.getcwd() + os.chdir(os.path.dirname(src_root)) + src_d = os.path.basename(src_root) + out_f = src_d + '.zip' + ZipDir(src_d, out_f) + os.chdir(current_dir) + +def generate_spec_file(tizen_studio, src_git, prop, sourceData, confData, ciData, app_type): + """ Use spec file template """ + + found_archs = [] + archs_lookup = [['i386', 'i486', 'i586', 'i686', 'x86'], \ + ['arm', 'armv7l'], \ + ['x86_64'], \ + ['aarch64']] + + internal_package_name = prop.get('sdk_package') + if app_type == 'wgt': + internal_package_name = prop.get('sdk_web_name') + + for fn in os.listdir(tizen_studio.builddir): + if app_type == 'tpk': + mtch = re.search(r'(.*)-([0-9.]+)-(.*)(? 0: + try: + src_git.push(repo = 'origin', src = '%s' % new_branch, force=True) + break + except Exception as err: + print 'git push exception: %s' % repr(err) + push_err_msg = push_err_msg + '\n' + str(err) + push_retry -= 1 + sleep(5) + if not push_retry: + postData['status'] = 'FAIL' + postData['error'] = 'Pushing binary error.\n%s' % push_error_msg + return postData + + message = '%s\n' \ + '[ABS] Ready.\n' \ + '- Original Tag: %s (%s)\n' \ + '- Original Commit: %s\n' \ + '- Requested By: %s\n' \ + % (prop.get('sr_message'), sourceData.get('TAG_NAME'), \ + prop.get('revision_number'), prop.get('commit_id'), sourceData.get('USER_EMAIL')) + try: + src_git.create_tag(name=new_tag, msg=message, commit=commit_id_new) + except Exception as err: + postData['status'] = 'FAIL' + postData['error'] = 'Creating tag error.\n%s' % repr(err) + return postData + + try: + revision_number_new = src_git.self.rev_parse(new_tag) + except: + revision_number_new = prop.get('commit_id') + + push_err_msg = '' + push_retry = 5 + while push_retry >0: + try: + src_git.push_tag('origin', new_tag) + break + except Exception as err: + print 'Cannot push tag %s (%s)' % (new_tag, str(err)) + push_err_msg = push_err_msg + '\n' + str(err) + push_retry -= 1 + sleep(5) + if not push_retry: + postData['status'] = 'FAIL' + postData['error'] = 'Pushing tag error.\n%s' % push_err_msg + return postData + + return postData + +def make_data(): + + # Fetch required variables... + sourceData = {'PROJECT' : os.getenv('PROJECT'), \ + 'GIT_PATH' : os.getenv('GIT_PATH'), \ + 'REVISION' : os.getenv('REVISION'), \ + 'BRANCH' : os.getenv('BRANCH'), \ + 'TAG_NAME' : os.getenv('TAG_NAME'), \ + 'USER_EMAIL': os.getenv('USER_EMAIL'), \ + 'PACKAGE_NAME_FROM_MAPPING': os.getenv('PACKAGE_NAME_FROM_MAPPING') \ + } + pprint(sourceData) + confData = {'ABS_CLI_SCRIPT_GIT' : os.getenv('ABS_CLI_SCRIPT_GIT'), \ + 'ROOTSTRAP_URL' : os.getenv('ROOTSTRAP_URL'), \ + 'SDK_SIGN_AUTHOR' : os.getenv('SDK_SIGN_AUTHOR'), \ + 'ABS_ROOTSTRAP_URL' : os.getenv('ABS_ROOTSTRAP_URL'), \ + 'ABS_MAILINGLIST' : os.getenv('ABS_MAILINGLIST'), \ + 'SDK_SIGN_GIT' : os.getenv('SDK_SIGN_GIT'), \ + 'SDK_TOOL_GIT' : os.getenv('SDK_TOOL_GIT'), \ + 'SDK_SIGN_AUTHOR' : os.getenv('SDK_SIGN_AUTHOR'), \ + 'SDK_SIGN_DIST' : os.getenv('SDK_SIGN_DIST'), \ + 'ABS_SDK_PACKAGE_SERVER': os.getenv('ABS_SDK_PACKAGE_SERVER'), \ + 'BUILD_MODE' : os.getenv('BUILD_MODE'), \ + 'EXCLUDE_ARCH' : os.getenv('EXCLUDE_ARCH') \ + } + pprint(confData) + ciData = {'ABS_WORKSPACE': os.getenv('ABS_WORKSPACE'), \ + 'JOB_NAME' : os.getenv('JOB_NAME'), \ + 'BUILD_ID' : os.getenv('BUILD_ID'), \ + } + if os.getenv('BUILD_CAUSE') is None: + ciData['BUILD_CAUSE'] = 'GIT' + else: + ciData['BUILD_CAUSE'] = 'CHECK' + confData['ABS_ROOTSTRAP_URL'] = os.getenv('BUILD_CAUSE').split(',')[1] + pprint(ciData) + return sourceData, confData, ciData + +def make_work_directory(basedir, builddir): + + print 'BASEDIR: %s' % basedir + if os.path.exists(basedir): + shutil.rmtree(basedir) + os.makedirs(builddir) + if True: + try: + if os.path.isdir('/share/build'): + shutil.rmtree('/share/build') + elif os.path.islink('/share/build'): + os.unlink('/share/build') + else: + os.remove('/share/build') + except Exception as err: + print err + os.symlink(builddir, '/share/build') + +def make_property_data(tag, branch, src_git, int_tag): + + pData = {} + project_properties = retrieve_project_property(src_git, int_tag) + pData['PACKAGE_NAME'] = project_properties.get('sdk_package') + pData['PROFILE'] = project_properties.get('sdk_profile') + pData['NEW_BRANCH'] = branch + '_tpk' + if tag is not None: + pData['NEW_TAG'] = tag.replace(branch, pData.get('NEW_BRANCH')) + pData['COMMIT_ID'] = project_properties.get('commit_id') + pData['COMMITTER'] = project_properties.get('committer') + pData['COMMIT_DATE'] = project_properties.get('commit_date').strip() + pData['COMMIT_MESSAGE'] = project_properties.get('commit_message').strip() + pData['TAGGER'] = project_properties.get('submitter') + pData['TAG_REVISION'] = project_properties.get('revision_number') + pData['TAG_DATE'] = project_properties.get('submit_date').strip() + pData['TAG_MESSAGE'] = project_properties.get('sr_message').strip() + + if project_properties.get('ret_error') != '': + print 'You got property error:\n%s' % project_properties.get('ret_error') + return None + if 'sdk_license' not in project_properties or project_properties.get('sdk_license') is None: + print 'No LICENSE provided.' + return None + + return pData, project_properties; + +def rootstrap_update_process(tizenStudioInst, basedir, confData): + + if True: + # Update rootstrap + new_rs = confData.get('ABS_ROOTSTRAP_URL') + supported_profiles = ['mobile', 'wearable'] + #FIXME: Check if url exists... + new_rs = new_rs.replace('__RELEASE_ID__', 'latest') + session = requests.Session() + session.trust_env = False + _r = session.get(new_rs) + if _r.status_code != 200: + # Search previous snapshot if latest have no rootstraps + if '/latest/' in new_rs: + # Download parent page and parse links + rs_parent_url = new_rs.replace('latest/builddata/private-sdk-rootstrap/', '') + _r = session.get(rs_parent_url) + snapshot_list = [] + for sn in _r.text.split('\n'): + if 'alt="[DIR]">> Adding %s(%s) %s %s' % (prj_name, _brch, _obs_prj, _pkg) + list_all.append(item) + return list_all + +def find_build_list(): + # Make required paremeters as dictionary + events = [] + if os.getenv('BUILD_CAUSE') is None: + event = {} + event['BUILD_CAUSE'] = os.getenv('BUILD_CAUSE') + event['PROJECT'] = None + event['GIT_PATH'] = os.getenv('GERRIT_PROJECT') + event['TAG_NAME'] = os.getenv('GERRIT_REFNAME').replace('refs/tags/', '') + event['BRANCH'] = event.get('TAG_NAME')[len('submit/'):event.get('TAG_NAME').rfind('/', len('submit/'))] + event['REVISION'] = os.getenv('GERRIT_NEWREV') + event['USER_EMAIL'] = os.getenv('GERRIT_EVENT_ACCOUNT_EMAIL') + # Find project mappings + obs_target_prjs = git_obs_map(event.get('GIT_PATH'), \ + event.get('BRANCH')) + for item in obs_target_prjs: + if item.get('OBS_staging_project') != 'abs': + continue + event['PROJECT'] = item.get('OBS_project') + event['PACKAGE_NAME_FROM_MAPPING'] = item.get('OBS_package') + events.append(event) + elif os.getenv('BUILD_CAUSE') is not None: + mainProject = os.getenv('BUILD_CAUSE').split(',')[0] + obs_target_prjs = list_all_packages(mainProject) + for item in obs_target_prjs: + if item.get('OBS_staging_project') != 'abs': + continue + event = {} + event['BUILD_CAUSE'] = os.getenv('BUILD_CAUSE') + event['PROJECT'] = mainProject + event['GIT_PATH'] = item.get('Project_name') + event['TAG_NAME'] = None + event['BRANCH'] = item.get('Branch_name') + event['REVISION'] = None + event['USER_EMAIL'] = None + events.append(event) + + return events + +def parseConf(mainProject, index): + + confData = {} + + # Get script path from system config file + meta_script_map = ast.literal_eval(os.getenv('ABS_CLI_SCRIPT_GIT_MAP')) + meta_script = meta_script_map.get('default') + if meta_script_map.get(mainProject) is not None: + meta_script = meta_script_map.get(mainProject) + print 'meta_script:%s' % meta_script + + # Fetch project environment variables from config git + configPath = os.path.join(os.getenv('WORKSPACE'), 'abs_config_%d' % index) + if not clone_gitproject(meta_script.split(',')[0], \ + configPath, \ + git_cache_dir='non-exist-dir'): + print 'Wrapper script %s clone fail' % meta_script.split(',')[0] + sys.exit(3) + workingGit = Git(configPath) + workingGit.checkout(meta_script.split(',')[1]) + + # Save project environment variables + if not os.path.isfile(os.path.join(configPath, 'conf.%s' % mainProject)): + print 'No conf.%s' % mainProject + return None + with open(os.path.join(configPath, 'conf.%s' % mainProject), 'r') as rf: + confContents = rf.readlines() + print "%s" % confContents + for x in confContents: + confData[x.split('=')[0]] = x.split('=')[1].strip() + + confData['ABS_CLI_SCRIPT_GIT'] = meta_script + + return confData + +def main(argv): + """ + Script entry point. + """ + + # Regular Gerrit Event + # Required fields checking + for ev in ['GERRIT_REFNAME', 'GERRIT_PROJECT', 'GERRIT_NEWREV', 'GERRIT_EVENT_ACCOUNT_EMAIL']: + if ev not in os.environ and os.getenv('BUILD_CAUSE') is None: + print 'Required event parameters are missing %s' % ev + sys.exit(4) + + if is_ref_deleted(os.getenv('GERRIT_OLDREV'), os.getenv('GERRIT_NEWREV')): + print 'REF DELETED(%s)!' % os.getenv('GERRIT_OLDREV') + return + + fullList = find_build_list() + pprint(fullList) + + index = 1 + confData = None + for events in fullList: + + extra_params = {'BACKEND_SELECTION': 'ABS', 'ABS_WORKSPACE': '/home/build/workspace'} + extra_params.update(events) + + if events.get('BUILD_CAUSE') is None: + confData = parseConf(events.get('PROJECT'), index) + elif confData is None: + confData = parseConf(events.get('PROJECT'), index) + if confData is None: + continue + extra_params.update(confData) + + for key, value in extra_params.items(): + if value is None: + extra_params[key] = '' + pprint(extra_params) + + # Forward it to build job + trigger_next('ABS_BUILD_%d_%d_%s' \ + % (int(os.getenv('BUILD_NUMBER')), index, os.path.basename(events.get('GIT_PATH'))), \ + {'ABS_SERVER_TYPE': 'DOCKER'}, extra_params=extra_params) + index += 1 + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) + diff --git a/abs/job_abs_frontend.sh b/abs/job_abs_frontend.sh new file mode 100755 index 0000000..2c00a2a --- /dev/null +++ b/abs/job_abs_frontend.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +set -x + +DOCKER_VOL="docker_volume" +SCRIPT_PATH="/var/lib/jenkins/jenkins-scripts" + +mkdir ${DOCKER_VOL} +export -p > ${DOCKER_VOL}/envs +chmod +x ${DOCKER_VOL}/envs +sed --quiet 's/^\(export \)*//' ${DOCKER_VOL}/envs + +sudo docker pull ${ABS_TIZEN_STUDIO_DOCKER_IMAGE} +sudo docker run -t -d -w /share \ + -v `pwd`/`basename ${SCRIPT_PATH}`:${SCRIPT_PATH} \ + -v `pwd`/${DOCKER_VOL}:/share:rw \ + --name abs_${BUILD_NUMBER} ${ABS_TIZEN_STUDIO_DOCKER_IMAGE} + +echo """#!/bin/bash + +source /share/envs + +chown build:build -R /share + +su build -c '"python ${SCRIPT_PATH}/abs/job_abs_backend.py"' + +""" > ${DOCKER_VOL}/build.sh +chmod +x ${DOCKER_VOL}/build.sh + +cp -rf ~/.ssh ${DOCKER_VOL}/ +sudo docker exec abs_${BUILD_NUMBER} bash -c "cp -r /share/.ssh /home/build/.ssh; \ + chown -R build:build /home/build/.ssh; \ + chmod 600 /home/build/.ssh/${ABS_SSH_KEY}" + +sudo docker exec abs_${BUILD_NUMBER} bash -lc "./build.sh" + +sudo docker stop abs_${BUILD_NUMBER} +sudo docker rm abs_${BUILD_NUMBER} + +cp docker_volume/*.env . + diff --git a/common/tizen_studio_docker.py b/common/tizen_studio_docker.py new file mode 100644 index 0000000..60f19c3 --- /dev/null +++ b/common/tizen_studio_docker.py @@ -0,0 +1,240 @@ +#!/usr/bin/env python +# vim: ai ts=4 sts=4 et sw=4 +# +# Copyright (C) 2017 Samsung Electronics. Co,. Ltd. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; version 2 +# of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +""" +Interface of Tizen Studio +""" + +import os +import sys +import shutil +import re +import stat +import ast +import subprocess +import glob +from datetime import datetime +from random import randint + +sys.path.insert(1, os.path.join(sys.path[0], '..')) + +from common.utils import wget_noproxy, list_files_in_url, tail +from common.buildtrigger import trigger_info, trigger_next +from common.git import Git, clone_gitproject +from common.gerrit import GerritEnv + +class LocalError(Exception): + """Local error exception.""" + pass + +class TizenStudio(object): + + def __init__(self, baseDir=None, confData=None): + + if baseDir is not None: + self.basedir = baseDir + else: + self.basedir = os.path.join(os.getenv('WORKSPACE'), os.getenv('JOB_NAME')) + self.builddir = os.path.join(self.basedir, 'build') + + if confData is not None: + self.confData = confData + + self.build_root = '/home/build' + self.share_root = '/share/build' + + # Fetch ABS cli wrapper from gerrit + self.wrapper_path = self.fetch_resources(confData.get('ABS_CLI_SCRIPT_GIT').split(',')) + print 'wrapper_path:%s' % self.wrapper_path + + self.sdk_path = os.path.join(self.build_root, 'tizen-sdk-cli') + + def get_template(self, mode=None): + + with open(os.path.join(self.wrapper_path, '%s.template' % mode), 'r') as rf: + cmd_template = rf.read() + return cmd_template + + def run_tizen_studio(self, cmd_to_run): + + with open(os.path.join(self.builddir, 'run'), 'w') as fcmdl: + fcmdl.write('%s' % cmd_to_run) + os.chmod(os.path.join(self.builddir, 'run'), 0777) + + if True: + print 'DOCKER running...' + cmd = os.path.join(self.builddir, 'run') + print 'Running inside docker [%s]' % cmd + + scr = subprocess.call(cmd, stdout=sys.stdout, + stderr=sys.stderr, shell=True) + + #read testresult from file + return int(scr) + + def fetch_resources(self, rc_repo_branch=[]): + + rc_git_path, rc_git_branch = rc_repo_branch + print 'fetch_resources: [%s] [%s]' % (rc_git_path, rc_git_branch) + rc_dir = os.path.join(self.share_root, os.path.basename(rc_git_path)) + print 'rc_dir:%s' % rc_dir + if not clone_gitproject(rc_git_path, rc_dir): + print 'Error clone project %s' % rc_git_path + rc_git = Git(rc_dir) + rc_git.checkout(rc_git_branch) + return rc_git.path + + def update_rootstrap(self, rootstrap_src_url, rootstrap_version, profiles=[]): + + # Fetch sdk-tool from gerrit + sdk_tool_path = self.fetch_resources(self.confData.get('SDK_TOOL_GIT').split(',')) + print 'sdk_tool_path:%s' % sdk_tool_path + + # Fetch hash-sign package from gerrit + hash_sign_path = self.fetch_resources(self.confData.get('SDK_SIGN_GIT').split(',')) + print 'hash_sign_path:%s' % hash_sign_path + certificate_path = hash_sign_path + author_cert_path = self.confData.get('SDK_SIGN_AUTHOR').split(',')[0] + author_cert_pass = self.confData.get('SDK_SIGN_AUTHOR').split(',')[1] + dist_cert_path = self.confData.get('SDK_SIGN_DIST').split(',')[0] + dist_cert_pass = self.confData.get('SDK_SIGN_DIST').split(',')[1] + + update_cmd = self.get_template('update') + update_cmd = update_cmd.replace( \ + '__SDK_PATH__', self.sdk_path).replace( \ + '__SHARE_ROOT__', self.share_root).replace( \ + '__TOOL_PATH__', os.path.join(self.share_root, os.path.basename(sdk_tool_path))).replace( \ + '__WRAPPER_PATH__', os.path.join(self.share_root, os.path.basename(self.wrapper_path))).replace( \ + '__PROFILE__', ' '.join(profiles)).replace( \ + '__PACKAGE_SERVER__', os.getenv('ABS_SDK_PACKAGE_SERVER')).replace( \ + '__ROOTSTRAP_URL__', rootstrap_src_url).replace( \ + '__CERTIFICATE_PATH__', certificate_path).replace( \ + '__AUTHOR_CERTIFICATE__', author_cert_path).replace( \ + '__AUTHOR_PASSWORD__', author_cert_pass).replace( \ + '__DIST_CERTIFICATE__', dist_cert_path).replace( \ + '__DIST_PASSWORD__', dist_cert_pass) + + update_cmd = update_cmd.replace('su - build -c', 'sudo su - build -c') + print 'UPDATE_CMD:\n%s' % update_cmd + + #### Running QEMU to launch Tizen Studio #### + print '[ TizenStudio START ] %s' % (str(datetime.now())) + sys.stdout.flush() + ret = self.run_tizen_studio(update_cmd) + print '[ TizenStudio END ] %s' % (str(datetime.now())) + + def gather_build_result(self): + + # Get installed rootstrap version + built_version = None + with open(os.path.join(self.builddir, 'rsver')) as rsverfile: + built_version = rsverfile.read().replace(' ', ', ').replace('\n', ' ').encode('utf8') + built_version = built_version.split('.')[-1] + print 'Installed RS version... %s' % built_version + if built_version is None: + print 'Not able detect installed Rootstrap version' + + tizen_studio_version = 'Tizen CLI 0.0.0' + tizen_studio_distribution = 'default' + with open(os.path.join(self.builddir, 'tizen_studio_version')) as tizenstudioverfile: + tizen_studio_version = tizenstudioverfile.read().strip() + print 'Installed Tizen Studio version... %s' % tizen_studio_version + with open(os.path.join(self.builddir, 'tizen_studio_distribution')) as tizenstudiodistributionfile: + tizen_studio_distribution = tizenstudiodistributionfile.read().strip() + print 'Installed Tizen Studio Distribution... %s' % tizen_studio_distribution + + self.built_version = built_version + self.tizen_studio_version = tizen_studio_version + self.tizen_studio_distribution = tizen_studio_distribution + + # Get building log + self.buildlog = '' + mtime = lambda f: os.stat(os.path.join(self.builddir, f)).st_mtime + for filename in list(sorted(os.listdir(self.builddir), key=mtime)): + if re.match('build.*\.log', filename): + onefile = tail(os.path.join(self.builddir, filename), c=4096) + self.buildlog = onefile[:onefile.rfind("Finished build-native")] + + if self.build_result != 0: + return {'built_version': built_version, \ + 'title': 'FAIL::CLI BUILD', \ + 'buildlog': self.buildlog, \ + 'tizen_studio_version': tizen_studio_version, \ + 'tizen_studio_distribution': tizen_studio_distribution} + + # Get packaging log + sys.stdout.flush() + print glob.glob(os.path.join(self.builddir, '*.tpk')) + print glob.glob(os.path.join(self.builddir, '*.wgt')) + print os.listdir(self.builddir) + sys.stdout.flush() + if not glob.glob(os.path.join(self.builddir, '*.tpk')) and not glob.glob(os.path.join(self.builddir, '*.wgt')): + for filename in os.listdir(self.builddir): + if re.match('pkg.*\.log', filename): + self.buildlog = self.buildlog + tail(os.path.join(self.builddir, filename)) + return {'built_version': built_version, \ + 'title': 'FAIL::CLI PKG', \ + 'buildlog': self.buildlog, \ + 'tizen_studio_version': tizen_studio_version, \ + 'tizen_studio_distribution': tizen_studio_distribution} + + return {'built_version': built_version, \ + 'title': None, \ + 'buildlog': None, \ + 'tizen_studio_version': tizen_studio_version, \ + 'tizen_studio_distribution': tizen_studio_distribution} + + def build_app_source(self, package, profile=None, gitpath=None, build_mode='Release', parallel_jobs='', build_type='default'): + + PROFILE = profile + + build_cmd = self.get_template('build') + print 'build_mode:%s' % build_mode + print 'package:%s' % package + print 'parrel_jobs:%s' % parallel_jobs + print 'profile:%s' % PROFILE + print 'sdk_path:%s' % self.sdk_path + print 'share_root:%s' % self.share_root + print 'build_root:%s' % self.build_root + build_cmd = build_cmd.replace( \ + '__SDK_PATH__', self.sdk_path).replace( \ + '__SHARE_ROOT__', self.share_root).replace( \ + '__BUILD_ROOT__', self.build_root).replace( \ + '__PROFILE__', PROFILE).replace( \ + '__PACKAGE__', package).replace( \ + '__PARALLEL_JOBS__', parallel_jobs).replace( \ + '__BUILD_MODE__', build_mode).replace( \ + '__BUILD_TYPE__', build_type) + + build_cmd = build_cmd.replace('su - build -c', 'sudo su - build -c') + print 'BUILD_CMD:\n%s' % build_cmd + + #### Running QEMU to launch Tizen Studio #### + print '[ TizenStudio START ] %s' % (str(datetime.now())) + sys.stdout.flush() + ret = self.run_tizen_studio(build_cmd) + print '[ TizenStudio END ] %s' % (str(datetime.now())) + + self.build_result = int(ret) + return self.build_result + +if __name__ == '__main__': + print 'This is not callable module' + sys.exit(-1) + diff --git a/debian/jenkins-scripts-abs.install b/debian/jenkins-scripts-abs.install index 8e95251..18a2f88 100644 --- a/debian/jenkins-scripts-abs.install +++ b/debian/jenkins-scripts-abs.install @@ -1,2 +1,3 @@ debian/tmp/abs/* /var/lib/jenkins/jenkins-scripts/abs/ debian/tmp/common/tizen_studio.py /var/lib/jenkins/jenkins-scripts/common/ +debian/tmp/common/tizen_studio_docker.py /var/lib/jenkins/jenkins-scripts/common/ diff --git a/packaging/jenkins-scripts.spec b/packaging/jenkins-scripts.spec index e009a79..1001e9b 100755 --- a/packaging/jenkins-scripts.spec +++ b/packaging/jenkins-scripts.spec @@ -340,10 +340,14 @@ fi %dir %{destdir}/abs %{destdir}/abs/job_abs_build.py %{destdir}/abs/job_abs_update.py +%{destdir}/abs/job_abs_frontend.py +%{destdir}/abs/job_abs_frontend.sh +%{destdir}/abs/job_abs_backend.py %{destdir}/abs/job_update_abs_rbs_status_for_dashboard.py %{destdir}/abs/job_update_status_remote_for_dashboard.py %dir %{destdir}/common %{destdir}/common/tizen_studio.py +%{destdir}/common/tizen_studio_docker.py %{destdir}/abs/report_template %files tzs -- 2.7.4