From 4a901c2b46fa988a1401503c74d3c525bc820cb3 Mon Sep 17 00:00:00 2001 From: SoonKyu Park Date: Wed, 20 Sep 2017 16:52:19 +0900 Subject: [PATCH] Add job_generate_gbs_full_build_script.py which generate gbs fullbuild bash script in dashboard Change-Id: I21c03a91fec866941d8afe33a70263bee43225f9 --- job_generate_gbs_full_build_script.py | 271 ++++++++++++++++++++++++++++++++++ 1 file changed, 271 insertions(+) create mode 100644 job_generate_gbs_full_build_script.py diff --git a/job_generate_gbs_full_build_script.py b/job_generate_gbs_full_build_script.py new file mode 100644 index 0000000..4a64da6 --- /dev/null +++ b/job_generate_gbs_full_build_script.py @@ -0,0 +1,271 @@ +#!/usr/bin/python +# +# Copyright (C) 2010, 2011, 2012, 2013, 2014 Intel, Inc. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +import xml.etree.ElementTree as ET +import sys +import os +import re +import json +import urllib2 +import subprocess + +from common.utils import sync + + +# prepare related global variables +workspace = os.getenv('WORKSPACE') +basic_url = 'http://download.tizen.org/snapshots/tizen/' +gbs_meta_default_profile = os.getenv('GBS_META_DEFAULT_PROFILE') +GBS_META_SUPPORT_TIZEN_VER = os.getenv('GBS_META_SUPPORT_TIZEN_VER') +PROJECT = os.getenv('PROJECT') +SNAPSHOT = os.getenv('SNAPSHOT') +REPOSITORY = os.getenv('REPOSITORY') +ARCHITECTURE = os.getenv('ARCHITECTURE') +BUILD_ARGS = os.getenv('BUILD_ARGS') +BUILD_WORKSPACE = os.getenv('BUILD_WORKSPACE') +USER_NAME = os.getenv('USER_NAME') +FILE_NAME = os.getenv('FILE_NAME') +#gbs_ref_fullbuild_root = workspace +gbs_default_build_arg='timeout 6h gbs build --threads=16 --define "jobs 8" --define "_smp_mflags -j8" --baselibs --clean-once' +fullbuild_script='' +SRC_ROOT=BUILD_WORKSPACE+'/'+'SRC-ROOT/' +BUILD_ROOT=BUILD_WORKSPACE+'/'+'GBS-ROOT/' +t_ver_profile={} + + +class LocalError(Exception): + """Local error exception.""" + pass + + +def isNumber(s): + try: + float(s) + return True + except ValueError: + return False + + +def get_tizen_version_profile(): + print '---get_tizen_version_profile start----------------------------------------' + + latest_ver = _get_latest_tizen_ver() + if isNumber(PROJECT.split(':')[1]): + t_ver_profile['version'] = PROJECT.split(':')[1] + else: + t_ver_profile['version'] = latest_ver + + t_ver_profile['profile'] = PROJECT.split(':')[-1].lower() + + +def setup_build_environment(): + print '---setup_build_environment start----------------------------------------' + + partial_script = '' + + #initial bash + partial_script += '#!/bin/bash\n\n' + #create workspace dir + partial_script += '#create workspace\n' + partial_script += 'mkdir -p '+BUILD_WORKSPACE+'\n' + partial_script += 'cd '+BUILD_WORKSPACE+'\n' + + partial_script += '\n#create SRC_ROOT where all tizen sources are downloaded\n' + partial_script += 'mkdir -p '+SRC_ROOT+'\n' + + print 'setup_build_environment script:\n%s' %partial_script + return partial_script + + +def _get_latest_tizen_ver(): + latest_ver = GBS_META_SUPPORT_TIZEN_VER.split(' ')[0] + for ver in GBS_META_SUPPORT_TIZEN_VER: + if (latest_ver < ver): + latest_ver = ver + + return latest_ver + + +def do_repo_init(): + print '---create_repo_init_arg start----------------------------------------' + + print t_ver_profile + partial_script = '' + + #default arg + partial_script += '\n#repo init\n' + partial_script += '/usr/bin/repo init -u https://git.tizen.org/cgit/scm/manifest' + + #add '-b' option + latest_tizen_ver = _get_latest_tizen_ver() + if t_ver_profile['version'] == latest_tizen_ver: + tizen_branch = 'tizen' + else: + tizen_branch = 'tizen_'+t_ver_profile['version'] + partial_script += ' -b '+tizen_branch + + #add '-m' option + xml_file = t_ver_profile['profile'].split(':')[-1].lower()+'_'+REPOSITORY + partial_script += ' -m '+xml_file+'.xml\n' + + print 'generate_repo_init_arg script:\n%s' %partial_script + return partial_script + + +def _get_snapshot_info(): + snapshot_info = {} + + latest_tizen_ver = _get_latest_tizen_ver() + if t_ver_profile['version'] == latest_tizen_ver: + tizen_ver = '' + else: + tizen_ver = t_ver_profile['version']+'-' + + ver_profile = tizen_ver+t_ver_profile['profile'] +# build_id = 'tizen-'+ver_profile+'_'+SNAPSHOT + build_id = SNAPSHOT + manifest_url = basic_url+'/'+ver_profile+'/'+build_id+'/builddata/manifest/' + + print manifest_url + res = urllib2.urlopen(manifest_url) + for index in res.read().split('a href="'): + if index.find(REPOSITORY+'.xml') != -1: + manifest_file=index.split('">')[0] + manifest_file_url = manifest_url + manifest_file + + snapshot_info['ver_profile'] = ver_profile + snapshot_info['build_id'] = build_id + snapshot_info['manifest_file_url'] = manifest_file_url + + print 'snapshot information: %s' %snapshot_info + return snapshot_info + + +def replace_snapshot_manifest(): + print '---replace_snapshot_manifest start----------------------------------------' + + partial_script = '' + snapshot = {} + + #download manifest file + snapshot = _get_snapshot_info() + partial_script += '\n#replace projects.xml file with manifest file in %s\n' %snapshot['build_id'] + partial_script += 'wget -O '+SRC_ROOT+'/.repo/manifests/'+t_ver_profile['profile']\ + +'/'+REPOSITORY+'/projects.xml '+snapshot['manifest_file_url'] + + print 'replace_snapshot_manifest:\n%s' %partial_script + return partial_script + +def do_repo_sync(): + print '---do_repo_sync start----------------------------------------' + + partial_script = '\n\n#repo sync\n' + partial_script += '/usr/bin/repo sync\n' + + return partial_script + + +def find_j_threads_no(): + print '---find_j_threads_no start----------------------------------------' + + #check number of cpus + partial_script = '\n#find number of "j" and number of "threads"\n' + + #calculate no of j and number of threads + partial_script += 'no_of_cpu=$(grep -c processor /proc/cpuinfo)\n' + partial_script += 'let no_of_j=$(expr $no_of_cpu/5+1)\n' + partial_script += 'let no_of_thread=$(expr $no_of_cpu/2+1)\n' + partial_script += 'if [ $no_of_j -le 1 ];then no_of_j=1 no_of_thread=$cpu_no;fi\n' + + return partial_script + + +def do_gbs_build(): + print '---do_gbs_build start----------------------------------------' + + partial_script = '\n#gbs build\n' + + #gbs default argument + partial_script += 'gbs build '+BUILD_ARGS + + #Add '-A' option + partial_script += ' -A '+ARCHITECTURE + + #Add '-B' option + partial_script += ' -B '+BUILD_ROOT + + #Add number of 'j' + partial_script += ' --define "jobs "$no_of_j"" --define "_smp_mflags -j"$no_of_j""' + + #Add number of 'threads' + partial_script += ' --threads="$no_of_thread"\n' + + return partial_script + + +def get_fullbuild_script(): + print '---get_fullbuild_script start----------------------------------------' + + fullbuild_script = '' + + get_tizen_version_profile() + fullbuild_script += setup_build_environment() + fullbuild_script += '\npushd '+SRC_ROOT+'\n' + fullbuild_script += do_repo_init() + fullbuild_script += replace_snapshot_manifest() + fullbuild_script += do_repo_sync() + fullbuild_script += find_j_threads_no() + fullbuild_script += do_gbs_build() + fullbuild_script += '\npopd'+'\n' + + print '=======================================================================' + print 'Result GBS fullbuild script is like below' + print '=======================================================================' + + print fullbuild_script + + return fullbuild_script + + +def generate_gbs_full_build_script(target_dir): + + script_out_text = get_fullbuild_script() + + target_filename = os.path.join(target_dir, "%s" + % (FILE_NAME)) + + if not os.path.exists(target_dir): + os.makedirs(target_dir) + with open(target_filename, "w") as f: + f.write(script_out_text) + +def main(): + """ + Script entry point. + """ + + print '----[JOB STARTED----' + + target_dir = ".dashboard/gbs_full_build_scripts" + generate_gbs_full_build_script(target_dir) + + sync_dest = os.path.join(os.getenv("IMG_SYNC_DEST_BASE"), "snapshots", target_dir) + sync(target_dir, sync_dest) + +if __name__ == '__main__': + sys.exit(main()) + -- 2.7.4