From 13e2e27bbd557aecb06e1433a4668f2c667f7563 Mon Sep 17 00:00:00 2001 From: SoonKyu Park Date: Mon, 29 Jan 2018 11:15:55 +0900 Subject: [PATCH] Add jenkins jobs and utils for gbs build on dashboard Change-Id: I53b0bebf3dd7cd28ba722c314da7b0312fc546e4 --- common/gbsutils.py | 625 ++++++++++++++++++++++++++++++++++ debian/jenkins-scripts-common.install | 1 + debian/jenkins-scripts.install | 2 + job_gbs_dashboard_build.py | 112 ++++++ job_gbsdbbuild_create_snapshot.py | 338 ++++++++++++++++++ packaging/jenkins-scripts.spec | 3 + 6 files changed, 1081 insertions(+) create mode 100644 common/gbsutils.py create mode 100644 job_gbs_dashboard_build.py create mode 100755 job_gbsdbbuild_create_snapshot.py diff --git a/common/gbsutils.py b/common/gbsutils.py new file mode 100644 index 0000000..3362247 --- /dev/null +++ b/common/gbsutils.py @@ -0,0 +1,625 @@ +#!/usr/bin/python +# +# Copyright (C) 2010, 2011, 2012, 2013, 2014 Intel, Inc. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# + +import sys +import os +import subprocess +import re +import shutil +import tempfile +import atexit +import urllib2 +import glob +from common.git import Git, clone_gitproject +from common.utils import sync + +class RuntimeException(Exception): + """Local error handler""" + pass + +class UtilsError(Exception): + """Local error handler""" + pass + +class LocalError(Exception): + """Local error exception.""" + pass + + +def gbs_conf_prefix(t_ver,profile,build_repository): + #[general] section + gbs_conf_text = '[general]\nfallback_to_native = true\nprofile = profile.'+t_ver+profile+'_'+build_repository+'\n\n\n' + + #Profile section + gbs_conf_text += '################## Profile Section ##################\n\n' + + return gbs_conf_text + + +class GBSBuild: + """A class which supports with statement""" + + def __init__(self, gbsdbbuild_project_id, obs_prj, basic_url, snapshot_num, full_dep_sel, new_pkg_list, rm_pkg_list, gbsbuild_workspace): + """ Initialize GBSBuild class""" + print "-----------------------------------------------------" + print "Initialize GBS Build Class" + print "-----------------------------------------------------" + + workspace = os.getenv('WORKSPACE') + + # prepare separate temp directory for each build + git_prj = 'scm/meta/obs' + git_branch="master" + tmpdir = tempfile.mkdtemp(prefix=workspace+'/') + atexit.register(shutil.rmtree, tmpdir) + prjdir = os.path.join(tmpdir, git_prj) + + # clone gerrit project to local dir + if not clone_gitproject(git_prj, prjdir): + print >> sys.stderr, 'Error cloning %s' %git_prj + return 1 + mygit = Git(prjdir) + mygit.checkout(git_branch) + + if len(re.findall('\D', obs_prj.split(':')[1].replace('.',''))) != 0: + t_ver = '' + t_ver_path = '/' + t_branch = 'tizen' + profile_array = obs_prj.split(':')[1:] + else: + t_ver = obs_prj.split(':')[1] + '-' + t_ver_path = '/' + obs_prj.split(':')[1] + '/' + t_branch = 'tizen_'+obs_prj.split(':')[-2] + profile_array = obs_prj.split(':')[2:] + + #find profile name + i=0 + profile = '' + while i < len(profile_array): + profile += profile_array[i].lower() + '-' + i += 1 + profile = profile.rstrip('-') + + basic_snapshot_url = basic_url + t_ver + profile + "/" + if snapshot_num == 'latest': + snapshot_url = basic_snapshot_url + snapshot_num + "/" + else: + snapshot_url = basic_snapshot_url + "tizen-" + t_ver + profile + "_" + snapshot_num + "/" + + if snapshot_url.split('/')[-1]: + snapshot_build_id = snapshot_url.split('/')[-1] + else: + snapshot_build_id = snapshot_url.split('/')[-2] + + repository = [] + arch_list= {} + path_prj_list= {} + obs_meta_file = prjdir + '/' + obs_prj + '/_meta' + lines = open(obs_meta_file).readlines() + for line in lines: + if line.find('repository name=') != -1: + repo_tmp=line.split('"')[1] + repository.append(repo_tmp) + arch_list[repo_tmp] = [] + path_prj_list[repo_tmp] = [] + if line.find('') != -1: + arch_tmp = line.split("")[1].split("")[0] + arch_list[repo_tmp].append(arch_tmp) + if line.find('path project=') != -1: + path_prj_arch = line.split('"')[1]+'##'+line.split('"')[3] + path_prj_list[repo_tmp].append(path_prj_arch) + + prj_src_root = os.path.join(gbsbuild_workspace,'SRC-ROOT',obs_prj) + build_root = os.path.join(gbsbuild_workspace,'GBS-ROOT') + live_root = os.path.join(gbsbuild_workspace,'live') + + if full_dep_sel == 'full build': + is_fullbuild = 'true' + else: + is_fullbuild = 'false' + + build_pkg_list = {} + if new_pkg_list: + item_is_git_name = 'true' + for item in new_pkg_list.split(' '): + if item_is_git_name == 'true': + git_name = item + item_is_git_name = 'false' + else: + commit_id = item + item_is_git_name = 'true' + build_pkg_list[git_name] = commit_id + + rm_pkg_list_array = [] + if rm_pkg_list: + for git_name in rm_pkg_list.split(' '): + rm_pkg_list_array.append(git_name) + + self.profile = profile + self.obs_prj = obs_prj + self.t_ver = t_ver + self.t_ver_path = t_ver_path + self.t_branch = t_branch + self.basic_snapshot_url = basic_snapshot_url + self.snapshot_num = snapshot_num + self.snapshot_url = snapshot_url + self.snapshot_build_id = snapshot_build_id + self.repository = repository + self.arch_list = arch_list + self.path_prj_list = path_prj_list + self.prj_src_root = prj_src_root + self.build_root = build_root + self.live_root = live_root + self.basic_url = basic_url + self.is_fullbuild = is_fullbuild + self.build_pkg_list = build_pkg_list + self.rm_pkg_list_array = rm_pkg_list_array + self.gbsdbbuild_project_id = gbsdbbuild_project_id + + print 'profile: %s,obs_prj: %s,t_ver: %s,t_ver_path: %s,t_branch: %s,\ + basic_snapshot_url: %s,snapshot_num: %s,snapshot_url: %s,snapshot_build_id: %s,\ + repository: %s,arch_list: %s, path_prj_list: %s,prj_src_root: %s\ + build_root: %s, basic_url: %s, is_fullbuild: %s, build_pkg_list: %s'\ + %(profile,obs_prj,t_ver,t_ver_path,t_branch,basic_snapshot_url,snapshot_num,\ + snapshot_url,snapshot_build_id,repository,arch_list,path_prj_list,prj_src_root,\ + build_root,basic_url,is_fullbuild,build_pkg_list) + + def convert_gbsbuild_to_dictionary(self): + dic = {} + + dic['profile'] = self.profile + dic['obs_prj'] = self.obs_prj + dic['t_ver'] = self.t_ver + dic['t_ver_path'] = self.t_ver_path + dic['t_branch'] = self.t_branch + dic['basic_snapshot_url'] = self.basic_snapshot_url + dic['snapshot_num'] = self.snapshot_num + dic['snapshot_url'] = self.snapshot_url + dic['snapshot_build_id'] = self.snapshot_build_id + dic['repository'] = self.repository + dic['arch_list'] = self.arch_list + dic['path_prj_list'] = self.path_prj_list + dic['prj_src_root'] = self.prj_src_root + dic['build_root'] = self.build_root + dic['live_root'] = self.live_root + dic['basic_url'] = self.basic_url + dic['is_fullbuild'] = self.is_fullbuild + dic['build_pkg_list'] = self.build_pkg_list + dic['rm_pkg_list_array'] = self.rm_pkg_list_array + dic['gbsdbbuild_project_id'] = self.gbsdbbuild_project_id + + return dic + + + def ___build_pkg_list_apply(self,manifest_content): + print '----start ___build_pkg_list_apply-----------------------------------' + + build_pkg_list_new = {} + build_pkg_list_replace = {} + + # build_pkg_list_new : git_name & commit_id for new packages + # build_pkg_list_replace : git_name & commit_id for already existing packages --> commit id has to be replaced + for git_name in self.build_pkg_list.keys(): + if manifest_content.find(git_name) != -1: + build_pkg_list_replace[git_name] = self.build_pkg_list[git_name] + else: + build_pkg_list_new[git_name] = self.build_pkg_list[git_name] + + # for build_pkg_list_replace, line which contains git_name has to be replaced to contain new commit id + replaced_manifest_content = '' + for each_line in manifest_content.split('\n'): + for git_name in build_pkg_list_replace.keys(): + if each_line.find(git_name) != -1: + each_line = ' '\ + %(git_name,git_name,build_pkg_list_replace[git_name]) + break + + rm_pkg_found = 'false' + for git_name in self.rm_pkg_list_array: + if each_line.find(git_name) != -1: + rm_pkg_found = 'true' + break + if rm_pkg_found == 'true': + continue + + if each_line.find('') != -1: + continue + + replaced_manifest_content += each_line+'\n' + + # for build_pkg_list_new, line has to be added newly + for git_name in build_pkg_list_new.keys(): + replaced_manifest_content += ' \n'\ + %(git_name,git_name,build_pkg_list_new[git_name]) + + replaced_manifest_content += '' + + return replaced_manifest_content + + + def __manifest_replace(self,repo): + print '----start __manifest_replace-----------------------------------' + manifest_file = self.prj_src_root+'/'+repo+'/.repo/manifests/'+self.profile+'/'+repo+'/projects.xml' + snapshot_manifest_url = self.snapshot_url+'/builddata/manifest/'+self.snapshot_build_id+'_'+repo+'.xml' + print 'snapshot_manifest_url : %s' %snapshot_manifest_url + res = urllib2.urlopen(snapshot_manifest_url) + snapshot_manifest_content = res.read() + new_snapshot_manifest_content = self.___build_pkg_list_apply(snapshot_manifest_content) + f = open(manifest_file,'wb') + f.write(new_snapshot_manifest_content) + f.close() + + + def _get_base_repo_in_gbs_conf(self,path_prj_arch,repo): + """ Get base repo url from obs project & repository""" + print "-------_get_base_repo_in_gbs_conf start-----------------" + + daily_release_url= os.getenv('URL_PUBLIC_REPO_BASE') + '/releases/daily/tizen/' + basic_url = self.basic_url + + base_repo = {} + obs_prj=path_prj_arch.split('##')[0] + path_arch=path_prj_arch.split('##')[1] + path_prj=obs_prj.replace('Tizen:','').replace(':','-').lower() + + if obs_prj.find(':ref:') != -1: + # Case of specific base snapshot number used for path project is specified in obs meta + ref_build_id=obs_prj.split(':ref:')[1] + obs_prj=obs_prj.split(':ref:')[0] + snapshot_id=obs_prj.replace('Tizen:','').replace(':','-').lower() + temp_id = '/'+snapshot_id+'/tizen-'+snapshot_id+'_'+ref_build_id+'/repos/'+path_arch + url_candidates = [] + url_candidates.append(basic_url+temp_id) + url_candidates.append(daily_release_url+temp_id) + url_candidates.append(public_basic_url+temp_id) + url_candidates.append(public_daily_release_url+temp_id) + for url in url_candidates: + try: + urllib2.urlopen(url) + except: + continue + else: + base_url = url + break + else: + # Case of Tizen:Base / Tizen:4.0:Base / Tizen 3.0:Base + # Must find base snapshot id which is lower than snapshot_num + snapshot_id=path_prj + res = urllib2.urlopen(basic_url + '/'+snapshot_id+'/') + base_url_list = res.read() + prev_build_id = '0' + base_build_id = '0' + for line in base_url_list.split('\n'): + if line.find('= float(self.snapshot_num): + if prev_build_id == '0': + base_build_id = build_id + else: + base_build_id = prev_build_id + break + else: + base_build_id = build_id + prev_build_id = build_id + base_url = basic_url+'/'+snapshot_id+'/tizen-'+snapshot_id+'_'+base_build_id+'/repos/'+path_arch + + print 'obs_prj: %s, snapshot_id: %s' %(obs_prj,snapshot_id) + print 'path_prj: %s, path_arch: %s, base_url: %s' %(path_prj,path_arch,base_url) + + base_repo['repo_name']='repo.'+self.t_ver+self.profile+'_'+repo+'__'+path_prj+'_'+path_arch + base_repo['debug_repo_name']='repo.'+self.t_ver+self.profile+'_'+repo+'__'+path_prj+'_'+path_arch+'_debug' + base_repo['repo_url']=base_url+'/packages/' + base_repo['debug_repo_url']=base_url+'/debug/' + + print 'base_repo: %s' %base_repo + + return base_repo + + + def ___get_profile_repo_section_for_one_profile_repo(self,repo): + #profile section + profile = self.profile + t_ver = self.t_ver + t_ver_path = self.t_ver_path + snapshot_url = self.snapshot_url + + gbs_conf_text = '############# '+ t_ver+profile+' #############\n' + gbs_conf_text += '[profile.'+t_ver+profile+'_'+repo+']\n' + if self.is_fullbuild == 'true': + gbs_conf_text += 'buildconf=./scm/meta/build-config'\ + +t_ver_path+profile+'/'+repo+'_build.conf\n' + gbs_conf_text += 'repos = ' + for path_prj_arch in self.path_prj_list[repo]: + base_repo = self._get_base_repo_in_gbs_conf(path_prj_arch,repo) + gbs_conf_text += base_repo['repo_name']+','+base_repo['debug_repo_name']+',' + if self.is_fullbuild == 'false': + gbs_conf_text += 'repo.'+t_ver+profile+'_'+repo+',' + gbs_conf_text += 'repo.'+t_ver+profile+'_'+repo+'_debug,' + gbs_conf_text = gbs_conf_text[:-1] + + gbs_conf_text += '\n\n' + + ####repo section#### + gbs_conf_text += '\n####################### Repo Section#######################\n' + + #base repo + gbs_conf_text += '\n############# base #############\n' + + for path_prj_arch in self.path_prj_list[repo]: + base_repo = self._get_base_repo_in_gbs_conf(path_prj_arch,repo) + gbs_conf_text += '['+base_repo['repo_name']+']\n' + gbs_conf_text += 'url = '+base_repo['repo_url']+'\n' + gbs_conf_text += '['+base_repo['debug_repo_name']+']\n' + gbs_conf_text += 'url = '+base_repo['debug_repo_url']+'\n\n' + + #profile repo + gbs_conf_text += '\n############# '+ t_ver+profile+' #############\n' + gbs_conf_text += '[repo.'+t_ver+profile+'_'+repo+']\n' + gbs_conf_text += 'url = '+snapshot_url+'repos/'+repo+'/packages/\n' + gbs_conf_text += '[repo.'+t_ver+profile+'_'+repo+'_debug]\n' + gbs_conf_text += 'url = '+snapshot_url+'repos/'+repo+'/debug/\n\n' + + return gbs_conf_text + + + def __update_gbs_conf_for_one_profile_repo(self,repo): + print '----start __update_gbs_conf-----------------------------------' + + gbs_conf_file = self.prj_src_root+'/'+repo+'/.gbs.conf' + if os.path.exists(gbs_conf_file): + os.remove(gbs_conf_file) + + gbs_conf_text = gbs_conf_prefix(self.t_ver,self.profile,repo) + gbs_conf_text += self.___get_profile_repo_section_for_one_profile_repo(repo) + + print gbs_conf_text + + f = open(gbs_conf_file,'wb') + f.write(gbs_conf_text) + f.close() + + + def _do_repo_init_sync(self,repo): + print '----start _do_repo_init_sync-----------------------------------' + repo_src_root=self.prj_src_root+'/'+repo + if not os.path.exists(repo_src_root): + os.mkdir(repo_src_root) + os.chdir(repo_src_root) + + #add '-u' option +#psk repo_init_arg = ' -u ssh://%s:%s/scm/manifest' %(os.getenv('GERRIT_HOSTNAME_EXTERNAL'), + repo_init_arg = ' -u ssh://%s:%s/scm/manifest' %(os.getenv('GERRIT_HOSTNAME_EXTERNAL'), + os.getenv('GERRIT_SSHPORT')) + #add '-b' option + repo_init_arg += ' -b '+self.t_branch + #add '-m' option + repo_init_arg += ' -m '+self.profile+'_'+repo+'.xml' + + cmd = 'repo init' + repo_init_arg + + #do repo init + cmd = 'repo init' + repo_init_arg + print 'repo init cmd: %s' %cmd + ret = subprocess.call(cmd, stdout=sys.stdout,stderr=sys.stderr, shell=True) + if ret != 0: + raise LocalError('repo int failed') + + self.__manifest_replace(repo) + + #do repo sync + print 'do repo sync' + cmd = 'repo sync' + ret = subprocess.call(cmd, stdout=sys.stdout,stderr=sys.stderr, shell=True) + if ret != 0: + raise LocalError('repo sync failed') + + self.__update_gbs_conf_for_one_profile_repo(repo) + + + def __is_gbs_fullbuild_result_fail(self,repo,arch): + print '----start __is_gbs_fullbuild_result_fail-----------------------------------' + + profile_list=os.listdir(self.build_root+'/local/repos/') + for profile in profile_list: + if profile.find(repo) != -1: + profile_path=profile + + index_file=self.build_root+'/local/repos/'+profile_path+'/'+arch+'/index.html' + + f = open(index_file,'rb') + build_result=f.read() + f.close() + + #summary is contents between 'Build Status Summary' and 'Build Statis Details' + summary=build_result.split('Build Status Summary')[1].split('Build Statis Details')[0] + + total_pkg_num=summary.split('')[1] + succeeded_pkg_num=summary.split('')[2] + + if total_pkg_num == succeeded_pkg_num: + print 'GBS fullbuild succeeded' + return 0 + else: + print 'There are errors on gbs fullbuild' + return 1 + + + def __find_binary_list_arg(self,repo): + """ Find argument of --binary-list : which is same as to find spec file name in build_pkg_list""" + print '----start __find_binary_list_arg-----------------------------------' + binary_list = '' + for git_name in self.build_pkg_list.keys(): + for each_file in os.listdir(os.path.join(self.prj_src_root,repo,git_name,'packaging')): + if each_file.endswith('.spec'): + binary_list += each_file.rstrip('.spec')+',' + binary_list = binary_list.rstrip(',') + + print 'build spec file list: %s' %binary_list + return binary_list + + + def _do_repo_arch_gbs_fullbuild(self,repo,arch): + print '----start _do_repo_arch_gbs_fullbuild-----------------------------------' + + gbs_default_build_arg='timeout 6h gbs build --threads=16 --define "jobs 8" --define "_smp_mflags -j8" --baselibs --clean-once' + + #add arch + gbs_build_arg = ' -A '+arch + + #add build root + gbs_build_arg += ' -B '+self.build_root + + #for dependency build, add --rdeps and --binary-list + if self.is_fullbuild == 'false': + binary_list_arg = self.__find_binary_list_arg(repo) + gbs_build_arg += ' --rdeps --binary-list='+binary_list_arg + + cmd = gbs_default_build_arg+gbs_build_arg\ + +' | awk \'{ print strftime("%Y-%m-%d %H:%M:%S"), $0; fflush(); }\'' + print 'gbs build argument is %s' %cmd + + subprocess.call(cmd, stdout=sys.stdout,stderr=sys.stderr, shell=True) + + if self.__is_gbs_fullbuild_result_fail(repo,arch): + # TRIGGER NEXT BUILD-MONITOR +# update_message="GBS Fullbuild Failed" +# if len(update_message) < 119: +# trigger_next("BUILD-MONITOR", \ +# {'bm_stage':'GBSFULLBUILD_SNAPSHOT', +# 'snapshot_name':ref_profile['ref_snapshot_build_id'], +# 'gbsfullbuild_string': update_message}) + + raise LocalError('There are errors on GBS fullbuild for repo:%s, arch:%s' %(repo,arch)) + + + def do_gbs_build(self): + print '----start do_gbs_build-----------------------------------' + + + if not os.path.exists(self.prj_src_root): + os.mkdir(self.prj_src_root) + + for repo in self.repository: +# for repo in ['standard']: + self._do_repo_init_sync(repo) + for arch in self.arch_list[repo]: +# for arch in ['armv7l']: + print 'OBS Project: %s, repository: %s, architecture: %s gbs build start'\ + %(self.obs_prj,repo,arch) + self._do_repo_arch_gbs_fullbuild(repo,arch) + + + def copy_build_results_to_dl_server(self): + print '----start copy_build_results_to_dl_server-----------------------------------' + + path_live_repo_base = os.getenv('PATH_LIVE_REPO_BASE').rstrip('/') + RSYNC_LIVE = os.getenv('IMG_SYNC_DEST_BASE')+'/'+os.path.basename(path_live_repo_base) + rpm_arch_type_list = 'aarch64 armv7l i586 i686 noarch vanish x86_64' + + live_root = self.live_root + #reconstruct gbs build results as similar stucture of OBS live repo + if os.path.exists(live_root): + cmd = 'sudo rm -rf '+live_root + subprocess.call(cmd, stdout=sys.stdout,stderr=sys.stderr, shell=True) + + os.mkdir(live_root) + + obs_liverepo=live_root + sync_dest=os.path.join(RSYNC_LIVE,os.getenv('GBSDBBUILD_DL_POSTFIX')) + for subdir in self.obs_prj.split(':'): + if subdir != self.obs_prj.split(':')[-1]: + obs_liverepo=os.path.join(obs_liverepo,subdir+':') + sync_dest=os.path.join(sync_dest,subdir+':') + else: + obs_liverepo=os.path.join(obs_liverepo,subdir) + sync_dest=os.path.join(sync_dest,subdir) + if not os.path.exists(obs_liverepo): + os.mkdir(obs_liverepo) + + obs_liverepo=os.path.join(obs_liverepo,self.gbsdbbuild_project_id) + if not os.path.exists(obs_liverepo): + os.mkdir(obs_liverepo) + + sync_dest=os.path.join(sync_dest,self.gbsdbbuild_project_id) + + buildlogs_root = os.path.join(obs_liverepo,'buildlogs') + os.mkdir(buildlogs_root) + + sync_out_dir=obs_liverepo + + profile_list=os.listdir(self.build_root+'/local/repos/') + for profile in profile_list: + build_result_repo_path=self.build_root+'/local/repos/'+profile+'/' + repository=profile.split('_')[-1] + live_repo_root=obs_liverepo+'/'+repository + if not os.path.exists(live_repo_root): + os.mkdir(live_repo_root) + ###########copy arch.rpm files############## + print '--------start copy arch.rpm files-----------------' + for arch in rpm_arch_type_list.split(' '): + if not os.path.exists(live_repo_root+'/'+arch): + os.mkdir(live_repo_root+'/'+arch) + for gbs_build_arch in os.listdir(build_result_repo_path): + print 'copy source file dir: %s' %(build_result_repo_path+'/'+gbs_build_arch+'/RPMS/*.'+arch+'.rpm') + print 'copy destination dir: %s' %(live_repo_root+'/'+arch+'/') + for file_name in glob.glob(build_result_repo_path+'/'+gbs_build_arch+'/RPMS/*.'+arch+'.rpm'): + shutil.copy(file_name, live_repo_root+'/'+arch+'/') + #Remove folder if there is no file in arch directory + if len(os.walk(live_repo_root+'/'+arch).next()[2]) == 0: + os.rmdir(live_repo_root+'/'+arch) + ###############copy src.rpm files############## + print '--------start copy src.rpm files-----------------' + for gbs_build_arch in os.listdir(build_result_repo_path): + if not os.path.exists(live_repo_root+'/src/'): + os.mkdir(live_repo_root+'/src/') + for file_name in glob.glob(build_result_repo_path+'/'+gbs_build_arch+'/SRPMS/*.src.rpm'): + shutil.copy(file_name, live_repo_root+'/src/') + ###############copy buildlog files############## + print '--------start copy buildlog files-----------------' + buildlogs_repository = os.path.join(buildlogs_root,repository) + os.mkdir(buildlogs_repository) + for gbs_build_arch in os.listdir(build_result_repo_path): + os.mkdir(os.path.join(buildlogs_repository,gbs_build_arch)) + success_log_dest=os.path.join(buildlogs_repository,gbs_build_arch,'succeeded') + fail_log_dest=os.path.join(buildlogs_repository,gbs_build_arch,'failed') + os.mkdir(success_log_dest) + os.mkdir(fail_log_dest) + #succeeded packages + success_log_root=os.path.join(build_result_repo_path,gbs_build_arch,'logs/success') + fail_log_root=os.path.join(build_result_repo_path,gbs_build_arch,'logs/fail') + for success_pkg in os.listdir(success_log_root): + src_file=success_log_root+'/'+success_pkg+'/log.txt' + print 'success_log_dest: %s ,success_pkg: %s' %(success_log_dest,success_pkg) + dest_file=success_log_dest+'/'+re.findall('\D*\d*\D+\d*[-]',success_pkg)[0].rstrip('-')+'.buildlog.txt' + shutil.copy(src_file,dest_file) + #failed packages + for fail_pkg in os.listdir(fail_log_root): + src_file=fail_log_root+'/'+fail_pkg+'/log.txt' + dest_file=fail_log_dest+'/'+re.findall('\D*\d*\D+\d*[-]',fail_pkg)[0].rstrip('-')+'.buildlog.txt' + shutil.copy(src_file,dest_file) + #Remove folder if there is no file in arch directory + if len(os.walk(success_log_dest).next()[2]) == 0: + os.rmdir(success_log_dest) + if len(os.walk(fail_log_dest).next()[2]) == 0: + os.rmdir(fail_log_dest) + + #Finally, rsync live folder to download server + print 'rsync sync_out_dir: %s, sync_dest: %s' %(sync_out_dir, sync_dest) + sync(sync_out_dir, sync_dest) + + diff --git a/debian/jenkins-scripts-common.install b/debian/jenkins-scripts-common.install index 8591eec..90d81de 100644 --- a/debian/jenkins-scripts-common.install +++ b/debian/jenkins-scripts-common.install @@ -44,3 +44,4 @@ debian/tmp/scripts/nuget.exe /var/lib/jenkins/jenkins-scripts/scripts debian/tmp/common/aws_ec2.py /var/lib/jenkins/jenkins-scripts/common/ debian/tmp/common/workflow.py /var/lib/jenkins/jenkins-scripts/common/ debian/tmp/common/get_name_from_specfile.py /var/lib/jenkins/jenkins-scripts/common/ +debian/tmp/common/gbsutils.py /var/lib/jenkins/jenkins-scripts/common/ diff --git a/debian/jenkins-scripts.install b/debian/jenkins-scripts.install index dba7df5..99c392d 100644 --- a/debian/jenkins-scripts.install +++ b/debian/jenkins-scripts.install @@ -48,3 +48,5 @@ debian/tmp/job_obs_worker_auto_scailing.py /var/lib/jenkins/jenkins-scripts/ debian/tmp/job_config_snapshot_repo_conf.py /var/lib/jenkins/jenkins-scripts/ debian/tmp/job_notify_buildstatus.py /var/lib/jenkins/jenkins-scripts/ debian/tmp/job_obs_project_manager.py /var/lib/jenkins/jenkins-scripts/ +debian/tmp/job_gbs_dashboard_build.py /var/lib/jenkins/jenkins-scripts/ +debian/tmp/job_gbsdbbuild_create_snapshot.py /var/lib/jenkins/jenkins-scripts/ diff --git a/job_gbs_dashboard_build.py b/job_gbs_dashboard_build.py new file mode 100644 index 0000000..43349e4 --- /dev/null +++ b/job_gbs_dashboard_build.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python + +import sys +import os +import tempfile +import atexit +import shutil +import urllib2 +import subprocess +import gzip +import re +import glob +from datetime import datetime +from common.git import Git, clone_gitproject +from common.gerrit import Gerrit, get_gerrit_event, GerritError, is_ref_deleted +from common.buildservice import BuildService +from common.utils import sync +from common.gbsutils import GBSBuild +from common.buildtrigger import trigger_next + + +# prepare related global variables +workspace = os.getenv('WORKSPACE') +basic_url= os.getenv('URL_PUBLIC_REPO_BASE') + '/snapshots/tizen/' +#basic_url = 'http://download.tizen.org/snapshots/tizen/' +gbs_meta_default_profile = os.getenv('GBS_META_DEFAULT_PROFILE') +SUPPORT_PROFILES = os.getenv('SUPPORT_PROFILES') +ARCHITECTURE = os.getenv('ARCHITECTURE') +gbsbuild_workspace = '/srv/gbs/gbsbuild-ROOT/' +BUILD_ROOT = gbsbuild_workspace + '/GBS-ROOT/' +src_root = gbsbuild_workspace + '/SRC-ROOT/' +LIVE_ROOT = gbsbuild_workspace + '/live/' +gbs_default_build_arg='timeout 6h gbs build --threads=16 --define "jobs 8" --define "_smp_mflags -j8" --baselibs --clean-once' +mic_default_build_arg='sudo mic --non-interactive cr auto -o '+workspace+'/mic/out ' +ref_profile={} +ALL_REPO_ARCH_BUILD = os.getenv('ALL_REPO_ARCH_BUILD') +rpm_arch_type_list = 'aarch64 armv7l i586 i686 noarch vanish x86_64' +RSYNC_LIVE = os.getenv('IMG_SYNC_DEST_BASE') + '/live' + +class LocalError(Exception): + """Local error exception.""" + pass + + +def _check_mount_existance(): + print '----start _check_mount_existance-----------------------------------' + umount_list = '' + + cmd='mount' + result=subprocess.check_output(cmd,shell=True) + for line in result.split('\n'): + if line.find(BUILD_ROOT.replace('//','/')) != -1: + umount_list+=line.split('on ')[1].split(' type')[0]+' ' + + if umount_list: + print 'There exist mount which is not unmouted by previous gbs fullbuild job.\n and these lists are like below:%s' %umount_list + cmd='sudo umount '+umount_list + subprocess.call(cmd, stdout=sys.stdout,stderr=sys.stderr, shell=True) + else: + print 'No mount is left by gbsfullbuild' + + + now = datetime.now() + + +def main(): + """script entry point""" + + print '---[JOB STARTED]----------------------------------------' + + obs_prj = os.getenv('PROJECT') + snapshot_num = os.getenv('SNAPSHOT') + full_dep_sel = os.getenv('FULL_DEP_SEL') + new_pkg_list = os.getenv('NEW_PKG_LIST') + rm_pkg_list = os.getenv('RM_PKG_LIST') + rm_from_img = os.getenv('RM_FROM_IMG') + + now = datetime.now() + gbsdbbuild_project_id = '%04d%02d%02d.%02d%02d%02d' %(now.year,now.month,now.day,now.hour,now.minute,now.second) + _check_mount_existance() + + if os.path.exists(BUILD_ROOT): + cmd = 'sudo rm -rf '+BUILD_ROOT + subprocess.call(cmd, stdout=sys.stdout,stderr=sys.stderr, shell=True) + + if not os.path.exists(gbsbuild_workspace): + os.mkdir(gbsbuild_workspace) + + if not os.path.exists(src_root): + os.mkdir(src_root) + + original_dir=os.getcwd() + + print 'gbs fullbuild start which links to obs project: %s' %obs_prj + + gbsbuild = GBSBuild(gbsdbbuild_project_id, obs_prj, basic_url, snapshot_num, full_dep_sel, new_pkg_list, rm_pkg_list, gbsbuild_workspace) + gbsbuild.do_gbs_build() + gbsbuild.copy_build_results_to_dl_server() + + os.chdir(original_dir) + + gbsbuild_dic = gbsbuild.convert_gbsbuild_to_dictionary() + print "gbsbuild_dic : %s" %gbsbuild_dic + + # TRIGGER NEXT GBSFULLBUILD-CREATE-SNAPSHOT + trigger_next("gbsdbbuild-create-snapshot",\ + {'gbsbuild_dic':gbsbuild_dic}) + + +if __name__ == '__main__': + sys.exit(main()) + diff --git a/job_gbsdbbuild_create_snapshot.py b/job_gbsdbbuild_create_snapshot.py new file mode 100755 index 0000000..898006f --- /dev/null +++ b/job_gbsdbbuild_create_snapshot.py @@ -0,0 +1,338 @@ +#!/usr/bin/env python +# vim: ai ts=4 sts=4 et sw=4 +# +# Copyright (C) 2010, 2011, 2012, 2013, 2014 Intel, Inc. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; version 2 +# of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +""" +This code is called by jenkins jobs triggered by OBS events. +""" + +import os +import sys +import re +import datetime +import ast + +from common.repomaker import find_files, RepoMaker, RepoMakerError +from common.buildtrigger import trigger_info, trigger_next +from common.buildservice import BuildService +from common.backenddb import BackendDB +from common.snapshot import Snapshot, SnapshotError, snapshot_project_enabled +from common.utils import make_latest_link +from common.send_mail import prepare_mail +from common.gbsutils import GBSBuild + +class LocalError(Exception): + """Local error exception.""" + pass + + +def replace_ks_file_for_gbsdbbuild(kickstart,build_id): + gbsdbbuild_dl_postfix = os.getenv('GBSDBBUILD_DL_POSTFIX') + replaced_ks='' + + snapshot_build_id = '' + for x in build_id.split('_')[:-1]: + snapshot_build_id = snapshot_build_id+x+'_' + snapshot_build_id = snapshot_build_id.rstrip('_') + + gbsdbbuild_project_id = build_id.split('_')[-1] + + for line in kickstart.split('\n'): + if line.find('snapshots/tizen') != -1: + if line.find('base/') != -1: + line+=' --priority=99' + else: + repo_name=line.split('--name=')[1].split(' ')[0] + orig_line=line.replace('/@BUILD_ID@/repos/','/'+snapshot_build_id+'/repos/')+' --priority=99\n' + add_line=line.replace(repo_name,'gbs_repo').replace('/snapshots/tizen/','/'+gbsdbbuild_dl_postfix+'/tizen/').replace('/@BUILD_ID@/repos/','/'+snapshot_build_id+'/'+build_id+'/repos/')+' --priority=1' + line=orig_line + add_line + replaced_ks+=line+'\n' + + return replaced_ks + + +def prepare_trigger_data(images, build_id, path_repo, project, + url_pub_base): + """ + prepare_trigger_data: + Prepare the trigger data + Args: + images_ks (truple list): [(ks_file_name, ks_file_content),] + build_id (str): the prerelease repo build_id + """ + download_host = os.getenv('DOWNLOAD_HOST') + trigger_data = {} #trigger job_imager + trigger_snapdiff = {} + + print 'prepare_trigger_data, images: %s' %images + for repo_name in images: + trigger_data[repo_name] = [] + trigger_snapdiff[repo_name] = [] + for ksname, kickstart in images[repo_name].ksi.items(): + print 'ksname: %s\nkickstart:%s' %(ksname,kickstart) + #temprary +# if ksname.find('mobile-wayland-armv7l-tm1') == -1 and ksname.find('iot-headless-2parts-armv7l-rpi3') == -1: +# if ksname.find('iot-headless-2parts-armv7l-rpi3') == -1: +# continue + #end of temprary + name = ksname.replace('.ks', '') + + kickstart = replace_ks_file_for_gbsdbbuild(kickstart,build_id) + + if download_host: + # replace host name + kickstart = re.sub('^(repo .*--baseurl=[^:]+://)[^/]+(/.*)', + '\\1%s\\2' % download_host, kickstart, + count=1, flags=re.MULTILINE) + data = {'name': name, + 'kickstart': kickstart, + 'buildid': build_id, + 'images_path': os.path.join("images", repo_name, name), + 'project': project, + 'repo': repo_name, + 'repo_path': path_repo, + 'url_pub_base': url_pub_base + } + + trigger_data[repo_name].append(data) + data_snapdiff = data.copy() + data_snapdiff.pop('kickstart') + trigger_snapdiff[repo_name].append(data_snapdiff) + + return trigger_data, trigger_snapdiff + + +def trigger_image_creation(trigger_data): + """Trigger the image_creation jobs""" + count = 0 + for repo in trigger_data.keys(): + for index, data in enumerate(trigger_data[repo]): + trigger_next('gbsfullbuild_image_trigger_%s_%s' % (repo, index), data) + count += 1 + # Request number of imager nodes + if os.getenv("ONDEMAND_SLAVE_CONFIGURATION_ENABLED", "0") == "1": + if count > 0: + trigger_next("SLAVE_BUILDER", {"data":"dummy"}, \ + extra_params={"ACTION": "REQUEST_WORKER", \ + "PURPOSE": "JENKINS_IMAGER", \ + "REQUESTED_NUM_EXECUTORS": "%d" % count}) + + +def create_local_targets(gbsbuild_dic): + local_targets = [] + for repository in gbsbuild_dic['arch_list']: + local_targets.append({'Name':repository}) + local_targets[-1]['Architectures'] = [] + tmp_arch_list = gbsbuild_dic['arch_list'][repository] + for arch in tmp_arch_list: + if arch.startswith('i') and arch.endswith('86'): + local_targets[-1]['Architectures'].append('ia32') + else: + local_targets[-1]['Architectures'].append(arch) + + return local_targets + + +def make_repo(project, backenddb, base_path, live_repo_base, gbsbuild_dic): + """ + make repo. + + Args: + project (str): OBS prerelease project name + repo (str): name of the OBS live repository + backenddb (BackendDB): backenddb instance + base_path (str): path to the location of snapshot + live_repo_base (str): path to live repo + Raises: + LocalError if can't create repos or can't find image configurations + """ + + try: +# snapshot = Snapshot(backenddb, base_path, obs_project=project) + + # Increment snapshot +# snapshot.inc() + + # Store variables into local rather than accessing Snapshot() instance. +# local_build_id = snapshot.build_id +# local_targets = snapshot.targets +# local_path = snapshot.path +# local_dir = snapshot.dir + local_build_id = gbsbuild_dic['snapshot_build_id']+'_'+gbsbuild_dic['gbsdbbuild_project_id'] + local_targets = create_local_targets(gbsbuild_dic) + local_dir = os.path.join(os.getenv('GBSDBBUILD_DL_POSTFIX'),'tizen',gbsbuild_dic['t_ver']+gbsbuild_dic['profile']) + local_path = os.path.join(base_path,local_dir,gbsbuild_dic['snapshot_build_id'],local_build_id) + + # Delete Snapshot() instance. +# del snapshot + + print 'We are working on:' + print '\t%s\n\t%s\n\t%s\n\t%s\n\n' % (local_build_id, local_targets, local_path, local_dir) + sys.stdout.flush() + except SnapshotError, err: + raise LocalError("Error getting snapshot info: %s" % str(err)) + + targets = local_targets + live_repo_path = os.path.join(live_repo_base, + project.replace(':', ':/'),gbsbuild_dic['gbsdbbuild_project_id']) + parent_snapshot_path = os.path.join(base_path,'snapshots/tizen', + gbsbuild_dic['t_ver']+gbsbuild_dic['profile'], + gbsbuild_dic['snapshot_build_id']) + repos = {} + imagedatas = {} + + # Convert live repo to download structure + for repo in targets: + repomaker = RepoMaker(local_build_id, local_path) + + try: + repomaker.add_repo(live_repo_path, repo['Name'], + repo['Architectures'],move=False) + except RepoMakerError, err: + #raise LocalError("Unable to create download repo: %s" % err) + print "Unable to create download repo: %s" % repr(err) + repos.update({repo['Name']: {'archs': list(set(repo['Architectures']))}}) + continue + + # Assuming that there can be just one image-configurations- + # rpm in the repo + if not repomaker.has_images(): + # repomaker did not found image-configurations in pre_release repo, + # let's take it from target repo, only one package repo is enough + + # Add image configuration to prerelease repo + img_conf = find_files(os.path.join(parent_snapshot_path, 'repos', + repo['Name']), + prefix="image-configurations-", + suffix='noarch.rpm') + print 'parent_snapshot_path: %s' %parent_snapshot_path + print "image-conf: %s" %img_conf + img_conf_list = list(img_conf) + # whether exist package of image-configuration + if not img_conf_list: + #raise LocalError("Image configuration not found in %s" % + # snapshot.path) + print "Image configuration not found in %s, repo:%s" %(snapshot_path, repo['Name']) + continue + + for rpm in img_conf_list: + repomaker.load_imagedata(rpm) + + # whether exist ks poin to the repo + if not repomaker.has_images(): + continue + + repos.update(repomaker.repos) + imagedatas[repo['Name']] = repomaker.imagedata + + # Generate image info to builddata/ dir + repomaker.gen_image_info() + + # Generate repo manifest + manifest_items = repomaker.gen_manifest_info(repo['Name'], + os.getenv('GERRIT_FETCH_URL'), + os.getenv('GERRIT_REVIEW_URL')) + repomaker.gen_manifest_info_app_from_rpm(repo['Name'], + os.getenv('GERRIT_FETCH_URL'), + os.getenv('GERRIT_REVIEW_URL'), + live_repo_path, repo['Architectures']) + # Check duplicated items +# path_list = [ t.keys()[0] for t in manifest_items if t.keys()[0] ] +# duplicated = set([x for x in path_list if path_list.count(x) > 1]) +# if len(duplicated) >= 1: +# prepare_mail('error_report_manifest.env', \ +# 'Snapshot %s have duplicated packages' % (local_build_id), \ +# '\nDuplicated items: %s' % duplicated, \ +# os.getenv('NOREPLY_EMAIL_SENDER'), \ +# os.getenv('MAILINGLIST_SYSTEM').split(',')) + + print 'psk-test\n%s' %imagedatas + return {'project': project, + 'repo': repos, + 'repo_path': os.path.join(local_dir,gbsbuild_dic['snapshot_build_id'],local_build_id), + 'build_id': local_build_id, + 'imagedata': imagedatas + } + +def main(): + """Script entry point.""" + + print '---[JOB STARTED: %s ]-------------------------' + global buildmonitor_enabled + buildmonitor_enabled = os.getenv("BUILDMONITOR_ENABLED", "0") != "0" + print 'buildmonitor_enabled(%s)\n' % (buildmonitor_enabled) + if buildmonitor_enabled: + global bm_start_datetime + bm_start_datetime = datetime.datetime.now() + + base_url = os.getenv("URL_PUBLIC_REPO_BASE") + base_path = os.getenv('PATH_REPO_BASE') + live_repo_base = os.path.join(os.getenv('PATH_LIVE_REPO_BASE'),os.getenv('GBSDBBUILD_DL_POSTFIX')) + + fields = trigger_info(os.getenv("TRIGGER_INFO")) + gbsbuild_dic = fields['gbsbuild_dic'] + snapshot_name = gbsbuild_dic['snapshot_build_id'] + project = gbsbuild_dic['obs_prj'] + + # Init backend database + redis_host = os.getenv("REDIS_HOST") + redis_port = int(os.getenv("REDIS_PORT")) + backenddb = BackendDB(redis_host, redis_port) + + repo_data = make_repo(project, backenddb, base_path, live_repo_base, gbsbuild_dic) + + # prepare trigger data for image creation jobs and snapdiff sync jobs + trigger_data, trigger_snapdiff = prepare_trigger_data( + repo_data['imagedata'], + repo_data['build_id'], + repo_data['repo_path'], + project, base_url) + + # trigger image creation jobs + trigger_image_creation(trigger_data) + + # trigger post snapshot creation job with repo data + data = repo_data.copy() + # remove unused item + data.pop('imagedata') + trigger_next("gbsfullbuild-buildlogs", data) + + # TRIGGER NEXT BUILD-MONITOR-Success +# update_message="Succeeded" +# if len(update_message) < 119: +# trigger_next("BUILD-MONITOR-Success", \ +# {'bm_stage':'GBSFULLBUILD_SNAPSHOT', +# 'snapshot_name':snapshot_name, +# 'gbsfullbuild_string': update_message}) + + # TRIGGER NEXT BUILD-MONITOR-Fail +# update_message="Image Creation Failed" +# if len(update_message) < 119: +# trigger_next("BUILD-MONITOR-Failed", \ +# {'bm_stage':'GBSFULLBUILD_SNAPSHOT', +# 'snapshot_name':snapshot_name, +# 'gbsfullbuild_string': update_message}) + + +if __name__ == '__main__': + try: + sys.exit(main()) + except LocalError, error: + print error + sys.exit(1) + diff --git a/packaging/jenkins-scripts.spec b/packaging/jenkins-scripts.spec index 1ac6644..0ecaedd 100644 --- a/packaging/jenkins-scripts.spec +++ b/packaging/jenkins-scripts.spec @@ -182,6 +182,8 @@ fi %{destdir}/job_config_snapshot_repo_conf.py %{destdir}/job_notify_buildstatus.py %{destdir}/job_obs_project_manager.py +%{destdir}/job_gbs_dashboard_build.py +%{destdir}/job_gbsdbbuild_create_snapshot.py %files common %defattr(-,jenkins,jenkins) @@ -236,6 +238,7 @@ fi %{destdir}/common/aws_ec2.py %{destdir}/common/workflow.py %{destdir}/common/get_name_from_specfile.py +%{destdir}/common/gbsutils.py %files dependsgraph %defattr(-,jenkins,jenkins) -- 2.7.4