From: SoonKyu Park Date: Fri, 9 Mar 2018 23:45:13 +0000 (+0900) Subject: 1. Deploy 'Daily GBS Build' X-Git-Tag: submit/trunk/20190927.012743~113 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=8e6d1c295ce38b4dc62ce710c7120db13810d08c;p=services%2Fjenkins-scripts.git 1. Deploy 'Daily GBS Build' 2. Handle '_aggregate' package 3. Change db-query handling method to speed-up 'gbs_build_package' db query Change-Id: I179a16e41f431350fa60faa0dc12a7de8d0fb8ed --- diff --git a/common/gbsutils.py b/common/gbsutils.py index d6d8b86..30ae964 100644 --- a/common/gbsutils.py +++ b/common/gbsutils.py @@ -159,6 +159,23 @@ class GBSBuild: path_prj_arch = line.split('"')[1]+'##'+line.split('"')[3] path_prj_list[repo_tmp].append(path_prj_arch) + #handle for _aggregate file + aggregate_list = [] + for root, dirs, files in os.walk(prjdir+'/'+obs_prj): + for each_file in files: + if each_file == "_aggregate": + aggregate_file=os.path.join(root,each_file) + lines = open(aggregate_file).readlines() + for line in lines: + if line.find('') != -1: + binary_rpm = line.split('')[1].split('')[0] + aggregate_list.append({'binary_rpm':binary_rpm}) + aggregate_list[-1]['repository_source_target_list'] = [] + if line.find(' int(last_snapshot_num): + last_snapshot_num = snapshot_num + snapshot_num_used = curr_date+"."+last_snapshot_num + + snapshot_id=snapshot_id_prefix+"_"+snapshot_num_used + tag_name="RELEASE/"+snapshot_id_prefix.upper()+"/"+snapshot_num_used + + return (snapshot_id,tag_name) + +def release_build(): + print "Start release build (periodic build)" + + for obs_prj in os.getenv("DAILY_BUILD_OBS_PROJECTS").split(' '): + (snapshot,tag) = get_snapshot_id(obs_prj) + print "snapshot:%s tag:%s" %(snapshot,tag) + if not snapshot: + print "Skip release build for %s because there is no snapshot created today" %obs_prj + continue + + bypass_data = { + "gbs_type" : "Release Build", + "tag" : tag, + "source_snapshot": snapshot, + "main_project" : obs_prj, + "mode" : "queued", + } + gbs_update_dashboard(bypass_data, via='direct_call') + trigger_next("gbs-dashboard-build_%s" % tag.replace('/', '_'), bypass_data) + def main(): """The main body""" + # If it is not triggered by trigger_next, consider this build as release buil + if os.getenv("TRIGGER_INFO") == None: + release_build() + return + contents = trigger_info(os.getenv("TRIGGER_INFO")) if contents.get("menu", None) == "Abort": diff --git a/job_gbs_dashboard_build.py b/job_gbs_dashboard_build.py index 044bb46..aa7644b 100644 --- a/job_gbs_dashboard_build.py +++ b/job_gbs_dashboard_build.py @@ -54,16 +54,18 @@ def main(): trigger_category = contents.get("gbs_type") if not new_pkg_list: new_pkg_list = '' - for pkg in contents.get("packages"): - if pkg.values()[0]['build'] == 'include': - new_pkg_list += (pkg.keys()[0]+' '+pkg.values()[0]['commit']+' ') - new_pkg_list.rstrip(' ') + if contents.get("packages"): + for pkg in contents.get("packages"): + if pkg.values()[0]['build'] == 'include': + new_pkg_list += (pkg.keys()[0]+' '+pkg.values()[0]['commit']+' ') + new_pkg_list.rstrip(' ') if not rm_pkg_list: rm_pkg_list = '' - for pkg in contents.get("packages"): - if pkg.values()[0]['build'] != 'include': - rm_pkg_list += (pkg.keys()[0]+' ') - rm_pkg_list.rstrip(' ') + if contents.get("packages"): + for pkg in contents.get("packages"): + if pkg.values()[0]['build'] != 'include': + rm_pkg_list += (pkg.keys()[0]+' ') + rm_pkg_list.rstrip(' ') gbsbuild_tag = contents.get("tag") diff --git a/job_gbsdbbuild_create_snapshot.py b/job_gbsdbbuild_create_snapshot.py index 045b63b..d177f7d 100755 --- a/job_gbsdbbuild_create_snapshot.py +++ b/job_gbsdbbuild_create_snapshot.py @@ -26,6 +26,7 @@ import sys import re import datetime import ast +import subprocess from common.repomaker import find_files, RepoMaker, RepoMakerError from common.buildtrigger import trigger_info, trigger_next @@ -49,7 +50,7 @@ def bm_update_gbsdbbuild_snapshot_failed(status_reason,gbsbuild_tag): } trigger_next("BUILD-MONITOR-4-%s" %bm_stage, bm_data) -def replace_ks_file_for_gbsdbbuild(kickstart,build_id): +def replace_ks_file_for_gbsdbbuild(kickstart,build_id,gbsbuild_trigger_category): gbsdbbuild_dl_postfix = os.getenv('GBSDBBUILD_DL_POSTFIX') replaced_ks='' @@ -65,15 +66,21 @@ def replace_ks_file_for_gbsdbbuild(kickstart,build_id): else: repo_name=line.split('--name=')[1].split(' ')[0] orig_line=line.replace('/@BUILD_ID@/repos/','/'+snapshot_build_id+'/repos/')+' --priority=99\n' - add_line=line.replace(repo_name,'gbs_repo').replace('/snapshots/tizen/','/'+gbsdbbuild_dl_postfix+'/tizen/').replace('/@BUILD_ID@/repos/','/'+snapshot_build_id+'/'+build_id+'/repos/')+' --priority=1' - line=orig_line + add_line + if gbsbuild_trigger_category.lower() == 'release build': + add_line=line.replace(repo_name,'gbs_repo').replace('/snapshots/tizen/','/'+gbsdbbuild_dl_postfix+'/releases/tizen/').replace('/@BUILD_ID@/repos/','/'+build_id+'/repos/')+' --priority=1' + else: + add_line=line.replace(repo_name,'gbs_repo').replace('/snapshots/tizen/','/'+gbsdbbuild_dl_postfix+'/tizen/').replace('/@BUILD_ID@/repos/','/'+snapshot_build_id+'/'+build_id+'/repos/')+' --priority=1' + if gbsbuild_trigger_category.lower() == 'dependency build': + line=orig_line + add_line + else: + line=add_line replaced_ks+=line+'\n' return replaced_ks def prepare_trigger_data(images, build_id, path_repo, project, - url_pub_base,gbsbuild_tag): + url_pub_base,gbsbuild_tag,gbsbuild_trigger_category): """ prepare_trigger_data: Prepare the trigger data @@ -98,7 +105,7 @@ def prepare_trigger_data(images, build_id, path_repo, project, #end of temprary name = ksname.replace('.ks', '') - kickstart = replace_ks_file_for_gbsdbbuild(kickstart,build_id) + kickstart = replace_ks_file_for_gbsdbbuild(kickstart,build_id,gbsbuild_trigger_category) if download_host: # replace host name @@ -188,10 +195,16 @@ def make_repo(project, backenddb, base_path, live_repo_base, gbsbuild_dic): # local_targets = snapshot.targets # local_path = snapshot.path # local_dir = snapshot.dir - local_build_id = gbsbuild_dic['snapshot_build_id']+'_'+gbsbuild_dic['gbsbuild_tag'].split('/')[1] - local_targets = create_local_targets(gbsbuild_dic) - local_dir = os.path.join(os.getenv('GBSDBBUILD_DL_POSTFIX'),'tizen',gbsbuild_dic['t_ver']+gbsbuild_dic['profile']) - local_path = os.path.join(base_path,local_dir,gbsbuild_dic['snapshot_build_id'],local_build_id) + if gbsbuild_dic['gbsbuild_tag'].startswith("RELEASE"): + local_build_id = gbsbuild_dic['snapshot_build_id'] + local_targets = create_local_targets(gbsbuild_dic) + local_dir = os.path.join(os.getenv('GBSDBBUILD_DL_POSTFIX'),'releases/tizen',gbsbuild_dic['t_ver']+gbsbuild_dic['profile']) + local_path = os.path.join(base_path,local_dir,local_build_id) + else: + local_build_id = gbsbuild_dic['snapshot_build_id']+'_'+gbsbuild_dic['gbsbuild_tag'].split('/')[1] + local_targets = create_local_targets(gbsbuild_dic) + local_dir = os.path.join(os.getenv('GBSDBBUILD_DL_POSTFIX'),'tizen',gbsbuild_dic['t_ver']+gbsbuild_dic['profile']) + local_path = os.path.join(base_path,local_dir,gbsbuild_dic['snapshot_build_id'],local_build_id) # Delete Snapshot() instance. # del snapshot @@ -207,7 +220,7 @@ def make_repo(project, backenddb, base_path, live_repo_base, gbsbuild_dic): targets = local_targets live_repo_path = os.path.join(live_repo_base, - project.replace(':', ':/'),gbsbuild_dic['gbsbuild_tag'].split('/')[1]) + project.replace(':', ':/'),gbsbuild_dic['gbsbuild_tag'].split('/')[-1]) parent_snapshot_path = os.path.join(base_path,'snapshots/tizen', gbsbuild_dic['t_ver']+gbsbuild_dic['profile'], gbsbuild_dic['snapshot_build_id']) @@ -276,9 +289,13 @@ def make_repo(project, backenddb, base_path, live_repo_base, gbsbuild_dic): os.getenv('GERRIT_REVIEW_URL'), live_repo_path, repo['Architectures']) + if gbsbuild_dic['trigger_category'].lower() == 'release build': + repo_path = os.path.join(local_dir,local_build_id) + else: + repo_path = os.path.join(local_dir,gbsbuild_dic['snapshot_build_id'],local_build_id) return {'project': project, 'repo': repos, - 'repo_path': os.path.join(local_dir,gbsbuild_dic['snapshot_build_id'],local_build_id), + 'repo_path': repo_path, 'build_id': local_build_id, 'imagedata': imagedatas } @@ -303,10 +320,16 @@ def main(): snapshot_name = gbsbuild_dic['snapshot_build_id'] project = gbsbuild_dic['obs_prj'] - snapshot_name = gbsbuild_dic['snapshot_build_id']+'_'+gbsbuild_dic['gbsbuild_tag'].split('/')[1] - snapshot_url = os.path.join(os.getenv('URL_PUBLIC_REPO_BASE'),os.getenv('GBSDBBUILD_DL_POSTFIX'),\ - 'tizen',gbsbuild_dic['t_ver']+gbsbuild_dic['profile'],\ - gbsbuild_dic['snapshot_build_id'],snapshot_name) + if gbsbuild_dic['gbsbuild_tag'].startswith("RELEASE"): + snapshot_name = gbsbuild_dic['snapshot_build_id'] + snapshot_url = os.path.join(os.getenv('URL_PUBLIC_REPO_BASE'),os.getenv('GBSDBBUILD_DL_POSTFIX'),\ + 'releases/tizen',gbsbuild_dic['t_ver']+gbsbuild_dic['profile'],\ + snapshot_name) + else: + snapshot_name = gbsbuild_dic['snapshot_build_id']+'_'+gbsbuild_dic['gbsbuild_tag'].split('/')[1] + snapshot_url = os.path.join(os.getenv('URL_PUBLIC_REPO_BASE'),os.getenv('GBSDBBUILD_DL_POSTFIX'),\ + 'tizen',gbsbuild_dic['t_ver']+gbsbuild_dic['profile'],\ + gbsbuild_dic['snapshot_build_id'],snapshot_name) bm_start_datetime = datetime.datetime.now() ## gbs_remote_jenkins_build_job({'bm_stage':'GBSDBBuild_Snapshot_Start', @@ -329,7 +352,7 @@ def main(): print "build_id_liverepo = %s" %build_id_liverepo cmd = 'rm -rf `find '+build_id_liverepo+\ ' ! -name '+os.path.basename(build_id_liverepo)+\ - ' | grep -v '+gbsbuild_dic['gbsbuild_tag'].split('/')[1]+'`' + ' | grep -v '+gbsbuild_dic['gbsbuild_tag'].split('/')[-1]+'`' print "clear live root command: %s" %cmd subprocess.call(cmd, stdout=sys.stdout,stderr=sys.stderr, shell=True) @@ -355,7 +378,7 @@ def main(): repo_data['imagedata'], repo_data['build_id'], repo_data['repo_path'], - project, base_url, gbsbuild_dic['gbsbuild_tag']) + project, base_url, gbsbuild_dic['gbsbuild_tag'], gbsbuild_dic['trigger_category']) # trigger image creation jobs trigger_image_creation(trigger_data) diff --git a/job_gbsfullbuild_buildlogs.py b/job_gbsfullbuild_buildlogs.py index c8a7aec..fcb9fd4 100755 --- a/job_gbsfullbuild_buildlogs.py +++ b/job_gbsfullbuild_buildlogs.py @@ -44,7 +44,7 @@ def main(): if gbsbuild_dic: live_repo_base = os.path.join(os.getenv('PATH_LIVE_REPO_BASE'),os.getenv('GBSDBBUILD_DL_POSTFIX')) sync_src = os.path.join(live_repo_base,obs_prj.replace(':',':/'), - gbsbuild_dic['gbsbuild_tag'].split('/')[1],'buildlogs') + gbsbuild_dic['gbsbuild_tag'].split('/')[-1],'buildlogs') else: live_repo_base = os.path.join(os.getenv('PATH_LIVE_REPO_BASE'),os.getenv('GBSFULLBUILD_DL_POSTFIX')) sync_src = os.path.join(live_repo_base,obs_prj.replace(':',':/'),'buildlogs') diff --git a/job_gbsfullbuild_image_creator.py b/job_gbsfullbuild_image_creator.py index d45a298..a7b30c5 100755 --- a/job_gbsfullbuild_image_creator.py +++ b/job_gbsfullbuild_image_creator.py @@ -379,7 +379,7 @@ def main(): mode = "image_finished" gbs_update_data = {"tag" : fields['gbsbuild_tag'], "mode" : mode, - "reason" : {"ks_name":fields['name'],"repo":fields['repo'],"status":gbs_img_status}} + "reason" : {"ks_name":fields['name'],"status":gbs_img_status}} gbs_update_dashboard(gbs_update_data,via='trigger',trigger_name=mode)