From d9d7fb4d0a05033426759a9ebf878388ed0f2eec Mon Sep 17 00:00:00 2001 From: SoonKyu Park Date: Fri, 23 Mar 2018 17:55:30 +0900 Subject: [PATCH] 1. 'git_ref_mapping_info' parameter (infomation from git-ref-mapping git) will be used in reference gbs fullbuild 2. Base working for 'removing package in image' Change-Id: Ic6d29b720fb3e30443f7b62261fcbfe6190c9499 --- common/gbsutils.py | 3 ++- job_gbs_build_dispatcher.py | 7 ++++--- job_gbs_dashboard_build.py | 21 ++++++++++++++++----- job_gbsdbbuild_create_snapshot.py | 7 +++++-- job_gbsdbbuild_one_repoarch_build.py | 3 ++- job_gbsdbbuild_update_meta.py | 26 ++++++++++++++------------ 6 files changed, 43 insertions(+), 24 deletions(-) diff --git a/common/gbsutils.py b/common/gbsutils.py index 5f6f201..030cc6a 100644 --- a/common/gbsutils.py +++ b/common/gbsutils.py @@ -95,7 +95,7 @@ def find_spec_name_from_build_log(buildlog_file): class GBSBuild: """A class which supports with statement""" - def __init__(self, gbsbuild_tag, obs_prj, basic_url, snapshot_num, trigger_category, new_pkg_list, rm_pkg_list, gbsbuild_workspace): + def __init__(self, gbsbuild_tag, obs_prj, basic_url, snapshot_num, trigger_category, new_pkg_list, rm_pkg_list, rm_from_img_list, gbsbuild_workspace): """ Initialize GBSBuild class""" print "-----------------------------------------------------" print "Initialize GBS Build Class" @@ -334,6 +334,7 @@ class GBSBuild: if each_line.find(git_name) != -1: rm_pkg_found = 'true' break + #if rm package is found in manifest file, don't perform adding each_line to manifest file if rm_pkg_found == 'true': continue diff --git a/job_gbs_build_dispatcher.py b/job_gbs_build_dispatcher.py index c24bbfc..82d051a 100755 --- a/job_gbs_build_dispatcher.py +++ b/job_gbs_build_dispatcher.py @@ -86,9 +86,9 @@ def release_snapshot(contents): gbs_update_dashboard(bypass_data, via='direct_call') def get_snapshot_id(obs_prj): - #psk temprary for building test - curr_date=datetime.today().strftime("%Y%m%d") - #curr_date='20180305' + curr_date = os.getenv('SPECIFIC_DATE_BUILD') + if not curr_date: + curr_date=datetime.today().strftime("%Y%m%d") basic_url= os.getenv('URL_PUBLIC_REPO_BASE') + '/snapshots/tizen/' profile = obs_prj.replace("Tizen:","").replace(":","-").lower() profile_url = basic_url+profile+'/' @@ -153,6 +153,7 @@ def ref_fullbuild(contents): "source_snapshot": snapshot, "main_project" : obs_prj, "mode" : "queued", + "git_ref_mapping_info": contents['git_ref_mapping_info'], } # submitter='sk7.park@samsung.com' # transform_packages=[{'platform/core/dotnet/build-tools': {'commit': '5bc00509597ae289c30bf4498bbb96e51dfe13b5', 'image': 'include', 'build': 'include'}}] diff --git a/job_gbs_dashboard_build.py b/job_gbs_dashboard_build.py index aa7644b..8e9bfb7 100644 --- a/job_gbs_dashboard_build.py +++ b/job_gbs_dashboard_build.py @@ -43,7 +43,7 @@ def main(): trigger_category = os.getenv('TRIGGER_CATEGORY') new_pkg_list = os.getenv('NEW_PKG_LIST') rm_pkg_list = os.getenv('RM_PKG_LIST') - rm_from_img = os.getenv('RM_FROM_IMG') + rm_from_img_list = os.getenv('RM_FROM_IMG') # If there is no direct input from input parameters, let build parameters come from trigger_info if not obs_prj: @@ -66,10 +66,17 @@ def main(): if pkg.values()[0]['build'] != 'include': rm_pkg_list += (pkg.keys()[0]+' ') rm_pkg_list.rstrip(' ') + if not rm_from_img_list: + rm_from_img_list = '' + if contents.get("packages"): + for pkg in contents.get("packages"): + if pkg.values()[0]['build'] == 'include' and pkg.values()[0]['image'] != 'include': + rm_from_img_list += (pkg.keys()[0]+' ') + rm_from_img_list.rstrip(' ') gbsbuild_tag = contents.get("tag") - print "Build Parameters:\nobs_prj: %s\nsnapshot_num: %s\ntrigger_category: %s\nnew_pkg_list: %s\nrm_pkg_list: %s\ngbsbuild_tag:%s\n" %(obs_prj,snapshot_num,trigger_category,new_pkg_list,rm_pkg_list,gbsbuild_tag) + print "Build Parameters:\nobs_prj: %s\nsnapshot_num: %s\ntrigger_category: %s\nnew_pkg_list: %s\nrm_pkg_list: %s\nrm_from_img_list: %s\ngbsbuild_tag:%s\n" %(obs_prj,snapshot_num,trigger_category,new_pkg_list,rm_pkg_list,rm_from_img_list,gbsbuild_tag) global buildmonitor_enabled buildmonitor_enabled = os.getenv("BUILDMONITOR_ENABLED", "0") != "0" @@ -77,7 +84,7 @@ def main(): print 'gbs fullbuild start which links to obs project: %s' %obs_prj - gbsbuild = GBSBuild(gbsbuild_tag, obs_prj, basic_url, snapshot_num, trigger_category, new_pkg_list, rm_pkg_list, gbsbuild_workspace) + gbsbuild = GBSBuild(gbsbuild_tag, obs_prj, basic_url, snapshot_num, trigger_category, new_pkg_list, rm_pkg_list, rm_from_img_list, gbsbuild_workspace) gbsbuild_dic = gbsbuild.convert_gbsbuild_to_dictionary() print "gbsbuild_dic : %s" %gbsbuild_dic @@ -102,7 +109,8 @@ def main(): 'repository':repo, 'architecture':arch, 'new_pkg_list':new_pkg_list, - 'rm_pkg_list':rm_pkg_list}) + 'rm_pkg_list':rm_pkg_list, + 'rm_from_img_list':rm_from_img_list}) if buildmonitor_enabled: gbs_update_data = {"tag" : gbsbuild_dic['gbsbuild_tag'], @@ -116,8 +124,11 @@ def main(): gbs_update_dashboard(gbs_update_data,via='trigger',trigger_name=mode) # TRIGGER NEXT GBSFULLBUILD-CREATE-SNAPSHOT + git_ref_mapping_info = contents.get('git_ref_mapping_info',{}) trigger_next("gbsdbbuild-create-snapshot",\ - {'gbsbuild_dic':gbsbuild_dic}) + {'gbsbuild_dic':gbsbuild_dic, + 'git_ref_mapping_info':git_ref_mapping_info + }) if __name__ == '__main__': diff --git a/job_gbsdbbuild_create_snapshot.py b/job_gbsdbbuild_create_snapshot.py index e538e38..f38a39e 100755 --- a/job_gbsdbbuild_create_snapshot.py +++ b/job_gbsdbbuild_create_snapshot.py @@ -35,7 +35,7 @@ from common.backenddb import BackendDB from common.snapshot import Snapshot, SnapshotError, snapshot_project_enabled from common.utils import make_latest_link from common.send_mail import prepare_mail -from common.gbsutils import GBSBuild,gbs_remote_jenkins_build_job,gbs_update_dashboard +from common.gbsutils import gbs_remote_jenkins_build_job,gbs_update_dashboard class LocalError(Exception): """Local error exception.""" @@ -449,8 +449,11 @@ def main(): 'snapshot_name':gbsbuild_dic['snapshot_build_id'], 'gbsfullbuild_string': update_message}) + # trigger gbsdbbuild update meta in case of reference snapshot building + git_ref_mapping_info = fields.get('git_ref_mapping_info',{}) if gbsbuild_dic["gbsbuild_tag"].startswith('REF') and overall_gbsbuild_status == 'S': - gbsmeta_data = {"triggered_from": 'gbsdbbuild_create_snapshot' + gbsmeta_data = {"triggered_from" : 'gbsdbbuild_create_snapshot', + "git_ref_mapping_info" : git_ref_mapping_info } trigger_next("gbsdbbuild-update-meta",gbsmeta_data) diff --git a/job_gbsdbbuild_one_repoarch_build.py b/job_gbsdbbuild_one_repoarch_build.py index 8831c43..d5f77de 100644 --- a/job_gbsdbbuild_one_repoarch_build.py +++ b/job_gbsdbbuild_one_repoarch_build.py @@ -140,6 +140,7 @@ def main(): architecture = content['architecture'] new_pkg_list = content['new_pkg_list'] rm_pkg_list = content['rm_pkg_list'] + rm_from_img_list = content['rm_from_img_list'] global buildmonitor_enabled buildmonitor_enabled = os.getenv("BUILDMONITOR_ENABLED", "0") != "0" @@ -177,7 +178,7 @@ def main(): gbsbuild_sub = GBSBuild(gbsbuild_dic['gbsbuild_tag'], gbsbuild_dic['obs_prj'], basic_url,\ gbsbuild_dic['snapshot_num'], gbsbuild_dic['trigger_category'], new_pkg_list, rm_pkg_list,\ - gbsbuild_dic['gbsbuild_workspace']) + rm_from_img_list, gbsbuild_dic['gbsbuild_workspace']) gbsbuild_sub_dic = gbsbuild_sub.convert_gbsbuild_to_dictionary() print "gbsbuild_sub_dic : %s" %gbsbuild_sub_dic diff --git a/job_gbsdbbuild_update_meta.py b/job_gbsdbbuild_update_meta.py index f2cbbfd..a26f205 100755 --- a/job_gbsdbbuild_update_meta.py +++ b/job_gbsdbbuild_update_meta.py @@ -484,12 +484,12 @@ def _update_scm_manifest(ref_list,base_prj_list,commit_msg,is_staging): print '%s, branch %s has nothing to commit' %(git_prj,git_branch) -def prepare_ref_binary(): +def prepare_ref_binary(git_ref_mapping_info): """ prepare_ref_binary_by_git""" print "----------prepare_ref_binary_by_git start--------" - git_prj = 'scm/git-ref-mapping' - git_branch = GIT_REF_MAPPING_BRANCH + git_prj = git_ref_mapping_info['git_prj'] + git_branch = git_ref_mapping_info['git_branch'] filename = 'git-ref-mapping.xml' tmpdir = tempfile.mkdtemp(prefix=workspace+'/') atexit.register(shutil.rmtree, tmpdir) @@ -501,8 +501,8 @@ def prepare_ref_binary(): return 1 mygit = Git(prjdir) mygit.checkout(git_branch) - commit_msg=mygit.show(git_branch).split('\n')[4].replace(" ","") - commit_log=mygit.show(git_branch) + commit_msg=mygit.show(git_ref_mapping_info['newrev']).split('\n')[4].replace(" ","") + commit_log=mygit.show(git_ref_mapping_info['newrev']) ref_binary = {} base_prj_list = {} @@ -539,7 +539,7 @@ def main(): print '---[JOB STARTED]----------------------------------------' - git_info = {} + git_ref_mapping_info = {} #comment out because this job is triggered by gerrit event event = get_gerrit_event() # event = trigger_info(os.getenv("TRIGGER_INFO")) @@ -559,19 +559,20 @@ def main(): "event %s! Only ref-updated events are allowed" \ %(event['event_type'], event['project']) return 1 - if event['refname'] != GIT_REF_MAPPING_BRANCH: - print "Configuration error: git branch of scm/git-ref-mapping mismatched!!" - return 1 + git_ref_mapping_info['git_prj'] = event['project'] + git_ref_mapping_info['git_branch'] = event['refname'] + git_ref_mapping_info['newrev'] = event['newrev'] else: print 'gbsdbbuild-update-meta job is triggered by other jenkins job\n\ gbsdbbuild-create-snapshot' - content = trigger_info(os.getenv("TRIGGER_INFO")) + contents = trigger_info(os.getenv("TRIGGER_INFO")) is_staging = '' - if content['triggered_from'] != 'gbsdbbuild_create_snapshot': + if contents['triggered_from'] != 'gbsdbbuild_create_snapshot': print 'Trigger error: gbsdbbuild-update-meta job is not triggered by gbsdbbuild-create-snapshot!!' return 1 + git_ref_mapping_info = contents['git_ref_mapping_info'] - (ref_binary,commit_msg,build_profile_snapshot_id_list) = prepare_ref_binary() + (ref_binary,commit_msg,build_profile_snapshot_id_list) = prepare_ref_binary(git_ref_mapping_info) print "commit_msg='%s'" %commit_msg print 'Each reference snapshot numbers are like below' @@ -593,6 +594,7 @@ def main(): for build_profile_snapshot_id in build_profile_snapshot_id_list: trigger_next("gbs_build_dispatcher",\ {"menu" : "Ref Fullbuild", + "git_ref_mapping_info" : git_ref_mapping_info, 'build_profile_snapshot_id': build_profile_snapshot_id}) if __name__ == '__main__': -- 2.7.4