From 9882eab3038b91fd8d17c8162c6ebe7252a607ac Mon Sep 17 00:00:00 2001 From: SungHun Hwang Date: Fri, 11 Nov 2016 13:54:40 +0900 Subject: [PATCH] [BuildMonitor] support the trbs project and enable the multi env 1. support the trbs project 2. enable the multi env and rearrange the job order 3. fixed the bm_img_size error at job_imager.py Change-Id: I90b09d7e233f5f9a704992e2cceb536b6406889c Signed-off-by: SungHun Hwang --- job_buildmonitor.py | 82 +++++++++++++++++++++++++++++++++++--------------- job_create_snapshot.py | 26 +++++++++------- job_imager.py | 13 ++++---- job_post_image.py | 21 +++++++------ job_pre_release_obs.py | 17 +++++------ job_request.py | 3 +- job_submit.py | 3 +- 7 files changed, 101 insertions(+), 64 deletions(-) diff --git a/job_buildmonitor.py b/job_buildmonitor.py index 16e988c..0f5aa40 100644 --- a/job_buildmonitor.py +++ b/job_buildmonitor.py @@ -43,6 +43,8 @@ TABLE : sr_status import os import sys import datetime + +from time import sleep from common import buildmonitor_db from common.buildtrigger import trigger_info, trigger_next @@ -165,6 +167,10 @@ def package_build_for_sr_detail_sr_stage(git_tag, bm_start_datetime, curr_sr_status_id = buildmonitor_db.get_value_from_query_data(query, query_data) #print '[%s] curr_sr_status_id(%s)\n' % (__file__, curr_sr_status_id) + # Wait 60 seconds to find the info_project_id + print '[%s] wait 60 sec\n' % (__file__) + sleep(60) + ### for sr_status_detail & sr_stage # bm_src_project_lst is needed for multi profile print '[%s] bm_src_project_lst(%s)\n' % (__file__, bm_src_project_lst) @@ -175,18 +181,25 @@ def package_build_for_sr_detail_sr_stage(git_tag, bm_start_datetime, query = "SELECT id FROM info_project WHERE project_name = %s" query_data = (bm_src_project,) info_project_id = buildmonitor_db.get_value_from_query_data(query, query_data) - ###================== [16_1110][TEMP] build_project - print '[%s][HSH] TEMP!!\n' % (__file__) - query = "INSERT INTO build_project (info_project_id, status) VALUES(%s, %s)" - query_data = (info_project_id, 'C') - buildmonitor_db.do_query(query, query_data) - ###================== [16_1110][TEMP] build_project query = "SELECT id FROM build_project WHERE info_project_id = %s" query_data = (info_project_id,) pre_build_project_id = buildmonitor_db.get_value_from_query_data(query, query_data) print '[%s] info_project_id(%s), pre_build_project_id(%s)\n' \ % (__file__, info_project_id, pre_build_project_id) + ###================== [16_1110][TEMP] build_project + if pre_build_project_id == 9876: + print '[%s][HSH] cannot find build_project_id!! Adding NOW!!\n' % (__file__) + query = "INSERT INTO build_project (info_project_id, status) VALUES(%s, %s)" + query_data = (info_project_id, 'C') + buildmonitor_db.do_query(query, query_data) + query = "SELECT id FROM build_project WHERE info_project_id = %s" + query_data = (info_project_id,) + pre_build_project_id = buildmonitor_db.get_value_from_query_data(query, query_data) + print '[%s][After] info_project_id(%s), pre_build_project_id(%s)\n' \ + % (__file__, info_project_id, pre_build_project_id) + ###================== [16_1110][TEMP] build_project + # get sr_status_detail_id for checking group submit query = "SELECT id FROM sr_status_detail WHERE sr_status_id = %s " \ "AND pre_build_project_id = %s" @@ -333,11 +346,18 @@ def start_pre_create_snapshot_for_build_snapshot(project, bm_start_datetime): #print '[%s] pre_info_project_id(%s), pre_build_project_id(%s)\n' \ # % (__file__, pre_info_project_id, pre_build_project_id) + ### to check duplicate build_snapshot_id + query = "SELECT id FROM build_snapshot WHERE build_project_id = %s" + query_data = (pre_build_project_id,) + pre_build_snapshot_id = buildmonitor_db.get_value_from_query_data(query, query_data) ### for build_snapshot - query = "INSERT INTO build_snapshot (build_project_id, start_time, status) " \ - "VALUES(%s, %s, %s)" - query_data = (pre_build_project_id, bm_start_datetime, 'R') - buildmonitor_db.do_query(query, query_data) + if pre_build_snapshot_id == 9876: + query = "INSERT INTO build_snapshot (build_project_id, start_time, status) " \ + "VALUES(%s, %s, %s)" + query_data = (pre_build_project_id, bm_start_datetime, 'R') + buildmonitor_db.do_query(query, query_data) + else: + print '[%s] duplicated build_snapshot_id!! skip insert\n' % (__file__) def create_snapshot_packages_for_build_snapshot_package(project, bm_repo, bm_arch, bm_pkg_url, @@ -407,7 +427,8 @@ def end_pre_create_snapshot_for_sr_stage(project, bm_git_tag, buildmonitor_db.do_query(query, query_data) def end_pre_create_snapshot_for_build_snapshot(project, bm_snapshot_name, - bm_snapshot_url, bm_end_datetime): + bm_snapshot_url, bm_snapshot_num, + bm_end_datetime): print '[%s] enter end_pre_create_snapshot_for_build_snapshot\n' % (__file__) # get pre_info_project_id -> get pre_build_project_id @@ -422,9 +443,9 @@ def end_pre_create_snapshot_for_build_snapshot(project, bm_snapshot_name, ### for build_snapshot query = "UPDATE build_snapshot SET snapshot_name = %s, snapshot_url = %s, " \ - "end_time = %s, status = %s WHERE build_project_id = %s" - query_data = (bm_snapshot_name, bm_snapshot_url, bm_end_datetime, - 'S', pre_build_project_id) + "snapshot_num = %s, end_time = %s, status = %s WHERE build_project_id = %s" + query_data = (bm_snapshot_name, bm_snapshot_url, bm_snapshot_num, + bm_end_datetime, 'S', pre_build_project_id) buildmonitor_db.do_query(query, query_data) #=============================================================================== @@ -435,7 +456,9 @@ def update_fail_create_image_for_sr_stage(fields, bm_start_datetime): bm_prj_name = fields["project"] bm_split_data = fields["project"].split(':')[-1] - bm_git_tag = 'submit/tizen/' + bm_split_data + bm_branch_name = fields["project"].split(':')[-2] + #bm_git_tag = 'submit/tizen/' + bm_split_data + bm_git_tag = 'submit/' + bm_branch_name + '/' + bm_split_data bm_url = os.path.join(fields.get('url_pub_base', ''), fields['repo_path'], fields['images_path']) #print '[%s] bm_prj_name(%s), bm_split_data(%s), bm_git_tag(%s), bm_url(%s)\n' \ @@ -444,8 +467,8 @@ def update_fail_create_image_for_sr_stage(fields, bm_start_datetime): bm_end_datetime = datetime.datetime.now() #if 'prerelease' in bm_url: - if not ((bm_url.find("prerelease")) == -1): - + #if not ((bm_url.find("prerelease")) == -1): + if not ((bm_url.find("prerelease")) == -1) or not ((bm_url.find("trbs")) == -1): # [pre_build] get current_sr_status_id query = "SELECT id FROM sr_status WHERE sr = %s" query_data = (bm_git_tag,) @@ -524,7 +547,8 @@ def create_image_for_build_image(fields, bm_start_datetime, def end_create_image_for_sr_stage(bm_start_datetime, project): print '[%s] enter end_create_image_for_sr_stage project(%s)\n' % (__file__, project) - if project.startswith("home:prerelease:"): + #if project.startswith("home:prerelease:"): + if project.startswith("home:prerelease:") or project.startswith("home:trbs:"): bm_end_datetime = datetime.datetime.now() bm_git_tag = project.split(":")[-3] + '/' + project.split(":")[-2] + '/' + project.split(":")[-1] #print '[%s] bm_git_tag(%s)\n' % (__file__, bm_git_tag) @@ -922,7 +946,15 @@ def main(): Script entry point. """ bm_connect_db() - content = trigger_info(os.getenv('TRIGGER_INFO')) + #content = trigger_info(os.getenv('TRIGGER_INFO')) + trigger_info_data = os.getenv('TRIGGER_INFO') + if trigger_info_data: + content = trigger_info(os.getenv('TRIGGER_INFO')) + else: + print '[%s] trigger_info_data is empty!!(%s)\n' \ + % (__file__, trigger_info_data) + return 1 + bm_stage = content.get("bm_stage") #======================================================= @@ -970,9 +1002,11 @@ def main(): # get vars project = content.get("project") bm_git_tag = content.get("bm_git_tag") - bm_start_datetime = bm_end_datetime = datetime.datetime.now() + bm_start_datetime = content.get("bm_start_datetime") + bm_end_datetime = content.get("bm_end_datetime") bm_snapshot_name = content.get("bm_snapshot_name") bm_snapshot_url = content.get("bm_snapshot_url") + bm_snapshot_num = content.get("bm_snapshot_num") # func call end_pre_create_snapshot_for_sr_stage(project, bm_git_tag, @@ -981,6 +1015,7 @@ def main(): end_pre_create_snapshot_for_build_snapshot(project, bm_snapshot_name, bm_snapshot_url, + bm_snapshot_num, bm_end_datetime) elif bm_stage == 'Pre_Snap_packages': @@ -1017,13 +1052,14 @@ def main(): bm_start_datetime = content.get("bm_start_datetime") bm_end_datetime = content.get("bm_end_datetime") build_id = content.get("build_id") + bm_img_size = content.get("bm_img_size") #print '[HSH][%s] fields(%s), bm_start_datetime(%s), build_id(%s)\n' \ # % (__file__, fields, bm_start_datetime, build_id) # func call if image_status == 'success': create_image_for_build_image(fields, bm_start_datetime, - bm_end_datetime, build_id) + bm_end_datetime, build_id, bm_img_size) elif image_status == 'failed': update_fail_create_image_for_sr_stage(fields, bm_start_datetime) #======================================================= @@ -1116,15 +1152,11 @@ def main(): # print 'no bm_stage\n' #bm_disconnect_db() - ''' - if __name__ == '__main__': try: - #sys.exit(main(os.path.basename(sys.argv[1]))) sys.exit(main()) except LocalError, err: print err - #print '[HSH] except!!\n' #bm_disconnect_db() sys.exit(1) diff --git a/job_create_snapshot.py b/job_create_snapshot.py index 9ba2eb1..53b5c33 100755 --- a/job_create_snapshot.py +++ b/job_create_snapshot.py @@ -117,13 +117,14 @@ def make_repo(project, backenddb, base_path, live_repo_base): #buildmonitor.start_create_snapshot_for_post_build_snapshot(project, # bm_snapshot_name, # bm_start_datetime) - bm_data = {"bm_stage" : 'Post_Snap_snapshot', + bm_stage = 'Post_Snap_snapshot' + bm_data = {"bm_stage" : bm_stage, "bm_stage_func" : 'start_create_snapshot_for_post_build_snapshot', "project" : project, "bm_snapshot_name" : bm_snapshot_name, "bm_start_datetime" : str(bm_start_datetime) } - trigger_next("BUILD-MONITOR", bm_data) + trigger_next("BUILD-MONITOR-2-%s" % bm_stage, bm_data) # Convert live repo to download structure for repo in targets: @@ -147,12 +148,12 @@ def make_repo(project, backenddb, base_path, live_repo_base): if buildmonitor_enabled: print '[%s][LocalError] bm_project(%s)\n' % (__file__, bm_project) #buildmonitor.update_fail_create_snapshot_for_sr_stage(bm_project) - bm_data = {"bm_stage" : 'Post_Snap_Fail', + bm_stage = 'Post_Snap_Fail' + bm_data = {"bm_stage" : bm_stage, "bm_stage_func" : 'update_fail_create_snapshot_for_sr_stage', "bm_project" : bm_project } - trigger_next("BUILD-MONITOR", bm_data) - + trigger_next("BUILD-MONITOR-5-%s" % bm_stage, bm_data) raise LocalError("Image configuration not found in %s" % snapshot.path) @@ -201,7 +202,8 @@ def make_repo(project, backenddb, base_path, live_repo_base): # repo['Architectures'][0], # base_path, # snapshot) - bm_data = {"bm_stage" : 'Post_Snap_packages', + bm_stage = 'Post_Snap_packages' + bm_data = {"bm_stage" : bm_stage, "bm_stage_func" : 'create_snapshot_packages_for_post_build_snapshot_package', "project" : project, "bm_repo" : bm_repo, @@ -211,7 +213,7 @@ def make_repo(project, backenddb, base_path, live_repo_base): "bm_pkg_mdate_lst" : bm_pkg_mdate_lst, "bm_pkg_size_lst" : bm_pkg_size_lst, } - trigger_next("BUILD-MONITOR", bm_data) + trigger_next("BUILD-MONITOR-3-%s" % bm_stage, bm_data) # Generate image info to builddata/ dir repomaker.gen_image_info() @@ -259,12 +261,13 @@ def main(): global bm_project # for update_fail_xxx bm_project = project #buildmonitor.start_create_snapshot_for_sr_stage(project, bm_start_datetime) - bm_data = {"bm_stage" : 'Post_Snap_Start', + bm_stage = 'Post_Snap_Start' + bm_data = {"bm_stage" : bm_stage, "bm_stage_func" : 'start_create_snapshot_for_sr_stage', "project" : project, "bm_start_datetime" : str(bm_start_datetime) } - trigger_next("BUILD-MONITOR", bm_data) + trigger_next("BUILD-MONITOR-1-%s" % bm_stage, bm_data) build = BuildService(obs_api, obs_user, obs_passwd) @@ -322,7 +325,8 @@ def main(): # bm_end_datetime, # bm_snapshot_url, # repo_data['build_id']) - bm_data = {"bm_stage" : 'Post_Snap_End', + bm_stage = 'Post_Snap_End' + bm_data = {"bm_stage" : bm_stage, "bm_stage_func" : 'end_create_snapshot_create_images_for_sr_stage', "project" : project, "bm_start_datetime" : str(bm_start_datetime), @@ -330,7 +334,7 @@ def main(): "bm_snapshot_url" : bm_snapshot_url, "repo_data_build_id" : repo_data['build_id'] } - trigger_next("BUILD-MONITOR", bm_data) + trigger_next("BUILD-MONITOR-4-%s" % bm_stage, bm_data) if __name__ == '__main__': try: diff --git a/job_imager.py b/job_imager.py index e4b69ea..c415cfa 100755 --- a/job_imager.py +++ b/job_imager.py @@ -337,27 +337,28 @@ def main(): trigger_next("IMAGE-TESTING", fields) if buildmonitor_enabled: - bm_base_path = os.getenv('PATH_REPO_BASE') - bm_img_url = os.path.join(bm_base_path, fields['repo_path'], fields['images_path']) - bm_img_path = bm_img_url + '/' + build_id + '_' + bm_device_name + '.tar.gz' + bm_img_url = os.path.join(sync_src, fields['images_path']) + bm_img_path = bm_img_url + '/' + build_id + '_' + fields['name'] + '.tar.gz' bm_img_size = os.path.getsize(bm_img_path) #print '[%s] bm_img_path(%s), bm_img_size(%s)\n' \ # % (__file__, bm_img_path, bm_img_size) bm_end_datetime = datetime.datetime.now() - bm_data = {"bm_stage": 'Image', + bm_stage = 'Image' + bm_data = {"bm_stage" : bm_stage, "status" : status, "fields" : fields, "bm_start_datetime" : str(bm_start_datetime), "bm_end_datetime" : str(bm_end_datetime), - "build_id" : build_id, + "build_id": build_id, "bm_img_size" : bm_img_size } trigger_next("BUILD-MONITOR", bm_data) else: if buildmonitor_enabled: bm_end_datetime = datetime.datetime.now() - bm_data = {"bm_stage": 'Image', + bm_stage = 'Image' + bm_data = {"bm_stage" : bm_stage, "status" : status, "fields" : fields, "bm_start_datetime": str(bm_start_datetime), diff --git a/job_post_image.py b/job_post_image.py index 2225305..59679dc 100755 --- a/job_post_image.py +++ b/job_post_image.py @@ -79,16 +79,17 @@ def main(): if buildmonitor_enabled: saveinfo = build.get_info(project) - if len(saveinfo['images']) == int(saveinfo['images_count']): - print '[%s] last image(%s)\n' \ - % (__file__, len(saveinfo['images'])) - #buildmonitor.end_create_image_for_sr_stage(bm_start_datetime, - # project) - bm_data = {"bm_stage" : 'post_image', - "bm_start_datetime" : str(bm_start_datetime), - "project" : project, - } - trigger_next("BUILD-MONITOR", bm_data) + if len(saveinfo['images']) == saveinfo['images_count']: + print '[%s] last image(%s)\n' \ + % (__file__, len(saveinfo['images'])) + #buildmonitor.end_create_image_for_sr_stage(bm_start_datetime, + # project) + bm_stage = 'Post_Image' + bm_data = {"bm_stage" : bm_stage, + "bm_start_datetime" : str(bm_start_datetime), + "project" : project, + } + trigger_next("BUILD-MONITOR", bm_data) if __name__ == "__main__": sys.exit(main()) diff --git a/job_pre_release_obs.py b/job_pre_release_obs.py index b8342c0..70c77c2 100755 --- a/job_pre_release_obs.py +++ b/job_pre_release_obs.py @@ -195,10 +195,10 @@ def make_repo(project, repo, backenddb, base_url, base_path, print '[%s][LocalError] bm_git_tag(%s)\n' % (__file__, bm_git_tag) #buildmonitor.update_fail_status_for_sr_stage(project, bm_git_tag) bm_stage = 'Pre_Snap_Fail' - bm_data = {"bm_stage": bm_stage, + bm_data = {"bm_stage" : bm_stage, "bm_git_tag" : bm_git_tag, } - trigger_next("BUILD-MONITOR-%s" % bm_stage, bm_data) + trigger_next("BUILD-MONITOR-4-%s" % bm_stage, bm_data) raise LocalError("Image configuration not found in %s" % snapshot.path) @@ -257,8 +257,7 @@ def make_repo(project, repo, backenddb, base_url, base_path, "bm_pkg_mdate_lst" : bm_pkg_mdate_lst, "bm_pkg_size_lst" : bm_pkg_size_lst } - #trigger_next("BUILD-MONITOR", bm_data) - trigger_next("BUILD-MONITOR-%s" % bm_stage, bm_data) + trigger_next("BUILD-MONITOR-2-%s" % bm_stage, bm_data) # Generate image info to builddata/ dir repomaker.gen_image_info(images_ks) @@ -392,14 +391,13 @@ def main(action): bm_git_tag = info['git_tag'] #buildmonitor.start_pre_create_snapshot_for_sr_stage(project, bm_git_tag, bm_start_datetime) #buildmonitor.start_pre_create_snapshot_for_build_snapshot(project, bm_start_datetime) - bm_stage = 'Pre_Snap_Start' + bm_stage = 'Pre_Snap_Start' # (+Pre_Snap_snapshot) bm_data = {"bm_stage" : bm_stage, "project" : project, "bm_git_tag" : bm_git_tag, "bm_start_datetime": str(bm_start_datetime) } - #trigger_next("BUILD-MONITOR", bm_data) - trigger_next("BUILD-MONITOR-%s" % bm_stage, bm_data) + trigger_next("BUILD-MONITOR-1-%s" % bm_stage, bm_data) buildstatus = build.getbuildstatus(project) print 'buildstatus=%s' %(buildstatus) @@ -439,7 +437,7 @@ def main(action): # bm_snapshot_url, # bm_end_datetime) bm_stage = 'Pre_Snap_End' - bm_data = {"bm_stage": bm_stage, + bm_data = {"bm_stage" : bm_stage, "project" : project, "bm_git_tag": bm_git_tag, "bm_start_datetime": str(bm_start_datetime), @@ -447,8 +445,7 @@ def main(action): "bm_snapshot_name" : bm_snapshot_name, "bm_snapshot_url" : bm_snapshot_url, } - #trigger_next("BUILD-MONITOR", bm_data) - trigger_next("BUILD-MONITOR-%s" % bm_stage, bm_data) + trigger_next("BUILD-MONITOR-3-%s" % bm_stage, bm_data) if __name__ == '__main__': try: diff --git a/job_request.py b/job_request.py index 4feffbf..50239cf 100755 --- a/job_request.py +++ b/job_request.py @@ -436,7 +436,8 @@ def main(): # bm_start_datetime, # bm_end_datetime, # data["GIT_TAG"]) - bm_data = {"bm_stage" : 'SR_Accept', + bm_stage = 'SR_Accept' + bm_data = {"bm_stage" : bm_stage, "event_fields" : event_fields, "bm_start_datetime" : str(bm_start_datetime), "bm_end_datetime" : str(bm_end_datetime), diff --git a/job_submit.py b/job_submit.py index 25ffcb6..34093ce 100755 --- a/job_submit.py +++ b/job_submit.py @@ -691,7 +691,8 @@ def main(build_type): #buildmonitor.package_build_for_sr_detail_sr_stage(git_tag, bm_start_datetime, # bm_end_datetime, # submit_info['pre_created']) - bm_data = {"bm_stage": 'Submit', + bm_stage = 'Submit' + bm_data = {"bm_stage" : bm_stage, "commit_date" : commit_date, "commit_msg" : commit_msg, "submit_date" : submit_date, -- 2.7.4