From 75ad09515385261c748306296669cce3a0379073 Mon Sep 17 00:00:00 2001 From: hyokeun Date: Thu, 14 Dec 2017 13:05:25 +0900 Subject: [PATCH] TPE-489 Cycle check for main projects Change-Id: I8d5002ba6af586d719670a16765419914a210095 --- common/buildservice.py | 30 ++++++++++++++++++++++++++++++ job_buildmonitor.py | 17 +++++++++++++++++ job_create_snapshot.py | 26 ++++++++++++++++++++++++-- job_jobs_dispatcher.py | 28 ++++++++++++++++------------ job_pre_release_obs.py | 32 +------------------------------- 5 files changed, 88 insertions(+), 45 deletions(-) diff --git a/common/buildservice.py b/common/buildservice.py index c6cab71..b18c041 100755 --- a/common/buildservice.py +++ b/common/buildservice.py @@ -1453,4 +1453,34 @@ class BuildService(OSC): sleep(1) retry_count -= 1 + def get_cycle_packages(self, project, repo_arch): + for repo in repo_arch: + for idx, item in enumerate(repo_arch[repo]): + cycle_packages = [] + dep_xml = self.get_dependson(project, repo, item.get('arch')) + dep_tree = ElementTree.fromstring(dep_xml) + for cycle in dep_tree.findall('cycle'): + for cycle_child in cycle.findall('package'): + cycle_packages.append(cycle_child.text) + repo_arch[repo][idx]['cycles'].extend(list(set(cycle_packages))) + return repo_arch + + def cycle_build_added_for_link_project(self, link_project, source_project): + + link_repo_arch = {} + for x in self.get_targets(link_project): + repo, arch = x.split('/') + if repo not in link_repo_arch: + link_repo_arch[repo] = [] + link_repo_arch[repo].append({'arch': arch, 'cycles': []}) + + link_repo_arch = self.get_cycle_packages(link_project, link_repo_arch) + + for repo in link_repo_arch: + for arch in link_repo_arch[repo]: + if len(arch.get('cycles')) >= 1: + return link_repo_arch + + return None + diff --git a/job_buildmonitor.py b/job_buildmonitor.py index c5ed74b..86d0c7d 100644 --- a/job_buildmonitor.py +++ b/job_buildmonitor.py @@ -1391,6 +1391,22 @@ def update_duplicated_package_warning(bm_snapshot_name, duplicated): buildmonitor_db.do_query(query, query_data) sys.stdout.flush() +def update_build_fail_reason_warning(bm_snapshot_name, bm_buildfailreason): + + if bm_buildfailreason is None or bm_buildfailreason == '': + return + + query = "SELECT status_info FROM build_snapshot WHERE snapshot_name=%s " + query_data = (bm_snapshot_name, ) + existing_info = buildmonitor_db.get_value_from_query_data(query, query_data) + if existing_info == 0 or existing_info is None: + info_str = "cycles" + else: + info_str = "%s cycles" % (existing_info) + query = "UPDATE build_snapshot SET status_info=%s WHERE snapshot_name=%s LIMIT 1" + query_data = (info_str, bm_snapshot_name) + buildmonitor_db.do_query(query, query_data) + def update_sr_submit_log(sr_submit_log_id, sr_tag, status, status_reason): print "[%s] update_str_submit_log %s, %s, %s\n" % (__file__, sr_submit_log_id, sr_tag, status_reason) @@ -2724,6 +2740,7 @@ def main(): bm_snapshot_url, bm_snapshot_name) update_duplicated_package_warning(bm_snapshot_name, list(set(bm_duplicated))) + update_build_fail_reason_warning(bm_snapshot_name, bm_buildfailreason) elif bm_stage == 'Post_Snap_Fail': print '[%s][Post_Snap_Fail]\n' % (__file__) diff --git a/job_create_snapshot.py b/job_create_snapshot.py index e813290..39c1566 100755 --- a/job_create_snapshot.py +++ b/job_create_snapshot.py @@ -324,15 +324,36 @@ def main(): build = BuildService(obs_api, obs_user, obs_passwd) + unresolvable_broken_packages = {} + BuildFailReason = 'Succeeded' + if buildmonitor_enabled: global bm_project # for update_fail_xxx + + unresolvable_broken_packages = build.get_package_build_result(project, ["unresolvable", "broken", "failed"]) + + if content.get('build_cycle_check', 0) == 1: + try: + cycle_packages = build.cycle_build_added_for_link_project(project, None) + if cycle_packages is not None: + print 'Cycle build detected. %s \n' % cycle_packages + BuildFailReason = 'Cycles' + # Update cycle info + for cp_repo in cycle_packages: + for cp_item in cycle_packages[cp_repo]: + for cp_pkg in cp_item.get('cycles'): + unresolvable_broken_packages[cp_repo][cp_item.get('arch')][cp_pkg] = 'cycle' + except Exception as err: + print repr(err) + bm_project = project #buildmonitor.start_create_snapshot_for_sr_stage(project, bm_start_datetime) bm_stage = 'Post_Snap_Start' bm_data = {"bm_stage" : bm_stage, "project" : project, + "BuildFailReason" : BuildFailReason, "bm_start_datetime" : str(bm_start_datetime), - "unresolvable_broken_packages": build.get_package_build_result(project, ["unresolvable", "broken", "failed"]) + "unresolvable_broken_packages": unresolvable_broken_packages } trigger_next("BUILD-MONITOR-1-%s" % bm_stage, bm_data) @@ -394,7 +415,8 @@ def main(): "bm_end_datetime" : str(bm_end_datetime), "bm_snapshot_url" : bm_snapshot_url, "bm_snapshot_name" : repo_data['build_id'], - "bm_duplicated" : repo_data['duplicated'] + "bm_duplicated" : repo_data['duplicated'], + "bm_buildfailreason": BuildFailReason } trigger_next("BUILD-MONITOR-4-%s" % bm_stage, bm_data) diff --git a/job_jobs_dispatcher.py b/job_jobs_dispatcher.py index cf55982..0c272a0 100755 --- a/job_jobs_dispatcher.py +++ b/job_jobs_dispatcher.py @@ -63,6 +63,19 @@ def make_prefix_trigger_file(target_project): return _prefix.upper() return '' +def is_build_cycle_check_enabled(project_name): + ret_data = 0 + try: + buildmonitor_db.connect_db() + query = 'SELECT build_cycle_check FROM main_project ' \ + 'WHERE name=%s ;' + ret_data = buildmonitor_db.get_value_from_query_data(query, (project_name,)) + except Exception as err: + print repr(err) + + buildmonitor_db.disconnect_db() + return int(ret_data) + def main(obs_event_fields): """The main body""" @@ -82,18 +95,8 @@ def main(obs_event_fields): if(isReadyForPrerelease(build, project)): print 'All the repositories are published for project %s.' \ 'Triggering the prerelease.' % (project) - target_project_name = '' - try: - target_project_name = get_info_from_prerelease_name(project)[0] - buildmonitor_db.connect_db() - query = 'SELECT build_cycle_check FROM main_project ' \ - 'WHERE name=%s ;' - ret_data = buildmonitor_db.get_value_from_query_data(query, (target_project_name,)) - obs_event_fields['build_cycle_check'] = int(ret_data) - buildmonitor_db.disconnect_db() - except Exception as err: - buildmonitor_db.disconnect_db() - print repr(err) + target_project_name = get_info_from_prerelease_name(project)[0] + obs_event_fields['build_cycle_check'] = is_build_cycle_check_enabled(target_project_name) trigger_next('%s#PRERELEASE#%s#%s' \ % (make_prefix_trigger_file(target_project_name), project, event_type), \ obs_event_fields) @@ -106,6 +109,7 @@ def main(obs_event_fields): if(isReadyForCreateSnapshot(build, backenddb, project)): print 'All the repositories are published for project %s.' \ 'Triggering the creating snapshot job..'%(project) + obs_event_fields['build_cycle_check'] = is_build_cycle_check_enabled(project) trigger_next('%s#SNAP#%s#%s' \ % (make_prefix_trigger_file(project), project, event_type), \ obs_event_fields) diff --git a/job_pre_release_obs.py b/job_pre_release_obs.py index c0edae2..a24d4f2 100755 --- a/job_pre_release_obs.py +++ b/job_pre_release_obs.py @@ -484,36 +484,6 @@ def check_build_fail(unresolvable_broken_failed_status): return True return False -def get_cycle_packages(build, project, repo_arch): - for repo in repo_arch: - for idx, item in enumerate(repo_arch[repo]): - cycle_packages = [] - dep_xml = build.get_dependson(project, repo, item.get('arch')) - dep_tree = ElementTree.fromstring(dep_xml) - for cycle in dep_tree.findall('cycle'): - for cycle_child in cycle.findall('package'): - cycle_packages.append(cycle_child.text) - repo_arch[repo][idx]['cycles'].extend(list(set(cycle_packages))) - return repo_arch - -def cycle_build_added(build, prerelease_project, source_project): - - prerelease_repo_arch = {} - for x in build.get_targets(prerelease_project): - repo, arch = x.split('/') - if repo not in prerelease_repo_arch: - prerelease_repo_arch[repo] = [] - prerelease_repo_arch[repo].append({'arch': arch, 'cycles': []}) - - prerelease_repo_arch = get_cycle_packages(build, prerelease_project, prerelease_repo_arch) - - for repo in prerelease_repo_arch: - for arch in prerelease_repo_arch[repo]: - if len(arch.get('cycles')) >= 1: - return prerelease_repo_arch - - return None - def get_unresolvable_broken_packages(unresolvable_broken_failed_status): unresolvable_broken_packages = {} @@ -579,7 +549,7 @@ def main(action): unresolvable_broken_packages = get_unresolvable_broken_packages(unresolvable_broken_failed_status) if content.get('build_cycle_check', 0) == 1: try: - cycle_packages = cycle_build_added(build, project, info.get('obs_target_prj')) + cycle_packages = build.cycle_build_added_for_link_project(project, info.get('obs_target_prj')) if cycle_packages is not None: print 'Cycle build detected. %s \nSet bBuildFail True' % cycle_packages bBuildFail = True -- 2.7.4