From 2994a66d5e36f4dc1b23cfe64f2dc3a287160adc Mon Sep 17 00:00:00 2001 From: Yonghee Han Date: Sat, 11 Jun 2016 15:55:24 +0900 Subject: [PATCH] 1) New workflow about Snapshot. 2) Add submitter on prorelease project. 3) Add chksnap option on prerelease project. 4) Unknown published status skipping prerelease repository creation as OBS Change-Id: Ie0a1b3fc43770b3c7e88f4747eb10d109027341d --- common/buildservice.py | 25 ++++++++++++++++++++ job_create_snapshot.py | 15 ------------ job_jobs_dispatcher.py | 63 ++++++++++++++++++++++++++++++++++++++++++++++---- job_post_image.py | 5 ++++ job_pre_release_obs.py | 28 ++++++++++++---------- job_submit.py | 9 ++++++-- 6 files changed, 112 insertions(+), 33 deletions(-) diff --git a/common/buildservice.py b/common/buildservice.py index aed77c6..20c3a14 100755 --- a/common/buildservice.py +++ b/common/buildservice.py @@ -359,6 +359,25 @@ class BuildService(OSC): ] = result.get('state') return targets + def get_repo_state_summary(self, project): + """Get repo's state summary of the project""" + targets = {} + query = [] + query.append('view=summary') + + u = core.makeurl(self.apiurl, ['build', project, '_result'],query=query) + try: + f = core.http_GET(u) + tree = ElementTree.fromstring(''.join(f.readlines())) + except urllib2.HTTPError, err: + + raise err + + for result in tree.findall('result'): + targets[('/'.join((result.get('repository'), result.get('arch')))) \ + ] = result.get('state') + return targets + def get_repo_config(self, project, repo): """ get_repo_config(project) -> string @@ -537,10 +556,16 @@ class BuildService(OSC): projects = saved_info.get('projects') or [] images = saved_info.get('images') or [] + submitter = saved_info.get('submitter') or [] saved_info.update(info) if 'projects' in info: saved_info['projects'] = list(set(projects + info['projects'])) + if 'submitter' in info: + if not info['submitter'] in submitter: + saved_info['submitter'] = (submitter + ',' + info['submitter']) + else: + saved_info['submitter'] = submitter if 'images' in info: if info['images']: # remove the old one if already exist diff --git a/job_create_snapshot.py b/job_create_snapshot.py index ddbdc8c..d5f475f 100755 --- a/job_create_snapshot.py +++ b/job_create_snapshot.py @@ -177,21 +177,6 @@ def main(): redis_port = int(os.getenv("REDIS_PORT")) backenddb = BackendDB(redis_host, redis_port) - # Check whether all the repos published, we skip it if any one of - # them not published - repo_status = build.get_repo_state(project) - for target_arch, status in repo_status.items(): - if status not in ['published', 'unpublished', 'unknown']: - print "The project: %s target: %s repo is still under %s, " \ - "skip now" % (project, target_arch, status) - - return 0 - - # whether project is not enabled - if not snapshot_project_enabled(backenddb, project): - print "Skipping, project isn't found in repo.yaml" - return 0 - repo_data = make_repo(project, backenddb, base_path, live_repo_base) # update/create the latest repo link diff --git a/job_jobs_dispatcher.py b/job_jobs_dispatcher.py index 69f2571..7b3af61 100755 --- a/job_jobs_dispatcher.py +++ b/job_jobs_dispatcher.py @@ -25,13 +25,68 @@ import sys import os from common.buildtrigger import trigger_next, trigger_info +from common.buildservice import BuildService +from common.backenddb import BackendDB +from common.snapshot import snapshot_project_enabled +from common.prerelease import is_prerelease_project +def isReadyForPrerelease(build, project): + + # whether Prerelease project is not enabled + repo_status = build.get_repo_state_summary(project) + for target_arch, status in repo_status.items(): + if status not in ['published', 'unpublished']: + print "Skipping prerelease repository creation as OBS "\ + "repository: %s/%s is still under %s, skip now." \ + % (project,target_arch, status) + return False + return True + +def isReadyForCreateSnapshot(build, backenddb, project): + + # whether project is not enabled + if not snapshot_project_enabled(backenddb, project): + print "Skipping creating snapshot, project isn't found in repo.yaml" + return False + + # Check whether all the repos published, we skip it if any one of + # them not published + repo_status = build.get_repo_state_summary(project) + for target_arch, status in repo_status.items(): + if status not in ['published', 'unpublished']: + print "The project: %s target: %s repo is still under %s, " \ + "skip creating snapshot now" % (project, target_arch, status) + return False + + return True def main(obs_event_fields): """The main body""" - trigger_next('#%s#%s' %(obs_event_fields.get('project') or - obs_event_fields.get('sourceproject', ''), - obs_event_fields['event_type']), - obs_event_fields) + + project = obs_event_fields.get('project') or obs_event_fields.get('sourceproject', '') + event_type = os.getenv('event_type') + if(event_type == 'OBS_REPO_PUBLISHED'): + obs_api = os.getenv("OBS_API_URL") + obs_user = os.getenv("OBS_API_USERNAME") + obs_passwd = os.getenv("OBS_API_PASSWD") + + build = BuildService(obs_api, obs_user, obs_passwd) + # Init backend database + redis_host = os.getenv("REDIS_HOST") + redis_port = int(os.getenv("REDIS_PORT")) + backenddb = BackendDB(redis_host, redis_port) + if(is_prerelease_project(project)): + if(isReadyForPrerelease(build, project)): + print 'All the repositories are published for project %s.' \ + 'Triggering the prerelease.' % (project) + trigger_next('#PRERELEASE#%s#%s' %(project,event_type),obs_event_fields) + else: + if(isReadyForCreateSnapshot(build, backenddb, project)): + print 'All the repositories are published for project %s.' \ + 'Triggering the creating snapshot job..'%(project) + trigger_next('#SNAP#%s#%s' %(project,event_type),obs_event_fields) + else: + trigger_next('#%s#%s' %(project,obs_event_fields['event_type']), + obs_event_fields) if __name__ == '__main__': # Jenkins task dispatcher, receive events from OBS jenkins notify plugin diff --git a/job_post_image.py b/job_post_image.py index bdc5c7f..b67de2f 100755 --- a/job_post_image.py +++ b/job_post_image.py @@ -63,6 +63,11 @@ def main(): "url": url}] } + saveinfo = build.get_info(project) + #print "images count : %s %s \n" % ( len(saveinfo['images']), saveinfo['images_count']) + if len(saveinfo['images']) + 1 >= saveinfo['images_count']: + # clear chksnap flag + info['chksnap'] = '0' build.update_info(info, project) if __name__ == "__main__": diff --git a/job_pre_release_obs.py b/job_pre_release_obs.py index 61673c2..88c4b60 100755 --- a/job_pre_release_obs.py +++ b/job_pre_release_obs.py @@ -88,7 +88,9 @@ def trigger_image_creation(images_ks, build_id, path_repo, images_ks (truple list): [(ks_file_name, ks_file_content),] build_id (str): the prerelease repo build_id """ + count = 0 for index, (ksname, kickstart) in enumerate(images_ks.items()): + count += 1 name = ksname.replace('.ks', '') data = {'name': name, 'kickstart': kickstart, @@ -113,6 +115,7 @@ def trigger_image_creation(images_ks, build_id, path_repo, }) trigger_next('%s/image_trigger_%s_%s' % (os.getenv('WORKSPACE'), repo_name, index), data) + return count def make_repo(project, repo, backenddb, base_url, base_path, live_repo_base, build): @@ -126,6 +129,7 @@ def make_repo(project, repo, backenddb, base_url, base_path, Raises: LocalError if can't create repos or can't find image configurations """ + images_count = 0 # Make build id from latest snapshot + project suffix target_project, tstamp = get_info_from_prerelease_name(project) @@ -186,12 +190,13 @@ def make_repo(project, repo, backenddb, base_url, base_path, repomaker.gen_image_info(images_ks) # trigger image creation jobs - trigger_image_creation(images_ks, prerelease.build_id, + images_count += trigger_image_creation(images_ks, prerelease.build_id, os.path.join(prerelease.dir, prerelease.build_id), project, base_url, repo['Name']) # reset 'images' and add download_url meta info - build.update_info({'images': [], + build.update_info({'images_count': images_count, + 'images': [], 'download_url': os.path.join(base_url, prerelease.dir, prerelease.build_id)}, @@ -295,17 +300,16 @@ def main(action): if action == 'create_images': repo = content.get("repo") - - # Check whether all the repos published, we skip it if any one of - # them not published - repo_status = build.get_repo_state(project) - for target_arch, status in repo_status.items(): - if status not in ['published', 'unpublished', 'unknown']: - print "Skipping prerelease repository creation as OBS "\ - "repository: %s/%s/%s is still under %s, skip now." \ - % (project, repo, target_arch, status) - + info = build.get_info(project) + try: + if (info['chksnap'] == '1'): + print "Already created snapshot....." return 0 + elif (info['chksnap'] == '0'): + build.update_info({'chksnap': '1'}, project) + print "create snapshot." + except: + build.update_info({'chksnap': '1'}, project) # cleanup src.rpm files on Live repo # we don't used src.rpm files on prerelease project. diff --git a/job_submit.py b/job_submit.py index d4ede1c..7c5d630 100755 --- a/job_submit.py +++ b/job_submit.py @@ -125,11 +125,13 @@ def create_project(git_url, git_project, git_tag, git_revision, build, 'git_commit': git_revision, 'obs_url': os.path.join(os.getenv('OBS_URL'), \ 'project/show?project=%s' % obs_project), + 'chksnap' : '0', 'images': []} - if submitter: - info['submitter'] = escape(submitter) if build.exists(obs_project): + if submitter: + info['submitter'] = submitter + # update project info build.update_info(info, obs_project) # unlink the project to upload packages @@ -138,6 +140,9 @@ def create_project(git_url, git_project, git_tag, git_revision, build, except ObsError, error: print 'Modify the meta conf to unlink failed: %s' % error else: + if submitter: + info['submitter'] = escape(submitter) + if not build.exists(obs_target_prj): raise LocalError("Target project %s doesn't exist" % obs_target_prj) try: -- 2.7.4