From f5ec3353671fed1a7e98e260899d77f0b61de9b4 Mon Sep 17 00:00:00 2001 From: hyokeun Date: Thu, 20 Jul 2017 08:59:26 +0900 Subject: [PATCH] Split obs creation task from pre-release-gerrit job Change-Id: Ia49ab8c249bf6e4d955147f55dc6ce6554cf0477 --- common/git.py | 1 + job_submit.py | 117 ++++++++++++++++++++++++++++++++++++---------------------- 2 files changed, 73 insertions(+), 45 deletions(-) diff --git a/common/git.py b/common/git.py index bf9d3aa..30b3f34 100644 --- a/common/git.py +++ b/common/git.py @@ -70,6 +70,7 @@ class Git(GitRepository): try: tag['commitid'] = self.rev_parse('%s^{commit}' % tag_name) + tag['tagrevision'] = self.rev_parse('%s' % tag_name) except GitRepositoryError, err: raise GitError('Error running rev parse %s^{commit}: %s' % \ (tag_name, str(err))) diff --git a/job_submit.py b/job_submit.py index 9d8acad..4d11dc0 100644 --- a/job_submit.py +++ b/job_submit.py @@ -44,7 +44,7 @@ from common.gerrit import is_ref_deleted from common.buildservice import BuildService from common.buildtrigger import trigger_info, trigger_next, get_jenkins_instance from common.backenddb import BackendDB -from common.prerelease import get_prerelease_project_name, prerelease_enabled +from common.prerelease import get_prerelease_project_name, prerelease_enabled, get_info_from_prerelease_name from common.iris_rest_client import IrisRestClient from common.send_mail import prepare_mail from common.gerrit import Gerrit, get_gerrit_event, GerritError, is_ref_deleted @@ -413,22 +413,58 @@ def get_base_project(build, _project): return path.get('project') return None -def create_project(build, obs_project, args): - """Create prerelease OBS project and upload sources for the build.""" +def create_project(build, obs_project, submit_list): + """ + Create prerelease OBS project and upload sources for the build. + Parameters: + @ build (buildservice.BuildService): Instance of BuildService + @ obs_project (str): Project name to be created + @ submit_list (list): + [ + { + 'project'(O) : Prerelease project name + 'package'(M) : OBS package name + 'gerrit_project'(M) : Git path + 'git_tag'(M) : Git tag name + 'gerrit_newrev'(O) : Revision of the tag + 'submitter'(M) : Author of the tag + 'build_flag'(O) : Enable/Disable OBS build after create + 'obs_target_prj'(O) : Target OBS project + 'ref_obs_target_prj'(O): Reference target OBS project to be linked + 'url'(O) : Hostname of the gerrit server for _service file + }, + ] + """ # Create review project if it doesn't exist print "Creating temporary review OBS project %s" % obs_project - git_url = args[0]['url'] - git_project = [ t['gerrit_project'] for t in (u for u in args) ] - git_tag = args[0]['git_tag'] - git_revision = [ t['gerrit_newrev'] for t in (u for u in args) ] - obs_target_prj = args[0]['obs_target_prj'] - ref_obs_target_prj = args[0]['ref_obs_target_prj'] - submitter = ','.join(list(set([ t['submitter'] for t in (u for u in args) ]))) - package = [ t['package'] for t in (u for u in args) ] - build_flag = args[0]['build_flag'] + _git_url = 'ssh://%s:%s' % (os.getenv('GERRIT_HOSTNAME_EXTERNAL'), + os.getenv('GERRIT_SSHPORT')) + git_revision = [] + _obs_target_prj, tstamp = get_info_from_prerelease_name(obs_project) + + git_url = submit_list[0].get('url', _git_url) + git_project = [ t['gerrit_project'] for t in (u for u in submit_list) ] + git_tag = submit_list[0].get('git_tag') + for gt in submit_list: + # Group submit should have the same tag name + assert gt['git_tag'] == git_tag + for u in submit_list: + if 'gerrit_newrev' in u: + git_revision.append(u.get('gerrit_newrev')) + else: + # Find revision of the tag + _rev = Git(os.path.join(os.getenv('GIT_CACHE_DIR'), \ + u.get('gerrit_project'))).get_tag(git_tag)['tagrevision'] + git_revision.append(_rev) + obs_target_prj = submit_list[0].get('obs_target_prj', _obs_target_prj) + ref_obs_target_prj = submit_list[0].get('ref_obs_target_prj', obs_target_prj) + submitter = ','.join(list(set([ t['submitter'] for t in (u for u in submit_list) ]))) + package = [ t['package'] for t in (u for u in submit_list) ] + build_flag = submit_list[0].get('build_flag', True) print '\nCREATING....%s\n%s\n%s\n%s\n%s\n' % (obs_project, git_project, git_revision, submitter, package) + sys.stdout.flush() info = {'projects': git_project, 'packages': package, @@ -470,7 +506,6 @@ def create_project(build, obs_project, args): #disable publish flag build.disable_build_flag(obs_project, repo = None, flag="publish", status="disable") - #disable build flag build.disable_build_flag(obs_project, repo = None, flag="build", status="disable") @@ -488,8 +523,6 @@ def create_project(build, obs_project, args): if build_flag != True: return - #build.link_project(obs_project, src=obs_target_prj, linktype="localdep") - #create the 'link' package that relate the original package for idx, val in enumerate(package): create_related_packages(build, obs_target_prj, obs_project, package[idx]) @@ -506,7 +539,6 @@ def create_project(build, obs_project, args): sleep(1) build.link_project(obs_project, src=ref_obs_target_prj, linktype="localdep") - def get_branch_name(tag): """Get branch name by parsing info from submit tag name. @@ -541,25 +573,13 @@ def send_mail_sr_message(info, mygerrit): if info['submitted'].get('submitter_email'): submitter += ' <%s>' % info['submitted'].get('submitter_email') if submitter: -# email_to = [] -# if True: -# grps = mygerrit.ls_groups(['--project %s' % info['submitted'].get('gitpath')]) -# for dg in [s for s in grps if " - Maintainers" in s]: -# for line in mygerrit.ls_members(['\'\"%s\"\'' % dg, '--recursive']): -# email_to.append(line.split('\t')[3]) -# email_to = [x for x in email_to if x != 'n/a'] -# -# print 'email_to : %s ' % (email_to) -# for user in email_to: -# submitter += '%s,' %(user) - prepare_mail("%s.env" % os.getenv('BUILD_TAG'), title, msg, os.getenv('NOREPLY_EMAIL_SENDER'), submitter) queued_requests = {} # key would be project name (home:prerelease:...) def enqueue_request(url, gerrit_project, git_tag, gerrit_newrev, build, obs_target_prj, ref_obs_target_prj, project, submitter, package, build_flag=True): - #TODO: Check group submits... + # Check group submits... enqueue_item = {'url': url, \ 'gerrit_project': gerrit_project, \ 'git_tag': git_tag, \ @@ -576,22 +596,15 @@ def enqueue_request(url, gerrit_project, git_tag, gerrit_newrev, queued_requests[project] = [enqueue_item] print 'QUEUED... %s\n' % enqueue_item -def process_requests(build): +def process_requests(build, request_q): fail_count = 0 - for x in queued_requests: - print '\n[%s]:\n %s' % (x, queued_requests[x]) - - for x in queued_requests: - args = queued_requests[x] + for x in request_q['requests']: + args = x[x.keys()[0]] retry_count = 3 while retry_count > 0: - #try: - create_project(build, x, args) - break - #except Exception, err: - # print '******* obs operation failed %s, retrying...' % err - # sleep(10) + create_project(build, x.keys()[0], args) + break if not retry_count: print 'retrying failed' fail_count += 1 @@ -615,6 +628,12 @@ def entry(option): build = BuildService(obs_api, obs_user, obs_passwd) + # Split obs creation job + if len(option) > 2 and option[2] == 'request': + content = trigger_info(os.getenv('TRIGGER_INFO')) + process_requests(build, content) + return + # First store current build and all the rest queued builds build_queue = [] job = os.getenv('JOB_NAME') @@ -652,7 +671,7 @@ def entry(option): curr_data[p['name']] = p['value'] build_queue.append({'%d-%d' % (up_id, int(ret_obj['queueId'])): curr_data}) - #TODO: Keep original process + # Keep original process if up_id != 0: jq = jenkinsinst.get_queue() for pending_build in jq.get_queue_items_for_job(os.getenv('JOB_NAME')): @@ -678,7 +697,7 @@ def entry(option): sr_count = 1 for k in build_queue: item = k[k.keys()[0]] - #TODO: Just forward it to main routine. (Merging group submits need target project name) + # Just forward it to main routine. (Merging group submits need target project name) print item['GERRIT_PROJECT'], item['GERRIT_NEWREV'] event.update({'project': item['GERRIT_PROJECT'], \ 'refname': item['GERRIT_REFNAME'], \ @@ -693,7 +712,16 @@ def entry(option): main(option[1], build, event, sr_count) sr_count += 1 - return process_requests(build) + for x in queued_requests: + print '\n[%s]:\n %s' % (x, queued_requests[x]) + + # Split obs creation job + queued_requests_list = [] + for x in queued_requests: + queued_requests_list.append({x: queued_requests[x]}) + trigger_next('PRE_RELEASE_REQUEST', {'requests': queued_requests_list, 'reason': 'gerrit'}) + + return 0 def main(build_type, build, event, sr_count): """ @@ -887,7 +915,6 @@ def main(build_type, build, event, sr_count): submit_info['pre_created'].append(project) else: submit_info['pre_created'] = products - elif build_type == 'snapshot': if not is_ref_deleted(event['oldrev'], event['newrev']): if build.exists(obs_target_prj): -- 2.7.4