From 4e9f9684034edad394544b1c62bc1fabac584f5a Mon Sep 17 00:00:00 2001 From: hyokeun Date: Wed, 9 Aug 2017 17:34:08 +0900 Subject: [PATCH] SR-SYNC: submit job changes For SR-SYNC mode, adding submissions fields. Do not update other fields because its not a real SR. Change-Id: Id539e8ad5f344fc235c60b8737034ddaadbe9dbc --- common/workflow.py | 71 +++++++++++++++++++++++++++++++++++++++++++++++------- job_submit.py | 16 ++++++++++-- 2 files changed, 76 insertions(+), 11 deletions(-) diff --git a/common/workflow.py b/common/workflow.py index 4005536..7af1039 100644 --- a/common/workflow.py +++ b/common/workflow.py @@ -51,6 +51,12 @@ class LocalError(Exception): """Local error exception.""" pass +HIGHLIGHT = '+' +GREYEDOUT = '-' +EMPTY_REV = '0' +MODE_NORMAL = 'NORMAL' +MODE_SRSYNC = 'SR-SYNC' + WRONG_DATE_MSG = '- The date %s in tag does NOT follow correct format.\n You can'\ ' use shell command "date --utc +%%Y%%m%%d.%%H%%M%%S" to '\ 'generate it, like 20120801.083113.' @@ -369,7 +375,7 @@ def get_base_project(build, _project): return path.get('project') return None -def create_project(build, obs_project, submit_list): +def create_project(build, obs_project, submit_list, mode=MODE_NORMAL): """ Create prerelease OBS project and upload sources for the build. Parameters: @@ -395,28 +401,46 @@ def create_project(build, obs_project, submit_list): # Create review project if it doesn't exist print "Creating temporary review OBS project %s" % obs_project + print '\nsubmit_list:\n' + for s in submit_list: + print '++++' + for u in s: + print '[%s]: %s' % (u, s[u]) + _git_url = 'ssh://%s:%s' % (os.getenv('GERRIT_HOSTNAME_EXTERNAL'), os.getenv('GERRIT_SSHPORT')) git_revision = [] - _obs_target_prj, tstamp = get_info_from_prerelease_name(obs_project) + _dest_obs_prj, tstamp = get_info_from_prerelease_name(obs_project) git_url = submit_list[0].get('url', _git_url) git_project = [ t['gerrit_project'] for t in (u for u in submit_list) ] + if mode == MODE_SRSYNC: + git_project = [ t['gerrit_project'][1:] for t in (u for u in submit_list) ] git_tag = submit_list[0].get('git_tag') + # Group submit should have the same tag name for gt in submit_list: - # Group submit should have the same tag name - assert gt['git_tag'] == git_tag + if mode == MODE_NORMAL: + assert gt['git_tag'] == git_tag for u in submit_list: + gerrit_path = u.get('gerrit_project') if 'gerrit_newrev' in u: git_revision.append(u.get('gerrit_newrev')) else: + _rev = EMPTY_REV # Find revision of the tag - _rev = Git(os.path.join(os.getenv('GIT_CACHE_DIR'), \ - u.get('gerrit_project'))).get_tag(git_tag)['tagrevision'] + if mode == MODE_NORMAL: + _path = os.path.join(os.getenv('GIT_CACHE_DIR'), gerrit_path)+'.git' + _rev = Git(_path).get_tag(u.get('git_tag'))['tagrevision'] + elif mode == MODE_SRSYNC: + if gerrit_path.startswith(HIGHLIGHT): + _path = os.path.join(os.getenv('PUBLIC_GIT_CACHE_DIR'), gerrit_path[1:]) + _rev = Git(_path).get_tag(u.get('git_tag'))['tagrevision'] git_revision.append(_rev) - obs_target_prj = submit_list[0].get('obs_target_prj', _obs_target_prj) + obs_target_prj = submit_list[0].get('obs_target_prj', _dest_obs_prj) ref_obs_target_prj = submit_list[0].get('ref_obs_target_prj', obs_target_prj) - submitter = ','.join(list(set([ t['submitter'] for t in (u for u in submit_list) ]))) + if mode == MODE_SRSYNC: + obs_target_prj = ref_obs_target_prj = _dest_obs_prj + submitter = ','.join(list(set([ t['submitter'] for t in (u for u in submit_list) if 'submitter' in t ]))) package = [ t['package'] for t in (u for u in submit_list) ] build_flag = submit_list[0].get('build_flag', True) print '\nCREATING....%s\n%s\n%s\n%s\n%s\n' % (obs_project, git_project, git_revision, submitter, package) @@ -433,6 +457,26 @@ def create_project(build, obs_project, submit_list): 'images': [], 'base': get_base_project(build, ref_obs_target_prj)} + if mode == MODE_SRSYNC: + submissions = [] + for x in submit_list: + # Lets add submit tag name only w/o git path + submissions.append({x['git_tag']: []}) + #TODO: Adding project git path and its package name delimited by = + #bFound = False + #for idx, t in enumerate(submissions): + # if t.keys()[0] == x['git_tag']: + # bFound = True + # submissions[idx][submissions[idx].keys()[0]].append(x['gerrit_project'] + '=' + x['package']) + # break + #if bFound == False: + # submissions.append({x['git_tag']: [x['gerrit_project'] + '=' + x['package']]}) + info['projects'] = [] + info['packages'] = [] + info['git_tag'] = obs_project.split(':'+obs_target_prj+':')[-1].replace(':','/') + info['git_commit'] = [] + info['submissions'] = submissions + if build.exists(obs_project): if submitter: info['submitter'] = submitter @@ -467,6 +511,14 @@ def create_project(build, obs_project, submit_list): try: for idx, val in enumerate(git_project): + if mode == MODE_SRSYNC and \ + (not git_project[idx] or not git_tag or \ + not package[idx] or git_revision[idx] == EMPTY_REV): + print '**** SKIP FORKED PACKAGE: %s,%s,%s,%s' \ + % (git_project[idx], package[idx], git_tag, git_revision[idx]) + continue + print '<< %s >>' % package[idx] + sys.stdout.flush() upload_obs_service(git_url, git_project[idx], git_tag, git_revision[idx], obs_project, build, package[idx]) except UploadError, err: @@ -481,7 +533,8 @@ def create_project(build, obs_project, submit_list): #create the 'link' package that relate the original package for idx, val in enumerate(package): - create_related_packages(build, obs_target_prj, obs_project, package[idx]) + if val and package[idx]: + create_related_packages(build, obs_target_prj, obs_project, package[idx]) #Wait 10 seconds to upload the package to the OBS sleep(10) diff --git a/job_submit.py b/job_submit.py index 8f73000..9be7a47 100644 --- a/job_submit.py +++ b/job_submit.py @@ -58,6 +58,9 @@ from common.workflow import create_project, find_submit_tag, check_tag_format, f reload(sys) sys.setdefaultencoding('utf-8') # pylint: disable-msg=E1101 +MODE_NORMAL = 'NORMAL' +MODE_SRSYNC = 'SR-SYNC' + TITLE_FAILED = '[Submit Request Failed]: tag: %s in %s' TITLE_SUCCESS = '[Submit Request Success]: tag: %s in %s' @@ -83,6 +86,10 @@ def send_mail(title, msg, receiver): prepare_mail("%s.env" % os.getenv('BUILD_TAG'), title, msg, os.getenv('NOREPLY_EMAIL_SENDER'), receiver['email']) +class LocalError(Exception): + """Local error exception.""" + pass + def get_branch_name(tag): """Get branch name by parsing info from submit tag name. @@ -145,9 +152,14 @@ def process_requests(build, request_q): for x in request_q['requests']: args = x[x.keys()[0]] + # Re-arrange SR-SYNC + if request_q['reason'] == MODE_SRSYNC: + for a in args: + a['url'] = request_q['url'] + a['obs_target_prj'] = request_q['obs_target_prj'] retry_count = 3 while retry_count > 0: - create_project(build, x.keys()[0], args) + create_project(build, x.keys()[0], args, mode=request_q['reason']) break if not retry_count: print 'retrying failed' @@ -263,7 +275,7 @@ def entry(option): queued_requests_list = [] for x in queued_requests: queued_requests_list.append({x: queued_requests[x]}) - trigger_next('PRE_RELEASE_REQUEST', {'requests': queued_requests_list, 'reason': 'gerrit'}) + trigger_next('PRE_RELEASE_REQUEST', {'requests': queued_requests_list, 'reason': 'NORMAL'}) return 0 -- 2.7.4