From 055016cf695d986f0e244ddb0306c618fbcf3799 Mon Sep 17 00:00:00 2001 From: Yonghee Han Date: Wed, 13 Sep 2017 17:34:13 +0900 Subject: [PATCH] TPE-326 Connect with Github of Tizen CI Change-Id: I5c594ffa141239343ce2589edf1dee98552a1415 --- common/workflow.py | 8 ++- job_submit.py | 141 +++++++++++++++++++++++++++++++++++++++++++---------- 2 files changed, 121 insertions(+), 28 deletions(-) diff --git a/common/workflow.py b/common/workflow.py index 25ce3c9..451ab32 100644 --- a/common/workflow.py +++ b/common/workflow.py @@ -536,7 +536,13 @@ def create_project(build, obs_project, submit_list, mode=MODE_NORMAL): 'obs_url': os.path.join(os.getenv('OBS_URL_EXTERNAL'), \ 'project/show?project=%s' % obs_project), 'images': [], - 'base': get_base_project(build, ref_obs_target_prj)} + 'base': get_base_project(build, ref_obs_target_prj) + } + # Enable Github Connection + if submit_list[0].get('github_type'): + info['github_type'] = {'github_fetch_url': submit_list[0].get('github_fetch_url'), \ + 'github_full_name': submit_list[0].get('github_full_name'), \ + } if mode == MODE_SRSYNC: submissions = [] diff --git a/job_submit.py b/job_submit.py index bd94ba7..9a9beb3 100644 --- a/job_submit.py +++ b/job_submit.py @@ -104,7 +104,7 @@ def get_branch_name(tag): branch = 'master' return branch -def send_mail_sr_message(info, mygerrit): +def send_mail_sr_message(info): """ send mail about buildstatus """ #make a message @@ -129,19 +129,23 @@ def send_mail_sr_message(info, mygerrit): os.getenv('NOREPLY_EMAIL_SENDER'), submitter) queued_requests = {} # key would be project name (home:prerelease:...) -def enqueue_request(url, gerrit_project, git_tag, gerrit_newrev, +def enqueue_request(event, url, git_tag, build, obs_target_prj, ref_obs_target_prj, project, submitter, package, build_flag=True): # Check group submits... enqueue_item = {'url': url, \ - 'gerrit_project': gerrit_project, \ + 'gerrit_project': event.get('project'), \ 'git_tag': git_tag, \ - 'gerrit_newrev': gerrit_newrev, \ + 'gerrit_newrev': event.get('newrev'), \ 'obs_target_prj': obs_target_prj, \ 'ref_obs_target_prj': ref_obs_target_prj, \ 'project': project, \ 'submitter': submitter, \ 'package': package, \ - 'build_flag': build_flag} + 'build_flag': build_flag, \ + 'github_type': event.get('github_type'), \ + 'github_fetch_url': event.get('github_fetch_url'), \ + 'github_full_name': event.get('github_full_name') + } if project in queued_requests: queued_requests[project].append(enqueue_item) else: @@ -169,16 +173,78 @@ def process_requests(build, request_q): return -1 return 0 +def entry_github(option): + """ Entry github """ + print '\n** Forward %s to group routine' % os.getenv('GERRIT_REFNAME') + + payload = json.loads(os.getenv('payload')) + payload_repository = payload.get('repository') + payload_sender = payload.get('sender') + + if payload is None or \ + (payload.get('deleted') is not None and payload.get('deleted') == 'true'): + print 'skip this job \n palyload = %s ' %(payload) + return + + git_tag = payload.get('ref') + git_branch = get_branch_name(git_tag) + + git_cache = os.getenv("GIT_CACHE_DIR") + git_full_name = payload_repository.get('full_name') + prjdir = os.path.join(git_cache, git_full_name) + giturl, gitproject = payload_repository.get('ssh_url').split('/') + gitproject = gitproject.split('.git')[0] + gitorg = giturl.split(':')[1] + git_cache_dir = os.path.join(git_cache, gitorg) + + if not clone_gitproject(gitproject, prjdir, giturl=giturl, git_cache_dir=git_cache_dir): + print >> sys.stderr, 'Error cloning %s' % payload['project'] + return 1 + mygit = Git(prjdir) + mygit.checkout(git_tag) + #TODO: Find commit id + commit_id = mygit.rev_parse('HEAD') + print 'commit_id: %s' % commit_id + + result_str, cd_err, cd_ret = mygit._git_inout('for-each-ref', \ + ['--format=%(tagger)', 'refs/tags/%s' %(git_tag)]) + lparen = result_str.find('<') + rparen = result_str.find('>') + account_name = result_str[0:lparen] + account_email = result_str[lparen+1:rparen] + info_data = "" + info_data += '%s=%s\n' % ('GERRIT_EVENT_TYPE', 'ref-updated') + info_data += '%s=%s\n' % ('GERRIT_EVENT_HASH', payload_repository.get('id')) + info_data += '%s=%s\n' % ('GERRIT_REFNAME', 'refs/tags/' + payload.get('ref')) + info_data += '%s=%s\n' % ('GERRIT_PROJECT', gitproject) + info_data += '%s=%s\n' % ('GERRIT_OLDREV', '0000000000000000000000000000000000000000') + info_data += '%s=%s\n' % ('GERRIT_NEWREV', commit_id) + info_data += '%s=%s\n' % ('GERRIT_EVENT_ACCOUNT', '%s <%s>' %(account_name, account_email)) + info_data += '%s=%s\n' % ('GERRIT_EVENT_ACCOUNT_NAME', account_name) + info_data += '%s=%s\n' % ('GERRIT_EVENT_ACCOUNT_EMAIL', account_email) + info_data += '%s=%s\n' % ('GERRIT_GITHUB_FETCH_URL', giturl ) + info_data += '%s=%s\n' % ('GERRIT_GITHUB_FULL_NAME', git_full_name ) + info_data += '%s=%s\n' % ('GERRIT_GITHUB_TYPE', True ) + + with open('PRE-RELEASE-SUBMIT_%d.env' % int(os.getenv('BUILD_NUMBER')), 'w') as info_f: + info_f.write(info_data) + print info_data + return + def entry(option): print option if len(option) > 2 and option[2] == 'bypass': print '\n** Forward %s to group routine' % os.getenv('GERRIT_REFNAME') with open('PRE-RELEASE-SUBMIT-GROUP_%d.env' % int(os.getenv('BUILD_NUMBER')), 'w') as info_f: for x in ['EVENT_TYPE', 'EVENT_HASH', 'REFNAME', 'PROJECT', 'OLDREV', 'NEWREV', \ - 'EVENT_ACCOUNT', 'EVENT_ACCOUNT_NAME', 'EVENT_ACCOUNT_EMAIL']: + 'EVENT_ACCOUNT', 'EVENT_ACCOUNT_NAME', 'EVENT_ACCOUNT_EMAIL', \ + 'GITHUB_TYPE', 'GITHUB_FETCH_URL', 'GITHUB_FULL_NAME']: info_f.write('%s=%s\n' % ('GERRIT_' + x, os.getenv('GERRIT_' + x))) print ' %s=%s' % (x, os.getenv('GERRIT_' + x)) return + elif len(option) > 2 and option[2] == 'github': + entry_github(option) + return obs_api = os.getenv("OBS_API_URL") obs_user = os.getenv("OBS_API_USERNAME") @@ -264,7 +330,11 @@ def entry(option): 'event_account_name': item['GERRIT_EVENT_ACCOUNT_NAME'], \ 'event_account_email' : item['GERRIT_EVENT_ACCOUNT_EMAIL'], \ 'event_type': item['GERRIT_EVENT_TYPE'], \ - 'event_hash': item['GERRIT_EVENT_HASH']}) + 'event_hash': item['GERRIT_EVENT_HASH'], \ + 'github_type': item['GERRIT_GITHUB_TYPE'], \ + 'github_fetch_url': item['GERRIT_GITHUB_FETCH_URL'], \ + 'github_full_name': item['GERRIT_GITHUB_FULL_NAME']} + ) print "option?%s" + option[1] main(option[1], build, event, sr_count) sr_count += 1 @@ -305,10 +375,20 @@ def main(build_type, build, event, sr_count): # prepare separate temp directory for each build git_cache = os.getenv("GIT_CACHE_DIR") - prjdir = os.path.join(git_cache, event['project']) + + # Enable Github Connection + if event.get('github_type'): + giturl = event.get('github_fetch_url') + prjdir = os.path.join(git_cache, event.get('github_full_name')) + gitorg = giturl.split(':')[1] + git_cache_dir = os.path.join(git_cache, gitorg) + else: # GERRIT + prjdir = os.path.join(git_cache, event['project']) + giturl = None # os.getenv('GERRIT_FETCH_URL') + git_cache_dir = None # clone gerrit project to local dir - if not clone_gitproject(event['project'], prjdir): + if not clone_gitproject(event['project'], prjdir, giturl=giturl, git_cache_dir=git_cache_dir): print >> sys.stderr, 'Error cloning %s' % event['project'] return 1 mygit = Git(prjdir) @@ -346,9 +426,6 @@ def main(build_type, build, event, sr_count): # % (receiver.get('author'), event['refname']), receiver) return 0 - mygerrit = Gerrit(event['hostname'], event['username'], \ - event['sshport'], int(os.getenv('GERRIT_SILENT_MODE'))) - result_str, cd_err, cd_ret = mygit._git_inout('for-each-ref', ['--format=%(tagger)', event['refname']]) lparen = result_str.find('<') rparen = result_str.find('>') @@ -361,17 +438,22 @@ def main(build_type, build, event, sr_count): if gerrit_account_email: submitter += ' <%s>' % gerrit_account_email - # check whether tag meet format - resp = check_tag_format(mygit, mygerrit, event, tag) - if resp is not None: - if 'author' not in resp['tagger'] or 'email' not in resp['tagger']: - recevier = { 'author' : event['event_account_name'], - 'email' : event['event_account_email'] } - send_mail(TITLE_FAILED % (tag, event['project']), resp['message'], recevier) - else: - send_mail(TITLE_FAILED % (tag, event['project']), resp['message'], resp['tagger']) - print 'The check for the tag format is error, exit now\n' - return 0 + # Enable Github Connection. + if event.get('github_type') is None: + mygerrit = Gerrit(event['hostname'], event['username'], \ + event['sshport'], int(os.getenv('GERRIT_SILENT_MODE'))) + + # check whether tag meet format + resp = check_tag_format(mygit, mygerrit, event, tag) + if resp is not None: + if 'author' not in resp['tagger'] or 'email' not in resp['tagger']: + recevier = { 'author' : event['event_account_name'], + 'email' : event['event_account_email'] } + send_mail(TITLE_FAILED % (tag, event['project']), resp['message'], recevier) + else: + send_mail(TITLE_FAILED % (tag, event['project']), resp['message'], resp['tagger']) + print 'The check for the tag format is error, exit now\n' + return 0 packagingdir = utils.parse_link('%s/%s' % (prjdir, 'packaging')) print 'packaging dir is %s/%s' % (prjdir, packagingdir) @@ -406,8 +488,13 @@ def main(build_type, build, event, sr_count): "tag": git_tag, } submit_info['pre_created'] = [] - url = 'ssh://%s:%s' % (os.getenv('GERRIT_HOSTNAME_EXTERNAL'), - os.getenv('GERRIT_SSHPORT')) + + # Enable Github Connection. + if event.get('github_type'): + url = '%s' %( event.get('github_fetch_url')) + else: + url = 'ssh://%s:%s' % (os.getenv('GERRIT_HOSTNAME_EXTERNAL'), + os.getenv('GERRIT_SSHPORT')) for item in obs_target_prjs: enable_build = True obs_target_prj = item['OBS_project'] @@ -491,7 +578,7 @@ def main(build_type, build, event, sr_count): if gerrit_account_email: submitter += ' <%s>' % gerrit_account_email - enqueue_request(url, event['project'], git_tag, event['newrev'], + enqueue_request( event, url, git_tag, build, obs_target_prj, ref_obs_target_prj, project, submitter, package, build_flag=enable_build) if True: # prepare submit_info @@ -515,7 +602,7 @@ def main(build_type, build, event, sr_count): "skipping" % obs_target_prj # send mail if submit_info['pre_created']: - send_mail_sr_message(submit_info, mygerrit) + send_mail_sr_message(submit_info) if buildmonitor_enabled: bm_end_datetime = datetime.datetime.now() -- 2.7.4