From b521268242aebcb5f90915218e9a9b91ef5fdf92 Mon Sep 17 00:00:00 2001 From: Yonghee Han Date: Thu, 29 Mar 2018 17:59:24 +0900 Subject: [PATCH] Add function : Git Sync project PMB -> trigger -> git sync -> update DB Needed for auto git sync. Check branch and merge branch to merge. Change-Id: I19cf540e80206858c1d2985ac5486a847ade2912 --- common/buildmonitor_extention.py | 15 ++ job_git_sync_downstream.py | 414 +++++++++++++++++++++++++++++++++++++++ packaging/jenkins-scripts.spec | 1 + 3 files changed, 430 insertions(+) create mode 100644 job_git_sync_downstream.py diff --git a/common/buildmonitor_extention.py b/common/buildmonitor_extention.py index 698ff57..18f67a3 100644 --- a/common/buildmonitor_extention.py +++ b/common/buildmonitor_extention.py @@ -641,6 +641,21 @@ class BuildMonitorExtention(object): return self.get_device_name_id(device_name) + def update_gbm2pin_git_sync_history(self, id, log, error_string=None): + + if not self.is_connect(): + return + + if error_string is None or error_string == "": + status = "succeeded" + else: + status = "failed" + print "Err:%s" %(error_string) + + query = "UPDATE gbm2spin_sync_history SET log=%s, status=%s WHERE id=%s" + query_data = ( log, status, id) + do_query(query, query_data) + class GbsDashboard(BuildMonitorExtention): gbs_stage_lookup = [ diff --git a/job_git_sync_downstream.py b/job_git_sync_downstream.py new file mode 100644 index 0000000..5d64907 --- /dev/null +++ b/job_git_sync_downstream.py @@ -0,0 +1,414 @@ +#!/usr/bin/env python + +import os +import sys +import base64 +import json +import re +import shutil +import tempfile +import traceback +import urllib2 +import xml.etree.ElementTree as ET +import ast +from common import gerrit +from common import git +from common import utils +from common import runner +from common.buildtrigger import trigger_info, trigger_next +from common.workflow import MailSender +from common.buildmonitor_extention import BuildMonitorExtention + +GIT_URL = 'ssh://%s@%s:%s' % (os.getenv('GERRIT_USERNAME'), + os.getenv('GERRIT_HOSTNAME'), + os.getenv('GERRIT_SSHPORT')) + +class GitUpdateError(Exception): pass +class RevisionError(Exception): pass +class UpstreamError(Exception): pass +class OverwriteError(Exception): pass +class MergeError(Exception): pass +class SubmitError(Exception): pass +class SubmitRequestError(Exception): pass + +class GitType: + PLATFORM = 1 + FORKED = 2 + PRODUCT = 3 + + @classmethod + def type_name(cls, value): + type_name = '' + for variable in dir(cls): + if cls.__dict__[variable] == value: + type_name = variable + break + return type_name + +IS_SUBMIT_REQUEST=False +def submit_request(project, branch, commit, tagname): + """ + Submit Request + """ + result = True + message = "Auto Sync SR Tag" + try: + + tmpdir = tempfile.mkdtemp(prefix=os.getenv('WORKSPACE')+'/') + if not git.clone_gitproject(project, os.path.join(tmpdir, project)): + raise SubmitRequestError('Submit Request git update failed(%s)' % (project)) + + git_dir = os.path.join(tmpdir, project) + gitprj = git.Git(git_dir) + + gitprj.set_branch(branch) + print 'set branch' + gitprj.checkout(commit) + print 'checkout' + gitprj.create_tag(tagname, message) + print 'create_tag' + gitprj.push_tag("origin", tagname) + print 'push' + except Exception as err: + result = False + + return result + +def submit_gerrit_change(gerritobj, gitprj, project, branch, commit): + result = True + gerritobj.review(commit=commit, project=project, message='Submitted by system administrator', verified=1, codereview=2, submit=True) + gerrit_query = 'project:%s branch:%s commit:%s status:merged' % (project, branch, commit) + ret = gerritobj.query(gerrit_query) + if not ret: + print 'Warning: Submit failed, try direct push...' + ret = gitprj.push('-q', 'origin', '%s:refs/heads/%s' % (commit, branch)) + if ret == None: + result = False + return result + +def conv_rev2commit(gitprj, git_dir, revision): + commit = '' + try: + retcode, outs = runner.show('git --git-dir=%s/.git cat-file -t %s' % (git_dir, revision)) + if retcode != 0: + raise RevisionError('not found object type(%s)' % revision) + obj_type = outs.strip() + if obj_type == 'commit' and re.match(r'[0-9a-f]{5,40}', revision): + commit = revision + else: + rev_name = gitprj.name_rev('--name-only %s' % (revision)) + if rev_name == 'undefined': + rev_name = revision + commits = gitprj.show('-s', '--format=%H', rev_name) + if not commits or len(commits) != 1: + raise RevisionError('not found commit(%s)' % revision) + commit = commits[0] + except RevisionError, exc: + print 'Error:', exc + return commit + +def parse_pkg_info_xml(tmpdir, pkg_info_url, user=None, password=None): + """ Parse Package-info.xml """ + + pkginfo_list = {} + + if not pkg_info_url: + return pkginfo_list + + file_name=os.path.join(tmpdir,"package-info.xml") + + if os.path.isfile(file_name): + os.remove(file_name) + + print "URL : %s" %(pkg_info_url) + request = urllib2.Request(pkg_info_url) + if user and password: + base64etrine = base64.encodestring('%s:%s' % (user, password)).replace('\n', '') + request.add_header("Authorization", "Basic %s" % base64string) + tree = ET.parse(urllib2.urlopen(request)) + root = tree.getroot() + + #print root.read() + for info in root.iter('gerritinfo'): + for content in info.iter('content'): + project='' + commit='' + for prj in content.findall('project'): + project = prj.text + for commit_id in content.findall('commit_id'): + commit=commit_id.text + + if project and commit: + if ('product/wearable/fork/' in project) or \ + ('product/wearable/' in project): + pkginfo_list[project] = commit + + return pkginfo_list + +def get_git_type(git_repo): + if (git_repo.startswith('tizen4.0/product/') and not '/fork/' in git_repo): + git_type = GitType.PRODUCT + elif git_repo.startswith('tizen4.0/product/') and '/fork/' in git_repo: + git_type = GitType.FORKED + else: + git_type = GitType.PLATFORM + return git_type + +def convert_private_git_repo(git_repo): + """ Convert git repository """ + + return git_repo.replace('tizen4.0/product/', 'product/mcd/') + +def main(): + """ + Script entry point. + """ + + print '---[JOB STARTED]-------------------------' + + if os.getenv('TRIGGER_INFO', None): + content = trigger_info(os.getenv('TRIGGER_INFO')) + else: + print 'Error : Can not parse from package-info.xml' + return + """ + ------------ + TRIGGER_INFO + ------------ + "sr_tag": "project_name": "triggered_by": + "fork_repo_prefix": "sync_branch": "sync_id": + "package_list": [ + ], + "merge_branch": + "sync_history_id": + "package_info_url": + """ + tmpdir = tempfile.mkdtemp(prefix=os.getenv('WORKSPACE')+'/') + pkg_info_url = content.get('package_info_url') + sync_branch = content.get('sync_branch') + branch = content.get('merge_branch') + git_prefix = content.get('fork_repo_prefix') + sync_history_id = content.get('sync_history_id') + tagname = content.get('sr_tag') + pkg_sync_list = content.get('package_list').split(',') + + if not pkg_info_url: + print 'Error : Can not parse from package-info.xml' + return + if not sync_branch: + print ' Error : Can not parse from sync_branch' + + private_gerrit = gerrit.Gerrit(os.getenv('GERRIT_HOSTNAME'), + os.getenv('GERRIT_USERNAME'), + os.getenv('GERRIT_SSHPORT'), + int(os.getenv('GERRIT_SILENT_MODE'))) + + request_projects = [] + new_private_projects = [] + overwrite_alreadys = [] + overwrite_successs = [] + upstream_failures = [] + overwrite_failures = [] + merge_failures = [] + merge_successs = [] + submit_failures = [] + submit_request_successs = [] + submit_request_failures = [] + + private_projects = private_gerrit.ls_projects() + + # Parse from package-info.xml + pkg_info_list = parse_pkg_info_xml(tmpdir, pkg_info_url) + # Compare package list + for project in pkg_sync_list: + try: + if pkg_info_list.get(project): + private_git_dir = '' + commit = revision = pkg_info_list.get(project) + request_projects.append((project, revision)) + print 'Project:', project + + if not project or not revision: + raise UpstreamError('not found information(%s, %s)' % ( project, revision)) + + git_type = get_git_type(project) + private_project = convert_private_git_repo(project) + request_projects.append((private_project, revision)) + + print 'Private Project:', private_project + print 'Revision Commit:', revision + print 'branch : ', branch + print 'downstream branch : ', sync_branch + print 'git_type : ', git_type + + if not private_project in private_projects: + print '\n* Does not exist Private git repo %s' % (private_project) + new_private_projects.append(private_project) + continue + + print '\n* Update private git repo %s' % (private_project) + if not git.clone_gitproject(private_project, os.path.join(tmpdir, private_project)): + raise OverwriteError('private git update failed(%s)' % (private_project)) + + private_git_dir = os.path.join(tmpdir, private_project) + private_gitprj = git.Git(private_git_dir) + + if not private_gitprj.has_branch('origin/%s' % sync_branch, True): + raise UpstreamError('Present %s branch failed(%s)' % (sync_branch, private_project)) + + if not private_gitprj.has_branch('origin/%s' % branch, True): + print '\n* Create %s branch on %s repository as %s' % (branch, private_project, commit) + ret = private_gitprj._git_inout('push',['-q', '-f', '%s/%s' % (GIT_URL, private_project), \ + '%s:refs/heads/%s' % (commit, branch)]) + if ret == None: + raise UpstreamError('create %s branch failed(%s)' % (branch, private_project)) + merge_successs.append((private_project, '%s -> %s' %(commit, commit))) + + if IS_SUBMIT_REQUEST: + ret = submit_request(project=private_project, branch=branch, commit=commit, tagname=tagname) + if not ret: + raise SubmitRequestError('submit request failed(%s, %s)' % (private_project, commit)) + submit_request_successs.append((private_project, commit, tagname)) + + else: ## Merge + if branch in private_gitprj.branch_contains(commit): + print 'already merge %s branch..' % (branch) + continue + + print '\n* Merge %s %s %s' % (commit, branch, private_project) + #retcode = private_gitprj.checkout('origin/%s' % branch, '--')[0] + private_gitprj.checkout('origin/%s' % branch) + #if retcode != 0: + # raise MergeError('not found %s branch(%s)' % (branch, private_project)) + + retcode = private_gitprj._exec_git('merge', ['--no-ff', commit])[0] + if retcode != 0: + raise MergeError('merge %s branch failed(%s, %s)' % (branch, private_project, commit)) + + retcode, merge_commit = private_gitprj._exec_git('log', ['-n1', '--pretty=%H', 'HEAD', '--']) + if retcode != 0: + raise MergeError('not found merge commit(%s, %s)' % (branch, private_project)) + + #print '\n* Commit --amend --no-edit.. ' + ret = private_gitprj._exec_git('commit', ['--amend', '--no-edit']) + if ret == None: + raise OverwriteError('git commit --amend --no-edit failed(%s)' % (private_project)) + + ret, outs = private_gitprj._exec_git('log', ['-n1', '--pretty=%H']) + if ret == None: + raise OverwriteError('git log -n1 --pretty=%H failed(%s)' % (private_project)) + #change overwrite_commit + merge_commit = outs.strip() + + print '\n* Push %s %s %s' % (merge_commit, branch, private_project) + ret = private_gitprj._exec_git('push',['-q', 'origin', '%s:refs/for/%s' % (merge_commit, branch)]) + if ret == None: + raise MergeError('git push failed(%s, %s)' % (private_project, merge_commit)) + + print '\n* Submit', merge_commit + ret = submit_gerrit_change(gerritobj=private_gerrit, gitprj=private_gitprj, \ + project=private_project, \ + branch=branch, commit=merge_commit) + if not ret: + raise SubmitError('submit failed(%s, %s)' % (private_project, merge_commit)) + merge_successs.append((private_project, '%s -> %s' %(commit, merge_commit))) + + if IS_SUBMIT_REQUEST: + ret = submit_request(project=private_project, branch=branch, commit=merge_commit, tagname=tagname) + if not ret: + raise SubmitRequestError('submit request failed(%s, %s)' % (private_project, merge_commit)) + submit_request_successs.append((private_project, merge_commit, tagname)) + else: + raise UpstreamError('Not exist in Package-info.xml') + + except UpstreamError, exc: + upstream_failures.append((project, revision, exc)) + print 'Error:', exc + except OverwriteError, exc: + overwrite_failures.append((project, revision, exc)) + print 'Error:', exc + except MergeError, exc: + merge_failures.append((project, revision, exc)) + print 'Error:', exc + except SubmitError, exc: + submit_failures.append((project, revision, exc)) + print 'Error:', exc + except SubmitRequestError, exc: + submit_request_failures.append((project, revision, exc)) + print 'Error:', exc + finally: + if private_git_dir and os.path.exists(private_git_dir): + shutil.rmtree(private_git_dir) + + description='' + if request_projects: + description += '\n[ Request Mixed Git Repository List]\n' + description += '\n'.join(str(item) for item in request_projects) + description += '\n' + if new_private_projects: + description +='\n[Does not exist Private Git Repository List]\n' + description +='\n'.join(new_private_projects) + description += '\n' + if overwrite_alreadys: + description +='\n[ Already Overwrite into Branch List]\n' + description +='\n'.join(str(item) for item in overwrite_alreadys) + description += '\n' + if overwrite_successs: + description +='\n[Overwrite into Branch Success List]\n' + description +='\n'.join(str(item) for item in overwrite_successs) + description += '\n' + if upstream_failures: + description +='\n[Update Upstream Branch Failure List]\n' + description +='\n'.join(str(item) for item in upstream_failures) + description += '\n' + if overwrite_failures: + description +='\n[Overwrite into Branch Failure List]\n' + description +='\n'.join(str(item) for item in overwrite_failures) + description += '\n' + if merge_failures: + description +='\n[Merge Failure List]\n' + description +='\n'.join(str(item) for item in merge_failures) + description += '\n' + if merge_successs: + description +='\n[Merge Success List]\n' + description +='\n'.join(str(item) for item in merge_successs) + description += '\n' + if submit_failures: + description +='\n[Submit Failure List]\n' + description +='\n'.join(str(item) for item in submit_failures) + description += '\n' + if submit_request_successs: + description +='\n[SR(Submit Request) Success List]\n' + description +='\n'.join(str(item) for item in submit_request_successs) + description += '\n' + if submit_request_failures: + description +='\n[SR(Submit Request) Failure List]\n' + description +='\n'.join(str(item) for item in submit_request_failures) + description += '\n' + + print description + # Update gbm2spin_sync_history DB + + bm_ext = BuildMonitorExtention() + bm_ext.update_gbm2pin_git_sync_history(sync_history_id, description) + + + #### Send mail to maintainers + if False: + ### Init MailSender #### + my_mail = MailSender() + + my_mail.add_title('[Auto Git-Sync] MCDtoSPIN Git Sync %s->' %(sync_branch, branch)) + my_mail.add_message('Hello\n A Build System inform you about Auto Git-Sync results\n\n') + my_mail.add_message('Downstream Branch: %s\nTarget Branch: %s' \ + %(sync_branch, branch)) + + #print 'members:',email_to_members + # Add maintatiners to mail + #[my_mail.add_maintainers(private_gerrit, group_name = name) for name in email_to_members] + my_mail.add_message(description) + +if __name__ == '__main__': + sys.exit(main()) + + diff --git a/packaging/jenkins-scripts.spec b/packaging/jenkins-scripts.spec index d110dc5..eb8b966 100644 --- a/packaging/jenkins-scripts.spec +++ b/packaging/jenkins-scripts.spec @@ -186,6 +186,7 @@ fi %{destdir}/job_gbsdbbuild_create_snapshot.py %{destdir}/job_gbsdbbuild_one_repoarch_build.py %{destdir}/job_gbsdbbuild_update_meta.py +%{destdir}/job_git_sync_downstream.py %files common %defattr(-,jenkins,jenkins) -- 2.7.4