From aa8739a1a76f73fd055943021f2c3b79bde49765 Mon Sep 17 00:00:00 2001 From: Yonghee Han Date: Mon, 28 Aug 2017 10:15:52 +0900 Subject: [PATCH] New Feature : Trigger Git-Sync for requests list Description: Update SPIN remote on upstream git repository Update private git repository Push overwrite commit to branch Review verify and codereview and submit Send a e-mail to Maintainers Change-Id: I4c09dc934b274361e7552e455a864a2686be1525 --- common/gerrit.py | 15 +- common/git.py | 47 ++++++ debian/jenkins-scripts.install | 3 +- job_submit_request_git_sync.py | 364 +++++++++++++++++++++++++++++++++++++++++ packaging/jenkins-scripts.spec | 1 + 5 files changed, 425 insertions(+), 5 deletions(-) create mode 100644 job_submit_request_git_sync.py diff --git a/common/gerrit.py b/common/gerrit.py index c6a52b9..65c5275 100644 --- a/common/gerrit.py +++ b/common/gerrit.py @@ -19,6 +19,7 @@ import getpass import os +import sys import json from subprocess import Popen, PIPE @@ -85,9 +86,12 @@ class Gerrit: ret_list = [] - results = self._cmd_run('ls-projects', args) + #results = self._cmd_run('ls-projects', args) - for prj in results: + sys.stdin, out = os.popen2("ssh -p %s %s@%s 'gerrit ls-projects %s'" + %(self.port, self.username, self.host, + ' '.join(args))) + for prj in out.readlines(): ret_list.append(prj.strip()) return ret_list @@ -100,7 +104,7 @@ class Gerrit: self._cmd_run('create-project', args) def review(self, commit=None, project=None, change_id_set=None, - message='', verified=0, codereview=0): + message='', verified=0, codereview=0, submit=False): """A wrapper method used to execute gerrit review operation""" args = [] @@ -116,7 +120,10 @@ class Gerrit: args.append('--verified %s ' % verified) if codereview: - args.append('--codereview %s ' % codereview) + args.append('--code-review %s ' % codereview) + + if submit: + args.append('--submit ') if commit or change_id_set: if commit: diff --git a/common/git.py b/common/git.py index 30b3f34..f811c9d 100644 --- a/common/git.py +++ b/common/git.py @@ -25,6 +25,7 @@ import shutil from common import runner from common.utils import retry +from common.utils import Workdir from gbp.git.repository import GitRepository from gbp.git.repository import GitRepositoryError @@ -40,6 +41,27 @@ class Git(GitRepository): def __init__(self, path): GitRepository.__init__(self, path) log.setup('auto', log.DEBUG) + self._git_dir = os.path.join(path, '.git') + + def _exec_git(self, command, args=[]): + """Exec a git command and return the output + """ + + if self._git_dir: + cmd = ['git --git-dir=%s ' % self._git_dir, command] + args + else: + cmd = ['git ', command] + args + + cmdln = ' '.join(cmd) + #print('run command: %s' % cmdln) + + with Workdir(self.path): + ret, outs = runner.show(cmdln) + + #if ret: + # raise errors.GitError("command error for: %s" % cmdln) + + return ret, outs def get_tag(self, tag_name): """Get tag info though tag name""" @@ -94,6 +116,30 @@ class Git(GitRepository): print 'git branch contains %s' % branches return branches + def create_overwrite_commit(self, branch, commit): + ret, outs = self._exec_git('log', ['-n1', '--pretty=%H','origin/%s' %(branch), '--']) + if ret != 0: + print('not found commit of %s' % branch) + return (False, '') + deprecated_commit = outs.strip() + ret, outs = self._exec_git('diff', ['--stat', '--exit-code', deprecated_commit, commit]) + if ret == 0: + print('%s branch is same as %s, no create overwrite commit' % (branch, commit)) + return (True, '') + ret, outs = self._exec_git('log', ['-n1', '--pretty=%T', commit]) + if ret != 0: + print('not found tree object of %s' % commit) + return (False, '') + tree_object = outs.strip() + msg = '"[SCM] %s branch is overwritten by %s"' % (branch, commit) + ret, outs = self._exec_git('commit-tree', [tree_object, '-p', commit, '-p', deprecated_commit, '-m', msg]) + if ret != 0: + print('commit-tree failed(tree:%s, parent: %s %s)' % (tree_object, commit, deprecated_commit)) + return (False, '') + overwrite_commit = outs.strip() + return (True, overwrite_commit) + + def _update_gitproject(localdir, gitpath=None): """Fetch latest code to local dir""" @@ -208,3 +254,4 @@ def fetch_change(gerritprj, localdir, refspec, giturl=None, bare=False, gerrit_h git = Git.create(localdir, bare) git.fetch(repo=giturl, refspec=refspec) git.checkout('FETCH_HEAD') + diff --git a/debian/jenkins-scripts.install b/debian/jenkins-scripts.install index 8a01e90..5823eee 100644 --- a/debian/jenkins-scripts.install +++ b/debian/jenkins-scripts.install @@ -36,5 +36,6 @@ debian/tmp/job_update_public_git.py /var/lib/jenkins/jenkins-scripts/ debian/tmp/job_control_ondemand_slaves.groovy /var/lib/jenkins/jenkins-scripts/ debian/tmp/job_test_trigger_automation.py /var/lib/jenkins/jenkins-scripts/ debian/tmp/job_update_gbs_meta.py /var/lib/jenkins/jenkins-scripts/ -debian/tmp/job_submit_request_sync.py /var/lib/jenkins/jenkins-scripts/ +debian/tmp/job_submit_request_sync.py /var/lib/jenkins/jenkins-scripts/ +debian/tmp/job_submit_request_git_sync.py /var/lib/jenkins/jenkins-scripts/ debian/tmp/job_gbs_ref_fullbuild.py /var/lib/jenkins/jenkins-scripts/ diff --git a/job_submit_request_git_sync.py b/job_submit_request_git_sync.py new file mode 100644 index 0000000..3075c73 --- /dev/null +++ b/job_submit_request_git_sync.py @@ -0,0 +1,364 @@ +#!/usr/bin/env python + +import os +import sys +import base64 +import json +import re +import shutil +import tempfile +import traceback +import urllib2 +import xml.etree.ElementTree as ET +import ast +from common import gerrit +from common import git +from common import utils +from common import runner +from common.buildtrigger import trigger_info, trigger_next +from common.send_mail import prepare_mail +from time import sleep + +GIT_URL = 'ssh://%s@%s:%s' % (os.getenv('GERRIT_USERNAME'), + os.getenv('GERRIT_HOSTNAME'), + os.getenv('GERRIT_SSHPORT')) +HIGHLIGHT = '+' +GREYEDOUT = '-' + +class GitUpdateError(Exception): pass +class RevisionError(Exception): pass +class UpstreamError(Exception): pass +class OverwriteError(Exception): pass + +class MailSender(object): + email_title = '' + email_head = '' + email_body = '' + email_footer = '\n\n--------------------------------------------------------\n'\ + 'Automatically generated by backend service.\n'\ + 'Please DO NOT Reply!' + email_to = [] + + def __init__(self, receiver=None, title=None, body=None): + if receiver is not None: self.email_to = receiver + if title is not None: self.email_title = title + if body is not None: self.email_body = body + + def add_receiver(self, add_new): + """ add receiver """ + if type(add_new) == list: + self.email_to.extend(x for x in add_new) + elif type(add_new) == str: + self.email_to.append(add_new) + else: + print 'TYPE(%s) == %s' % (add_new, type(add_new)) + self.email_to = list(set(self.email_to)) + + def add_title(self, add_new): + """ add title """ + self.email_title = self.email_title + ' ' + add_new + + def add_message(self, add_new, top=None): + """ add message """ + if top is not None: + self.email_body = add_new + '\n' + self.email_body + else: + self.email_body = self.email_body + '\n' + add_new + + def add_maintainers(self, mygerrit, group_name=None, project=None): + """ add maintainers """ + if not mygerrit: + print 'mygerrit is Null...' + return + + if not project and group_name: + mbrs = mygerrit.ls_members(['\'\"%s\"\'' % group_name, '--recursive']) + for line in mbrs: + self.add_receiver(line.split('\t')[3]) + return + + if project: + grps = mygerrit.ls_groups(['--project %s' % project]) + dest_grp = [s for s in grps if " - Maintainers" in s] + for dg in dest_grp: + mbrs = mygerrit.ls_members(['\'\"%s\"\'' % dg, '--recursive']) + for line in mbrs: + self.add_receiver(line.split('\t')[3]) + return + + def send_mail(self): + """ send mail """ + self.email_body = self.email_head + self.email_body + self.email_footer + + self.email_to = [x for x in self.email_to if x != 'n/a'] + print '\n\n' + print self.email_title + #m_body = '' + #for m in self.email_body.splitlines(): + # m_body += '\n'.join(m[i:i+128] for i in xrange(0, len(m), 128)) + '\n' + #self.email_body = m_body + print self.email_body + print self.email_to + + prepare_mail("%s.env" % os.getenv('BUILD_TAG'), \ + self.email_title, \ + self.email_body, \ + os.getenv('NOREPLY_EMAIL_SENDER'), \ + self.email_to) + +def conv_rev2commit(gitprj, git_dir, revision): + commit = '' + try: + retcode, outs = runner.show('git --git-dir=%s/.git cat-file -t %s' % (git_dir, revision)) + if retcode != 0: + raise RevisionError('not found object type(%s)' % revision) + obj_type = outs.strip() + if obj_type == 'commit' and re.match(r'[0-9a-f]{5,40}', revision): + commit = revision + else: + rev_name = gitprj.name_rev('--name-only %s' % (revision)) + if rev_name == 'undefined': + rev_name = revision + commits = gitprj.show('-s', '--format=%H', rev_name) + if not commits or len(commits) != 1: + raise RevisionError('not found commit(%s)' % revision) + commit = commits[0] + except RevisionError, exc: + print 'Error:', exc + return commit + +def update_sync_for_git_sync(tmpdir, private_gerrit, target, requests, request_snapshot): + """ + Update sync for git sync. + """ + + target_project = target.get('target') or [] + target_fork_prefix = target.get('fork') or [] + branch = target.get('branch') or [] # tizen_4.0_mcd_z4 + source_branch = target.get('source_branch') or [] # public_tizen + email_to_members = target.get('notify') or [] + + GIT_PREFIX = target_fork_prefix + UPSTREAM_BRANCH = "public_" + source_branch + overwrite_branches = [branch] + request_projects = [] + new_private_projects = [] + overwrite_alreadys = [] + overwrite_successs = [] + upstream_failures = [] + overwrite_failures = [] + + private_projects = private_gerrit.ls_projects() + + upstream_git_dir = '' + private_git_dir = '' + + for item in requests[requests.keys()[0]]: + project = item['gerrit_project'] + revision = item['cid'] + submitter = item['submitter'] + git_tag = item['git_tag'] + + if project.startswith(HIGHLIGHT): + continue + else: + project = project[1:] + request_projects.append((project, revision, submitter, git_tag)) + + try: + print '\n-----[Manifest Information]-------------------------------' + print 'GIT_PREFIX:', GIT_PREFIX + print 'UPSTREAM_BRANCH: ', UPSTREAM_BRANCH + print 'Project:', project + print 'Revision:', revision + + if not project or not revision: + raise UpstreamError('not found information(%s, %s)' % ( project, revision)) + + print '\n* Update upstream git repo %s' % (project) + if not git.clone_gitproject(project, os.path.join(tmpdir, project)): + raise UpstreamError('upstream git update failed(%s)' % (project)) + + upstream_git_dir = os.path.join(tmpdir, project) + upstream_gitprj = git.Git(upstream_git_dir) + + commit = conv_rev2commit(upstream_gitprj, upstream_git_dir, revision) + if not commit: + raise UpstreamError('failure to convert revision to commit(%s)' % revision) + + private_project = '%s%s' % (GIT_PREFIX, project) + + if not private_project in private_projects: + print '\n* Does not exist Private git repo %s' % (private_project) + #private_gerrit.create_project(aname=private_project, parent='Tizen-Projects', empty_commit=True) + #private_gerrit.create_project(private_project, '-p', os.getenv('GERRIT_PARENT_PROJECT')) + new_private_projects.append(private_project) + continue + + print '\n* Push %s %s %s' % (commit, UPSTREAM_BRANCH, private_project) + ret = upstream_gitprj._git_inout('push',['-q', '-f', '%s/%s' % (GIT_URL, private_project), \ + '%s:refs/heads/%s' % (commit, UPSTREAM_BRANCH)]) + if ret == None: + raise UpstreamError('git push failed(%s)' % (private_project)) + + print '\n* Overwrite Branches:', overwrite_branches + if not overwrite_branches: + print 'not found overwrite branch information of %s, skip overwrite..' % (private_project) + continue + + print '\n* Update private git repo %s' % (private_project) + if not git.clone_gitproject(private_project, os.path.join(tmpdir, private_project)): + raise OverwriteError('private git update failed(%s)' % (private_project)) + + private_git_dir = os.path.join(tmpdir, private_project) + private_gitprj = git.Git(private_git_dir) + # ERROR: missing Change-Id in commit message footer + retcode, outs = runner.show('scp -p -P %s %s@%s:hooks/commit-msg %s/.git/hooks/' \ + %(os.getenv('GERRIT_SSHPORT'), os.getenv('GERRIT_USERNAME'), + os.getenv('GERRIT_HOSTNAME'), private_git_dir)) + if retcode != 0: + raise RevisionError('error') + + for branch in overwrite_branches: + print '\n* Overwrite %s %s %s' % (commit, branch, private_project) + ret, overwrite_commit = private_gitprj.create_overwrite_commit(branch, commit) + if not ret: + raise OverwriteError('overwrite %s branch failed(%s, %s)' % (branch, private_project, commit)) + + if not overwrite_commit: + print 'already overwrite %s branch..' % (branch) + overwrite_alreadys.append((private_project, commit)) + continue + + print '\n* checkout %s ' %(overwrite_commit) + private_gitprj.checkout(overwrite_commit) + + #print '\n* Commit --amend --no-edit.. ' + ret = private_gitprj._exec_git('commit', ['--amend', '--no-edit']) + if ret == None: + raise OverwriteError('git commit --amend --no-edit failed(%s)' % (private_project)) + + ret, outs = private_gitprj._exec_git('log', ['-n1', '--pretty=%H']) + if ret == None: + raise OverwriteError('git log -n1 --pretty=%H failed(%s)' % (private_project)) + #change overwrite_commit + overwrite_commit = outs.strip() + + print '\n* Push %s %s %s' % (overwrite_commit, branch, private_project) + ret = private_gitprj._exec_git('push',['-q', 'origin', '%s:refs/for/%s' % (overwrite_commit, branch)]) + if ret == None: + raise OverwriteError('git push failed(%s)' % (private_project)) + print '\n* Submit', overwrite_commit + private_gerrit.review(commit = overwrite_commit, project =private_project, \ + message = 'Submitted by system administrator', verified = 1, \ + codereview = 2, submit = True) + overwrite_successs.append((private_project, '%s -> %s' %(commit, overwrite_commit))) + except UpstreamError, exc: + upstream_failures.append((project, revision, exc)) + print 'Error:', exc + except OverwriteError, exc: + overwrite_failures.append((project, revision, exc)) + print 'Error:', exc + finally: + if upstream_git_dir and os.path.exists(upstream_git_dir): + shutil.rmtree(upstream_git_dir) + if private_git_dir and os.path.exists(private_git_dir): + shutil.rmtree(private_git_dir) + + if request_projects: + print '\n-----[ Request Mixed Git Repository List]----------' + print '\n'.join(str(item) for item in request_projects) + + if new_private_projects: + print '\n-----[Does not exist Private Git Repository List]----------' + print '\n'.join(new_private_projects) + + if overwrite_alreadys: + print '\n----[ Already Overwrite into %s Branch List]-----------'%(branch) + print '\n'.join(str(item) for item in overwrite_alreadys) + + if overwrite_successs: + print '\n----[Overwrite into %s Branch Success List]-----------'%(branch) + print '\n'.join(str(item) for item in overwrite_successs) + + if upstream_failures: + print '\n-----[Update Upstream %s Branch Failure List]----------------'%(UPSTREAM_BRANCH) + print '\n'.join(str(item) for item in upstream_failures) + + if overwrite_failures: + print '\n-----[Overwrite into %s Branch Failure List]----------'%(branch) + print '\n'.join(str(item) for item in overwrite_failures) + + #### Send mail to maintainers + if True: + ### Init MailSender #### + my_mail = MailSender() + + my_mail.add_title('[Auto Git-Sync] Target Project:%s Branch:%s' %(target_project, branch)) + my_mail.add_message('Hello\n A Build System infrom you about Auto Git-Sync results\n\n') + my_mail.add_message('Public Source Snapshot: %s Upstream Branch: %s\nTarget Project: %s\nTarget Branch: %s' \ + %(request_snapshot, UPSTREAM_BRANCH, target_project, branch)) + + print 'members:',email_to_members + # Add maintatiners to mail + [my_mail.add_maintainers(private_gerrit, group_name = name) for name in email_to_members] + + if new_private_projects: + my_mail.add_message('\n-----[Does not exist Private Git Repository List]-----------') + [my_mail.add_message(str(item)) for item in new_private_projects] + + if overwrite_alreadys: + my_mail.add_message('\n----[ Already Overwrite into %s Branch List]-----------'%(branch)) + [my_mail.add_message(str(item)) for item in overwrite_alreadys] + + if overwrite_successs: + my_mail.add_message('\n-----[Overwrite into %s Branch Success List]-----------'%(branch)) + [my_mail.add_message(str(item)) for item in overwrite_successs] + + if upstream_failures: + my_mail.add_message('\n-----[Update Upstream %s Branch Failure List]-----------'%(UPSTREAM_BRANCH)) + [my_mail.add_message(str(item)) for item in upstream_failures] + + if overwrite_failures: + my_mail.add_message('\n-----[Overwrite into %s Branch Failure List]----------'%(branch)) + [my_mail.add_message(str(item)) for item in overwrite_failures] + + # Send mail to maintainers + my_mail.send_mail() + + +def main(): + """ + Script entry point. + """ + + print '---[JOB STARTED]-------------------------' + + if os.getenv('TRIGGER_INFO', None): + content = trigger_info(os.getenv('TRIGGER_INFO')) + else: + return + + if not os.getenv('SR_SYNC_PROJECTS'): + print 'No SR_SYNC_PROJECTS' + return + + enabled_projects = ast.literal_eval(os.getenv('SR_SYNC_PROJECTS')) + + private_gerrit = gerrit.Gerrit(os.getenv('GERRIT_HOSTNAME'), + os.getenv('GERRIT_USERNAME'), + os.getenv('GERRIT_SSHPORT'), + int(os.getenv('GERRIT_SILENT_MODE'))) + + #print enabled_projects + for target in enabled_projects[content['obs_target_prj']]: + if target.get('git-sync') == 'Yes': + tmpdir = tempfile.mkdtemp(prefix=os.getenv('WORKSPACE')+'/') + update_sync_for_git_sync(tmpdir, private_gerrit, target, content['requests'][0], content['snapshot']) + else: + print '%s project : git-sync option is not enabled..' %(target.get('target')) + +if __name__ == '__main__': + sys.exit(main()) + + diff --git a/packaging/jenkins-scripts.spec b/packaging/jenkins-scripts.spec index c689392..699979d 100644 --- a/packaging/jenkins-scripts.spec +++ b/packaging/jenkins-scripts.spec @@ -165,6 +165,7 @@ fi %{destdir}/job_test_trigger_automation.py %{destdir}/job_update_gbs_meta.py %{destdir}/job_submit_request_sync.py +%{destdir}/job_submit_request_git_sync.py %{destdir}/job_gbs_ref_fullbuild.py %files common -- 2.7.4