From bee1a68020aae24383d67fa22ad7d0ac4498d34e Mon Sep 17 00:00:00 2001 From: Yonghee Han Date: Fri, 22 Jul 2016 10:23:31 +0900 Subject: [PATCH] Feature : Check Human Error about SR tag 1) check whether tag follow proper format : tag name format,date format 2) check whether tag is an annotated tag 3) Check if change is still under review 4) Check specfile exist under packaging 5) Check parse specfile 6) Report an e-mail message about the error to the submitter Change-Id: Ibd722848e87c2148de730aa766c97883e842efca --- job_submit.py | 346 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 345 insertions(+), 1 deletion(-) diff --git a/job_submit.py b/job_submit.py index 8827898..37571a3 100755 --- a/job_submit.py +++ b/job_submit.py @@ -33,15 +33,240 @@ from time import sleep from osc import core from gitbuildsys.errors import ObsError +from common import utils from common.mapping import git_obs_map -from common.git import clone_gitproject +from common.git import Git, clone_gitproject from common.upload_service import upload_obs_service, UploadError from common.gerrit import is_ref_deleted from common.buildservice import BuildService from common.backenddb import BackendDB from common.prerelease import get_prerelease_project_name, prerelease_enabled from common.iris_rest_client import IrisRestClient +from common.send_mail import prepare_mail +from common.gerrit import Gerrit, get_gerrit_event, GerritError, is_ref_deleted +from gbp.rpm import SpecFile +from gbp.git.repository import GitRepositoryError +from gbp.errors import GbpError + +WRONG_DATE_MSG = '- The date %s in tag does NOT follow correct format.\n You can'\ + ' use shell command "date --utc +%%Y%%m%%d.%%H%%M%%S" to '\ + 'generate it, like 20120801.083113.' + +UNDER_REVIEW_MSG = '- Submission %s has been rejected because tagged commit %s'\ + ' is still under review in gerrit.\n Please re-trigger '\ + 'submission after your change is accepted.' + +WRONG_COMMIT_MSG = '- The commit %s tag attached does NOT exist in git tree or'\ + ' gerrit open change.\n Please make sure the commit has been '\ + 'pushed to gerrit and correct magical ref refs/for/branch, '\ + 'then re-submit tag.' + +UNKNOWN_FORMAT_MSG = '- Unknown tag format,\n please follow the format '\ + 'submit/{version}/{date.time}.' + +NOT_ANNOTATED_MSG = '- Tag should be annotated tag.' + +SUGGESTION = 'Suggest to use "gbs submit" to trigger submission\n'\ + +TITLE_FAILED = '[Submit Request Failed]: tag: %s in %s' + +TITLE_SUCCESS = '[Submit Request Success]: tag: %s in %s' + +PRERELEASE_EMAIL_BODY = 'A SR (Submit Request) has been triggered to submit ' \ + 'the commit to OBS project.\n' \ + 'Please check the following details.\n\n' \ + 'Git branch : %s\n' \ + 'Git commit : %s\n\n' \ + 'Git path : %s\n' \ + 'Git tag : %s\n' \ + 'Submitter : %s\n' + +EMAIL_FOOTER = '\n\n--------------------------------------------------------\n'\ + 'Automatically generated by backend service.\n'\ + 'Please DO NOT Reply!' + +def send_mail(title, msg, receiver): + """ post message back to gerrit and send mail to tag owner """ + print 'msg %s' % msg + if 'author' in receiver and 'email' in receiver: + msg = 'Hi, %s,\n\n' % receiver['author'] + msg + EMAIL_FOOTER + prepare_mail("%s.env" % os.getenv('BUILD_TAG'), title, msg, + os.getenv('NOREPLY_EMAIL_SENDER'), receiver['email']) + +def parse_submit_tag(tag): + """parse info from submit tag name""" + + branch = None + date = None + + if tag.startswith('submit/'): + pos = tag.rfind('/', len('submit/')) + if pos != -1: + branch = tag[len('submit/'):pos] + if branch == 'trunk': + branch = 'master' + date = tag[pos+1:] + + return branch, date + +def find_submit_tag(event, mygit): + """find the corresponding submit tag for this event""" + + if event['event_type'] == 'ref-updated': + tag = event['refname'][len('refs/tags/'):] + event['branch'] = parse_submit_tag(tag)[0] + # Since patchset_revision is used in gerrit feedback, real tag check + # is needed; and the key point is parse_submit_tag can not ensure the + # tag exsisting too. + try: + event['patchset_revision'] = mygit.rev_parse('%s^{commit}' % tag) + except GitRepositoryError: + tag = None + elif event['event_type'] == 'change-merged': + # for chanage-merged, search submit tag on this commit + branch = event['branch'] + if event['branch'] == 'master': + branch = 'trunk' + try: + tag = mygit.describe(event['patchset_revision'], + pattern='submit/%s/*' % branch, + exact_match=True) + except GitRepositoryError: + # don'n find submit tag on this commit, return None + tag = None + + return tag + +def check_tag_format(git, mygerrit, event, tag): + """check whether tag follow proper format""" + + branch, date = parse_submit_tag(tag) + message = [] + psr = event['patchset_revision'] + + # check tag name format + if branch and date: + # check date format + pattern = re.compile(r'^[0-9]{8}\.[0-9]{6}$') + if not pattern.match(date): + message.append(WRONG_DATE_MSG % date) + + if not git.branch_contains(tag): + # Check if change is still under review + cmd = '--current-patch-set status: open project: %s commit: %s' % \ + (event['project'], psr) + gerritinfo = mygerrit.query(cmd) + if len(gerritinfo) == 1 and 'number' in gerritinfo[0] \ + and 'currentPatchSet' in gerritinfo[0]: + if gerritinfo[0]['branch'] == branch: + # the tagged commit still open, abort submit this time + message.append(UNDER_REVIEW_MSG % (tag, psr)) + else: + # cannot find tagged commit in git tree or gerrit open change + message.append(WRONG_COMMIT_MSG % psr) + else: + # wrong tag format + message.append(UNKNOWN_FORMAT_MSG) + + # check whether tag is an annotated tag + tagger = git.get_tag(tag) + if 'author' not in tagger or 'email' not in tagger: + message.append(NOT_ANNOTATED_MSG) + + # post comment to gerrit and send email if check failed + if message: + print message + + msg = 'The tag %s was pushed, but it was not completed because of '\ + 'the following reason(s):\n\n' % tag + '\n'.join(message) + + if len(message) != 1 or (message[0] != UNDER_REVIEW_MSG % (tag, psr) \ + and message[0] != WRONG_COMMIT_MSG % psr): + msg += '\n\n' + SUGGESTION + + try: + mygerrit.review(commit=psr, message=msg) + except GerritError, err: + print >> sys.stderr, 'Error posting review comment '\ + 'back to Gerrit: %s' % str(err) + # return 1 if this exception is not caused by invalid commit + if 'no such patch set' not in str(err): + return False + if 'author' not in tagger or 'email' not in tagger: + recevier = { 'author' : event['event_account_name'], + 'email' : event['event_account_email'] } + send_mail(TITLE_FAILED % (tag, event['project']), msg, recevier) + else: + send_mail(TITLE_FAILED % (tag, event['project']), msg, tagger) + + return False + + return True + +def find_specfile(prj_dir, packaging_dir, tag, event, tagger, pkg_name=None): + """search specfile under packaging directory""" + + msg = '' + + if pkg_name: + spec = '%s/%s/%s.spec' % (prj_dir, packaging_dir, pkg_name) + if not os.path.isfile(spec): + msg = "The tag %s pushed, but backend service can not find %s \n"\ + "under packaging directory, which is caused by mistake \n"\ + "OBS_PACKAGE parameter in scm/git-obs-mapping project. \n"\ + "Please correct it or contact system administrator for \n"\ + "more details." % (tag, os.path.basename(spec)) + else: + specs = utils.find_spec('%s/%s' % (prj_dir, packaging_dir)) + if not specs: + # no spec exist under packaging, use default name + msg = "The tag %s pushed, but packaging directory doesn't \n"\ + "contain any spec file. Please create one and \n"\ + "re-submit it." % tag + elif len(specs) == 1: + # only one spec exist under packaging + spec = specs[0] + else: + # multiple specs exist under packaging, use default name + spec = '%s/%s/%s.spec' % (prj_dir, packaging_dir, \ + os.path.basename(event['project'])) + if not os.path.isfile(spec): + msg = "The tag %s pushed, but packaging directory contains \n"\ + "multiply spec files, backend service can not decide \n"\ + "which spec file to use. Please use OBS_PACKAGE \n"\ + "parameter in scm/git-obs-mapping project to specify \n"\ + "the target spec file or contact system \n"\ + "administrator for more details." % tag + + if msg: + print msg + send_mail(TITLE_FAILED % (tag, event['project']), msg, tagger) + return None + else: + print 'specfile %s' % spec + return spec + + +def parse_specfile(specfile, tag, event, tagger): + """parse specfile""" + + spec = None + + try: + # use gbp to parse specfile + spec = SpecFile(specfile) + except GbpError, err: + print 'gbp parse spec failed. %s' % err + msg = 'The tag %s pushed, but backend service failed to parse %s. \n'\ + 'Please try "gbs export" on this tag and make sure it can '\ + 'work.\n\n'\ + 'Error message:\n'\ + '%s' % (tag, os.path.basename(specfile), err) + send_mail(TITLE_FAILED % (tag, event['project']), msg, tagger) + return None + + return spec class LocalError(Exception): """Local error exception.""" @@ -176,6 +401,9 @@ def create_project(git_url, git_project, git_tag, git_revision, build, #create the 'link' package that relate the original package create_related_packages(build, obs_target_prj, obs_project, package) + #Wait 10 seconds to upload the package to the OBS + sleep(10) + #default build flag build.default_build_flag(obs_project, repo = None, flag="build") @@ -196,6 +424,40 @@ def get_branch_name(tag): branch = 'master' return branch +def send_mail_sr_message(info, mygerrit): + """ send mail about buildstatus """ + + #make a message + title = TITLE_SUCCESS % ( info['submitted'].get('tag'), info['submitted'].get('gitpath')) + email_body = PRERELEASE_EMAIL_BODY % (info['submitted'].get('branch'), \ + info['submitted'].get('commit_id'), \ + info['submitted'].get('gitpath'), \ + info['submitted'].get('tag'), \ + info['submitted'].get('submitter') + ) + #add products project + email_body += 'obs_prj : %s \n' %(info['pre_created']) + + msg = 'Hello \n\n' + email_body + EMAIL_FOOTER + print title + print msg + submitter = info['submitted'].get('submitter') + if submitter: +# email_to = [] +# if True: +# grps = mygerrit.ls_groups(['--project %s' % info['submitted'].get('gitpath')]) +# for dg in [s for s in grps if " - Maintainers" in s]: +# for line in mygerrit.ls_members(['\'\"%s\"\'' % dg, '--recursive']): +# email_to.append(line.split('\t')[3]) +# email_to = [x for x in email_to if x != 'n/a'] +# +# print 'email_to : %s ' % (email_to) +# for user in email_to: +# submitter += '%s,' %(user) + + prepare_mail("%s.env" % os.getenv('BUILD_TAG'), title, msg, + os.getenv('NOREPLY_EMAIL_SENDER'), submitter) + def main(build_type): """ Script entry point. @@ -216,6 +478,8 @@ def main(build_type): gerrit_account_name = os.getenv("GERRIT_EVENT_ACCOUNT_NAME") gerrit_account_email = os.getenv("GERRIT_EVENT_ACCOUNT_EMAIL") + event = get_gerrit_event() + # Init backend database redis_host = os.getenv("REDIS_HOST") redis_port = int(os.getenv("REDIS_PORT")) @@ -241,9 +505,68 @@ def main(build_type): "tag": git_tag, }) + # prepare separate temp directory for each build + git_cache = os.getenv("GIT_CACHE_DIR") + prjdir = os.path.join(git_cache, event['project']) + + # clone gerrit project to local dir + if not clone_gitproject(event['project'], prjdir): + print >> sys.stderr, 'Error cloning %s' % event['project'] + return 1 + mygit = Git(prjdir) + + # check whether tag name is start with 'submit/' + if not event['refname'].startswith('refs/tags/submit/'): + print '\nREFNAME "%s" isn\'t start with refs/tags/submit, exit now'\ + % event['refname'] + return 0 + elif is_ref_deleted(event['oldrev'], event['newrev']): + print '\nREFNAME "%s" is deleted, exit now' % event['refname'] + return 0 + + mygerrit = Gerrit(event['hostname'], event['username'], \ + event['sshport'], int(os.getenv('GERRIT_SILENT_MODE'))) + + tag = find_submit_tag(event, mygit) + if not tag: + print '\nThis commit don\'t contain submit/*/* tag, exit now' + return 0 + + submitter = '' + if gerrit_account_name: + submitter = gerrit_account_name + if gerrit_account_email: + submitter += ' <%s>' % gerrit_account_email + + # check whether tag meet format + if not check_tag_format(mygit, mygerrit, event, tag): + print 'The check for the tag format is error, exit now\n' + return 0 + + packagingdir = utils.parse_link('%s/%s' % (prjdir, 'packaging')) + print 'packaging dir is %s/%s' % (prjdir, packagingdir) + + # checkout submit tag + mygit.checkout(tag) + + tagger = mygit.get_tag(tag) + # get list of target projects from git-obs-mapping obs_target_prjs = git_obs_map(os.getenv("GERRIT_PROJECT"), git_branch) build = BuildService(obs_api, obs_user, obs_passwd) + + # prepare submit_info + submit_info={} + submit_info['submitted'] = { + "branch": git_branch, + "commit_id": gerrit_newrev, + "gitpath": gerrit_project, + "submitter": gerrit_account_name, + "submitter_email": gerrit_account_email, + "tag": git_tag, + } + submit_info['pre_created'] = [] + for obs_target_prj, obs_stg_prj, obs_pkg in obs_target_prjs: url = 'ssh://%s:%s' % (os.getenv('GERRIT_HOSTNAME'), os.getenv('GERRIT_SSHPORT')) @@ -273,6 +596,17 @@ def main(build_type): submitter = gerrit_account_name if gerrit_account_email: submitter += ' <%s>' % gerrit_account_email + + # search specfile under packaging directory + specfile = find_specfile(prjdir, packagingdir, tag, event, tagger, obs_pkg) + if not specfile: + continue + + # parse specfile + spec = parse_specfile(specfile, tag, event, tagger) + if not spec: + continue + retry_count = 3 while retry_count > 0: try: @@ -285,6 +619,13 @@ def main(build_type): "product": obs_target_prj, "gitpath": gerrit_project, }) + # prepare submit_info + if project: + products = submit_info['pre_created'] or [] + if not obs_target_prj in products: + submit_info['pre_created'].append(project) + else: + submit_info['pre_created'] = products break except Exception, err: print 'obs operation failed, retrying...' @@ -306,6 +647,9 @@ def main(build_type): else: print "Warning: target project %s doesn't exist, " \ "skipping" % obs_target_prj + # send mail + if submit_info['pre_created']: + send_mail_sr_message(submit_info, mygerrit) if __name__ == '__main__': try: -- 2.7.4