From 99af0b359675c282579fb0e8e755bd28bb655f4b Mon Sep 17 00:00:00 2001 From: Lin Yang Date: Fri, 3 Aug 2012 17:59:14 +0800 Subject: [PATCH] continue support previous tag format 'build/*' --- git.py | 21 ++++-- submitobs.py | 50 ++++++------- submitobs_orig.py | 212 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 250 insertions(+), 33 deletions(-) create mode 100755 submitobs_orig.py diff --git a/git.py b/git.py index e97a786..ed835e2 100644 --- a/git.py +++ b/git.py @@ -251,13 +251,24 @@ class Git: def get_tagger(self, tag): ret, outs = self._exec_git('show', - ['-s', tag]) + ['-s --pretty=format:###', tag]) tagger = {} + msg = '' + msgstub = False for line in outs.splitlines(): - if line.startswith('Tagger: '): - tagger['author'] = line[len('Tagger: '):line.index(' <')] - tagger['email'] = line[line.index(' <')+2:line.index('>')] - break + if not line: + continue + if msgstub: + if line.startswith('###'): + tagger['message'] = msg[:-1] + break + else: + msg += '%s\n' % line + else: + if line.startswith('Tagger: '): + tagger['author'] = line[len('Tagger: '):line.index(' <')] + tagger['email'] = line[line.index(' <')+2:line.index('>')] + msgstub = True print 'git get_tagger %s' % tag, tagger return tagger diff --git a/submitobs.py b/submitobs.py index 736518e..b563266 100755 --- a/submitobs.py +++ b/submitobs.py @@ -102,17 +102,20 @@ if __name__ == '__main__': if branchs: # the tagged commit has been merged on gerrit if branch not in branchs: - checktagmsg = 'Tag %s only can be used to submit commit on %s branch to OBS. Please check and re-submit another proper tag. Recommend to use "gbs submit" to do this.' % (tag, branch) + title = '[Submit Request Failed]: improper tag: %s' % tag + checktagmsg = 'The tag %s pushed, but it only can be used to submit commit on %s branch to OBS. Please "gbs submit" to trigger the submission.' % (tag, branch) else: GERRIT_BRANCH = branch print 'submit tag : %s, on branch %s' % (tag, branch) else: # The tagged commmit still open, abort submit this time print '\nThis change is still open in gerrit, exit now' - checktagmsg = 'The commit tag %s attached is still open in gerrit. After reviewer accpet this commit, it will be submitted to OBS corresponding project.' % tag + title = '[Submit Request Failed]: tag: %s' % tag + checktagmsg = 'The commit tag %s attached is still under review in gerrit. After reviewer accpet this commit, it will be submitted to OBS corresponding project.' % tag else: print 'tag %s don\'t match the name format, exit now' % tag - checktagmsg = 'Backend service cannot find branch info in tag %s. Please create tag "submit//" and submit to gerrit. Recommend to use "gbs submit" to do this.' % tag + title = '[Submit Request Failed]: unknown tag: %s' % tag + checktagmsg = 'The tag %s pushed, but unknown tag format, please use "gbs submit" to trigger the submission.' % tag else: # Only when there is tag submit/*, it's for submit if GERRIT_BRANCH == 'master': @@ -128,34 +131,36 @@ if __name__ == '__main__': commitinfo = mygit.get_commit_info(tag) tagger = mygit.get_tagger(tag) + if GERRIT_EVENT_TYPE == "REF_UPDATED": + gerritinfo = utils.get_gerrit_info(GERRIT_PROJECT, commitinfo['id']) + print 'gerritinfo', gerritinfo + if gerritinfo: + GERRIT_CHANGE_NUMBER = gerritinfo['changenum'] + GERRIT_PATCHSET_NUMBER = gerritinfo['patchsetnum'] if not tagger: print '\ntag %s is not annotated tag, exit now' % tag end('failure') - if checktagmsg: + if checktagmsg and title: print checktagmsg checktagmsg = 'Hi, %s,\n\n' % tagger['author'] + checktagmsg + '\n\n----------------------------------------------------------------\nAutomatically generated by backend service.\nPlease DO NOT Reply!' - sendmail('[submit to OBS failed]: tag %s' % tag, checktagmsg, FROM_EMAIL, tagger['email']) + sendmail(title, checktagmsg, FROM_EMAIL, tagger['email']) end('failure') packagingdir = utils.parse_link('%s/%s' % (prjdir, 'packaging')) print('packaging dir is %s/%s' % (prjdir, packagingdir)) - msg = 'Tag: %s\nCommit: %s %s\nSubmitter: %s <%s>' % (tag, commitinfo['id'], commitinfo['subject'], tagger['author'], tagger['email']) + msg = 'Submitter: %s <%s>\nComments: %s\nGit project: %s\nTag:%s\nCommit: %s %s' % (tagger['author'], tagger['email'], tagger['message'], GERRIT_PROJECT, tag, commitinfo['id'], commitinfo['subject']) mygit.checkout(tag) mapping = utils.parse_mapping('%s/git/%s/git-obs-mapping.xml' % (JENKINS_HOME, mappingprj), GERRIT_PROJECT, GERRIT_BRANCH) print 'git-obs-mapping:', mapping - reqlist = [] - dstprjlist = [] for target in mapping: mygit.clean('-fd') (obs_dst_prj, obs_stg_prj, obs_pkg) = target if not obs_dst_prj: continue - else: - dstprjlist.append(obs_dst_prj) if not obs_stg_prj: obs_stg_prj = obs_dst_prj @@ -206,12 +211,11 @@ if __name__ == '__main__': print '\nSubmit local pkg to obs...' localpkg.commit(msg) if obs_stg_prj != obs_dst_prj: - for req in localpkg.query_req(obs_dst_prj): - msg += '\n\n%s' % req.description - print 'supersede previous SR %s' % req.reqid + #for req in localpkg.query_req(obs_dst_prj): + # msg += '\n\n%s' % req.description + # print 'supersede previous SR %s' % req.reqid print '\nsr msg:\n%s' % msg newreq = localpkg.submit_req(obs_dst_prj, msg=msg) - reqlist.append(newreq) print 'New request %s is created' % newreq break except errors.ObsError, exc: @@ -222,19 +226,9 @@ if __name__ == '__main__': if not retry_count: end('retry') - # post sr info back to gerrit - if reqlist: - if GERRIT_EVENT_TYPE == "REF_UPDATED": - gerritinfo = utils.get_gerrit_info(GERRIT_PROJECT, commitinfo['id']) - print 'gerritinfo', gerritinfo - if gerritinfo: - GERRIT_CHANGE_NUMBER = gerritinfo['changenum'] - GERRIT_PATCHSET_NUMBER = gerritinfo['patchsetnum'] - - requrl = '' - for req in reqlist: - requrl += ' %s/request/show/%s' % (OBS_URL, req) - comment = 'A request has been created to submit this commit to OBS %s project.\n- Commit: %s %s\n- Tag: %s\n- Submitter: %s <%s>\n- Request URL:%s' % (' '.join(dstprjlist), commitinfo['id'], commitinfo['subject'], tag, tagger['author'], tagger['email'], requrl) + # post sr info back to gerrit + requrl = ' %s/request/show/%s' % (OBS_URL, newreq) + comment = 'A SR (Submit Request) has been trigger to submit the commit to OBS %s project.\n- Submitter: %s <%s>\n- Comments: %s\n- Git project: %s\n- Tag:%s\n- Commit: %s %s\n- Request URL:%s' % (obs_dst_prj, tagger['author'], tagger['email'], tagger['message'], GERRIT_PROJECT, tag, commitinfo['id'], commitinfo['subject'], requrl) if GERRIT_CHANGE_NUMBER and GERRIT_CHANGE_NUMBER: print '%s %s %s,%s --message \'"%s"\'' % (gerritcmd, 'review', GERRIT_CHANGE_NUMBER, GERRIT_PATCHSET_NUMBER, comment) @@ -243,6 +237,6 @@ if __name__ == '__main__': print 'GERRIT_CHANGE_NUMBER %s and GERRIT_CHANGE_NUMBER %s is empty.' % (GERRIT_CHANGE_NUMBER, GERRIT_CHANGE_NUMBER) comment = 'Hi, %s,\n\n' % tagger['author'] + comment + '\n\n----------------------------------------------------------------\nAutomatically generated by backend service.\nPlease DO NOT Reply!' - sendmail('[submit to OBS success]: tag %s' % tag, comment, FROM_EMAIL, tagger['email']) + sendmail('[Submit Request]: changes to %s/%s' % (obs_dst_prj, spec.name), comment, FROM_EMAIL, tagger['email']) end('success') diff --git a/submitobs_orig.py b/submitobs_orig.py new file mode 100755 index 0000000..a3596c7 --- /dev/null +++ b/submitobs_orig.py @@ -0,0 +1,212 @@ +#!/usr/bin/env python +# vim: ai ts=4 sts=4 et sw=4 + +"""This script will sync a merged change in gerrit to OBS corresponding project. +""" + +import os +import tempfile +import glob +import shutil +import re + +import runner +import utils +import git +import obspkg +from envparas import * +import errors +import mysql +from send_mail import sendmail + +import gbp.rpm + +giturl = 'ssh://%s@%s:%s' % (GERRIT_USERNAME, GERRIT_HOSTNAME, GERRIT_SSHPORT) +gerritcmd = 'ssh -p %s %s@%s gerrit' % (GERRIT_SSHPORT, GERRIT_USERNAME, GERRIT_HOSTNAME) +ret = { 'rc' : {'success' : 0, 'failure' : 0, 'retry' : 1}, + 'dbstate': {'success' : 'TRIGGER_SUCCESS', + 'failure' : 'TRIGGER_FAILURE', + 'retry' : 'TRIGGER_RETRY'}, + 'dbtable': {'CHANGE_MERGED' : 'ChangeMerged_Event', + 'REF_UPDATED' : 'RefUpdated_Event'}, + 'dbkey': {'CHANGE_MERGED': {'changeNum' : GERRIT_CHANGE_NUMBER, + 'patchsetNum' : GERRIT_PATCHSET_NUMBER}, + 'REF_UPDATED':{'oldRev' : GERRIT_OLDREV, + 'newRev' : GERRIT_NEWREV}}, + } + +def end(result = 'success'): + print 'execute result: %s' % result + # cleanup workspace + shutil.rmtree(tmpdir) + exit(ret['rc'][result]) + +if __name__ == '__main__': + + print '---[JOB STARTED]----------------------------------------' + tmpdir = tempfile.mkdtemp(prefix=WORKSPACE+'/') + prjdir = os.path.join(tmpdir, GERRIT_PROJECT) + prjpath, prj = os.path.split(GERRIT_PROJECT) + + # check whether tag name is start with 'submit/' + if GERRIT_EVENT_TYPE == "REF_UPDATED": + if not GERRIT_REFNAME.startswith('refs/tags/build/'): + print '\nREFNAME "%s" isn\'t start with refs/tags/build, exit now' % GERRIT_REFNAME + end('success') + + # check whether exist git-obs-mapping.xml in local + if GERRIT_PROJECT == mappingprj or not os.path.isfile('%s/git/%s/git-obs-mapping.xml' % (JENKINS_HOME, mappingprj)): + print('Update %s/git-obs-mapping.xml to local.' % mappingprj) + if not utils.retry(git.update_git_project, (os.path.join(JENKINS_HOME, 'git'), mappingprj, giturl)): + end('retry') + if GERRIT_PROJECT == mappingprj: + print '\nPulled scm/git-obs-mapping change to local, exit now' + end('success') + + # update local git tree from remote + try: + if os.path.exists(os.path.join(JENKINS_HOME, 'git', GERRIT_PROJECT)): + print '\nuse local repo as reference to clone' + if runner.show('git clone %s/%s --reference %s %s' % (giturl, GERRIT_PROJECT, os.path.join(JENKINS_HOME, 'git', GERRIT_PROJECT), '%s/%s' % (tmpdir,GERRIT_PROJECT)))[0]: + print 'use local repo as reference to clone: Failed.' + shutil.rmtree(os.path.join(JENKINS_HOME, 'git', GERRIT_PROJECT)) + except Exception, ex: + print '\nExcept occur when use reference repo to clone code' + print ex + if not utils.retry(git.update_git_project, (tmpdir, GERRIT_PROJECT, giturl)): + end('retry') + + packagingdir = utils.parse_link('%s/%s' % (prjdir, 'packaging')) + print('packaging dir is %s/%s' % (prjdir, packagingdir)) + mygit = git.Git(prjdir) + + checktagmsg = '' + if GERRIT_EVENT_TYPE == "REF_UPDATED": + tag = GERRIT_REFNAME[len('refs/tags/'):] + title = '[Submit Request Failed]: improper tag: %s' % tag + checktagmsg = 'The submission tag %s pushed, which will not work several weeks later. Because submission process changed, please upgrade gbs and use \'gbs submit\' to trigger the submission.' % tag + else: + needsr = False + tag = mygit.describe('--exact-match --tags --match "build/*"', GERRIT_PATCHSET_REVISION) + if tag: + files = mygit.show('--pretty="format:" --name-only', GERRIT_PATCHSET_REVISION) + p = re.compile('^%s/.*\.changes$' % packagingdir) + for filename in files: + if p.match(filename): + needsr = True + break + print 'submit tag : %s, on branch %s' % (tag, GERRIT_BRANCH) + else: + print '\nThis change don\'t contain build/* tag, exit now' + if not needsr: + end('success') + + commitinfo = mygit.get_commit_info(tag) + tagger = mygit.get_tagger(tag) + if GERRIT_EVENT_TYPE == "REF_UPDATED": + gerritinfo = utils.get_gerrit_info(GERRIT_PROJECT, commitinfo['id']) + print 'gerritinfo', gerritinfo + if gerritinfo: + GERRIT_CHANGE_NUMBER = gerritinfo['changenum'] + GERRIT_PATCHSET_NUMBER = gerritinfo['patchsetnum'] + if checktagmsg and title: + if GERRIT_CHANGE_NUMBER and GERRIT_CHANGE_NUMBER: + print '%s %s %s,%s --message \'"%s"\'' % (gerritcmd, 'review', GERRIT_CHANGE_NUMBER, GERRIT_PATCHSET_NUMBER, checktagmsg) + runner.show('%s %s %s,%s --message \'"%s"\'' % (gerritcmd, 'review', GERRIT_CHANGE_NUMBER, GERRIT_PATCHSET_NUMBER, checktagmsg)) + if tagger: + print checktagmsg + checktagmsg = 'Hi, %s,\n\n' % tagger['author'] + checktagmsg + '\n\n----------------------------------------------------------------\nAutomatically generated by backend service.\nPlease DO NOT Reply!' + sendmail(title, checktagmsg, FROM_EMAIL, tagger['email']) + end('success') + + msg = 'Submitter: %s <%s>\nComments: %s\nGit project: %s\nTag:%s\nCommit: %s %s' % (tagger['author'], tagger['email'], tagger['message'], GERRIT_PROJECT, tag, commitinfo['id'], commitinfo['subject']) + + mygit.checkout(tag) + + mapping = utils.parse_mapping('%s/git/%s/git-obs-mapping.xml' % (JENKINS_HOME, mappingprj), GERRIT_PROJECT, GERRIT_BRANCH) + print 'git-obs-mapping:', mapping + + for target in mapping: + mygit.clean('-fd') + (obs_dst_prj, obs_stg_prj, obs_pkg) = target + if not obs_dst_prj: + continue + + if not obs_stg_prj: + obs_stg_prj = obs_dst_prj + + if obs_pkg: + specfile = '%s/%s/%s.spec' % (prjdir, packagingdir, obs_pkg) + else: + specfile = utils.guess_spec('%s/%s' % (prjdir, packagingdir)) + if not specfile: + specfile = '%s/%s/%s.spec' % (prjdir, packagingdir, prj) + print('specfile %s' % specfile) + + if not os.path.isfile(specfile): + continue + # use gbp to parse specfile + try: + spec = gbp.rpm.parse_spec(specfile) + except GbpError, exc: + print('gbp parse spec failed. %s' % exc) + end('failure') + + # use gbs export to generate tarball + outdir = tempfile.mkdtemp(prefix=tmpdir+'/') + with utils.Workdir(prjdir): + runner.show('gbs export --spec=%s -o %s' % (specfile, outdir)) + print '\ngbs export --spec=%s -o %s' % (specfile, outdir) + tarballdir = os.path.join(outdir, os.listdir(outdir)[0]) + + if obs_stg_prj != obs_dst_prj: + tmppkg = obspkg.ObsPackage(tmpdir, obs_stg_prj, 'tmp', + apiurl = OBS_API_URL, oscrc = OBS_OSCRC_PATH) + if tmppkg.is_new_pkg(): + tmppkg.commit("Leave an empty package in this project to prevent OBS delete it automatically when all request from here are accepted.") + + print '\nCheckout %s/%s to local' % (obs_stg_prj, spec.name) + localpkg = obspkg.ObsPackage(tmpdir, obs_stg_prj, spec.name, + apiurl = OBS_API_URL, oscrc = OBS_OSCRC_PATH) + oscworkdir = localpkg.get_workdir() + localpkg.remove_all() + for myfile in os.listdir(tarballdir): + shutil.copy2(os.path.join(tarballdir, myfile), os.path.join(oscworkdir, myfile)) + localpkg.update_local() + + retry_count = 3 + while retry_count > 0: + try: + # submit local pkg to obs and create sr + print '\nSubmit local pkg to obs...' + localpkg.commit(msg) + if obs_stg_prj != obs_dst_prj: + #for req in localpkg.query_req(obs_dst_prj): + # msg += '\n\n%s' % req.description + # print 'supersede previous SR %s' % req.reqid + print '\nsr msg:\n%s' % msg + newreq = localpkg.submit_req(obs_dst_prj, msg=msg) + print 'New request %s is created' % newreq + break + except errors.ObsError, exc: + print('obs operation failed, retrying...') + sleep(1) + retry_count -= 1 + + if not retry_count: + end('retry') + + # post sr info back to gerrit + requrl = ' %s/request/show/%s' % (OBS_URL, newreq) + comment = 'A SR (Submit Request) has been trigger to submit the commit to OBS %s project.\n- Submitter: %s <%s>\n- Comments: %s\n- Git project: %s\n- Tag:%s\n- Commit: %s %s\n- Request URL:%s' % (obs_dst_prj, tagger['author'], tagger['email'], tagger['message'], GERRIT_PROJECT, tag, commitinfo['id'], commitinfo['subject'], requrl) + + if GERRIT_CHANGE_NUMBER and GERRIT_CHANGE_NUMBER: + print '%s %s %s,%s --message \'"%s"\'' % (gerritcmd, 'review', GERRIT_CHANGE_NUMBER, GERRIT_PATCHSET_NUMBER, comment) + runner.show('%s %s %s,%s --message \'"%s"\'' % (gerritcmd, 'review', GERRIT_CHANGE_NUMBER, GERRIT_PATCHSET_NUMBER, comment)) + else: + print 'GERRIT_CHANGE_NUMBER %s and GERRIT_CHANGE_NUMBER %s is empty.' % (GERRIT_CHANGE_NUMBER, GERRIT_CHANGE_NUMBER) + + comment = 'Hi, %s,\n\n' % tagger['author'] + comment + '\n\n----------------------------------------------------------------\nAutomatically generated by backend service.\nPlease DO NOT Reply!' + sendmail('[Submit Request]: changes to %s/%s' % (obs_dst_prj, spec.name), comment, FROM_EMAIL, tagger['email']) + + end('success') -- 2.7.4