From: hyokeun Date: Sat, 29 Jul 2017 02:48:45 +0000 (+0900) Subject: Move common APIs to common/workflow module X-Git-Tag: submit/trunk/20190927.012743~363 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=255d23266dba9307cf581e9e732b01cb566931fd;p=services%2Fjenkins-scripts.git Move common APIs to common/workflow module Change-Id: I07bfa5013b4dc8113ad6a694d1c85b6762e0fa29 --- diff --git a/common/workflow.py b/common/workflow.py new file mode 100644 index 0000000..4005536 --- /dev/null +++ b/common/workflow.py @@ -0,0 +1,611 @@ +#!/usr/bin/env python +# vim: ai ts=4 sts=4 et sw=4 +# +# Copyright (C) 2017 Samsung Electronics. Co,. Ltd. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; version 2 +# of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +""" +Common part of the CI process +""" + +import os +import sys +import json +import re +import requests +import xml.etree.cElementTree as ElementTree +from xml.sax.saxutils import escape +from time import sleep + +from gitbuildsys.errors import ObsError + +from common import utils +from common.mapping import git_obs_map, git_virtual_branch_map +from common.upload_service import upload_obs_service, UploadError +from common.prerelease import get_info_from_prerelease_name +from common.git import Git, clone_gitproject +from common.gerrit import GerritEnv, GerritError + +from gbp.rpm import SpecFile +from gbp.git.repository import GitRepositoryError +from gbp.errors import GbpError + +# set default char-set endcoding to utf-8 +reload(sys) +sys.setdefaultencoding('utf-8') # pylint: disable-msg=E1101 + +class LocalError(Exception): + """Local error exception.""" + pass + +WRONG_DATE_MSG = '- The date %s in tag does NOT follow correct format.\n You can'\ + ' use shell command "date --utc +%%Y%%m%%d.%%H%%M%%S" to '\ + 'generate it, like 20120801.083113.' + +UNDER_REVIEW_MSG = '- Submission %s has been rejected because tagged commit %s'\ + ' is still under review in gerrit.\n Please re-trigger '\ + 'submission after your change is accepted.' + +WRONG_COMMIT_MSG = '- The commit %s tag attached does NOT exist in git tree or'\ + ' gerrit open change.\n Please make sure the commit has been '\ + 'pushed to gerrit and correct magical ref refs/for/branch, '\ + 'then re-submit tag.' + +UNKNOWN_FORMAT_MSG = '- Unknown tag format,\n please follow the format '\ + 'submit/{version}/{date.time}.' + +WRONG_FORMAT_MSG = '- Wrong tag format,\n please follow the format '\ + 'submit/{branch}/{date.time}. \n'\ + 'Git branch : %s. Tag branch: %s' + +NOT_ANNOTATED_MSG = '- Tag should be annotated tag.' + +SUGGESTION = 'Suggest to use "gbs submit" to trigger submission\n'\ + +def parse_submit_tag(tag): + """parse info from submit tag name""" + + branch = None + date = None + + if tag.startswith('submit/'): + pos = tag.rfind('/', len('submit/')) + if pos != -1: + branch = tag[len('submit/'):pos] + if branch == 'trunk': + branch = 'master' + date = tag[pos+1:] + + return branch, date + +def find_submit_tag(event, mygit): + """find the corresponding submit tag for this event""" + + if event['event_type'] == 'ref-updated': + tag = event['refname'][len('refs/tags/'):] + event['branch'] = parse_submit_tag(tag)[0] + # Since patchset_revision is used in gerrit feedback, real tag check + # is needed; and the key point is parse_submit_tag can not ensure the + # tag exsisting too. + try: + event['patchset_revision'] = mygit.rev_parse('%s^{commit}' % tag) + except GitRepositoryError: + tag = None + elif event['event_type'] == 'change-merged': + # for chanage-merged, search submit tag on this commit + branch = event['branch'] + if event['branch'] == 'master': + branch = 'trunk' + try: + tag = mygit.describe(event['patchset_revision'], + pattern='submit/%s/*' % branch, + exact_match=True) + except GitRepositoryError: + # don'n find submit tag on this commit, return None + tag = None + + return tag + +def check_tag_format(git, mygerrit, event, tag): + """check whether tag follow proper format""" + + branch, date = parse_submit_tag(tag) + message = [] + psr = event['patchset_revision'] + + # check tag name format + if branch and date: + # check date format + pattern = re.compile(r'^[0-9]{8}\.[0-9]{6}$') + if not pattern.match(date): + message.append(WRONG_DATE_MSG % date) + + if not git.branch_contains(tag): + # Check if change is still under review + cmd = '--current-patch-set status: open project: %s commit: %s' % \ + (event['project'], psr) + gerritinfo = mygerrit.query(cmd) + if len(gerritinfo) == 1 and 'number' in gerritinfo[0] \ + and 'currentPatchSet' in gerritinfo[0]: + if gerritinfo[0]['branch'] == branch: + # the tagged commit still open, abort submit this time + message.append(UNDER_REVIEW_MSG % (tag, psr)) + else: + # cannot find tagged commit in git tree or gerrit open change + message.append(WRONG_COMMIT_MSG % psr) + else: + # check for contains branch + contain = False + cbrch = git.branch_contains(tag) + if branch in cbrch: + print '%s branch is contains in %s branch' %(branch, cbrch) + contain = True + else: + vbrch = git_virtual_branch_map() + for items in vbrch: + if '%s' %(branch) in items and \ + items['%s' %(branch)] is not None \ + and items['%s' %(branch)] in cbrch: + print '%s branch is virtual %s branch' %(branch, items['%s' %(branch)]) + contain = True + if not contain: + # wrong tag format + print '%s branch is not contains in %s branch' %(branch, cbrch) + message.append(WRONG_FORMAT_MSG % (cbrch, branch)) + else: + # wrong tag format + message.append(UNKNOWN_FORMAT_MSG) + + # check whether tag is an annotated tag + tagger = git.get_tag(tag) + if 'author' not in tagger or 'email' not in tagger: + message.append(NOT_ANNOTATED_MSG) + + # post comment to gerrit and send email if check failed + if message: + print message + + msg = 'The tag %s was pushed, but it was not completed because of '\ + 'the following reason(s):\n\n' % tag + '\n'.join(message) + + if len(message) != 1 or (message[0] != UNDER_REVIEW_MSG % (tag, psr) \ + and message[0] != WRONG_COMMIT_MSG % psr): + msg += '\n\n' + SUGGESTION + + try: + mygerrit.review(commit=psr, message=msg) + except GerritError, err: + print >> sys.stderr, 'Error posting review comment '\ + 'back to Gerrit: %s' % str(err) + # return 1 if this exception is not caused by invalid commit + if 'no such patch set' not in str(err): + return None + + return {'message': msg, 'tagger': tagger} + + return None + +def find_specfile(prj_dir, packaging_dir, tag, event, tagger, pkg_name=None, debug_prj_name=None): + """search specfile under packaging directory""" + + msg = '' + + if True: + specs = utils.find_spec('%s/%s' % (prj_dir, packaging_dir)) + if not specs: + # no spec exist under packaging, use default name + msg = "The tag %s pushed, but packaging directory doesn't \n"\ + "contain any spec file. Please create one and \n"\ + "re-submit it." % tag + elif len(specs) == 1: + # only one spec exist under packaging + spec = specs[0] + print 'Single spec. Use %s' % os.path.basename(spec) + else: + # multiple specs exist under packaging, use default name + spec = None + if pkg_name: + spec = '%s/%s/%s.spec' % (prj_dir, packaging_dir, pkg_name) + print 'Multiple specs. Try %s' % os.path.basename(spec) + if spec is None or not os.path.isfile(spec): + spec = '%s/%s/%s.spec' % (prj_dir, packaging_dir, \ + os.path.basename(event['project'])) + print 'Multiple specs with no pkg_name.spec. Try %s' % os.path.basename(spec) + if not os.path.isfile(spec): + spec = sorted(specs)[0] + print 'Multiple sorted(specs)[0] %s' % os.path.basename(spec) + if not os.path.isfile(spec): + msg = "The tag %s pushed, but packaging directory contains \n"\ + "multiply spec files, backend service can not decide \n"\ + "which spec file to use. Please use OBS_PACKAGE \n"\ + "parameter in scm/git-obs-mapping project to specify \n"\ + "the target spec file or contact system \n"\ + "administrator for more details." % tag + + if msg: + if debug_prj_name is not None: + msg = "TARGET_PROJECT: %s\n%s" % (debug_prj_name, msg) + print msg + return {'message': msg, 'spec': None} + else: + print 'specfile %s' % spec + return {'message': None, 'spec': spec} + +def parse_specfile(specfile, tag, event, tagger): + """parse specfile""" + + spec = None + + try: + # use gbp to parse specfile + spec = SpecFile(specfile) + except GbpError, err: + print 'gbp parse spec failed. %s' % err + msg = 'The tag %s pushed, but backend service failed to parse %s. \n'\ + 'Please try "gbs export" on this tag and make sure it can '\ + 'work.\n\n'\ + 'Error message:\n'\ + '%s' % (tag, os.path.basename(specfile), err) + return {'message': msg, 'spec': None} + + return {'message': None, 'spec': spec} + +def change_release_name(build, project, git_tag): + """ + Change release name from project config in OBS + Add the datetime into release name. + Eg: 'Release: .' ----> 'Release: 20141010..' + """ + # get project config + config = build.get_project_config(project) + release_name = 'Release: %s' % (git_tag.split('/')[-1]) + res = re.findall(r'^Release: ?\S+$', config, flags=re.MULTILINE) + if res: + if git_tag.split('/')[-1] not in res[0]: + note = '#Insert time from submission into release name\n' + release_name = '%s.%s' % (release_name, + res[0].split('Release:')[1].strip()) + config = config.replace(res[0], '%s%s' % (note, release_name), 1) + else: + note = '#Add release name into prjconf\n' + config = note + '%s\n' % release_name + config + + #Add rpmbuild stage option + if os.getenv('PRERELEASE_RPMBUILD_STAGE'): + # Check if we've got required fields in TRIGGER_INFO + if not os.getenv('PRERELEASE_RPMBUILD_STAGE') in ('ba', 'bb'): + print 'Error: PRERELEASE_RPMBUILD_STAGE %s' % (os.getenv('PRERELEASE_RPMBUILD_STAGE')) + else: + rpmbuildstage = 'Rpmbuildstage: -%s' % (os.getenv('PRERELEASE_RPMBUILD_STAGE')) + res = re.findall(r'^Rpmbuildstage: ?\S+$', config, flags=re.MULTILINE) + if res: + config = config.replace(res[0], '%s' % (rpmbuildstage), 1) + else: + config = config + '#Add RpmbuildStage option into prjconf\n' + '%s\n' % (rpmbuildstage) + + # Add "CopyLinkedPackages: yes" for prerelease projects. + if not re.search("CopyLinkedPackages:", config): + config = config + "\nCopyLinkedPackages: yes\n"; + + # set project config + build.set_project_config(project, config) + +def copy_person_project_meta(build, obs_target_prj, obs_project): + """copy the maintainer list from obs_target_prj meta to corresponding + prerelease project + """ + src_meta_xml = build.get_meta(obs_target_prj) + src_xml_root = ElementTree.fromstringlist(src_meta_xml) + # get peron list from obs_target_prj meta + person_dict = {} + for person in src_xml_root.findall('person'): + if person.get('userid') in person_dict: + person_dict[person.get('userid')].append(person.get('role')) + else: + person_dict[person.get('userid')] = [person.get('role')] + # add person to prerelease project + if person_dict: + build.addPerson(obs_project, person_dict) + +def create_related_packages(build, obs_target_prj, obs_pre_prj, pre_package): + """create the 'link' package that relate the original package + obs_target_prj is the base project + obs_pre_prj is the prelease project + pre_package is the original package + """ + sourceinfo = build.get_sourceinfo_list(obs_target_prj) + for package in sourceinfo: + if sourceinfo[package]: + link_prj, link_pkg = sourceinfo[package][-1].split('/') + if link_prj == obs_target_prj and link_pkg == pre_package: + build.create_link_pac(obs_pre_prj, pre_package, \ + obs_pre_prj, package) + if re.search("_aggregate", package): + print "Copypac aggregate package: %s/%s" %(obs_pre_prj, package) + build.create_copy_pac(obs_target_prj, package, obs_pre_prj,\ + package) + aggregate_file_name="_aggregate" + build.get_source_file(obs_target_prj, package, aggregate_file_name) + content = "" + with open(aggregate_file_name, 'r') as f: + content = f.read() + content_xml_root = ElementTree.fromstringlist(content) + for element in content_xml_root.findall('aggregate'): + element.set('project',obs_pre_prj) + content = ElementTree.tostring(content_xml_root) + with open(aggregate_file_name, 'w') as f: + f.write(content) + commit_msg="uploaded to copy pac %s/%s from %s" % (obs_pre_prj, package, obs_target_prj) + try: + build.commit_files(obs_pre_prj, package, + [(aggregate_file_name, True)], commit_msg) + except ObsError, error: + raise UploadError("Unable to upload _aggregate to %s: %s" % \ + (obs_pre_prj, error)) + print "Copypac done." + +def get_base_project(build, _project): + _meta = build.get_meta(_project) + xml_meta = ElementTree.fromstringlist(_meta) + _base_prj = '' + for repos in xml_meta.findall('repository'): + for path in repos.findall('path'): + if 'base' not in path.get('project').lower(): continue + print 'Found base project %s for %s' % (path.get('project'), \ + _project) + return path.get('project') + return None + +def create_project(build, obs_project, submit_list): + """ + Create prerelease OBS project and upload sources for the build. + Parameters: + @ build (buildservice.BuildService): Instance of BuildService + @ obs_project (str): Project name to be created + @ submit_list (list): + [ + { + 'project'(O) : Prerelease project name + 'package'(M) : OBS package name + 'gerrit_project'(M) : Git path + 'git_tag'(M) : Git tag name + 'gerrit_newrev'(O) : Revision of the tag + 'submitter'(M) : Author of the tag + 'build_flag'(O) : Enable/Disable OBS build after create + 'obs_target_prj'(O) : Target OBS project + 'ref_obs_target_prj'(O): Reference target OBS project to be linked + 'url'(O) : Hostname of the gerrit server for _service file + }, + ] + """ + + # Create review project if it doesn't exist + print "Creating temporary review OBS project %s" % obs_project + + _git_url = 'ssh://%s:%s' % (os.getenv('GERRIT_HOSTNAME_EXTERNAL'), + os.getenv('GERRIT_SSHPORT')) + git_revision = [] + _obs_target_prj, tstamp = get_info_from_prerelease_name(obs_project) + + git_url = submit_list[0].get('url', _git_url) + git_project = [ t['gerrit_project'] for t in (u for u in submit_list) ] + git_tag = submit_list[0].get('git_tag') + for gt in submit_list: + # Group submit should have the same tag name + assert gt['git_tag'] == git_tag + for u in submit_list: + if 'gerrit_newrev' in u: + git_revision.append(u.get('gerrit_newrev')) + else: + # Find revision of the tag + _rev = Git(os.path.join(os.getenv('GIT_CACHE_DIR'), \ + u.get('gerrit_project'))).get_tag(git_tag)['tagrevision'] + git_revision.append(_rev) + obs_target_prj = submit_list[0].get('obs_target_prj', _obs_target_prj) + ref_obs_target_prj = submit_list[0].get('ref_obs_target_prj', obs_target_prj) + submitter = ','.join(list(set([ t['submitter'] for t in (u for u in submit_list) ]))) + package = [ t['package'] for t in (u for u in submit_list) ] + build_flag = submit_list[0].get('build_flag', True) + print '\nCREATING....%s\n%s\n%s\n%s\n%s\n' % (obs_project, git_project, git_revision, submitter, package) + sys.stdout.flush() + + info = {'projects': git_project, + 'packages': package, + 'obs_target_prj': obs_target_prj, + 'ref_obs_target_prj': ref_obs_target_prj, + 'git_tag': git_tag, + 'git_commit': git_revision, + 'obs_url': os.path.join(os.getenv('OBS_URL_EXTERNAL'), \ + 'project/show?project=%s' % obs_project), + 'images': [], + 'base': get_base_project(build, ref_obs_target_prj)} + + if build.exists(obs_project): + if submitter: + info['submitter'] = submitter + + # update project info + build.update_info(info, obs_project) + # unlink the project to upload packages + try: + if build_flag == True: + build.unlink_project(obs_project) + except ObsError, error: + print 'Modify the meta conf to unlink failed: %s' % error + else: + if submitter: + info['submitter'] = escape(submitter) + + if not build.exists(ref_obs_target_prj): + raise LocalError("Target project %s doesn't exist" % ref_obs_target_prj) + try: + build.create_project(obs_project, ref_obs_target_prj, + description=json.dumps(info)) + except ObsError, error: + LocalError("Unable to create project %s: %s" % (obs_project, error)) + + # change release name of project config in OBS + change_release_name(build, obs_project, git_tag) + + #disable publish flag + build.disable_build_flag(obs_project, repo = None, flag="publish", status="disable") + #disable build flag + build.disable_build_flag(obs_project, repo = None, flag="build", status="disable") + + try: + for idx, val in enumerate(git_project): + upload_obs_service(git_url, git_project[idx], git_tag, + git_revision[idx], obs_project, build, package[idx]) + except UploadError, err: + raise LocalError(err) + + # copy the maintainer list from obs_target_prj meta to corresponding + # prerelease project + copy_person_project_meta(build, ref_obs_target_prj, obs_project) + + if build_flag != True: + return + + #create the 'link' package that relate the original package + for idx, val in enumerate(package): + create_related_packages(build, obs_target_prj, obs_project, package[idx]) + + #Wait 10 seconds to upload the package to the OBS + sleep(10) + + #default build flag + build.default_build_flag(obs_project, repo = None, flag="build") + #default publish flag + build.default_build_flag(obs_project, repo = None, flag="publish") + + # Enabled link project + sleep(1) + build.link_project(obs_project, src=ref_obs_target_prj, linktype="localdep") + +def get_obs_package_name(git, branch, target_project, tag, gerrit_env=None): + + if not gerrit_env: + gerrit_env = GerritEnv() + git_cache = gerrit_env.gitcache + + prjdir = os.path.join(git_cache, git) + if not clone_gitproject(git, prjdir, \ + gerrit_hostname=gerrit_env.hostname, \ + gerrit_username=gerrit_env.username, \ + gerrit_sshport=gerrit_env.sshport, \ + git_cache_dir=git_cache): + print >> sys.stderr, 'Error cloning %s' % git + return 1 + mygit = Git(prjdir) + mygit.checkout(tag) + packagingdir = utils.parse_link('%s/%s' % (prjdir, 'packaging')) + + obs_target_prjs = git_obs_map(git, \ + gerrit_branch=branch, \ + gitcache=git_cache + ) + for item in obs_target_prjs: + obs_target_prj = item['OBS_project'] + obs_stg_prj = item['OBS_staging_project'] + obs_pkg = item['OBS_package'] + obs_use_specname = item['OBS_use_specname'] + + if obs_target_prj != target_project: + continue + if obs_stg_prj not in ['prerelease', 'abs']: + continue + + if obs_use_specname == 'yes': + resp = find_specfile(prjdir, packagingdir, tag, git, None) + if not resp['spec']: + return None + resp = parse_specfile(resp['spec'], tag, git, None) + if not resp['spec']: + return None + print 'spec name = %s' %(resp['spec'].name) + return resp['spec'].name + else: + if obs_pkg: + print 'package name from mapping = %s' % (obs_pkg) + return obs_pkg + else: + print 'package name from basename = %s' % (os.path.basename(git)) + return os.path.basename(git) + + return None + +def get_manifest_filelists_snapshot(profile, request_url, timeout=5, group=None): + """ get manifest filelists from snapshot""" + #p = re.compile(r'alt=\"\[(TXT|DIR| )]*\]\".*') + p = re.compile(r'.*') + + if not request_url: + return {} + print request_url + # get data from url + for loop in range(10): + try: + f = requests.get(request_url, + auth=(profile['snapshot_username'], + profile['snapshot_password']), + timeout=timeout) + if f.status_code == 200: + break + except requests.exceptions.Timeout as e: + print(e) + continue + except requests.exceptions.ConnectionError as e: + print(e) + continue + except Exception as e: + print(e) + raise Exception('exception from get manifest filelists') + + results = {} + exclude_pkgs = [] + found_links = p.findall(f.text) + for link in found_links: + if link == '../': + continue + if not link.endswith('.xml'): + continue + manifest_url = os.path.join(request_url, link) + f = requests.get(manifest_url, + auth=(profile['snapshot_username'], + profile['snapshot_password']), + timeout=timeout) + try: + tree = ElementTree.fromstring(f.text) + except ElementTree.ParseError: + raise ElementTree.ParseError + for result in tree.findall('project'): + if '_preloadapp.xml' in link: + exclude_pkgs.append(''.join(result.get('path'))) + else: + results[''.join(result.get('path'))] = result.get('revision') + + if group == 'abs': + preloadapp_pkgs = {} + for app in exclude_pkgs: + preloadapp_pkgs[app] = results[app] + return preloadapp_pkgs + + return results + +if __name__ == '__main__': + print 'Non-executable module' + sys.exit(0) + diff --git a/debian/jenkins-scripts-common.install b/debian/jenkins-scripts-common.install index 243cdb8..76d1a24 100644 --- a/debian/jenkins-scripts-common.install +++ b/debian/jenkins-scripts-common.install @@ -40,3 +40,4 @@ debian/tmp/scripts/check_section.sh /var/lib/jenkins/jenkins-scripts/scripts debian/tmp/scripts/get_git_desc_info.sh /var/lib/jenkins/jenkins-scripts/scripts debian/tmp/scripts/nuget.exe /var/lib/jenkins/jenkins-scripts/scripts debian/tmp/common/aws_ec2.py /var/lib/jenkins/jenkins-scripts/common/ +debian/tmp/common/workflow.py /var/lib/jenkins/jenkins-scripts/common/ diff --git a/job_submit.py b/job_submit.py index 4d11dc0..79985ec 100644 --- a/job_submit.py +++ b/job_submit.py @@ -52,34 +52,12 @@ from gbp.rpm import SpecFile from gbp.git.repository import GitRepositoryError from gbp.errors import GbpError +from common.workflow import create_project, find_submit_tag, check_tag_format, find_specfile, parse_specfile + # set default char-set endcoding to utf-8 reload(sys) sys.setdefaultencoding('utf-8') # pylint: disable-msg=E1101 -WRONG_DATE_MSG = '- The date %s in tag does NOT follow correct format.\n You can'\ - ' use shell command "date --utc +%%Y%%m%%d.%%H%%M%%S" to '\ - 'generate it, like 20120801.083113.' - -UNDER_REVIEW_MSG = '- Submission %s has been rejected because tagged commit %s'\ - ' is still under review in gerrit.\n Please re-trigger '\ - 'submission after your change is accepted.' - -WRONG_COMMIT_MSG = '- The commit %s tag attached does NOT exist in git tree or'\ - ' gerrit open change.\n Please make sure the commit has been '\ - 'pushed to gerrit and correct magical ref refs/for/branch, '\ - 'then re-submit tag.' - -UNKNOWN_FORMAT_MSG = '- Unknown tag format,\n please follow the format '\ - 'submit/{version}/{date.time}.' - -WRONG_FORMAT_MSG = '- Wrong tag format,\n please follow the format '\ - 'submit/{branch}/{date.time}. \n'\ - 'Git branch : %s. Tag branch: %s' - -NOT_ANNOTATED_MSG = '- Tag should be annotated tag.' - -SUGGESTION = 'Suggest to use "gbs submit" to trigger submission\n'\ - TITLE_FAILED = '[Submit Request Failed]: tag: %s in %s' TITLE_SUCCESS = '[Submit Request Success]: tag: %s in %s' @@ -105,440 +83,6 @@ def send_mail(title, msg, receiver): prepare_mail("%s.env" % os.getenv('BUILD_TAG'), title, msg, os.getenv('NOREPLY_EMAIL_SENDER'), receiver['email']) -def parse_submit_tag(tag): - """parse info from submit tag name""" - - branch = None - date = None - - if tag.startswith('submit/'): - pos = tag.rfind('/', len('submit/')) - if pos != -1: - branch = tag[len('submit/'):pos] - if branch == 'trunk': - branch = 'master' - date = tag[pos+1:] - - return branch, date - -def find_submit_tag(event, mygit): - """find the corresponding submit tag for this event""" - - if event['event_type'] == 'ref-updated': - tag = event['refname'][len('refs/tags/'):] - event['branch'] = parse_submit_tag(tag)[0] - # Since patchset_revision is used in gerrit feedback, real tag check - # is needed; and the key point is parse_submit_tag can not ensure the - # tag exsisting too. - try: - event['patchset_revision'] = mygit.rev_parse('%s^{commit}' % tag) - except GitRepositoryError: - tag = None - elif event['event_type'] == 'change-merged': - # for chanage-merged, search submit tag on this commit - branch = event['branch'] - if event['branch'] == 'master': - branch = 'trunk' - try: - tag = mygit.describe(event['patchset_revision'], - pattern='submit/%s/*' % branch, - exact_match=True) - except GitRepositoryError: - # don'n find submit tag on this commit, return None - tag = None - - return tag - -def check_tag_format(git, mygerrit, event, tag): - """check whether tag follow proper format""" - - branch, date = parse_submit_tag(tag) - message = [] - psr = event['patchset_revision'] - - # check tag name format - if branch and date: - # check date format - pattern = re.compile(r'^[0-9]{8}\.[0-9]{6}$') - if not pattern.match(date): - message.append(WRONG_DATE_MSG % date) - - if not git.branch_contains(tag): - # Check if change is still under review - cmd = '--current-patch-set status: open project: %s commit: %s' % \ - (event['project'], psr) - gerritinfo = mygerrit.query(cmd) - if len(gerritinfo) == 1 and 'number' in gerritinfo[0] \ - and 'currentPatchSet' in gerritinfo[0]: - if gerritinfo[0]['branch'] == branch: - # the tagged commit still open, abort submit this time - message.append(UNDER_REVIEW_MSG % (tag, psr)) - else: - # cannot find tagged commit in git tree or gerrit open change - message.append(WRONG_COMMIT_MSG % psr) - else: - # check for contains branch - contain = False - cbrch = git.branch_contains(tag) - if branch in cbrch: - print '%s branch is contains in %s branch' %(branch, cbrch) - contain = True - else: - vbrch = git_virtual_branch_map() - for items in vbrch: - if '%s' %(branch) in items and \ - items['%s' %(branch)] is not None \ - and items['%s' %(branch)] in cbrch: - print '%s branch is virtual %s branch' %(branch, items['%s' %(branch)]) - contain = True - if not contain: - # wrong tag format - print '%s branch is not contains in %s branch' %(branch, cbrch) - message.append(WRONG_FORMAT_MSG % (cbrch, branch)) - else: - # wrong tag format - message.append(UNKNOWN_FORMAT_MSG) - - # check whether tag is an annotated tag - tagger = git.get_tag(tag) - if 'author' not in tagger or 'email' not in tagger: - message.append(NOT_ANNOTATED_MSG) - - # post comment to gerrit and send email if check failed - if message: - print message - - msg = 'The tag %s was pushed, but it was not completed because of '\ - 'the following reason(s):\n\n' % tag + '\n'.join(message) - - if len(message) != 1 or (message[0] != UNDER_REVIEW_MSG % (tag, psr) \ - and message[0] != WRONG_COMMIT_MSG % psr): - msg += '\n\n' + SUGGESTION - - try: - mygerrit.review(commit=psr, message=msg) - except GerritError, err: - print >> sys.stderr, 'Error posting review comment '\ - 'back to Gerrit: %s' % str(err) - # return 1 if this exception is not caused by invalid commit - if 'no such patch set' not in str(err): - return False - if 'author' not in tagger or 'email' not in tagger: - recevier = { 'author' : event['event_account_name'], - 'email' : event['event_account_email'] } - send_mail(TITLE_FAILED % (tag, event['project']), msg, recevier) - else: - send_mail(TITLE_FAILED % (tag, event['project']), msg, tagger) - - return False - - return True - -def find_specfile(prj_dir, packaging_dir, tag, event, tagger, pkg_name=None, debug_prj_name=None): - """search specfile under packaging directory""" - - msg = '' - - if True: - specs = utils.find_spec('%s/%s' % (prj_dir, packaging_dir)) - if not specs: - # no spec exist under packaging, use default name - msg = "The tag %s pushed, but packaging directory doesn't \n"\ - "contain any spec file. Please create one and \n"\ - "re-submit it." % tag - elif len(specs) == 1: - # only one spec exist under packaging - spec = specs[0] - print 'Single spec. Use %s' % os.path.basename(spec) - else: - # multiple specs exist under packaging, use default name - spec = None - if pkg_name: - spec = '%s/%s/%s.spec' % (prj_dir, packaging_dir, pkg_name) - print 'Multiple specs. Try %s' % os.path.basename(spec) - if spec is None or not os.path.isfile(spec): - spec = '%s/%s/%s.spec' % (prj_dir, packaging_dir, \ - os.path.basename(event['project'])) - print 'Multiple specs with no pkg_name.spec. Try %s' % os.path.basename(spec) - if not os.path.isfile(spec): - spec = sorted(specs)[0] - print 'Multiple sorted(specs)[0] %s' % os.path.basename(spec) - if not os.path.isfile(spec): - msg = "The tag %s pushed, but packaging directory contains \n"\ - "multiply spec files, backend service can not decide \n"\ - "which spec file to use. Please use OBS_PACKAGE \n"\ - "parameter in scm/git-obs-mapping project to specify \n"\ - "the target spec file or contact system \n"\ - "administrator for more details." % tag - - if msg: - if debug_prj_name is not None: - msg = "TARGET_PROJECT: %s\n%s" % (debug_prj_name, msg) - print msg - send_mail(TITLE_FAILED % (tag, event['project']), msg, tagger) - return None - else: - print 'specfile %s' % spec - return spec - - -def parse_specfile(specfile, tag, event, tagger): - """parse specfile""" - - spec = None - - try: - # use gbp to parse specfile - spec = SpecFile(specfile) - except GbpError, err: - print 'gbp parse spec failed. %s' % err - msg = 'The tag %s pushed, but backend service failed to parse %s. \n'\ - 'Please try "gbs export" on this tag and make sure it can '\ - 'work.\n\n'\ - 'Error message:\n'\ - '%s' % (tag, os.path.basename(specfile), err) - send_mail(TITLE_FAILED % (tag, event['project']), msg, tagger) - return None - - return spec - -class LocalError(Exception): - """Local error exception.""" - pass - - -def change_release_name(build, project, git_tag): - """ - Change release name from project config in OBS - Add the datetime into release name. - Eg: 'Release: .' ----> 'Release: 20141010..' - """ - # get project config - config = build.get_project_config(project) - release_name = 'Release: %s' % (git_tag.split('/')[-1]) - res = re.findall(r'^Release: ?\S+$', config, flags=re.MULTILINE) - if res: - if git_tag.split('/')[-1] not in res[0]: - note = '#Insert time from submission into release name\n' - release_name = '%s.%s' % (release_name, - res[0].split('Release:')[1].strip()) - config = config.replace(res[0], '%s%s' % (note, release_name), 1) - else: - note = '#Add release name into prjconf\n' - config = note + '%s\n' % release_name + config - - #Add rpmbuild stage option - if os.getenv('PRERELEASE_RPMBUILD_STAGE'): - # Check if we've got required fields in TRIGGER_INFO - if not os.getenv('PRERELEASE_RPMBUILD_STAGE') in ('ba', 'bb'): - print 'Error: PRERELEASE_RPMBUILD_STAGE %s' % (os.getenv('PRERELEASE_RPMBUILD_STAGE')) - else: - rpmbuildstage = 'Rpmbuildstage: -%s' % (os.getenv('PRERELEASE_RPMBUILD_STAGE')) - res = re.findall(r'^Rpmbuildstage: ?\S+$', config, flags=re.MULTILINE) - if res: - config = config.replace(res[0], '%s' % (rpmbuildstage), 1) - else: - config = config + '#Add RpmbuildStage option into prjconf\n' + '%s\n' % (rpmbuildstage) - - # Add "CopyLinkedPackages: yes" for prerelease projects. - if not re.search("CopyLinkedPackages:", config): - config = config + "\nCopyLinkedPackages: yes\n"; - - # set project config - build.set_project_config(project, config) - -def copy_person_project_meta(build, obs_target_prj, obs_project): - """copy the maintainer list from obs_target_prj meta to corresponding - prerelease project - """ - src_meta_xml = build.get_meta(obs_target_prj) - src_xml_root = ElementTree.fromstringlist(src_meta_xml) - # get peron list from obs_target_prj meta - person_dict = {} - for person in src_xml_root.findall('person'): - if person.get('userid') in person_dict: - person_dict[person.get('userid')].append(person.get('role')) - else: - person_dict[person.get('userid')] = [person.get('role')] - # add person to prerelease project - if person_dict: - build.addPerson(obs_project, person_dict) - -def create_related_packages(build, obs_target_prj, obs_pre_prj, pre_package): - """create the 'link' package that relate the original package - obs_target_prj is the base project - obs_pre_prj is the prelease project - pre_package is the original package - """ - sourceinfo = build.get_sourceinfo_list(obs_target_prj) - for package in sourceinfo: - if sourceinfo[package]: - link_prj, link_pkg = sourceinfo[package][-1].split('/') - if link_prj == obs_target_prj and link_pkg == pre_package: - build.create_link_pac(obs_pre_prj, pre_package, \ - obs_pre_prj, package) - if re.search("_aggregate", package): - print "Copypac aggregate package: %s/%s" %(obs_pre_prj, package) - build.create_copy_pac(obs_target_prj, package, obs_pre_prj,\ - package) - aggregate_file_name="_aggregate" - build.get_source_file(obs_target_prj, package, aggregate_file_name) - content = "" - with open(aggregate_file_name, 'r') as f: - content = f.read() - content_xml_root = ElementTree.fromstringlist(content) - for element in content_xml_root.findall('aggregate'): - element.set('project',obs_pre_prj) - content = ElementTree.tostring(content_xml_root) - with open(aggregate_file_name, 'w') as f: - f.write(content) - commit_msg="uploaded to copy pac %s/%s from %s" % (obs_pre_prj, package, obs_target_prj) - try: - build.commit_files(obs_pre_prj, package, - [(aggregate_file_name, True)], commit_msg) - except ObsError, error: - raise UploadError("Unable to upload _aggregate to %s: %s" % \ - (obs_pre_prj, error)) - print "Copypac done." - -def get_base_project(build, _project): - _meta = build.get_meta(_project) - xml_meta = ElementTree.fromstringlist(_meta) - _base_prj = '' - for repos in xml_meta.findall('repository'): - for path in repos.findall('path'): - if 'base' not in path.get('project').lower(): continue - print 'Found base project %s for %s' % (path.get('project'), \ - _project) - return path.get('project') - return None - -def create_project(build, obs_project, submit_list): - """ - Create prerelease OBS project and upload sources for the build. - Parameters: - @ build (buildservice.BuildService): Instance of BuildService - @ obs_project (str): Project name to be created - @ submit_list (list): - [ - { - 'project'(O) : Prerelease project name - 'package'(M) : OBS package name - 'gerrit_project'(M) : Git path - 'git_tag'(M) : Git tag name - 'gerrit_newrev'(O) : Revision of the tag - 'submitter'(M) : Author of the tag - 'build_flag'(O) : Enable/Disable OBS build after create - 'obs_target_prj'(O) : Target OBS project - 'ref_obs_target_prj'(O): Reference target OBS project to be linked - 'url'(O) : Hostname of the gerrit server for _service file - }, - ] - """ - - # Create review project if it doesn't exist - print "Creating temporary review OBS project %s" % obs_project - - _git_url = 'ssh://%s:%s' % (os.getenv('GERRIT_HOSTNAME_EXTERNAL'), - os.getenv('GERRIT_SSHPORT')) - git_revision = [] - _obs_target_prj, tstamp = get_info_from_prerelease_name(obs_project) - - git_url = submit_list[0].get('url', _git_url) - git_project = [ t['gerrit_project'] for t in (u for u in submit_list) ] - git_tag = submit_list[0].get('git_tag') - for gt in submit_list: - # Group submit should have the same tag name - assert gt['git_tag'] == git_tag - for u in submit_list: - if 'gerrit_newrev' in u: - git_revision.append(u.get('gerrit_newrev')) - else: - # Find revision of the tag - _rev = Git(os.path.join(os.getenv('GIT_CACHE_DIR'), \ - u.get('gerrit_project'))).get_tag(git_tag)['tagrevision'] - git_revision.append(_rev) - obs_target_prj = submit_list[0].get('obs_target_prj', _obs_target_prj) - ref_obs_target_prj = submit_list[0].get('ref_obs_target_prj', obs_target_prj) - submitter = ','.join(list(set([ t['submitter'] for t in (u for u in submit_list) ]))) - package = [ t['package'] for t in (u for u in submit_list) ] - build_flag = submit_list[0].get('build_flag', True) - print '\nCREATING....%s\n%s\n%s\n%s\n%s\n' % (obs_project, git_project, git_revision, submitter, package) - sys.stdout.flush() - - info = {'projects': git_project, - 'packages': package, - 'obs_target_prj': obs_target_prj, - 'ref_obs_target_prj': ref_obs_target_prj, - 'git_tag': git_tag, - 'git_commit': git_revision, - 'obs_url': os.path.join(os.getenv('OBS_URL_EXTERNAL'), \ - 'project/show?project=%s' % obs_project), - 'images': [], - 'base': get_base_project(build, ref_obs_target_prj)} - - if build.exists(obs_project): - if submitter: - info['submitter'] = submitter - - # update project info - build.update_info(info, obs_project) - # unlink the project to upload packages - try: - if build_flag == True: - build.unlink_project(obs_project) - except ObsError, error: - print 'Modify the meta conf to unlink failed: %s' % error - else: - if submitter: - info['submitter'] = escape(submitter) - - if not build.exists(ref_obs_target_prj): - raise LocalError("Target project %s doesn't exist" % ref_obs_target_prj) - try: - build.create_project(obs_project, ref_obs_target_prj, - description=json.dumps(info)) - except ObsError, error: - LocalError("Unable to create project %s: %s" % (obs_project, error)) - - # change release name of project config in OBS - change_release_name(build, obs_project, git_tag) - - #disable publish flag - build.disable_build_flag(obs_project, repo = None, flag="publish", status="disable") - #disable build flag - build.disable_build_flag(obs_project, repo = None, flag="build", status="disable") - - try: - for idx, val in enumerate(git_project): - upload_obs_service(git_url, git_project[idx], git_tag, - git_revision[idx], obs_project, build, package[idx]) - except UploadError, err: - raise LocalError(err) - - # copy the maintainer list from obs_target_prj meta to corresponding - # prerelease project - copy_person_project_meta(build, ref_obs_target_prj, obs_project) - - if build_flag != True: - return - - #create the 'link' package that relate the original package - for idx, val in enumerate(package): - create_related_packages(build, obs_target_prj, obs_project, package[idx]) - - #Wait 10 seconds to upload the package to the OBS - sleep(10) - - #default build flag - build.default_build_flag(obs_project, repo = None, flag="build") - #default publish flag - build.default_build_flag(obs_project, repo = None, flag="publish") - - # Enabled link project - sleep(1) - build.link_project(obs_project, src=ref_obs_target_prj, linktype="localdep") - def get_branch_name(tag): """Get branch name by parsing info from submit tag name. @@ -786,7 +330,14 @@ def main(build_type, build, event, sr_count): submitter += ' <%s>' % gerrit_account_email # check whether tag meet format - if not check_tag_format(mygit, mygerrit, event, tag): + resp = check_tag_format(mygit, mygerrit, event, tag) + if resp is not None: + if 'author' not in resp['tagger'] or 'email' not in resp['tagger']: + recevier = { 'author' : event['event_account_name'], + 'email' : event['event_account_email'] } + send_mail(TITLE_FAILED % (tag, event['project']), resp['message'], recevier) + else: + send_mail(TITLE_FAILED % (tag, event['project']), resp['message'], resp['tagger']) print 'The check for the tag format is error, exit now\n' return 0 @@ -835,15 +386,17 @@ def main(build_type, build, event, sr_count): enable_build = False if obs_use_specname == 'yes': # search specfile under packaging directory - specfile = find_specfile(prjdir, packagingdir, tag, event, tagger) - if not specfile: + resp = find_specfile(prjdir, packagingdir, tag, event, tagger) + if not resp['spec']: + send_mail(TITLE_FAILED % (tag, event['project']), resp['message'], tagger) return 0 # parse specfile - spec = parse_specfile(specfile, tag, event, tagger) - if not spec: + resp = parse_specfile(resp['spec'], tag, event, tagger) + if not resp['spec']: + send_mail(TITLE_FAILED % (tag, event['project']), resp['message'], tagger) return 0 - package = spec.name - print 'spec name = %s' %(spec.name) + package = resp['spec'].name + print 'spec name = %s' %(resp['spec'].name) else: # get package name from xml files. if obs_pkg: diff --git a/packaging/jenkins-scripts.spec b/packaging/jenkins-scripts.spec index cf091da..1a3a111 100644 --- a/packaging/jenkins-scripts.spec +++ b/packaging/jenkins-scripts.spec @@ -214,6 +214,7 @@ fi %{destdir}/scripts/get_git_desc_info.sh %{destdir}/scripts/nuget.exe %{destdir}/common/aws_ec2.py +%{destdir}/common/workflow.py %files dependsgraph %defattr(-,jenkins,jenkins)