--- /dev/null
+#!/usr/bin/env python
+# vim: ai ts=4 sts=4 et sw=4
+#
+# Copyright (C) 2017 Samsung Electronics. Co,. Ltd.
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; version 2
+# of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+#
+"""
+Common part of the CI process
+"""
+
+import os
+import sys
+import json
+import re
+import requests
+import xml.etree.cElementTree as ElementTree
+from xml.sax.saxutils import escape
+from time import sleep
+
+from gitbuildsys.errors import ObsError
+
+from common import utils
+from common.mapping import git_obs_map, git_virtual_branch_map
+from common.upload_service import upload_obs_service, UploadError
+from common.prerelease import get_info_from_prerelease_name
+from common.git import Git, clone_gitproject
+from common.gerrit import GerritEnv, GerritError
+
+from gbp.rpm import SpecFile
+from gbp.git.repository import GitRepositoryError
+from gbp.errors import GbpError
+
+# set default char-set endcoding to utf-8
+reload(sys)
+sys.setdefaultencoding('utf-8') # pylint: disable-msg=E1101
+
+class LocalError(Exception):
+ """Local error exception."""
+ pass
+
+WRONG_DATE_MSG = '- The date %s in tag does NOT follow correct format.\n You can'\
+ ' use shell command "date --utc +%%Y%%m%%d.%%H%%M%%S" to '\
+ 'generate it, like 20120801.083113.'
+
+UNDER_REVIEW_MSG = '- Submission %s has been rejected because tagged commit %s'\
+ ' is still under review in gerrit.\n Please re-trigger '\
+ 'submission after your change is accepted.'
+
+WRONG_COMMIT_MSG = '- The commit %s tag attached does NOT exist in git tree or'\
+ ' gerrit open change.\n Please make sure the commit has been '\
+ 'pushed to gerrit and correct magical ref refs/for/branch, '\
+ 'then re-submit tag.'
+
+UNKNOWN_FORMAT_MSG = '- Unknown tag format,\n please follow the format '\
+ 'submit/{version}/{date.time}.'
+
+WRONG_FORMAT_MSG = '- Wrong tag format,\n please follow the format '\
+ 'submit/{branch}/{date.time}. \n'\
+ 'Git branch : %s. Tag branch: %s'
+
+NOT_ANNOTATED_MSG = '- Tag should be annotated tag.'
+
+SUGGESTION = 'Suggest to use "gbs submit" to trigger submission\n'\
+
+def parse_submit_tag(tag):
+ """parse info from submit tag name"""
+
+ branch = None
+ date = None
+
+ if tag.startswith('submit/'):
+ pos = tag.rfind('/', len('submit/'))
+ if pos != -1:
+ branch = tag[len('submit/'):pos]
+ if branch == 'trunk':
+ branch = 'master'
+ date = tag[pos+1:]
+
+ return branch, date
+
+def find_submit_tag(event, mygit):
+ """find the corresponding submit tag for this event"""
+
+ if event['event_type'] == 'ref-updated':
+ tag = event['refname'][len('refs/tags/'):]
+ event['branch'] = parse_submit_tag(tag)[0]
+ # Since patchset_revision is used in gerrit feedback, real tag check
+ # is needed; and the key point is parse_submit_tag can not ensure the
+ # tag exsisting too.
+ try:
+ event['patchset_revision'] = mygit.rev_parse('%s^{commit}' % tag)
+ except GitRepositoryError:
+ tag = None
+ elif event['event_type'] == 'change-merged':
+ # for chanage-merged, search submit tag on this commit
+ branch = event['branch']
+ if event['branch'] == 'master':
+ branch = 'trunk'
+ try:
+ tag = mygit.describe(event['patchset_revision'],
+ pattern='submit/%s/*' % branch,
+ exact_match=True)
+ except GitRepositoryError:
+ # don'n find submit tag on this commit, return None
+ tag = None
+
+ return tag
+
+def check_tag_format(git, mygerrit, event, tag):
+ """check whether tag follow proper format"""
+
+ branch, date = parse_submit_tag(tag)
+ message = []
+ psr = event['patchset_revision']
+
+ # check tag name format
+ if branch and date:
+ # check date format
+ pattern = re.compile(r'^[0-9]{8}\.[0-9]{6}$')
+ if not pattern.match(date):
+ message.append(WRONG_DATE_MSG % date)
+
+ if not git.branch_contains(tag):
+ # Check if change is still under review
+ cmd = '--current-patch-set status: open project: %s commit: %s' % \
+ (event['project'], psr)
+ gerritinfo = mygerrit.query(cmd)
+ if len(gerritinfo) == 1 and 'number' in gerritinfo[0] \
+ and 'currentPatchSet' in gerritinfo[0]:
+ if gerritinfo[0]['branch'] == branch:
+ # the tagged commit still open, abort submit this time
+ message.append(UNDER_REVIEW_MSG % (tag, psr))
+ else:
+ # cannot find tagged commit in git tree or gerrit open change
+ message.append(WRONG_COMMIT_MSG % psr)
+ else:
+ # check for contains branch
+ contain = False
+ cbrch = git.branch_contains(tag)
+ if branch in cbrch:
+ print '%s branch is contains in %s branch' %(branch, cbrch)
+ contain = True
+ else:
+ vbrch = git_virtual_branch_map()
+ for items in vbrch:
+ if '%s' %(branch) in items and \
+ items['%s' %(branch)] is not None \
+ and items['%s' %(branch)] in cbrch:
+ print '%s branch is virtual %s branch' %(branch, items['%s' %(branch)])
+ contain = True
+ if not contain:
+ # wrong tag format
+ print '%s branch is not contains in %s branch' %(branch, cbrch)
+ message.append(WRONG_FORMAT_MSG % (cbrch, branch))
+ else:
+ # wrong tag format
+ message.append(UNKNOWN_FORMAT_MSG)
+
+ # check whether tag is an annotated tag
+ tagger = git.get_tag(tag)
+ if 'author' not in tagger or 'email' not in tagger:
+ message.append(NOT_ANNOTATED_MSG)
+
+ # post comment to gerrit and send email if check failed
+ if message:
+ print message
+
+ msg = 'The tag %s was pushed, but it was not completed because of '\
+ 'the following reason(s):\n\n' % tag + '\n'.join(message)
+
+ if len(message) != 1 or (message[0] != UNDER_REVIEW_MSG % (tag, psr) \
+ and message[0] != WRONG_COMMIT_MSG % psr):
+ msg += '\n\n' + SUGGESTION
+
+ try:
+ mygerrit.review(commit=psr, message=msg)
+ except GerritError, err:
+ print >> sys.stderr, 'Error posting review comment '\
+ 'back to Gerrit: %s' % str(err)
+ # return 1 if this exception is not caused by invalid commit
+ if 'no such patch set' not in str(err):
+ return None
+
+ return {'message': msg, 'tagger': tagger}
+
+ return None
+
+def find_specfile(prj_dir, packaging_dir, tag, event, tagger, pkg_name=None, debug_prj_name=None):
+ """search specfile under packaging directory"""
+
+ msg = ''
+
+ if True:
+ specs = utils.find_spec('%s/%s' % (prj_dir, packaging_dir))
+ if not specs:
+ # no spec exist under packaging, use default name
+ msg = "The tag %s pushed, but packaging directory doesn't \n"\
+ "contain any spec file. Please create one and \n"\
+ "re-submit it." % tag
+ elif len(specs) == 1:
+ # only one spec exist under packaging
+ spec = specs[0]
+ print 'Single spec. Use %s' % os.path.basename(spec)
+ else:
+ # multiple specs exist under packaging, use default name
+ spec = None
+ if pkg_name:
+ spec = '%s/%s/%s.spec' % (prj_dir, packaging_dir, pkg_name)
+ print 'Multiple specs. Try %s' % os.path.basename(spec)
+ if spec is None or not os.path.isfile(spec):
+ spec = '%s/%s/%s.spec' % (prj_dir, packaging_dir, \
+ os.path.basename(event['project']))
+ print 'Multiple specs with no pkg_name.spec. Try %s' % os.path.basename(spec)
+ if not os.path.isfile(spec):
+ spec = sorted(specs)[0]
+ print 'Multiple sorted(specs)[0] %s' % os.path.basename(spec)
+ if not os.path.isfile(spec):
+ msg = "The tag %s pushed, but packaging directory contains \n"\
+ "multiply spec files, backend service can not decide \n"\
+ "which spec file to use. Please use OBS_PACKAGE \n"\
+ "parameter in scm/git-obs-mapping project to specify \n"\
+ "the target spec file or contact system \n"\
+ "administrator for more details." % tag
+
+ if msg:
+ if debug_prj_name is not None:
+ msg = "TARGET_PROJECT: %s\n%s" % (debug_prj_name, msg)
+ print msg
+ return {'message': msg, 'spec': None}
+ else:
+ print 'specfile %s' % spec
+ return {'message': None, 'spec': spec}
+
+def parse_specfile(specfile, tag, event, tagger):
+ """parse specfile"""
+
+ spec = None
+
+ try:
+ # use gbp to parse specfile
+ spec = SpecFile(specfile)
+ except GbpError, err:
+ print 'gbp parse spec failed. %s' % err
+ msg = 'The tag %s pushed, but backend service failed to parse %s. \n'\
+ 'Please try "gbs export" on this tag and make sure it can '\
+ 'work.\n\n'\
+ 'Error message:\n'\
+ '%s' % (tag, os.path.basename(specfile), err)
+ return {'message': msg, 'spec': None}
+
+ return {'message': None, 'spec': spec}
+
+def change_release_name(build, project, git_tag):
+ """
+ Change release name from project config in OBS
+ Add the datetime into release name.
+ Eg: 'Release: <CI_CNT>.<B_CNT>' ----> 'Release: 20141010.<CI_CNT>.<B_CNT>'
+ """
+ # get project config
+ config = build.get_project_config(project)
+ release_name = 'Release: %s' % (git_tag.split('/')[-1])
+ res = re.findall(r'^Release: ?\S+$', config, flags=re.MULTILINE)
+ if res:
+ if git_tag.split('/')[-1] not in res[0]:
+ note = '#Insert time from submission into release name\n'
+ release_name = '%s.%s' % (release_name,
+ res[0].split('Release:')[1].strip())
+ config = config.replace(res[0], '%s%s' % (note, release_name), 1)
+ else:
+ note = '#Add release name into prjconf\n'
+ config = note + '%s\n' % release_name + config
+
+ #Add rpmbuild stage option
+ if os.getenv('PRERELEASE_RPMBUILD_STAGE'):
+ # Check if we've got required fields in TRIGGER_INFO
+ if not os.getenv('PRERELEASE_RPMBUILD_STAGE') in ('ba', 'bb'):
+ print 'Error: PRERELEASE_RPMBUILD_STAGE %s' % (os.getenv('PRERELEASE_RPMBUILD_STAGE'))
+ else:
+ rpmbuildstage = 'Rpmbuildstage: -%s' % (os.getenv('PRERELEASE_RPMBUILD_STAGE'))
+ res = re.findall(r'^Rpmbuildstage: ?\S+$', config, flags=re.MULTILINE)
+ if res:
+ config = config.replace(res[0], '%s' % (rpmbuildstage), 1)
+ else:
+ config = config + '#Add RpmbuildStage option into prjconf\n' + '%s\n' % (rpmbuildstage)
+
+ # Add "CopyLinkedPackages: yes" for prerelease projects.
+ if not re.search("CopyLinkedPackages:", config):
+ config = config + "\nCopyLinkedPackages: yes\n";
+
+ # set project config
+ build.set_project_config(project, config)
+
+def copy_person_project_meta(build, obs_target_prj, obs_project):
+ """copy the maintainer list from obs_target_prj meta to corresponding
+ prerelease project
+ """
+ src_meta_xml = build.get_meta(obs_target_prj)
+ src_xml_root = ElementTree.fromstringlist(src_meta_xml)
+ # get peron list from obs_target_prj meta
+ person_dict = {}
+ for person in src_xml_root.findall('person'):
+ if person.get('userid') in person_dict:
+ person_dict[person.get('userid')].append(person.get('role'))
+ else:
+ person_dict[person.get('userid')] = [person.get('role')]
+ # add person to prerelease project
+ if person_dict:
+ build.addPerson(obs_project, person_dict)
+
+def create_related_packages(build, obs_target_prj, obs_pre_prj, pre_package):
+ """create the 'link' package that relate the original package
+ obs_target_prj is the base project
+ obs_pre_prj is the prelease project
+ pre_package is the original package
+ """
+ sourceinfo = build.get_sourceinfo_list(obs_target_prj)
+ for package in sourceinfo:
+ if sourceinfo[package]:
+ link_prj, link_pkg = sourceinfo[package][-1].split('/')
+ if link_prj == obs_target_prj and link_pkg == pre_package:
+ build.create_link_pac(obs_pre_prj, pre_package, \
+ obs_pre_prj, package)
+ if re.search("_aggregate", package):
+ print "Copypac aggregate package: %s/%s" %(obs_pre_prj, package)
+ build.create_copy_pac(obs_target_prj, package, obs_pre_prj,\
+ package)
+ aggregate_file_name="_aggregate"
+ build.get_source_file(obs_target_prj, package, aggregate_file_name)
+ content = ""
+ with open(aggregate_file_name, 'r') as f:
+ content = f.read()
+ content_xml_root = ElementTree.fromstringlist(content)
+ for element in content_xml_root.findall('aggregate'):
+ element.set('project',obs_pre_prj)
+ content = ElementTree.tostring(content_xml_root)
+ with open(aggregate_file_name, 'w') as f:
+ f.write(content)
+ commit_msg="uploaded to copy pac %s/%s from %s" % (obs_pre_prj, package, obs_target_prj)
+ try:
+ build.commit_files(obs_pre_prj, package,
+ [(aggregate_file_name, True)], commit_msg)
+ except ObsError, error:
+ raise UploadError("Unable to upload _aggregate to %s: %s" % \
+ (obs_pre_prj, error))
+ print "Copypac done."
+
+def get_base_project(build, _project):
+ _meta = build.get_meta(_project)
+ xml_meta = ElementTree.fromstringlist(_meta)
+ _base_prj = ''
+ for repos in xml_meta.findall('repository'):
+ for path in repos.findall('path'):
+ if 'base' not in path.get('project').lower(): continue
+ print 'Found base project %s for %s' % (path.get('project'), \
+ _project)
+ return path.get('project')
+ return None
+
+def create_project(build, obs_project, submit_list):
+ """
+ Create prerelease OBS project and upload sources for the build.
+ Parameters:
+ @ build (buildservice.BuildService): Instance of BuildService
+ @ obs_project (str): Project name to be created
+ @ submit_list (list):
+ [
+ {
+ 'project'(O) : Prerelease project name
+ 'package'(M) : OBS package name
+ 'gerrit_project'(M) : Git path
+ 'git_tag'(M) : Git tag name
+ 'gerrit_newrev'(O) : Revision of the tag
+ 'submitter'(M) : Author of the tag
+ 'build_flag'(O) : Enable/Disable OBS build after create
+ 'obs_target_prj'(O) : Target OBS project
+ 'ref_obs_target_prj'(O): Reference target OBS project to be linked
+ 'url'(O) : Hostname of the gerrit server for _service file
+ },
+ ]
+ """
+
+ # Create review project if it doesn't exist
+ print "Creating temporary review OBS project %s" % obs_project
+
+ _git_url = 'ssh://%s:%s' % (os.getenv('GERRIT_HOSTNAME_EXTERNAL'),
+ os.getenv('GERRIT_SSHPORT'))
+ git_revision = []
+ _obs_target_prj, tstamp = get_info_from_prerelease_name(obs_project)
+
+ git_url = submit_list[0].get('url', _git_url)
+ git_project = [ t['gerrit_project'] for t in (u for u in submit_list) ]
+ git_tag = submit_list[0].get('git_tag')
+ for gt in submit_list:
+ # Group submit should have the same tag name
+ assert gt['git_tag'] == git_tag
+ for u in submit_list:
+ if 'gerrit_newrev' in u:
+ git_revision.append(u.get('gerrit_newrev'))
+ else:
+ # Find revision of the tag
+ _rev = Git(os.path.join(os.getenv('GIT_CACHE_DIR'), \
+ u.get('gerrit_project'))).get_tag(git_tag)['tagrevision']
+ git_revision.append(_rev)
+ obs_target_prj = submit_list[0].get('obs_target_prj', _obs_target_prj)
+ ref_obs_target_prj = submit_list[0].get('ref_obs_target_prj', obs_target_prj)
+ submitter = ','.join(list(set([ t['submitter'] for t in (u for u in submit_list) ])))
+ package = [ t['package'] for t in (u for u in submit_list) ]
+ build_flag = submit_list[0].get('build_flag', True)
+ print '\nCREATING....%s\n%s\n%s\n%s\n%s\n' % (obs_project, git_project, git_revision, submitter, package)
+ sys.stdout.flush()
+
+ info = {'projects': git_project,
+ 'packages': package,
+ 'obs_target_prj': obs_target_prj,
+ 'ref_obs_target_prj': ref_obs_target_prj,
+ 'git_tag': git_tag,
+ 'git_commit': git_revision,
+ 'obs_url': os.path.join(os.getenv('OBS_URL_EXTERNAL'), \
+ 'project/show?project=%s' % obs_project),
+ 'images': [],
+ 'base': get_base_project(build, ref_obs_target_prj)}
+
+ if build.exists(obs_project):
+ if submitter:
+ info['submitter'] = submitter
+
+ # update project info
+ build.update_info(info, obs_project)
+ # unlink the project to upload packages
+ try:
+ if build_flag == True:
+ build.unlink_project(obs_project)
+ except ObsError, error:
+ print 'Modify the meta conf to unlink failed: %s' % error
+ else:
+ if submitter:
+ info['submitter'] = escape(submitter)
+
+ if not build.exists(ref_obs_target_prj):
+ raise LocalError("Target project %s doesn't exist" % ref_obs_target_prj)
+ try:
+ build.create_project(obs_project, ref_obs_target_prj,
+ description=json.dumps(info))
+ except ObsError, error:
+ LocalError("Unable to create project %s: %s" % (obs_project, error))
+
+ # change release name of project config in OBS
+ change_release_name(build, obs_project, git_tag)
+
+ #disable publish flag
+ build.disable_build_flag(obs_project, repo = None, flag="publish", status="disable")
+ #disable build flag
+ build.disable_build_flag(obs_project, repo = None, flag="build", status="disable")
+
+ try:
+ for idx, val in enumerate(git_project):
+ upload_obs_service(git_url, git_project[idx], git_tag,
+ git_revision[idx], obs_project, build, package[idx])
+ except UploadError, err:
+ raise LocalError(err)
+
+ # copy the maintainer list from obs_target_prj meta to corresponding
+ # prerelease project
+ copy_person_project_meta(build, ref_obs_target_prj, obs_project)
+
+ if build_flag != True:
+ return
+
+ #create the 'link' package that relate the original package
+ for idx, val in enumerate(package):
+ create_related_packages(build, obs_target_prj, obs_project, package[idx])
+
+ #Wait 10 seconds to upload the package to the OBS
+ sleep(10)
+
+ #default build flag
+ build.default_build_flag(obs_project, repo = None, flag="build")
+ #default publish flag
+ build.default_build_flag(obs_project, repo = None, flag="publish")
+
+ # Enabled link project
+ sleep(1)
+ build.link_project(obs_project, src=ref_obs_target_prj, linktype="localdep")
+
+def get_obs_package_name(git, branch, target_project, tag, gerrit_env=None):
+
+ if not gerrit_env:
+ gerrit_env = GerritEnv()
+ git_cache = gerrit_env.gitcache
+
+ prjdir = os.path.join(git_cache, git)
+ if not clone_gitproject(git, prjdir, \
+ gerrit_hostname=gerrit_env.hostname, \
+ gerrit_username=gerrit_env.username, \
+ gerrit_sshport=gerrit_env.sshport, \
+ git_cache_dir=git_cache):
+ print >> sys.stderr, 'Error cloning %s' % git
+ return 1
+ mygit = Git(prjdir)
+ mygit.checkout(tag)
+ packagingdir = utils.parse_link('%s/%s' % (prjdir, 'packaging'))
+
+ obs_target_prjs = git_obs_map(git, \
+ gerrit_branch=branch, \
+ gitcache=git_cache
+ )
+ for item in obs_target_prjs:
+ obs_target_prj = item['OBS_project']
+ obs_stg_prj = item['OBS_staging_project']
+ obs_pkg = item['OBS_package']
+ obs_use_specname = item['OBS_use_specname']
+
+ if obs_target_prj != target_project:
+ continue
+ if obs_stg_prj not in ['prerelease', 'abs']:
+ continue
+
+ if obs_use_specname == 'yes':
+ resp = find_specfile(prjdir, packagingdir, tag, git, None)
+ if not resp['spec']:
+ return None
+ resp = parse_specfile(resp['spec'], tag, git, None)
+ if not resp['spec']:
+ return None
+ print 'spec name = %s' %(resp['spec'].name)
+ return resp['spec'].name
+ else:
+ if obs_pkg:
+ print 'package name from mapping = %s' % (obs_pkg)
+ return obs_pkg
+ else:
+ print 'package name from basename = %s' % (os.path.basename(git))
+ return os.path.basename(git)
+
+ return None
+
+def get_manifest_filelists_snapshot(profile, request_url, timeout=5, group=None):
+ """ get manifest filelists from snapshot"""
+ #p = re.compile(r'alt=\"\[(TXT|DIR| )]*\]\".*<a href=\"(.*)\">')
+ p = re.compile(r'.*<a href=\"(.*)\">')
+
+ if not request_url:
+ return {}
+ print request_url
+ # get data from url
+ for loop in range(10):
+ try:
+ f = requests.get(request_url,
+ auth=(profile['snapshot_username'],
+ profile['snapshot_password']),
+ timeout=timeout)
+ if f.status_code == 200:
+ break
+ except requests.exceptions.Timeout as e:
+ print(e)
+ continue
+ except requests.exceptions.ConnectionError as e:
+ print(e)
+ continue
+ except Exception as e:
+ print(e)
+ raise Exception('exception from get manifest filelists')
+
+ results = {}
+ exclude_pkgs = []
+ found_links = p.findall(f.text)
+ for link in found_links:
+ if link == '../':
+ continue
+ if not link.endswith('.xml'):
+ continue
+ manifest_url = os.path.join(request_url, link)
+ f = requests.get(manifest_url,
+ auth=(profile['snapshot_username'],
+ profile['snapshot_password']),
+ timeout=timeout)
+ try:
+ tree = ElementTree.fromstring(f.text)
+ except ElementTree.ParseError:
+ raise ElementTree.ParseError
+ for result in tree.findall('project'):
+ if '_preloadapp.xml' in link:
+ exclude_pkgs.append(''.join(result.get('path')))
+ else:
+ results[''.join(result.get('path'))] = result.get('revision')
+
+ if group == 'abs':
+ preloadapp_pkgs = {}
+ for app in exclude_pkgs:
+ preloadapp_pkgs[app] = results[app]
+ return preloadapp_pkgs
+
+ return results
+
+if __name__ == '__main__':
+ print 'Non-executable module'
+ sys.exit(0)
+
from gbp.git.repository import GitRepositoryError
from gbp.errors import GbpError
+from common.workflow import create_project, find_submit_tag, check_tag_format, find_specfile, parse_specfile
+
# set default char-set endcoding to utf-8
reload(sys)
sys.setdefaultencoding('utf-8') # pylint: disable-msg=E1101
-WRONG_DATE_MSG = '- The date %s in tag does NOT follow correct format.\n You can'\
- ' use shell command "date --utc +%%Y%%m%%d.%%H%%M%%S" to '\
- 'generate it, like 20120801.083113.'
-
-UNDER_REVIEW_MSG = '- Submission %s has been rejected because tagged commit %s'\
- ' is still under review in gerrit.\n Please re-trigger '\
- 'submission after your change is accepted.'
-
-WRONG_COMMIT_MSG = '- The commit %s tag attached does NOT exist in git tree or'\
- ' gerrit open change.\n Please make sure the commit has been '\
- 'pushed to gerrit and correct magical ref refs/for/branch, '\
- 'then re-submit tag.'
-
-UNKNOWN_FORMAT_MSG = '- Unknown tag format,\n please follow the format '\
- 'submit/{version}/{date.time}.'
-
-WRONG_FORMAT_MSG = '- Wrong tag format,\n please follow the format '\
- 'submit/{branch}/{date.time}. \n'\
- 'Git branch : %s. Tag branch: %s'
-
-NOT_ANNOTATED_MSG = '- Tag should be annotated tag.'
-
-SUGGESTION = 'Suggest to use "gbs submit" to trigger submission\n'\
-
TITLE_FAILED = '[Submit Request Failed]: tag: %s in %s'
TITLE_SUCCESS = '[Submit Request Success]: tag: %s in %s'
prepare_mail("%s.env" % os.getenv('BUILD_TAG'), title, msg,
os.getenv('NOREPLY_EMAIL_SENDER'), receiver['email'])
-def parse_submit_tag(tag):
- """parse info from submit tag name"""
-
- branch = None
- date = None
-
- if tag.startswith('submit/'):
- pos = tag.rfind('/', len('submit/'))
- if pos != -1:
- branch = tag[len('submit/'):pos]
- if branch == 'trunk':
- branch = 'master'
- date = tag[pos+1:]
-
- return branch, date
-
-def find_submit_tag(event, mygit):
- """find the corresponding submit tag for this event"""
-
- if event['event_type'] == 'ref-updated':
- tag = event['refname'][len('refs/tags/'):]
- event['branch'] = parse_submit_tag(tag)[0]
- # Since patchset_revision is used in gerrit feedback, real tag check
- # is needed; and the key point is parse_submit_tag can not ensure the
- # tag exsisting too.
- try:
- event['patchset_revision'] = mygit.rev_parse('%s^{commit}' % tag)
- except GitRepositoryError:
- tag = None
- elif event['event_type'] == 'change-merged':
- # for chanage-merged, search submit tag on this commit
- branch = event['branch']
- if event['branch'] == 'master':
- branch = 'trunk'
- try:
- tag = mygit.describe(event['patchset_revision'],
- pattern='submit/%s/*' % branch,
- exact_match=True)
- except GitRepositoryError:
- # don'n find submit tag on this commit, return None
- tag = None
-
- return tag
-
-def check_tag_format(git, mygerrit, event, tag):
- """check whether tag follow proper format"""
-
- branch, date = parse_submit_tag(tag)
- message = []
- psr = event['patchset_revision']
-
- # check tag name format
- if branch and date:
- # check date format
- pattern = re.compile(r'^[0-9]{8}\.[0-9]{6}$')
- if not pattern.match(date):
- message.append(WRONG_DATE_MSG % date)
-
- if not git.branch_contains(tag):
- # Check if change is still under review
- cmd = '--current-patch-set status: open project: %s commit: %s' % \
- (event['project'], psr)
- gerritinfo = mygerrit.query(cmd)
- if len(gerritinfo) == 1 and 'number' in gerritinfo[0] \
- and 'currentPatchSet' in gerritinfo[0]:
- if gerritinfo[0]['branch'] == branch:
- # the tagged commit still open, abort submit this time
- message.append(UNDER_REVIEW_MSG % (tag, psr))
- else:
- # cannot find tagged commit in git tree or gerrit open change
- message.append(WRONG_COMMIT_MSG % psr)
- else:
- # check for contains branch
- contain = False
- cbrch = git.branch_contains(tag)
- if branch in cbrch:
- print '%s branch is contains in %s branch' %(branch, cbrch)
- contain = True
- else:
- vbrch = git_virtual_branch_map()
- for items in vbrch:
- if '%s' %(branch) in items and \
- items['%s' %(branch)] is not None \
- and items['%s' %(branch)] in cbrch:
- print '%s branch is virtual %s branch' %(branch, items['%s' %(branch)])
- contain = True
- if not contain:
- # wrong tag format
- print '%s branch is not contains in %s branch' %(branch, cbrch)
- message.append(WRONG_FORMAT_MSG % (cbrch, branch))
- else:
- # wrong tag format
- message.append(UNKNOWN_FORMAT_MSG)
-
- # check whether tag is an annotated tag
- tagger = git.get_tag(tag)
- if 'author' not in tagger or 'email' not in tagger:
- message.append(NOT_ANNOTATED_MSG)
-
- # post comment to gerrit and send email if check failed
- if message:
- print message
-
- msg = 'The tag %s was pushed, but it was not completed because of '\
- 'the following reason(s):\n\n' % tag + '\n'.join(message)
-
- if len(message) != 1 or (message[0] != UNDER_REVIEW_MSG % (tag, psr) \
- and message[0] != WRONG_COMMIT_MSG % psr):
- msg += '\n\n' + SUGGESTION
-
- try:
- mygerrit.review(commit=psr, message=msg)
- except GerritError, err:
- print >> sys.stderr, 'Error posting review comment '\
- 'back to Gerrit: %s' % str(err)
- # return 1 if this exception is not caused by invalid commit
- if 'no such patch set' not in str(err):
- return False
- if 'author' not in tagger or 'email' not in tagger:
- recevier = { 'author' : event['event_account_name'],
- 'email' : event['event_account_email'] }
- send_mail(TITLE_FAILED % (tag, event['project']), msg, recevier)
- else:
- send_mail(TITLE_FAILED % (tag, event['project']), msg, tagger)
-
- return False
-
- return True
-
-def find_specfile(prj_dir, packaging_dir, tag, event, tagger, pkg_name=None, debug_prj_name=None):
- """search specfile under packaging directory"""
-
- msg = ''
-
- if True:
- specs = utils.find_spec('%s/%s' % (prj_dir, packaging_dir))
- if not specs:
- # no spec exist under packaging, use default name
- msg = "The tag %s pushed, but packaging directory doesn't \n"\
- "contain any spec file. Please create one and \n"\
- "re-submit it." % tag
- elif len(specs) == 1:
- # only one spec exist under packaging
- spec = specs[0]
- print 'Single spec. Use %s' % os.path.basename(spec)
- else:
- # multiple specs exist under packaging, use default name
- spec = None
- if pkg_name:
- spec = '%s/%s/%s.spec' % (prj_dir, packaging_dir, pkg_name)
- print 'Multiple specs. Try %s' % os.path.basename(spec)
- if spec is None or not os.path.isfile(spec):
- spec = '%s/%s/%s.spec' % (prj_dir, packaging_dir, \
- os.path.basename(event['project']))
- print 'Multiple specs with no pkg_name.spec. Try %s' % os.path.basename(spec)
- if not os.path.isfile(spec):
- spec = sorted(specs)[0]
- print 'Multiple sorted(specs)[0] %s' % os.path.basename(spec)
- if not os.path.isfile(spec):
- msg = "The tag %s pushed, but packaging directory contains \n"\
- "multiply spec files, backend service can not decide \n"\
- "which spec file to use. Please use OBS_PACKAGE \n"\
- "parameter in scm/git-obs-mapping project to specify \n"\
- "the target spec file or contact system \n"\
- "administrator for more details." % tag
-
- if msg:
- if debug_prj_name is not None:
- msg = "TARGET_PROJECT: %s\n%s" % (debug_prj_name, msg)
- print msg
- send_mail(TITLE_FAILED % (tag, event['project']), msg, tagger)
- return None
- else:
- print 'specfile %s' % spec
- return spec
-
-
-def parse_specfile(specfile, tag, event, tagger):
- """parse specfile"""
-
- spec = None
-
- try:
- # use gbp to parse specfile
- spec = SpecFile(specfile)
- except GbpError, err:
- print 'gbp parse spec failed. %s' % err
- msg = 'The tag %s pushed, but backend service failed to parse %s. \n'\
- 'Please try "gbs export" on this tag and make sure it can '\
- 'work.\n\n'\
- 'Error message:\n'\
- '%s' % (tag, os.path.basename(specfile), err)
- send_mail(TITLE_FAILED % (tag, event['project']), msg, tagger)
- return None
-
- return spec
-
-class LocalError(Exception):
- """Local error exception."""
- pass
-
-
-def change_release_name(build, project, git_tag):
- """
- Change release name from project config in OBS
- Add the datetime into release name.
- Eg: 'Release: <CI_CNT>.<B_CNT>' ----> 'Release: 20141010.<CI_CNT>.<B_CNT>'
- """
- # get project config
- config = build.get_project_config(project)
- release_name = 'Release: %s' % (git_tag.split('/')[-1])
- res = re.findall(r'^Release: ?\S+$', config, flags=re.MULTILINE)
- if res:
- if git_tag.split('/')[-1] not in res[0]:
- note = '#Insert time from submission into release name\n'
- release_name = '%s.%s' % (release_name,
- res[0].split('Release:')[1].strip())
- config = config.replace(res[0], '%s%s' % (note, release_name), 1)
- else:
- note = '#Add release name into prjconf\n'
- config = note + '%s\n' % release_name + config
-
- #Add rpmbuild stage option
- if os.getenv('PRERELEASE_RPMBUILD_STAGE'):
- # Check if we've got required fields in TRIGGER_INFO
- if not os.getenv('PRERELEASE_RPMBUILD_STAGE') in ('ba', 'bb'):
- print 'Error: PRERELEASE_RPMBUILD_STAGE %s' % (os.getenv('PRERELEASE_RPMBUILD_STAGE'))
- else:
- rpmbuildstage = 'Rpmbuildstage: -%s' % (os.getenv('PRERELEASE_RPMBUILD_STAGE'))
- res = re.findall(r'^Rpmbuildstage: ?\S+$', config, flags=re.MULTILINE)
- if res:
- config = config.replace(res[0], '%s' % (rpmbuildstage), 1)
- else:
- config = config + '#Add RpmbuildStage option into prjconf\n' + '%s\n' % (rpmbuildstage)
-
- # Add "CopyLinkedPackages: yes" for prerelease projects.
- if not re.search("CopyLinkedPackages:", config):
- config = config + "\nCopyLinkedPackages: yes\n";
-
- # set project config
- build.set_project_config(project, config)
-
-def copy_person_project_meta(build, obs_target_prj, obs_project):
- """copy the maintainer list from obs_target_prj meta to corresponding
- prerelease project
- """
- src_meta_xml = build.get_meta(obs_target_prj)
- src_xml_root = ElementTree.fromstringlist(src_meta_xml)
- # get peron list from obs_target_prj meta
- person_dict = {}
- for person in src_xml_root.findall('person'):
- if person.get('userid') in person_dict:
- person_dict[person.get('userid')].append(person.get('role'))
- else:
- person_dict[person.get('userid')] = [person.get('role')]
- # add person to prerelease project
- if person_dict:
- build.addPerson(obs_project, person_dict)
-
-def create_related_packages(build, obs_target_prj, obs_pre_prj, pre_package):
- """create the 'link' package that relate the original package
- obs_target_prj is the base project
- obs_pre_prj is the prelease project
- pre_package is the original package
- """
- sourceinfo = build.get_sourceinfo_list(obs_target_prj)
- for package in sourceinfo:
- if sourceinfo[package]:
- link_prj, link_pkg = sourceinfo[package][-1].split('/')
- if link_prj == obs_target_prj and link_pkg == pre_package:
- build.create_link_pac(obs_pre_prj, pre_package, \
- obs_pre_prj, package)
- if re.search("_aggregate", package):
- print "Copypac aggregate package: %s/%s" %(obs_pre_prj, package)
- build.create_copy_pac(obs_target_prj, package, obs_pre_prj,\
- package)
- aggregate_file_name="_aggregate"
- build.get_source_file(obs_target_prj, package, aggregate_file_name)
- content = ""
- with open(aggregate_file_name, 'r') as f:
- content = f.read()
- content_xml_root = ElementTree.fromstringlist(content)
- for element in content_xml_root.findall('aggregate'):
- element.set('project',obs_pre_prj)
- content = ElementTree.tostring(content_xml_root)
- with open(aggregate_file_name, 'w') as f:
- f.write(content)
- commit_msg="uploaded to copy pac %s/%s from %s" % (obs_pre_prj, package, obs_target_prj)
- try:
- build.commit_files(obs_pre_prj, package,
- [(aggregate_file_name, True)], commit_msg)
- except ObsError, error:
- raise UploadError("Unable to upload _aggregate to %s: %s" % \
- (obs_pre_prj, error))
- print "Copypac done."
-
-def get_base_project(build, _project):
- _meta = build.get_meta(_project)
- xml_meta = ElementTree.fromstringlist(_meta)
- _base_prj = ''
- for repos in xml_meta.findall('repository'):
- for path in repos.findall('path'):
- if 'base' not in path.get('project').lower(): continue
- print 'Found base project %s for %s' % (path.get('project'), \
- _project)
- return path.get('project')
- return None
-
-def create_project(build, obs_project, submit_list):
- """
- Create prerelease OBS project and upload sources for the build.
- Parameters:
- @ build (buildservice.BuildService): Instance of BuildService
- @ obs_project (str): Project name to be created
- @ submit_list (list):
- [
- {
- 'project'(O) : Prerelease project name
- 'package'(M) : OBS package name
- 'gerrit_project'(M) : Git path
- 'git_tag'(M) : Git tag name
- 'gerrit_newrev'(O) : Revision of the tag
- 'submitter'(M) : Author of the tag
- 'build_flag'(O) : Enable/Disable OBS build after create
- 'obs_target_prj'(O) : Target OBS project
- 'ref_obs_target_prj'(O): Reference target OBS project to be linked
- 'url'(O) : Hostname of the gerrit server for _service file
- },
- ]
- """
-
- # Create review project if it doesn't exist
- print "Creating temporary review OBS project %s" % obs_project
-
- _git_url = 'ssh://%s:%s' % (os.getenv('GERRIT_HOSTNAME_EXTERNAL'),
- os.getenv('GERRIT_SSHPORT'))
- git_revision = []
- _obs_target_prj, tstamp = get_info_from_prerelease_name(obs_project)
-
- git_url = submit_list[0].get('url', _git_url)
- git_project = [ t['gerrit_project'] for t in (u for u in submit_list) ]
- git_tag = submit_list[0].get('git_tag')
- for gt in submit_list:
- # Group submit should have the same tag name
- assert gt['git_tag'] == git_tag
- for u in submit_list:
- if 'gerrit_newrev' in u:
- git_revision.append(u.get('gerrit_newrev'))
- else:
- # Find revision of the tag
- _rev = Git(os.path.join(os.getenv('GIT_CACHE_DIR'), \
- u.get('gerrit_project'))).get_tag(git_tag)['tagrevision']
- git_revision.append(_rev)
- obs_target_prj = submit_list[0].get('obs_target_prj', _obs_target_prj)
- ref_obs_target_prj = submit_list[0].get('ref_obs_target_prj', obs_target_prj)
- submitter = ','.join(list(set([ t['submitter'] for t in (u for u in submit_list) ])))
- package = [ t['package'] for t in (u for u in submit_list) ]
- build_flag = submit_list[0].get('build_flag', True)
- print '\nCREATING....%s\n%s\n%s\n%s\n%s\n' % (obs_project, git_project, git_revision, submitter, package)
- sys.stdout.flush()
-
- info = {'projects': git_project,
- 'packages': package,
- 'obs_target_prj': obs_target_prj,
- 'ref_obs_target_prj': ref_obs_target_prj,
- 'git_tag': git_tag,
- 'git_commit': git_revision,
- 'obs_url': os.path.join(os.getenv('OBS_URL_EXTERNAL'), \
- 'project/show?project=%s' % obs_project),
- 'images': [],
- 'base': get_base_project(build, ref_obs_target_prj)}
-
- if build.exists(obs_project):
- if submitter:
- info['submitter'] = submitter
-
- # update project info
- build.update_info(info, obs_project)
- # unlink the project to upload packages
- try:
- if build_flag == True:
- build.unlink_project(obs_project)
- except ObsError, error:
- print 'Modify the meta conf to unlink failed: %s' % error
- else:
- if submitter:
- info['submitter'] = escape(submitter)
-
- if not build.exists(ref_obs_target_prj):
- raise LocalError("Target project %s doesn't exist" % ref_obs_target_prj)
- try:
- build.create_project(obs_project, ref_obs_target_prj,
- description=json.dumps(info))
- except ObsError, error:
- LocalError("Unable to create project %s: %s" % (obs_project, error))
-
- # change release name of project config in OBS
- change_release_name(build, obs_project, git_tag)
-
- #disable publish flag
- build.disable_build_flag(obs_project, repo = None, flag="publish", status="disable")
- #disable build flag
- build.disable_build_flag(obs_project, repo = None, flag="build", status="disable")
-
- try:
- for idx, val in enumerate(git_project):
- upload_obs_service(git_url, git_project[idx], git_tag,
- git_revision[idx], obs_project, build, package[idx])
- except UploadError, err:
- raise LocalError(err)
-
- # copy the maintainer list from obs_target_prj meta to corresponding
- # prerelease project
- copy_person_project_meta(build, ref_obs_target_prj, obs_project)
-
- if build_flag != True:
- return
-
- #create the 'link' package that relate the original package
- for idx, val in enumerate(package):
- create_related_packages(build, obs_target_prj, obs_project, package[idx])
-
- #Wait 10 seconds to upload the package to the OBS
- sleep(10)
-
- #default build flag
- build.default_build_flag(obs_project, repo = None, flag="build")
- #default publish flag
- build.default_build_flag(obs_project, repo = None, flag="publish")
-
- # Enabled link project
- sleep(1)
- build.link_project(obs_project, src=ref_obs_target_prj, linktype="localdep")
-
def get_branch_name(tag):
"""Get branch name by parsing info
from submit tag name.
submitter += ' <%s>' % gerrit_account_email
# check whether tag meet format
- if not check_tag_format(mygit, mygerrit, event, tag):
+ resp = check_tag_format(mygit, mygerrit, event, tag)
+ if resp is not None:
+ if 'author' not in resp['tagger'] or 'email' not in resp['tagger']:
+ recevier = { 'author' : event['event_account_name'],
+ 'email' : event['event_account_email'] }
+ send_mail(TITLE_FAILED % (tag, event['project']), resp['message'], recevier)
+ else:
+ send_mail(TITLE_FAILED % (tag, event['project']), resp['message'], resp['tagger'])
print 'The check for the tag format is error, exit now\n'
return 0
enable_build = False
if obs_use_specname == 'yes':
# search specfile under packaging directory
- specfile = find_specfile(prjdir, packagingdir, tag, event, tagger)
- if not specfile:
+ resp = find_specfile(prjdir, packagingdir, tag, event, tagger)
+ if not resp['spec']:
+ send_mail(TITLE_FAILED % (tag, event['project']), resp['message'], tagger)
return 0
# parse specfile
- spec = parse_specfile(specfile, tag, event, tagger)
- if not spec:
+ resp = parse_specfile(resp['spec'], tag, event, tagger)
+ if not resp['spec']:
+ send_mail(TITLE_FAILED % (tag, event['project']), resp['message'], tagger)
return 0
- package = spec.name
- print 'spec name = %s' %(spec.name)
+ package = resp['spec'].name
+ print 'spec name = %s' %(resp['spec'].name)
else:
# get package name from xml files.
if obs_pkg:
projects / services / jenkins-scripts.git / commitdiff