From: ZhuoX Li Date: Fri, 8 Aug 2014 08:09:25 +0000 (+0800) Subject: Get rid of dependency of 'gbs'. X-Git-Tag: 1.0~92 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=93297f93f7769c19207b121443e1f04ee4ee8daf;p=services%2Fjenkins-scripts.git Get rid of dependency of 'gbs'. Jenkins scripts job_buildcheck_post.py job_policycheck.py is unused for us. They should be removed. And get rid of dependency of 'gbs', the test case about 'gbs' should be also removed. Fixes: #1546 Change-Id: I65e9596d35fba8c0bc6ad526cb2ea46a7176acf0 --- diff --git a/common/tempbuildpkg.py b/common/tempbuildpkg.py index 7e9ecbe..36ce530 100644 --- a/common/tempbuildpkg.py +++ b/common/tempbuildpkg.py @@ -24,23 +24,6 @@ class TempPackageBuild: else: self.package = package - def remote_build(self, dest_project, git_project, packagingdir, - message = ''): - """ Use gbs remotebuild to send local git tree to OBS as obs package """ - if not self._bs.get_targets(dest_project): - print '\n----------\nOBS %s project do not have repo setting, ' \ - 'quit build check against to it\n----------' % dest_project - return 'NoRepo' - with open('%s/packaging/git_info' % git_project, 'w') as _fh: - _fh.write(message) - - try: - return runner.show("gbs -v rb --include-all -B %s -T %s " \ - "--packaging-dir %s %s" % (dest_project, self.project, \ - packagingdir, git_project)) - finally: - os.remove("%s/packaging/git_info" % git_project) - def get_build_result(self): """ return build result of this package """ try: diff --git a/job_buildcheck_post.py b/job_buildcheck_post.py deleted file mode 100755 index 6312b6e..0000000 --- a/job_buildcheck_post.py +++ /dev/null @@ -1,119 +0,0 @@ -#!/usr/bin/env python -# vim: ai ts=4 sts=4 et sw=4 - -"""This job will be triggered by obs finished build events of buildcheck project -in OBS, get its build result and post it back to gerrit. If build fails, -it will send out build log as mail attachment to patch owner.""" - -import os -import tempfile -import shutil -import gzip -import sys -import atexit -from urllib2 import HTTPError - -# internal -from common import gerrit - -from common.tempbuildpkg import TempPackageBuild -from common.send_mail import prepare_mail -from common.buildtrigger import trigger_info - - -def main(): - """The main body""" - - print '---[JOB STARTED]----------------------------------------' - - work_space = os.getenv('WORKSPACE') - - tmpdir = tempfile.mkdtemp(prefix=(work_space + '/')) - atexit.register(shutil.rmtree, tmpdir) - - active_status = ['building', 'scheduled', 'dispatching', 'blocked'] - - event_dict = trigger_info(os.getenv('TRIGGER_INFO')) - - # check whether this is buildcheck project - if ':buildcheck:' not in event_dict['project']: - print 'This project %s is not buildcheck project, exit now' \ - % event_dict['project'] - return 0 - - if event_dict['event_type'] == 'OBS_REPO_PUBLISH_STATE' and \ - event_dict['state'] != 'published': - print 'repo is publishing, exit now' - return 0 - - project = event_dict['project'] - obs_api_username = os.getenv('OBS_API_USERNAME') - obs_dest_prj = project[len('home:%s:' % obs_api_username) : \ - project.index(':buildcheck:')] - - msg = '[BUILD CHECK RESULT] This change is built against OBS project '\ - '%s: \n\n %-15s%-15s%-15s' % (obs_dest_prj, "repository", \ - "arch", "result") - - try: - temp_build = TempPackageBuild(os.getenv('OBS_API_URL'), \ - obs_api_username, os.getenv('OBS_API_PASSWD'), \ - event_dict['project']) - except HTTPError, err: - if err.code == 404: - print 'This TempPackage has already been deleted, exit now' - return 0 - else: - raise err - - git_info = temp_build.get_git_info() - build_results = temp_build.get_build_result() - print 'build result: ', build_results - if not build_results: - print 'Build status is null, exit now' - return 0 - for build_repo in build_results.keys(): - for active in active_status: - if active in build_results[build_repo].values(): - print 'This package is still building, exit now' - return 0 - - for repo in build_results: - for arch in build_results[repo]: - msg = msg + '\n\n %-15s%-15s%-15s' % (repo, arch, \ - build_results[repo][arch]) - if build_results[repo][arch] == 'failed' and \ - 'OWNER_EMAIL' in git_info and git_info['OWNER_EMAIL']: - log = temp_build.get_build_log(repo, arch) - # write log to a file as attachment - buildlog_gz = os.path.join(work_space, \ - "buildlog-%s-%s-%s.log.gz" % \ - (repo, arch, build_results[repo][arch])) - log_file = gzip.open(buildlog_gz, "wb") - log_file.write(log) - log_file.close() - prepare_mail("%s.env" % os.getenv('BUILD_TAG'), \ - '[BUILD CHECK RESULT]: '\ - 'Your change for %s is %s build against %s' % (\ - git_info['GIT_PROJECT'], build_results[repo][arch], \ - obs_dest_prj), msg, os.getenv('NOREPLY_EMAIL_SENDER'), \ - '%s' % git_info['OWNER_EMAIL'], \ - attachment = buildlog_gz) - os.unlink(buildlog_gz) - - temp_build.del_itself() - - print msg - if git_info.has_key('COMMIT_ID') and git_info['COMMIT_ID']: - try: - gerr = gerrit.Gerrit(os.getenv('GERRIT_HOSTNAME'), - os.getenv('GERRIT_USERNAME'), - os.getenv('GERRIT_SSHPORT'), - int(os.getenv('GERRIT_SILENT_MODE'))) - gerr.review(commit = git_info['COMMIT_ID'], message = msg) - except gerrit.GerritError, err: - print >> sys.stderr, 'Error posting review comment '\ - 'back to Gerrit: %s' % str(err) - -if __name__ == '__main__': - sys.exit(main()) diff --git a/job_policycheck.py b/job_policycheck.py deleted file mode 100755 index b741e20..0000000 --- a/job_policycheck.py +++ /dev/null @@ -1,213 +0,0 @@ -#!/usr/bin/env python -# vim: ai ts=4 sts=4 et sw=4 - -"""This script will do policy check when patch is created in gerrit. -""" - -import os -import tempfile -import shutil -import sys -import atexit - -# internal module -from common import utils -from common.gerrit import get_gerrit_event -from common.git import Git, clone_gitproject -from common import gerrit -from common.mapping import Mapping, MappingError -from common.tempbuildpkg import TempPackageBuild -from common import runner - -from gbp.rpm import SpecFile - -def is_git_obs_mapping_exists(git_cache_dir, mapping_prj): - """check whether exist git-obs-mapping.xml in local""" - - if not os.path.isfile('%s/%s/git-obs-mapping.xml' % (git_cache_dir, \ - mapping_prj)): - print('Update %s/git-obs-mapping.xml to local.' % mapping_prj) - if not clone_gitproject(mapping_prj, os.path.join(git_cache_dir, - mapping_prj)): - return False - return True - -def specfile_chooser(prjdir, packagingdir, obs_pkg, specs, prj): - """Choose which specfile should be used when specfile(s) exist""" - - specfile = None - - if obs_pkg: - # use obs_pkg as specfile name if obs_pkg set - specfile = '%s/%s/%s.spec' % (prjdir, packagingdir, obs_pkg) - else: - if len(specs) == 1: - # use it if only one specfile exist under packaging dir - specfile = specs[0] - else: - # use gerrit project name as specfile name - specfile = '%s/%s/%s.spec' % (prjdir, packagingdir, prj) - - if specfile not in specs: - specfile = None - - print('specfile %s' % specfile) - - return specfile - -def export(prjdir, packagingdir, mygit, events, mygerrit): - """Check a git repo whether it can be executed `gbs export` correctly""" - mygit.checkout('FETCH_HEAD') - - pkgdir = os.path.join(prjdir, packagingdir) - if not os.path.exists(pkgdir): - msg = "Error: Source tree doesn't have packaging/ directory" - mygerrit.review(events['patchset_revision'], message=msg) - return False - - tempd = tempfile.mkdtemp(dir=pkgdir) - cmd = 'gbs export --packaging-dir %s -o %s' % (packagingdir, tempd) - - with utils.Workdir(prjdir): - rcode, outs = runner.show(cmd) - - shutil.rmtree(tempd) - - if rcode: - msg = "Use 'gbs export' to generate package on this patch failed, " \ - "please have a check.\n\n" \ - "gbs command: %s\n" \ - "%s" % (cmd[:cmd.rfind(' -o ')], outs) - mygerrit.review(events['patchset_revision'], message=msg) - return False - - return True - -def build_package(obs_dest_prj, specfile, prjdir, packagingdir, events, - mygerrit): - """Build package in OBS""" - - # import environment variables - obs_api_url = os.getenv('OBS_API_URL') - obs_api_username = os.getenv('OBS_API_USERNAME') - obs_api_passwd = os.getenv('OBS_API_PASSWD') - build_num = os.getenv('BUILD_NUMBER') - - # build the package under home project of backend service user - buildcheck_project = "home:%s:%s:buildcheck:%s" % (obs_api_username, \ - obs_dest_prj, build_num) - - try: - spec = SpecFile(specfile) - - # submit this change to OBS for build check - print 'submit this change to OBS for build check...' - temp_build = TempPackageBuild(obs_api_url, obs_api_username, \ - obs_api_passwd, buildcheck_project, spec.name) - result = temp_build.remote_build(obs_dest_prj, prjdir, packagingdir, \ - 'GIT_PROJECT: %s\nCOMMIT_ID: %s\nOWNER_EMAIL: %s' % \ - (events['project'], events['patchset_revision'], \ - events['change_owner_email'])) - if result == 'NoRepo': - msg = "[BUILD CHECK RESULT] Can not test build against to " \ - "project %s, which currently has no build targets " \ - "defined. Please contact the build system " \ - "administrator." % obs_dest_prj - mygerrit.review(events['patchset_revision'], message = msg) - elif result[0]: - print result[1] - return -1 - except Exception, err: - print '%s' % err - return -1 - -def main(): - """The main body""" - - print '---[JOB STARTED]----------------------------------------' - - # import environment variables - events = get_gerrit_event() - workspace = os.getenv('WORKSPACE') - git_cache_dir = os.getenv('GIT_CACHE_DIR') - mapping_prj = os.getenv('MAPPING_PRJ') - - # create temporary directory - tmpdir = tempfile.mkdtemp(prefix=(workspace + '/')) - atexit.register(shutil.rmtree, tmpdir) - prjdir = os.path.join(tmpdir, events['project']) - prj = os.path.split(events['project'])[1] - - # check whether exist git-obs-mapping.xml in local - if not is_git_obs_mapping_exists(git_cache_dir, mapping_prj): - return 1 - - # init module instance - try: - mygerrit = gerrit.Gerrit(events['hostname'], - events['username'], events['sshport'], - int(os.getenv('GERRIT_SILENT_MODE'))) - except gerrit.GerritError, err: - print >> sys.stderr, str(err) - - try: - mymapping = Mapping('%s/%s/git-obs-mapping.xml' % \ - (git_cache_dir, mapping_prj)) - except MappingError, err: - print >> sys.stderr, str(err) - - obstargets = mymapping.get_submit_mapping(events['project'], \ - events['branch']) - if not obstargets: - return 0 - - # clone gerrit project to local dir - if not clone_gitproject(events['project'], prjdir): - return 1 - - # fetch patch from remote and checkout it - mygit = Git(prjdir) - mygit.fetch(repo='origin', tags=True, refspec=events['refspec']) - mygit.checkout('FETCH_HEAD') - - # parse link if packaging dir is a link - packagingdir = utils.parse_link('%s/%s' % (prjdir, 'packaging')) - packagingpath = os.path.join(prjdir, packagingdir) - print('packaging dir is %s' % packagingpath) - if not os.path.exists(packagingpath): - msg = "Project doesn't contain packaging/ directory" - mygerrit.review(events['patchset_revision'], message=msg) - return 1 - - # search spec file under packaging dir - specs = utils.find_spec(packagingpath) - if not specs: - msg = "No .spec files found in packaging directory" - mygerrit.review(events['patchset_revision'], message=msg) - return 1 - - if not export(prjdir, packagingdir, mygit, events, mygerrit): - return 1 - - rc_codes = [] - - for target in obstargets: - (obs_dest_prj, _, obs_pkg) = target - - # exit if obs_dest_prj is not set - if not obs_dest_prj: - return - - # choose which specfile should be use - specfile = specfile_chooser(prjdir, packagingdir, obs_pkg, specs, prj) - if not specfile: - continue - - # push this patch to obs to check build resutl - rc_codes.append(build_package(obs_dest_prj, specfile, prjdir, \ - packagingdir, events, mygerrit)) - - return -1 if any(rc_codes) else 0 - -if __name__ == '__main__': - sys.exit(main()) diff --git a/job_submitobs.py b/job_submitobs.py index 2c8d0bb..dc1f5b7 100755 --- a/job_submitobs.py +++ b/job_submitobs.py @@ -214,12 +214,10 @@ def find_specfile(prj_dir, packaging_dir, tag, event, tagger, pkg_name = None): print('specfile %s' % spec) return spec -def parse_specfile(specfile, tmp_dir, prj_dir, packaging_dir, tag, event, \ - tagger): - """parse specfile and use "gbs export" to generate tarball""" +def parse_specfile(specfile, tag, event, tagger): + """parse specfile""" spec = None - tarball_dir = None try: # use gbp to parse specfile @@ -232,27 +230,9 @@ def parse_specfile(specfile, tmp_dir, prj_dir, packaging_dir, tag, event, \ 'Error message:\n'\ '%s' % (tag, os.path.basename(specfile), err) send_mail(TITLE_FAILED % (tag, event['project']), msg, tagger) - return None, None - - # use gbs export to generate tarball - outdir = tempfile.mkdtemp(prefix=tmp_dir+'/') - cmd = 'gbs export --packaging-dir %s -o %s' % (packaging_dir, outdir) - with utils.Workdir(prj_dir): - rcode, outs = runner.show(cmd) - - # gbs export failed, send error message to developer - if rcode != 0: - msg = 'The tag %s pushed, but gbs failed to export packaging files '\ - 'on this tag. Please try "gbs export" on this tag and make '\ - 'sure it can work.\n\n'\ - 'gbs command: "%s"\n'\ - '%s' % (tag, cmd[:cmd.rfind(' -o ')], outs) - send_mail(TITLE_FAILED % (tag, event['project']), msg, tagger) - return None, None - - tarball_dir = os.path.join(outdir, os.listdir(outdir)[0]) + return None - return spec, tarball_dir + return spec def check_obs_project(apiurl, apiuser, apipasswd, obs_prjs): """check whether the specific project exist in obs @@ -446,10 +426,9 @@ def main(): if not specfile: return 0 - # parse specfile and use "gbs export" to generate tarball - spec, tarballdir = parse_specfile(specfile, tmpdir, prjdir, \ - packagingdir, tag, event, tagger) - if not spec or not tarballdir: + # parse specfile + spec = parse_specfile(specfile, tag, event, tagger) + if not spec: return 0 retry_count = 3 diff --git a/packaging/jenkins-scripts.spec b/packaging/jenkins-scripts.spec index 64e474a..244e365 100644 --- a/packaging/jenkins-scripts.spec +++ b/packaging/jenkins-scripts.spec @@ -17,7 +17,6 @@ Source: %{name}-%{version}.tar.gz Requires: %{name}-common = %{version}-%{release} Requires: %{name}-submitobs = %{version}-%{release} Requires: %{name}-tzs = %{version}-%{release} -Requires: gbs >= 0.17 BuildArch: noarch %define destdir /var/lib/jenkins/%{name} @@ -39,7 +38,6 @@ Requires: python-requests >= 2.0.1 Requires: python-snapdiff Requires: python-yaml Requires: python-lxml -Requires: gbs-api %description common Common part of jenkins scripts @@ -55,7 +53,6 @@ Jenkins scripts for tzs customer appliance instance %package submitobs Summary: Temporary package to isolate job_submitobs Group: Development/Tools/Building -Requires: gbs >= 0.17 Requires: %{name}-common = %{version}-%{release} %description submitobs @@ -88,10 +85,8 @@ fi %defattr(-,jenkins,jenkins) %dir /var/lib/jenkins/ %dir %{destdir} -%{destdir}/job_buildcheck_post.py %{destdir}/job_post_image.py %{destdir}/job_local_cache_cleanups.py -%{destdir}/job_policycheck.py %{destdir}/job_pre_release_obs.py %{destdir}/job_publish_test_results.py %{destdir}/job_request.py diff --git a/tests/test_gbs.py b/tests/test_gbs.py deleted file mode 100644 index 609c6af..0000000 --- a/tests/test_gbs.py +++ /dev/null @@ -1,125 +0,0 @@ -#!/usr/bin/python -tt -# vim: ai ts=4 sts=4 et sw=4 -# -#This program is free software,testcase -# - -"""Tests for Gbs func""" - -import os -import subprocess -import shutil -import unittest - -SPEC_TEST = """Name: gbs-test\nSummary: gbs export test\n -Version: 1.0\nRelease: 1\nLicense: GPL v2\n -Source: %{name}_%{version}.tar.gz\n -%description\n -gbs interface test -""" - -GBS_CONF = """[general]\nprofile = profile.tizen -[profile.tizen]\nobs = obs.tizen -[obs.tizen] -url = https://newobs-testing.bj.intel.com:444 -user = Admin -""" - -class GbsTest(unittest.TestCase): - """Test for GBS functionality - """ - def test_gbs_export(self): - """Test gbs export, and parameter: --packaging-dir, -o - """ - # create a temporary - prjdir = tempdir() - # run gbs export - with Workspace(prjdir, os.getcwd()): - return_code = runner("""gbs export --include-all --packaging-dir repackaging -o .""", \ - back = False) - self.assertEqual(return_code, 0) - shutil.rmtree(prjdir) - - def test_gbs_rb(self): - """Test gbs remotebuild, and parameter: - -v, -B, -T, --include-all, --spec - """ - result_v = if_contain_parameter('gbs --help', ['-v']) - command = 'gbs rb --help' - parameter = ['--include-all', '-B', '-T', '--spec', '--packaging-dir'] - result_rb = if_contain_parameter(command, parameter) - self.assertEqual(result_rb, True) - self.assertEqual(result_v, True) - -def if_contain_parameter(command, parameter): - """return the result for different parameter - """ - output = runner(command) - r_out, w_out = output.communicate() - result = True - for para in parameter: - if not para in r_out: - result = False - return result - -def tempdir(): - """create a temporary include repackaging directory, - two specfiles, .git.conf, and a common file - """ - # check the git project exists, before create a git project - num = 1 - tmpdir = '/tmp/Jenkins_gbs_test%s'% num - while True: - tmpdir = '/tmp/Jenkins_gbs_test%s'% num - num += 1 - if os.path.exists(tmpdir): - continue - else: - os.makedirs(tmpdir) - break - # create a git project, include repackaging, two specfiles, .gbs.conf, 1.txx - with Workspace(tmpdir, os.getcwd()): - os.mkdir('repackaging') - runner('git init', back = False) - with Workspace(tmpdir + '/repackaging', os.getcwd()): - with open('gbs-test.spec', 'w') as spec_file: - spec_file.write(SPEC_TEST) - tag_name = 'submit/1.txt' - with open('1.txt', 'w') as test_file: - test_file.write('dfadfadadf') - with open('.gbs.conf', 'w') as conf_file: - conf_file.write(GBS_CONF) - runner('git add .', back = False) - runner('git commit -m"gbs export test"', back = False) - runner('git tag "%s" -m"%s"'% (tag_name, tag_name), back = False) - return tmpdir - -def runner(command, back = True): - """wrapper for most of subprocess - input: - if back: - only stdout - else: - both stdout and stderr - return output - """ - if back: - return subprocess.Popen(command, stdout = subprocess.PIPE, - stderr = subprocess.STDOUT, - shell = True) - else: - return subprocess.Popen(command, - shell = True, - stdout = subprocess.PIPE).wait() - -class Workspace(): - def __init__(self, tmp_path, path): - self.path = path - self.tmp_path = tmp_path - - def __enter__(self): - if self.path != self.tmp_path: - os.chdir(self.tmp_path) - - def __exit__(self, exception_type, exception_val, trace): - os.chdir(self.path)