From 6808c6d74363b1a7645e2a4fe331a365cd6210c7 Mon Sep 17 00:00:00 2001 From: Ed Bartosh Date: Wed, 18 Jul 2012 10:07:20 +0300 Subject: [PATCH] Implemented submitting to OBS without creating local copy. This change implements major redesign of the way gbs works with OBS. Currently it creates working copy, which requires a lot of code to support it. This change proposes much more simple way - without creating local copy. It drastically simplifies the code (+324, -1470) and illiminates buildservice and obspkg modules. Fixes #137 and #138 Change-Id: I2fe944246da89b2919b6148abe9713d69558f1e1 --- gitbuildsys/buildservice.py | 1215 ---------------------------------------- gitbuildsys/cmd_remotebuild.py | 144 ++--- gitbuildsys/obspkg.py | 193 ------- gitbuildsys/oscapi.py | 241 ++++++++ gitbuildsys/utils.py | 12 + 5 files changed, 330 insertions(+), 1475 deletions(-) delete mode 100644 gitbuildsys/buildservice.py delete mode 100644 gitbuildsys/obspkg.py create mode 100644 gitbuildsys/oscapi.py diff --git a/gitbuildsys/buildservice.py b/gitbuildsys/buildservice.py deleted file mode 100644 index 0ceedc4..0000000 --- a/gitbuildsys/buildservice.py +++ /dev/null @@ -1,1215 +0,0 @@ -# -# buildservice.py - Buildservice API support for Yabsc -# - -# Copyright (C) 2008 James Oakley -# Copyright (C) 2010, 2011, 2012 Intel, Inc. - -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - -import os -import string -import sys -import shutil -import tempfile -import time -import urllib2 -import M2Crypto -import xml.etree.cElementTree as ElementTree -from collections import defaultdict - -import errors -from osc import conf, core - -class ObsError(Exception): - pass - -# Injection code for osc.core to fix the empty XML bug -def solid_get_files_meta(self, revision='latest', skip_service=True): - from time import sleep - import msger - - retry_count = 3 - while retry_count > 0: - fm = core.show_files_meta(self.apiurl, self.prjname, self.name, - revision=revision, meta=self.meta) - try: - root = core.ET.fromstring(fm) - break - except: - msger.warning('corrupted or empty obs server response ,retrying ...') - sleep(1) - retry_count -= 1 - - if not retry_count: - # all the re-try failed, abort - raise ObsError('cannet fetch files meta xml from server') - - # look for "too large" files according to size limit and mark them - for e in root.findall('entry'): - size = e.get('size') - if size and self.size_limit and int(size) > self.size_limit \ - or skip_service and (e.get('name').startswith('_service:') or e.get('name').startswith('_service_')): - e.set('skipped', 'true') - return core.ET.tostring(root) - -core.Package.get_files_meta = solid_get_files_meta - -class _Metafile: - """ - _Metafile(url, input, change_is_required=False, file_ext='.xml') - - Implementation on osc.core.metafile that does not print to stdout - """ - - def __init__(self, url, input, change_is_required=False, file_ext='.xml'): - self.url = url - self.change_is_required = change_is_required - - (fd, self.filename) = tempfile.mkstemp(prefix = 'osc_metafile.', suffix = file_ext, dir = '/tmp') - - f = os.fdopen(fd, 'w') - f.write(''.join(input)) - f.close() - - self.hash_orig = core.dgst(self.filename) - - def sync(self): - hash = core.dgst(self.filename) - if self.change_is_required == True and hash == self.hash_orig: - os.unlink(self.filename) - return True - - # don't do any exception handling... it's up to the caller what to do in case - # of an exception - core.http_PUT(self.url, file=self.filename) - os.unlink(self.filename) - return True - -# helper functions for class _ProjectFlags -def _flag2bool(flag): - """ - _flag2bool(flag) -> Boolean - Returns a boolean corresponding to the string 'enable', or 'disable' - """ - - if flag == 'enable': - return True - elif flag == 'disable': - return False - -def _bool2flag(b): - """ - _bool2flag(b) -> String - - Returns 'enable', or 'disable' according to boolean value b - """ - if b == True: - return 'enable' - elif b == False: - return 'disable' - -class _ProjectFlags(object): - """ - _ProjectFlags(bs, project) - - Represents the flags in project through the BuildService object bs - """ - - def __init__(self, bs, project): - self.bs = bs - self.tree = ElementTree.fromstring(self.bs.getProjectMeta(project)) - - # The "default" flags, when undefined - self.defaultflags = {'build': True, - 'publish': True, - 'useforbuild': True, - 'debuginfo': False} - - # Figure out what arches and repositories are defined - self.arches = {} - self.repositories = {} - - # Build individual repository list - for repository in self.tree.findall('repository'): - repodict = {'arches': {}} - self.__init_flags_in_dict(repodict) - for arch in repository.findall('arch'): - repodict['arches'][arch.text] = {} - self.__init_flags_in_dict(repodict['arches'][arch.text]) - # Add placeholder in global arches - self.arches[arch.text] = {} - self.repositories[repository.get('name')] = repodict - - # Initialise flags in global arches - for archdict in self.arches.values(): - self.__init_flags_in_dict(archdict) - - # A special repository representing the global and global arch flags - self.allrepositories = {'arches': self.arches} - self.__init_flags_in_dict(self.allrepositories) - - # Now populate the structures from the xml data - for flagtype in ('build', 'publish', 'useforbuild', 'debuginfo'): - flagnode = self.tree.find(flagtype) - if flagnode: - for node in flagnode: - repository = node.get('repository') - arch = node.get('arch') - - if repository and arch: - self.repositories[repository]['arches'][arch][flagtype] = _flag2bool(node.tag) - elif repository: - self.repositories[repository][flagtype] = _flag2bool(node.tag) - elif arch: - self.arches[flagtype] = _flag2bool(node.tag) - else: - self.allrepositories[flagtype] = _flag2bool(node.tag) - - def __init_flags_in_dict(self, d): - """ - __init_flags_in_dict(d) - - Initialize all build flags to None in d - """ - d.update({'build': None, - 'publish': None, - 'useforbuild': None, - 'debuginfo': None}) - - def save(self): - """ - save() - - Save flags - """ - - for flagtype in ('build', 'publish', 'useforbuild', 'debuginfo'): - # Clear if set - flagnode = self.tree.find(flagtype) - if flagnode: - self.tree.remove(flagnode) - - # Generate rule nodes - rulenodes = [] - - # globals - if self.allrepositories[flagtype] != None: - rulenodes.append(ElementTree.Element(_bool2flag(self.allrepositories[flagtype]))) - for arch in self.arches: - if self.arches[arch][flagtype] != None: - rulenodes.append(ElementTree.Element(_bool2flag(self.arches[arch][flagtype]), arch=arch)) - - # repositories - for repository in self.repositories: - if self.repositories[repository][flagtype] != None: - rulenodes.append(ElementTree.Element(_bool2flag(self.repositories[repository][flagtype]), repository=repository)) - for arch in self.repositories[repository]['arches']: - if self.repositories[repository]['arches'][arch][flagtype] != None: - rulenodes.append(ElementTree.Element(_bool2flag(self.repositories[repository]['arches'][arch][flagtype]), arch=arch, repository=repository)) - - # Add nodes to tree - if rulenodes: - from pprint import pprint - pprint(rulenodes) - flagnode = ElementTree.Element(flagtype) - self.tree.insert(3, flagnode) - for rulenode in rulenodes: - flagnode.append(rulenode) - - print ElementTree.tostring(self.tree) - -class BuildService(object): - """Interface to Build Service API""" - - def __init__(self, apiurl=None, oscrc=None): - if oscrc: - try: - conf.get_config(override_conffile = oscrc) - except OSError, e: - if e.errno == 1: - # permission problem, should be the chmod(0600) issue - raise RuntimeError, 'Current user has no write permission for specified oscrc: %s' % oscrc - - raise # else - except urllib2.URLError: - raise errors.ObsError("invalid service apiurl: %s" % apiurl) - else: - conf.get_config() - - if apiurl: - self.apiurl = apiurl - else: - self.apiurl = conf.config['apiurl'] - - def getAPIServerList(self): - """getAPIServerList() -> list - - Get list of API servers configured in .oscrc - """ - apiservers = [] - for host in conf.config['api_host_options'].keys(): - apiurl = "%s://%s" % (conf.config['scheme'], host) - return apiservers - - # the following two alias api are added temporarily for compatible safe - def is_new_package(self, dst_project, dst_package): - return self.isNewPackage(dst_project, dst_package) - def gen_req_info(self, reqid, show_detail = True): - return self.genRequestInfo(reqid, show_detail) - - def isNewPackage(self, dst_project, dst_package): - """Check whether the dst pac is a new one - """ - - new_pkg = False - try: - core.meta_exists(metatype = 'pkg', - path_args = (core.quote_plus(dst_project), core.quote_plus(dst_package)), - create_new = False, - apiurl = self.apiurl) - except urllib2.HTTPError, e: - if e.code == 404: - new_pkg = True - else: - raise e - return new_pkg - - def isNewProject(self, project): - """Check whether the specified prject is a new one - """ - - new_prj = False - try: - core.meta_exists(metatype = 'prj', - path_args = (core.quote_plus(project)), - create_new = False, - apiurl = self.apiurl) - except urllib2.HTTPError, e: - if e.code == 404: - new_prj = True - else: - raise errors.ObsError("%s" % e) - except (urllib2.URLError, M2Crypto.m2urllib2.URLError, \ - M2Crypto.SSL.SSLError), e: - raise errors.ObsError("%s" % e) - return new_prj - - def genRequestInfo(self, reqid, show_detail = True): - """Generate formated diff info for request, - mainly used by notification mails from BOSS - """ - - def _gen_request_diff(): - """ Recommanded getter: request_diff can get req diff info even if req is accepted/declined - """ - reqdiff = '' - - try: - diff = core.request_diff(self.apiurl, reqid) - - try: - reqdiff += diff.decode('utf-8') - except UnicodeDecodeError: - try: - reqdiff += diff.decode('iso-8859-1') - except UnicodeDecodeError: - pass - - except (AttributeError, urllib2.HTTPError), e: - return None - - return reqdiff - - def _gen_server_diff(req): - """ Reserved getter: get req diff, if and only if the recommanded getter failed - """ - reqdiff = '' - - src_project = req.actions[0].src_project - src_package = req.actions[0].src_package - src_rev = req.actions[0].src_rev - try: - dst_project = req.actions[0].dst_project - dst_package = req.actions[0].dst_package - except AttributeError: - dst_project = req.actions[0].tgt_project - dst_package = req.actions[0].tgt_package - - # Check whether the dst pac is a new one - new_pkg = False - try: - core.meta_exists(metatype = 'pkg', - path_args = (core.quote_plus(dst_project), core.quote_plus(dst_package)), - create_new = False, - apiurl = self.apiurl) - except urllib2.HTTPError, e: - if e.code == 404: - new_pkg = True - else: - raise e - - if new_pkg: - src_fl = self.getSrcFileList(src_project, src_package, src_rev) - - spec_file = None - yaml_file = None - for f in src_fl: - if f.endswith(".spec"): - spec_file = f - elif f.endswith(".yaml"): - yaml_file = f - - reqdiff += 'This is a NEW package in %s project.\n' % dst_project - - reqdiff += 'The files in the new package:\n' - reqdiff += '%s/\n' % src_package - reqdiff += ' |__ ' + '\n |__ '.join(src_fl) - - if yaml_file: - reqdiff += '\n\nThe content of the YAML file, %s:\n' % (yaml_file) - reqdiff += '===================================================================\n' - reqdiff += self.getSrcFileContent(src_project, src_package, yaml_file, src_rev) - reqdiff += '\n===================================================================\n' - - if spec_file: - reqdiff += '\n\nThe content of the spec file, %s:\n' % (spec_file) - reqdiff += '===================================================================\n' - reqdiff += self.getSrcFileContent(src_project, src_package, spec_file, src_rev) - reqdiff += '\n===================================================================\n' - else: - reqdiff += '\n\nspec file NOT FOUND!\n' - - else: - try: - diff = core.server_diff(self.apiurl, - dst_project, dst_package, None, - src_project, src_package, src_rev, False) - - try: - reqdiff += diff.decode('utf-8') - except UnicodeDecodeError: - try: - reqdiff += diff.decode('iso-8859-1') - except UnicodeDecodeError: - pass - - except urllib2.HTTPError, e: - e.osc_msg = 'Diff not possible' - return '' - - return reqdiff - - #################################### - # function implementation start here - - req = core.get_request(self.apiurl, reqid) - try: - req.reviews = [] - reqinfo = unicode(req) - except UnicodeEncodeError: - reqinfo = u'' - - if show_detail: - diff = _gen_request_diff() - if diff is None: - diff = _gen_server_diff(req) - - reqinfo += diff - - # the result, in unicode string - return reqinfo - - def reqAccept(self, reqid, msg=''): - """ This method is called to accept a request - Success: return None - Failed: return string of error message - """ - - try: - core.change_request_state(self.apiurl, reqid, 'accepted', message=msg, supersed=None) - except Exception, e: - return str(e) - - return None - - def reqDecline(self, reqid, msg=''): - """ This method is called to decline a request - Success: return None - Failed: return string of error message - """ - - try: - core.change_request_state(self.apiurl, reqid, 'declined', message=msg, supersed=None) - except Exception, e: - return str(e) - - return None - - def reqRevoke(self, reqid, msg=''): - """ This method is called to revoke a request - Success: return None - Failed: return string of error message - """ - - try: - core.change_request_state(self.apiurl, reqid, 'revoked', message=msg, supersed=None) - except Exception, e: - return str(e) - - return None - - def reqReview(self, reqid, user='', group='', msg=''): - """ This method is called to add review msg to a request - Success: return None - Failed: return string of error message - """ - try: - query = { 'cmd': 'addreview' } - if user: - query['by_user'] = user - if group: - query['by_group'] = group - u = core.makeurl(self.apiurl, ['request', reqid], query=query) - f = core.http_POST(u, data=msg) - root = ElementTree.parse(f).getroot() - root.get('code') - except Exception, e: - return str(e) - - return None - - def getSrcFileList(self, project, package, revision=None): - """ get source file list of prj/pac - """ - - return core.meta_get_filelist(self.apiurl, project, package, expand=True, revision=revision) - - def getSrcFileContent(self, project, package, path, revision=None): - """ Cat remote file - """ - - rev = core.show_upstream_xsrcmd5(self.apiurl, project, package, revision=revision) - if rev: - query = { 'rev': rev } - else: - query = None - - u = core.makeurl(self.apiurl, ['source', project, package, core.pathname2url(path)], query=query) - - content = '' - for buf in core.streamfile(u, core.http_GET, core.BUFSIZE): - content += buf - - # return unicode str - return content.decode('utf8') - - def getSrcFileChecksum(self, project, package, path, revision=None): - """ getSrcFileChecksum(project, package, path, revision=None) -> string - returns source md5 of a source file - """ - - query = {} - query['expand'] = 1 - if revision: - query['rev'] = revision - - u = core.makeurl(self.apiurl, ['source', project, package], query=query) - f = core.http_GET(u) - root = ElementTree.parse(f).getroot() - - for node in root.findall('entry'): - if node.get('name') == path: - return node.get('md5') - - return None - - def getPackageChecksum(self, project, package, revision=None): - """ getPackageChecksum(project, package, revision=None) -> string - returns srcmd5 of a package - """ - - query = {} - query['expand'] = 1 - if revision: - query['rev'] = revision - - u = core.makeurl(self.apiurl, ['source', project, package], query=query) - f = core.http_GET(u) - root = ElementTree.parse(f).getroot() - - return root.get('srcmd5') - - def getLinkinfo(self, project, package, revision=None): - """ getLinkinfo(project, package, revision=None) -> (linked_prj, linked_pkg, linked_srcmd5) - returns link info of a prj/pkg - """ - - query = {} - query['expand'] = 1 - if revision: - query['rev'] = revision - - u = core.makeurl(self.apiurl, ['source', project, package], query=query) - f = core.http_GET(u) - root = ElementTree.parse(f).getroot() - - for node in root.findall('linkinfo'): - return (node.get('project'), node.get('package'), node.get('srcmd5')) - - return None - - def getUserData(self, user, *tags): - """getUserData() -> str - - Get the user data - """ - return core.get_user_data(self.apiurl, user, *tags) - - def getUserName(self): - """getUserName() -> str - - Get the user name associated with the current API server - """ - return conf.config['api_host_options'][self.apiurl]['user'] - - def getProjectList(self): - """getProjectList() -> list - - Get list of projects - """ - return [project for project in core.meta_get_project_list(self.apiurl) if project != 'deleted'] - - def getWatchedProjectList(self): - """getWatchedProjectList() -> list - - Get list of watched projects - """ - username = self.getUserName() - tree = ElementTree.fromstring(''.join(core.get_user_meta(self.apiurl, username))) - projects = [] - watchlist = tree.find('watchlist') - if watchlist: - for project in watchlist.findall('project'): - projects.append(project.get('name')) - homeproject = 'home:%s' % username - if not homeproject in projects and homeproject in self.getProjectList(): - projects.append(homeproject) - return projects - - def watchProject(self, project): - """ - watchProject(project) - - Watch project - """ - username = self.getUserName() - data = core.meta_exists('user', username, create_new=False, apiurl=self.apiurl) - url = core.make_meta_url('user', username, self.apiurl) - - person = ElementTree.fromstring(''.join(data)) - watchlist = person.find('watchlist') - if not watchlist: - watchlist = ElementTree.SubElement(person, 'watchlist') - ElementTree.SubElement(watchlist, 'project', name=str(project)) - - f = _Metafile(url, ElementTree.tostring(person)) - f.sync() - - def unwatchProject(self, project): - """ - watchProject(project) - - Watch project - """ - username = self.getUserName() - data = core.meta_exists('user', username, create_new=False, apiurl=self.apiurl) - url = core.make_meta_url('user', username, self.apiurl) - - person = ElementTree.fromstring(''.join(data)) - watchlist = person.find('watchlist') - for node in watchlist: - if node.get('name') == str(project): - watchlist.remove(node) - break - - f = _Metafile(url, ElementTree.tostring(person)) - f.sync() - - def getRepoState(self, project): - targets = {} - tree = ElementTree.fromstring(''.join(core.show_prj_results_meta(self.apiurl, project))) - for result in tree.findall('result'): - targets[('/'.join((result.get('repository'), result.get('arch'))))] = result.get('state') - return targets - - def getResults(self, project): - """getResults(project) -> (dict, list) - - Get results of a project. Returns (results, targets) - - results is a dict, with package names as the keys, and lists of result codes as the values - - targets is a list of targets, corresponding to the result code lists - """ - results = {} - targets = [] - tree = ElementTree.fromstring(''.join(core.show_prj_results_meta(self.apiurl, project))) - for result in tree.findall('result'): - targets.append('/'.join((result.get('repository'), result.get('arch')))) - for status in result.findall('status'): - package = status.get('package') - code = status.get('code') - if not package in results: - results[package] = [] - results[package].append(code) - return (results, targets) - - def getDiff(self, sprj, spkg, dprj, dpkg, rev): - diff = '' - diff += core.server_diff(self.apiurl, sprj, spkg, None, - dprj, dpkg, rev, False, True) - return diff - - def getTargets(self, project): - """ - getTargets(project) -> list - - Get a list of targets for a project - """ - targets = [] - tree = ElementTree.fromstring(''.join(core.show_project_meta(self.apiurl, project))) - for repo in tree.findall('repository'): - for arch in repo.findall('arch'): - targets.append('%s/%s' % (repo.get('name'), arch.text)) - return targets - - def getPackageStatus(self, project, package): - """ - getPackageStatus(project, package) -> dict - - Returns the status of a package as a dict with targets as the keys and status codes as the - values - """ - status = {} - tree = ElementTree.fromstring(''.join(core.show_results_meta(self.apiurl, project, package))) - for result in tree.findall('result'): - target = '/'.join((result.get('repository'), result.get('arch'))) - statusnode = result.find('status') - code = statusnode.get('code') - details = statusnode.find('details') - if details is not None: - code += ': ' + details.text - status[target] = code - return status - - def getProjectDiff(self, src_project, dst_project): - diffs = [] - - packages = self.getPackageList(src_project) - for src_package in packages: - diff = core.server_diff(self.apiurl, - dst_project, src_package, None, - src_project, src_package, None, False) - diffs.append(diff) - - return '\n'.join(diffs) - - def getPackageList(self, prj, deleted=None): - query = {} - if deleted: - query['deleted'] = 1 - - u = core.makeurl(self.apiurl, ['source', prj], query) - f = core.http_GET(u) - root = ElementTree.parse(f).getroot() - return [ node.get('name') for node in root.findall('entry') ] - - def getBinaryList(self, project, target, package): - """ - getBinaryList(project, target, package) -> list - - Returns a list of binaries for a particular target and package - """ - - (repo, arch) = target.split('/') - return core.get_binarylist(self.apiurl, project, repo, arch, package) - - def getBinary(self, project, target, package, file, path): - """ - getBinary(project, target, file, path) - - Get binary 'file' for 'project' and 'target' and save it as 'path' - """ - - (repo, arch) = target.split('/') - core.get_binary_file(self.apiurl, project, repo, arch, file, target_filename=path, package=package) - - def getBuildLog(self, project, target, package, offset=0): - """ - getBuildLog(project, target, package, offset=0) -> str - - Returns the build log of a package for a particular target. - - If offset is greater than 0, return only text after that offset. This allows live streaming - """ - - (repo, arch) = target.split('/') - u = core.makeurl(self.apiurl, ['build', project, repo, arch, package, '_log?nostream=1&start=%s' % offset]) - return core.http_GET(u).read() - - def getWorkerStatus(self): - """ - getWorkerStatus() -> list of dicts - - Get worker status as a list of dictionaries. Each dictionary contains the keys 'id', - 'hostarch', and 'status'. If the worker is building, the dict will additionally contain the - keys 'project', 'package', 'target', and 'starttime' - """ - - url = core.makeurl(self.apiurl, ['build', '_workerstatus']) - f = core.http_GET(url) - tree = ElementTree.parse(f).getroot() - workerstatus = [] - for worker in tree.findall('building'): - d = {'id': worker.get('workerid'), - 'status': 'building'} - for attr in ('hostarch', 'project', 'package', 'starttime'): - d[attr] = worker.get(attr) - d['target'] = '/'.join((worker.get('repository'), worker.get('arch'))) - d['started'] = time.asctime(time.localtime(float(worker.get('starttime')))) - workerstatus.append(d) - for worker in tree.findall('idle'): - d = {'id': worker.get('workerid'), - 'hostarch': worker.get('hostarch'), - 'status': 'idle'} - workerstatus.append(d) - return workerstatus - - def getWaitStats(self): - """ - getWaitStats() -> list - - Returns the number of jobs in the wait queue as a list of (arch, count) - pairs - """ - - url = core.makeurl(self.apiurl, ['build', '_workerstatus']) - f = core.http_GET(url) - tree = ElementTree.parse(f).getroot() - stats = [] - for worker in tree.findall('waiting'): - stats.append((worker.get('arch'), int(worker.get('jobs')))) - return stats - - def getSubmitRequests(self): - """ - getSubmitRequests() -> list of dicts - - """ - - url = core.makeurl(self.apiurl, ['search', 'request', '?match=submit']) - f = core.http_GET(url) - tree = ElementTree.parse(f).getroot() - submitrequests = [] - for sr in tree.findall('request'): - if sr.get('type') != "submit": - continue - - d = {'id': int(sr.get('id'))} - sb = sr.findall('submit')[0] - src = sb.findall('source')[0] - d['srcproject'] = src.get('project') - d['srcpackage'] = src.get('package') - dst = sb.findall('target')[0] - d['dstproject'] = dst.get('project') - d['dstpackage'] = dst.get('package') - d['state'] = sr.findall('state')[0].get('name') - - submitrequests.append(d) - submitrequests.sort(key=lambda x: x['id']) - return submitrequests - - def rebuild(self, project, package, target=None, code=None): - """ - rebuild(project, package, target, code=None) - - Rebuild 'package' in 'project' for 'target'. If 'code' is specified, - all targets with that code will be rebuilt - """ - - if target: - (repo, arch) = target.split('/') - else: - repo = None - arch = None - return core.rebuild(self.apiurl, project, package, repo, arch, code) - - def abortBuild(self, project, package=None, target=None): - """ - abort(project, package=None, target=None) - - Abort build of a package or all packages in a project - """ - - if target: - (repo, arch) = target.split('/') - else: - repo = None - arch = None - return core.abortbuild(self.apiurl, project, package, arch, repo) - - def getBuildHistory(self, project, package, target): - """ - getBuildHistory(project, package, target) -> list - - Get build history of package for target as a list of tuples of the form - (time, srcmd5, rev, versrel, bcnt) - """ - - (repo, arch) = target.split('/') - u = core.makeurl(self.apiurl, ['build', project, repo, arch, package, '_history']) - f = core.http_GET(u) - root = ElementTree.parse(f).getroot() - - r = [] - for node in root.findall('entry'): - rev = int(node.get('rev')) - srcmd5 = node.get('srcmd5') - versrel = node.get('versrel') - bcnt = int(node.get('bcnt')) - t = time.localtime(int(node.get('time'))) - t = time.strftime('%Y-%m-%d %H:%M:%S', t) - - r.append((t, srcmd5, rev, versrel, bcnt)) - return r - - def getCommitLog(self, project, package, revision=None): - """ - getCommitLog(project, package, revision=None) -> list - - Get commit log for package in project. If revision is set, get just the - log for that revision. - - Each log is a tuple of the form (rev, srcmd5, version, time, user, - comment) - """ - - u = core.makeurl(self.apiurl, ['source', project, package, '_history']) - f = core.http_GET(u) - root = ElementTree.parse(f).getroot() - - r = [] - revisions = root.findall('revision') - revisions.reverse() - for node in revisions: - rev = int(node.get('rev')) - if revision and rev != int(revision): - continue - srcmd5 = node.find('srcmd5').text - version = node.find('version').text - user = node.find('user').text - try: - comment = node.find('comment').text - except: - comment = '' - t = time.localtime(int(node.find('time').text)) - t = time.strftime('%Y-%m-%d %H:%M:%S', t) - - r.append((rev, srcmd5, version, t, user, comment)) - return r - - def getProjectMeta(self, project): - """ - getProjectMeta(project) -> string - - Get XML metadata for project - """ - - return ''.join(core.show_project_meta(self.apiurl, project)) - - def getProjectData(self, project, tag): - """ - getProjectData(project, tag) -> list - - Return a string list if node has text, else return the values dict list - """ - - data = [] - tree = ElementTree.fromstring(self.getProjectMeta(project)) - nodes = tree.findall(tag) - if nodes: - for node in nodes: - node_value = {} - for key in node.keys(): - node_value[key] = node.get(key) - - if node_value: - data.append(node_value) - else: - data.append(node.text) - - return data - - def getProjectPersons(self, project, role): - """ - getProjectPersons(project, role) -> list - - Return a userid list in this project with this role - """ - - userids = [] - persons = self.getProjectData(project, 'person') - for person in persons: - if person.has_key('role') and person['role'] == role: - userids.append(person['userid']) - - return userids - - def getProjectDevel(self, project): - """ - getProjectDevel(project) -> tuple (devel_prj, devel_pkg) - - Return the devel tuple of a project if it has the node, else return None - """ - - devels = self.getProjectData(project, 'devel') - for devel in devels: - if devel.has_key('project') and devel.has_key('package'): - return (devel['project'], devel['package']) - - return None - - def getProjectLink(self, project): - """ - getProjectLink(project) -> string - - Return the linked project of a project if it has the node, else return None - """ - - links = self.getProjectData(project, 'link') - for link in links: - if link.has_key('project'): - return link['project'] - - return None - - def deleteProject(self, project): - """ - deleteProject(project) - - Delete the specific project - """ - - try: - core.delete_project(self.apiurl, project) - except Exception: - return False - - return True - - def getPackageMeta(self, project, package): - """ - getPackageMeta(project, package) -> string - - Get XML metadata for package in project - """ - - return ''.join(core.show_package_meta(self.apiurl, project, package)) - - def getPackageData(self, project, package, tag): - """ - getPackageData(project, package, tag) -> list - - Return a string list if node has text, else return the values dict list - """ - - data = [] - tree = ElementTree.fromstring(self.getPackageMeta(project, package)) - nodes = tree.findall(tag) - if nodes: - for node in nodes: - node_value = {} - for key in node.keys(): - node_value[key] = node.get(key) - - if node_value: - data.append(node_value) - else: - data.append(node.text) - - return data - - def getPackagePersons(self, project, package, role): - """ - getPackagePersons(project, package, role) -> list - - Return a userid list in the package with this role - """ - - userids = [] - persons = self.getPackageData(project, package, 'person') - for person in persons: - if person.has_key('role') and person['role'] == role: - userids.append(person['userid']) - - return userids - - def getPackageDevel(self, project, package): - """ - getPackageDevel(project, package) -> tuple (devel_prj, devel_pkg) - - Return the devel tuple of a package if it has the node, else return None - """ - - devels = self.getPackageData(project, package, 'devel') - for devel in devels: - if devel.has_key('project') and devel.has_key('package'): - return (devel['project'], devel['package']) - - return None - - def deletePackage(self, project, package): - """ - deletePackage(project, package) - - Delete the specific package in project - """ - - try: - core.delete_package(self.apiurl, project, package) - except Exception: - return False - - return True - - def projectFlags(self, project): - """ - projectFlags(project) -> _ProjectFlags - - Return a _ProjectFlags object for manipulating the flags of project - """ - - return _ProjectFlags(self, project) - - def checkout(self, prj, pkg, rev='latest'): - """ checkout the package to current dir with link expanded - """ - - core.checkout_package(self.apiurl, prj, pkg, rev, prj_dir=prj, expand_link=True) - - def findPac(self, wd='.'): - """Get the single Package object for specified dir - the 'wd' should be a working dir for one single pac - """ - - if core.is_package_dir(wd): - return core.findpacs([wd])[0] - else: - return None - - def mkPac(self, prj, pkg): - """Create empty package for new one under CWD - """ - - core.make_dir(self.apiurl, prj, pkg, pathname = '.') - - pkg_path = os.path.join(prj, pkg) - shutil.rmtree(pkg_path, ignore_errors = True) - os.chdir(prj) - core.createPackageDir(pkg) - - def submit(self, msg, wd='.'): - if not core.is_package_dir(wd): - # TODO show some error message - return - - pac = core.findpacs([wd])[0] - prj = os.path.normpath(os.path.join(pac.dir, os.pardir)) - pac_path = os.path.basename(os.path.normpath(pac.absdir)) - files = {} - files[pac_path] = pac.todo - try: - core.Project(prj).commit(tuple([pac_path]), msg=msg, files=files) - except urllib2.HTTPError, e: - raise errors.ObsError('%s' % e) - core.store_unlink_file(pac.absdir, '_commit_msg') - - def branchPkg(self, src_project, src_package, rev=None, target_project=None, target_package=None): - """Create branch package from `src_project/src_package` - arguments: - rev: revision of src project/package - target_project: name of target proj, use default one if None - target_package: name of target pkg, use the same as asrc if None - """ - - if target_project is None: - target_project = 'home:%s:branches:%s' \ - % (conf.get_apiurl_usr(self.apiurl), src_project) - - if target_package is None: - target_package = src_package - - exists, targetprj, targetpkg, srcprj, srcpkg = \ - core.branch_pkg(self.apiurl, - src_project, - src_package, - rev=rev, - target_project=target_project, - target_package=target_package, - force=True) - - return (targetprj, targetpkg) - - def get_buildconfig(self, prj, repository): - return core.get_buildconfig(self.apiurl, prj, repository) - - def get_ArchitectureList(self, prj, target): - """ - return the list of Archictecture of the target of the projectObsName for a OBS server. - """ - url = core.makeurl(self.apiurl,['build', prj, target]) - f = core.http_GET(url) - if f == None: - return None - - aElement = ElementTree.fromstring(''.join(f.readlines())) - result = [] - for directory in aElement: - for entry in directory.getiterator(): - result.append(entry.get("name")) - - return result - - def get_results(self, prj, package): - try: - results = defaultdict(dict) - build_status = core.get_results(self.apiurl, prj, package) - for res in build_status: - repo, arch, status = res.split() - results[repo][arch] = status - return results - except (M2Crypto.m2urllib2.URLError, M2Crypto.SSL.SSLError), err: - raise errors.ObsError(str(err)) - - def get_buildlog(self, prj, package, repository, arch, offset = 0): - """prints out the buildlog on stdout""" - all_bytes = string.maketrans('', '') - remove_bytes = all_bytes[:10] + all_bytes[11:32] - try: - log = self.getBuildLog(prj, '%s/%s' % (repository, arch), package) - sys.stdout.write(log.translate(all_bytes, remove_bytes)) - except (M2Crypto.m2urllib2.URLError, M2Crypto.SSL.SSLError), err: - raise errors.ObsError(str(err)) diff --git a/gitbuildsys/cmd_remotebuild.py b/gitbuildsys/cmd_remotebuild.py index b53a9b1..6216022 100644 --- a/gitbuildsys/cmd_remotebuild.py +++ b/gitbuildsys/cmd_remotebuild.py @@ -20,16 +20,16 @@ """ import os -import tempfile import glob +import shutil import msger -from conf import configmgr -import buildservice -import obspkg import errors import utils +from conf import configmgr +from oscapi import OSC, OSCError + import gbp.rpm from gbp.scripts.buildpackage_rpm import main as gbp_build from gbp.rpm.git import GitRepositoryError, RpmGitRepository @@ -88,7 +88,7 @@ def do(opts, args): utils.gitStatusChecker(repo, opts) workdir = repo.path - tmpdir = os.path.join(workdir, 'packaging') + tmpdir = os.path.join(workdir, 'packaging', '.export') if not os.path.exists(tmpdir): os.makedirs(tmpdir) @@ -109,6 +109,7 @@ def do(opts, args): if not spec.name: msger.error('can\'t get correct name.') + package = spec.name if opts.base_obsprj is None: # TODO, get current branch of git to determine it @@ -133,59 +134,65 @@ def do(opts, args): tmpf = utils.Temp(dirn=tmpdir, prefix='.oscrc', content=oscrc) oscrcpath = tmpf.path - if opts.buildlog: - bs = buildservice.BuildService(apiurl=APISERVER, oscrc=oscrcpath) - archlist = [] - status = bs.get_results(target_prj, spec.name) - for build_repo in status.keys(): - for arch in status[build_repo]: - archlist.append('%-15s%-15s' % (build_repo, arch)) - if not obs_repo or not obs_arch or obs_repo not in status.keys() or \ - obs_arch not in status[obs_repo].keys(): - msger.error('no valid repo / arch specified for buildlog, '\ - 'valid arguments of repo and arch are:\n%s' % \ - '\n'.join(archlist)) - if status[obs_repo][obs_arch] not in ['failed', 'succeeded', \ - 'building']: - msger.error('build status of %s for %s/%s is %s, no build log.' % \ - (spec.name, obs_repo, obs_arch, status[obs_repo][obs_arch])) - bs.get_buildlog(target_prj, spec.name, obs_repo, obs_arch) - return 0 - - if opts.status: - bs = buildservice.BuildService(apiurl=APISERVER, oscrc=oscrcpath) - results = [] - status = bs.get_results(target_prj, spec.name) - for build_repo in status.keys(): - for arch in status[build_repo]: - stat = status[build_repo][arch] - results.append('%-15s%-15s%-15s' % (build_repo, arch, stat)) - msger.info('build results from build server:\n%s' % '\n'.join(results)) - return 0 - - prj = obspkg.ObsProject(target_prj, apiurl = APISERVER, oscrc = oscrcpath) - msger.info('checking status of obs project: %s ...' % target_prj) - if prj.is_new(): - # FIXME: How do you know that a certain user does not have permission to - # create any project, anywhewre? - if opts.target_obsprj and not target_prj.startswith('home:%s:' % USER): - msger.error('no permission to create project %s, only sub projects'\ - 'of home:%s are allowed ' % (target_prj, USER)) - msger.info('creating %s for package build ...' % target_prj) - prj.branch_from(base_prj) - - msger.info('checking out %s/%s to %s ...' % (target_prj, spec.name, tmpdir)) - - target_prj_path = os.path.join(tmpdir, target_prj) - if os.path.exists(target_prj_path) and \ - not os.access(target_prj_path, os.W_OK|os.R_OK|os.X_OK): - msger.error('No access permission to %s, please check' \ - % target_prj_path) - - localpkg = obspkg.ObsPackage(tmpdir, target_prj, spec.name, - APISERVER, oscrcpath) - oscworkdir = localpkg.get_workdir() - localpkg.remove_all() + api = OSC(APISERVER, oscrc=oscrcpath) + + try: + if opts.buildlog: + archlist = [] + status = api.get_results(target_prj, package) + + for build_repo in status.keys(): + for arch in status[build_repo]: + archlist.append('%-15s%-15s' % (build_repo, arch)) + if not obs_repo or not obs_arch or obs_repo not in status.keys() \ + or obs_arch not in status[obs_repo].keys(): + msger.error('no valid repo / arch specified for buildlog, '\ + 'valid arguments of repo and arch are:\n%s' % \ + '\n'.join(archlist)) + if status[obs_repo][obs_arch] not in ['failed', 'succeeded', + 'building', 'finishing']: + msger.error('build status of %s for %s/%s is %s, no build log.'\ + % (package, obs_repo, obs_arch, + status[obs_repo][obs_arch])) + msger.info('build log for %s/%s/%s/%s' % (target_prj, package, + obs_repo, obs_arch)) + print api.get_buildlog(target_prj, package, obs_repo, obs_arch) + + return 0 + + if opts.status: + results = [] + + status = api.get_results(target_prj, package) + + for build_repo in status.keys(): + for arch in status[build_repo]: + stat = status[build_repo][arch] + results.append('%-15s%-15s%-15s' % (build_repo, arch, stat)) + msger.info('build results from build server:\n%s' \ + % '\n'.join(results)) + return 0 + + msger.info('checking status of obs project: %s ...' % target_prj) + if not api.exists(target_prj): + # FIXME: How do you know that a certain user does not have + # permissions to create any project, anywhewre? + if opts.target_obsprj and \ + not target_prj.startswith('home:%s:' % USER): + msger.error('no permission to create project %s, only sub '\ + 'projects of home:%s are allowed ' % (target_prj, USER)) + + msger.info('copying settings of %s to %s' % (base_prj, target_prj)) + api.copy_project(base_prj, target_prj) + + if api.exists(target_prj, package): + msger.info('cleaning existing package') + api.remove_files(target_prj, package) + else: + msger.info('creating new package %s/%s' % (target_prj, package)) + api.create_package(target_prj, package) + except OSCError, err: + msger.error(str(err)) with utils.Workdir(workdir): if opts.commit: @@ -198,7 +205,7 @@ def do(opts, args): try: if gbp_build(["argv[0] placeholder", "--git-export-only", "--git-ignore-new", "--git-builder=osc", - "--git-export-dir=%s" % oscworkdir, + "--git-export-dir=%s" % tmpdir, "--git-packaging-dir=packaging", "--git-specfile=%s" % relative_spec, "--git-export=%s" % commit]): @@ -206,19 +213,22 @@ def do(opts, args): except GitRepositoryError, excobj: msger.error("Repository error: %s" % excobj) - localpkg.update_local() - try: commit_msg = repo.get_commit_info('HEAD')['subject'] - msger.info('commit packaging files to build server ...') - localpkg.commit (commit_msg) - except errors.ObsError, exc: - msger.error('commit packages fail: %s, please check the permission '\ - 'of target project:%s' % (exc,target_prj)) except GitRepositoryError, exc: msger.error('failed to get commit info: %s' % exc) + msger.info('commit packaging files to build server ...') + try: + api.commit_files(target_prj, package, + glob.glob("%s/*" % tmpdir), commit_msg) + except errors.ObsError, exc: + msger.error('commit packages fail: %s, please check the permission '\ + 'of target project:%s' % (exc, target_prj)) + + shutil.rmtree(tmpdir) + msger.info('local changes submitted to build server successfully') msger.info('follow the link to monitor the build progress:\n' ' %s/package/show?package=%s&project=%s' \ - % (APISERVER.replace('api', 'build'), spec.name, target_prj)) + % (APISERVER.replace('api', 'build'), package, target_prj)) diff --git a/gitbuildsys/obspkg.py b/gitbuildsys/obspkg.py deleted file mode 100644 index 8fa0090..0000000 --- a/gitbuildsys/obspkg.py +++ /dev/null @@ -1,193 +0,0 @@ -#!/usr/bin/python -tt -# vim: ai ts=4 sts=4 et sw=4 -# -# Copyright (c) 2012 Intel, Inc. -# -# This program is free software; you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the Free -# Software Foundation; version 2 of the License -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY -# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License -# for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., 59 -# Temple Place - Suite 330, Boston, MA 02111-1307, USA. - -from __future__ import with_statement -import os -import shutil -import buildservice -import runner -import msger -import errors -from utils import Workdir - -class ObsPackage(object): - """ Wrapper class of local package dir of OBS - """ - - def __init__(self, basedir, prj, pkg, apiurl=None, oscrc=None): - """Arguments: - basedir: the base local dir to store obs packages - prj: obs project - pkg: obs package - apiurl: optional, the api url of obs service - if not specified, the one from oscrc will be used - oscrc: optional, the path of customized oscrc - if not specified, ~/.oscrc will be used - """ - - if oscrc: - self._oscrc = oscrc - else: - self._oscrc = os.path.expanduser('~/.oscrc') - - self._bs = msger.PrintBufWrapper(buildservice.BuildService, #class - msger.verbose, # func_for_stdout - msger.warning, # func_for_stderr - apiurl, oscrc) # original args - - self._apiurl = self._bs.apiurl - - self._bdir = os.path.abspath(os.path.expanduser(basedir)) - self._prj = prj - self._pkg = pkg - self._pkgpath = os.path.join(self._bdir, prj, pkg) - - if not os.path.exists(self._bdir): - os.makedirs(self._bdir) - - with Workdir(self._bdir): - shutil.rmtree(prj, ignore_errors = True) - - if self._bs.isNewPackage(prj, pkg): - # to init new package in local dir - self._mkpac() - else: - # to checkout server stuff - self._checkout_latest() - - def _mkpac(self): - with Workdir(self._bdir): - self._bs.mkPac(self._prj, self._pkg) - - @msger.waiting - def _checkout_latest(self): - """ checkout the 'latest' revision of package with link expanded - """ - - with Workdir(self._bdir): - try: - self._bs.checkout(self._prj, self._pkg) - except buildservice.ObsError, err: - raise errors.ObsError(str(err)) - - def get_workdir(self): - return self._pkgpath - - def remove_all(self): - """Remove all files under pkg dir - """ - - with Workdir(self._pkgpath): - runner.quiet('/bin/rm -f *') - - def update_local(self): - """Do the similar work of 'osc addremove', - remove all deleted files and added all new files - """ - - with Workdir(self._pkgpath): - pac = self._bs.findPac() - # FIXME, if pac.to_be_added are needed to be considered. - pac.todo = list(set(pac.filenamelist + pac.filenamelist_unvers)) - for filename in pac.todo: - if os.path.isdir(filename): - continue - # ignore foo.rXX, foo.mine for files which are in 'C' state - if os.path.splitext(filename)[0] in pac.in_conflict: - continue - state = pac.status(filename) - if state == '?': - pac.addfile(filename) - elif state == '!': - pac.delete_file(filename) - - def add_file(self, fpath): - # copy the file in - runner.quiet('/bin/cp -f %s %s' % (fpath, self._pkgpath)) - - # add it into local pac - with Workdir(self._pkgpath): - pac = self._bs.findPac() - if pac: - pac.addfile(os.path.basename(fpath)) - else: - msger.warning('Invalid pac working dir, skip') - - @msger.waiting - def commit(self, msg): - with Workdir(self._pkgpath): - self._bs.submit(msg) - -class ObsProject(object): - """ Wrapper class of project in OBS - """ - - def __init__(self, prj, apiurl=None, oscrc=None): - """Arguments: - prj: name of obs project - apiurl: optional, the api url of obs service - if not specified, the one from oscrc will be used - oscrc: optional, the path of customized oscrc - if not specified, ~/.oscrc will be used - """ - - if oscrc: - self._oscrc = oscrc - else: - self._oscrc = os.path.expanduser('~/.oscrc') - - self._bs = buildservice.BuildService(apiurl, oscrc) - self._apiurl = self._bs.apiurl - self._prj = prj - - def is_new(self): - return self._bs.isNewProject(self._prj) - - def create(self): - """Create an empty project""" - # TODO - pass - - def branch_from(self, src_prj): - """Create a new branch project of `src_prj` - """ - - if self._bs.isNewProject(src_prj): - raise errors.ObsError('project: %s do not exists' % src_prj) - - if not self.is_new(): - msger.warning('branched project: %s exists' % self._prj) - return - - # pick the 1st valid package inside src prj FIXME - pkglist = self._bs.getPackageList(src_prj) - if len(pkglist) == 0: - raise errors.ObsError('base project %s is empty.' % src_prj) - dumb_pkg = pkglist[0] - - # branch out the new one - target_prj, target_pkg = self._bs.branchPkg(src_prj, dumb_pkg, - target_project = self._prj, - target_package = 'dumb_pkg') - - if target_prj != self._prj: - raise errors.ObsError('branched prj: %s is not the expected %s' \ - % (target_prj, self._prj)) - - # remove the dumb pkg - self._bs.deletePackage(target_prj, target_pkg) diff --git a/gitbuildsys/oscapi.py b/gitbuildsys/oscapi.py new file mode 100644 index 0000000..8fe04de --- /dev/null +++ b/gitbuildsys/oscapi.py @@ -0,0 +1,241 @@ +#!/usr/bin/python -tt +# vim: ai ts=4 sts=4 et sw=4 +# +# Copyright (c) 2012 Intel, Inc. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation; version 2 of the License +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., 59 +# Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +""" +This module provides wrapper class around OSC API. +Only APIs which are required by cmd_remotebuild present here. + +""" + +import os +import urllib2 +import M2Crypto +import ssl + +from collections import defaultdict +from urllib import quote_plus, pathname2url + +import msger +import errors +from utils import hexdigest + +from osc import conf, core + + +class OSCError(Exception): + """Local exception class.""" + pass + + +class OSC(object): + """Interface to OSC API""" + + def __init__(self, apiurl=None, oscrc=None): + if oscrc: + try: + conf.get_config(override_conffile = oscrc) + except OSError, err: + if err.errno == 1: + # permission problem, should be the chmod(0600) issue + raise RuntimeError('Current user has no write permission '\ + 'for specified oscrc: %s' % oscrc) + + raise # else + except urllib2.URLError: + raise errors.ObsError("invalid service apiurl: %s" % apiurl) + else: + conf.get_config() + + if apiurl: + self.apiurl = apiurl + else: + self.apiurl = conf.config['apiurl'] + + @staticmethod + def core_http(method, url, data=None, filep=None): + """Wrapper above core. to catch exceptions.""" + try: + return method(url, data=data, file=filep) + except (urllib2.URLError, M2Crypto.m2urllib2.URLError, + M2Crypto.SSL.SSLError, ssl.SSLError), err: + raise OSCError(str(err)) + + def copy_project(self, src, target, rewrite=False): + """ + Create new OBS project based on existing project. + Copy config and repositories from src project to target + """ + + if self.exists(target): + msger.warning('target project: %s exists' % target) + if rewrite: + msger.warning('rewriting target project %s' % target) + else: + return + + # Create target meta + meta = ''\ + '' % \ + (target, conf.get_apiurl_usr(self.apiurl)) + + # Collect source repos + repos = defaultdict(list) + for repo in core.get_repos_of_project(self.apiurl, src): + repos[repo.name].append(repo.arch) + + # Copy repos to target + for name in repos: + meta += '' % name + meta += '' % (src, name) + for arch in repos[name]: + meta += "%s\n" % arch + meta += "\n" + meta += "\n" + + try: + # Create project and set its meta + core.edit_meta('prj', path_args=quote_plus(target), data=meta) + except (urllib2.URLError, M2Crypto.m2urllib2.URLError, + M2Crypto.SSL.SSLError), err: + raise OSCError("Can't set meta for %s: %s" % (target, str(err))) + + # copy project config + try: + config = core.show_project_conf(self.apiurl, src) + except (urllib2.URLError, M2Crypto.m2urllib2.URLError, + M2Crypto.SSL.SSLError), err: + raise OSCError("Can't get config from project %s: %s" \ + % (src, str(err))) + + url = core.make_meta_url("prjconf", quote_plus(target), + self.apiurl, False) + try: + self.core_http(core.http_PUT, url, data=config) + except OSCError, err: + raise OSCError("can't copy config from %s to %s: %s" \ + % (src, target, err)) + + def exists(self, prj, pkg=''): + """Check if project or package exists.""" + + metatype = 'prj' + path_args = [core.quote_plus(prj)] + if pkg: + metatype = 'pkg' + path_args.append(core.quote_plus(pkg)) + err = None + try: + core.meta_exists(metatype = metatype, path_args = tuple(path_args), + create_new = False, apiurl = self.apiurl) + except urllib2.HTTPError, err: + if err.code == 404: + return False + except (urllib2.URLError, M2Crypto.m2urllib2.URLError, \ + M2Crypto.SSL.SSLError), err: + pass + if err: + raise OSCError("can't check if %s/%s exists: %s" % (prj, pkg, err)) + + return True + + def commit_files(self, prj, pkg, files, message): + """Commits files to OBS.""" + + query = {'cmd' : 'commitfilelist', + 'user' : conf.get_apiurl_usr(self.apiurl), + 'comment': message} + url = core.makeurl(self.apiurl, ['source', prj, pkg], query=query) + + xml = "" + for fpath in files: + with open(fpath) as fhandle: + xml += '' % \ + (os.path.basename(fpath), hexdigest(fhandle)) + xml += "" + + try: + self.core_http(core.http_POST, url, data=xml) + for fpath in files: + put_url = core.makeurl( + self.apiurl, ['source', prj, pkg, + pathname2url(os.path.basename(fpath))], + query="rev=repository") + self.core_http(core.http_PUT, put_url, filep=fpath) + self.core_http(core.http_POST, url, data=xml) + except OSCError, err: + raise OSCError("can't commit files to %s/%s: %s" % (prj, pkg, err)) + + def remove_files(self, prj, pkg, fnames=None): + """ + Remove file[s] from the package. + If filenames are not provided remove all files. + """ + if not fnames: + url = core.makeurl(self.apiurl, ['source', prj, pkg]) + fnames = [entry.get('name') for entry in \ + core.ET.fromstring(core.http_GET(url).read())] + for fname in fnames: + query = 'rev=upload' + url = core.makeurl(self.apiurl, + ['source', prj, pkg, pathname2url(fname)], + query=query) + try: + self.core_http(core.http_DELETE, url) + except OSCError, err: + raise OSCError("can\'t remove files from %s/%s: %s" \ + % (prj, pkg, err)) + + def create_package(self, prj, pkg): + """Create package in the project.""" + + meta = ''\ + '<description/></package>' % (prj, pkg) + url = core.make_meta_url("pkg", (quote_plus(prj), quote_plus(pkg)), + self.apiurl, False) + try: + self.core_http(core.http_PUT, url, data=meta) + except OSCError, err: + raise OSCError("can't create %s/%s: %s" % (prj, pkg, err)) + + def get_results(self, prj, pkg): + """Get package build results.""" + results = defaultdict(dict) + try: + build_status = core.get_results(self.apiurl, prj, pkg) + except (urllib2.URLError, M2Crypto.m2urllib2.URLError, + M2Crypto.SSL.SSLError), err: + raise OSCError("can't get %s/%s build results: %s" \ + % (prj, pkg, str(err))) + + for res in build_status: + repo, arch, status = res.split() + results[repo][arch] = status + return results + + def get_buildlog(self, prj, pkg, repo, arch): + """Get package build log from OBS.""" + + url = core.makeurl(self.apiurl, ['build', prj, repo, arch, pkg, + '_log?nostream=1&start=0']) + try: + log = self.core_http(core.http_GET, url).read() + except OSCError, err: + raise OSCError("can't get %s/%s build log: %s" % (prj, pkg, err)) + + return log.translate(None, "".join([chr(i) \ + for i in range(10) + range(11,32)])) diff --git a/gitbuildsys/utils.py b/gitbuildsys/utils.py index 92e73ec..7774748 100644 --- a/gitbuildsys/utils.py +++ b/gitbuildsys/utils.py @@ -22,6 +22,7 @@ import tempfile import shutil import pycurl import urlparse +import hashlib # cElementTree can be standard or 3rd-party depending on python version try: @@ -330,3 +331,14 @@ def gitStatusChecker(git, opts): if uncommitted_files: msger.info('the following uncommitted changes would be included'\ ':\n %s' % '\n '.join(uncommitted_files)) + +def hexdigest(fhandle, block_size=4096): + """Calculates hexdigest of file content.""" + md5obj = hashlib.new('md5') + while True: + data = fhandle.read(block_size) + if not data: + break + md5obj.update(data) + return md5obj.hexdigest() + -- 2.7.4