From: Zhang Qiang Date: Tue, 26 Jun 2012 13:41:55 +0000 (+0800) Subject: parse repos and fetch correct build conf from repos X-Git-Tag: 0.7.1~10 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=75e781143bc36a3942ea29f272fabfba0e6769f2;p=tools%2Fgbs.git parse repos and fetch correct build conf from repos If no --dist specified, gbs would try to get build config from repos. Currently, the position of build config is located at builddata directory, and build.xml is the index file of build config file. If no build config file found from repos, build config from ~/.gbs.conf would be used. With this patch, a new format of repo is acceptable, which contains 'builddata/build.xml', which contains repos and archs info, also build conf can be avaliable from this file. --- diff --git a/distfiles/debian/control b/distfiles/debian/control index e5c1814..44ebf48 100644 --- a/distfiles/debian/control +++ b/distfiles/debian/control @@ -16,7 +16,8 @@ Depends: ${misc:Depends}, ${python:Depends}, qemu-arm-static (>= 0.14.1) | qemu-user-static, binfmt-support, sudo, - git-buildpackage-rpm + git-buildpackage-rpm, + python-pycurl Description: The command line tools for Tizen package developers The command line tools for Tizen package developers will be used to do packaging related tasks. diff --git a/distfiles/gbs.spec b/distfiles/gbs.spec index 0223631..c2b7cf4 100644 --- a/distfiles/gbs.spec +++ b/distfiles/gbs.spec @@ -9,6 +9,7 @@ BuildArch: noarch URL: http://www.tizen.org Source0: %{name}-%{version}.tar.bz2 Requires: python >= 2.5 +Requires: python-pycurl Requires: git-core Requires: osc >= 0.131 Requires: build >= 2011.10.10 diff --git a/gitbuildsys/cmd_build.py b/gitbuildsys/cmd_build.py index 669d4f8..b2eaf5d 100644 --- a/gitbuildsys/cmd_build.py +++ b/gitbuildsys/cmd_build.py @@ -25,6 +25,7 @@ import subprocess import urlparse import re import tempfile +import base64 import msger import utils @@ -168,6 +169,7 @@ def get_env_proxies(): proxies.append('%s=%s' % (name, value)) return proxies + def get_reops_conf(): repos = set() @@ -256,9 +258,6 @@ def do(opts, args): msger.error('no spec file found under /packaging sub-directory') specfile = utils.guess_spec(workdir, opts.spec) - distconf = configmgr.get('distconf', 'build') - if opts.dist: - distconf = opts.dist build_cmd = configmgr.get('build_cmd', 'build') build_root = configmgr.get('build_root', 'build') @@ -266,7 +265,6 @@ def do(opts, args): build_root = opts.buildroot cmd = [ build_cmd, '--root='+build_root, - '--dist='+distconf, '--arch='+buildarch ] build_jobs = get_processors() @@ -279,15 +277,76 @@ def do(opts, args): repos_urls_conf, repo_auth_conf = get_reops_conf() + repos = {} if opts.repositories: for repo in opts.repositories: - cmd += ['--repository='+repo] + (scheme, host, path, parm, query, frag) = \ + urlparse.urlparse(repo.rstrip('/') + '/') + repos[repo] = {} + if '@' in host: + try: + user_pass, host = host.split('@', 1) + except ValueError, e: + raise errors.ConfigError('Bad URL: %s' % repo) + userpwd = user_pass.split(':', 1) + repos[repo]['user'] = userpwd[0] + if len(userpwd) == 2: + repos[repo]['passwd'] = userpwd[1] + else: + repos[repo]['passwd'] = None + else: + repos[repo]['user'] = None + repos[repo]['passwd'] = None + elif repos_urls_conf: for url in repos_urls_conf: - cmd += ['--repository=' + url ] + repos[url] = {} + if repo_auth_conf: + repo_auth = {} + for item in repo_auth_conf.split(';'): + key, val = item.split(':', 1) + if key == 'passwdx': + key = 'passwd' + val = base64.b64decode(val).decode('bz2') + repo_auth[key] = val + if 'user' in repo_auth: + repos[url]['user'] = repo_auth['user'] + if 'passwd' in repo_auth: + repos[url]['passwd'] = repo_auth['passwd'] + else: + repos[url]['passwd'] = None + repos[url]['user'] = None else: msger.error('No package repository specified.') + cachedir = os.path.join(configmgr.get('tmpdir'), 'gbscache') + if not os.path.exists(cachedir): + os.makedirs(cachedir) + msger.info('generate repositories ...') + repoparser = utils.RepoParser(repos, cachedir) + repourls = repoparser.get_repos_by_arch(buildarch) + if not repourls: + msger.error('no repositories found for arch: %s under the following '\ + 'repos:\n %s' % (buildarch, '\n'.join(repos.keys()))) + for url in repourls: + cmd += ['--repository=%s' % url] + + if opts.dist: + distconf = opts.dist + else: + distconf = repoparser.buildconf + if distconf is None: + msger.info('failed to get build conf, use default build conf') + distconf = configmgr.get('distconf', 'build') + else: + msger.info('build conf has been downloaded at:\n %s\n '\ + 'you can save it and use -D to specify it, which can '\ + 'prevent downloading it everytime ' % distconf) + + if distconf is None: + msger.error('No build config file specified, please specify in '\ + '~/.gbs.conf or command line using -D') + cmd += ['--dist=%s' % distconf] if opts.noinit: cmd += ['--no-init'] if opts.ccache: diff --git a/gitbuildsys/errors.py b/gitbuildsys/errors.py index 4995984..ea6b8e8 100644 --- a/gitbuildsys/errors.py +++ b/gitbuildsys/errors.py @@ -65,3 +65,5 @@ class GBSError(Exception): def __str__(self): return self.msg +class UrlError(CmdError): + keyword = '' diff --git a/gitbuildsys/utils.py b/gitbuildsys/utils.py index 3c2e91a..130895b 100644 --- a/gitbuildsys/utils.py +++ b/gitbuildsys/utils.py @@ -20,7 +20,16 @@ import os import glob import tempfile import shutil +import pycurl +import urlparse +# cElementTree can be standard or 3rd-party depending on python version +try: + from xml.etree import cElementTree as ET +except ImportError: + import cElementTree as ET + +import errors import msger class Workdir(object): @@ -82,3 +91,162 @@ class TempCopy(object): def __del__(self): if os.path.exists(self.name): os.unlink(self.name) + +def urlgrab(url, filename, user = None, passwd = None): + + outfile = open(filename, 'w') + curl = pycurl.Curl() + curl.setopt(pycurl.URL, url) + curl.setopt(pycurl.WRITEDATA, outfile) + curl.setopt(pycurl.FAILONERROR, True) + curl.setopt(pycurl.FOLLOWLOCATION, True) + curl.setopt(pycurl.SSL_VERIFYPEER, False) + curl.setopt(pycurl.SSL_VERIFYHOST, False) + if user: + userpwd = user + if passwd: + userpwd = '%s:%s' % (user, passwd) + curl.setopt(pycurl.USERPWD, userpwd) + + try: + curl.perform() + except pycurl.error, e: + errcode = e.args[0] + if errcode == pycurl.E_OPERATION_TIMEOUTED: + raise errors.UrlError('timeout on %s: %s' % (self.url, e)) + elif errcode == pycurl.E_FILESIZE_EXCEEDED: + raise errors.UrlError('max download size exceeded on %s'\ + % self.url) + else: + errmsg = 'pycurl error %s - "%s"' % (errcode, str(e.args[1])) + raise errors.UrlError(errmsg) + finally: + outfile.close() + curl.close() + + return filename + +class RepoParser(object): + """ Repository parser for generate real repourl and build config + """ + def __init__(self, repos, cachedir): + self.repos = repos + self.cachedir = cachedir + self.archs = [] + self.localrepos = [] + self.repourls = {} + self.buildmeta = None + self.buildconf = None + self.parse() + + def get_buildconf(self): + elementTree = ET.parse(self.buildmeta) + root = elementTree.getroot() + buildElem = root.find('buildconf') + if buildElem is None: + return None + buildconf = buildElem.text.strip() + + return buildconf + + def build_repos_from_buildmeta(self, baseurl): + if not (self.buildmeta and os.path.exists(self.buildmeta)): + return + + elementTree = ET.parse(self.buildmeta) + root = elementTree.getroot() + archs = [] + repos = [] + repo_items = root.find('repos') + if repo_items: + for repo in repo_items.findall('repo'): + repos.append(repo.text.strip()) + arch_items = root.find('archs') + if arch_items: + for arch in arch_items.findall('arch'): + archs.append(arch.text.strip()) + for arch in archs: + repourls = [os.path.join(baseurl, 'repos', repo, arch, 'packages') \ + for repo in repos] + self.repourls[arch] = repourls + self.archs = archs + + def parse(self): + for repo in self.repos: + # Check if repo is standard repo with repodata/repomd.xml exist + repomd_url = os.path.join(repo, 'repodata/repomd.xml') + repomd_file = os.path.join(self.cachedir, 'repomd.xml') + try: + urlgrab(repomd_url, repomd_file, self.repos[repo]['user'], \ + self.repos[repo]['passwd']) + # Try to download build.xml + buildxml_url = urlparse.urljoin(repo.rstrip('/') + '/', \ + '../../../../builddata/build.xml') + self.buildmeta = os.path.join(self.cachedir, \ + os.path.basename(buildxml_url)) + urlgrab(buildxml_url, self.buildmeta, \ + self.repos[repo]['user'], \ + self.repos[repo]['passwd']) + # Try to download build conf + if self.buildconf is None: + build_conf = self.get_buildconf() + buildconf_url = buildxml_url.replace(os.path.basename \ + (buildxml_url), build_conf) + self.buildconf = os.path.join(self.cachedir, \ + os.path.basename(buildconf_url)) + urlgrab(buildconf_url, self.buildconf, \ + self.repos[repo]['user'],\ + self.repos[repo]['passwd']) + # buildconf downloaded succeed, break! + break + except errors.UrlError: + # if it's standard repo, that means buildconf fails to be + # downloaded, so reset buildconf and break + if self.buildmeta: + self.buildconf = None + break + pass + + # Check if it's repo with builddata/build.xml exist + buildxml_url = os.path.join(repo, 'builddata/build.xml') + self.buildmeta = os.path.join(self.cachedir, 'build.xml') + try: + urlgrab(buildxml_url, self.buildmeta, self.repos[repo]['user'],\ + self.repos[repo]['passwd']) + except errors.UrlError: + self.buildmeta = None + continue + + # Generate repos from build.xml + self.build_repos_from_buildmeta(repo) + + try: + # download build conf + build_conf = self.get_buildconf() + buildconf_url = urlparse.urljoin(repo.rstrip('/') + '/', \ + 'builddata/%s' % build_conf) + self.buildconf = os.path.join(self.cachedir, \ + os.path.basename(buildconf_url)) + urlgrab(buildconf_url, self.buildconf, \ + self.repos[repo]['user'], \ + self.repos[repo]['passwd']) + except errors.UrlError: + self.buildconf = None + + # Split out local repo + for repo in self.repos: + if repo.startswith('/') and os.path.exists(repo): + self.localrepos.append(repo) + + def get_repos_by_arch(self, arch): + # return directly for standard repos + if not self.repourls: + return self.repos.keys() + self.localrepos + + if arch in ['ia32', 'i686', 'i586']: + arch = 'ia32' + + if arch in self.repourls: + return self.repourls[arch] + self.localrepos + + return None