From: Ed Bartosh Date: Tue, 17 Jul 2012 08:17:56 +0000 (+0300) Subject: Used --repo build options instead of GBS_BUILD_REPOAUTH hack X-Git-Tag: 0.9~34 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=7e3ccb55bd645e461baf712b5a6436f3542e0ed8;p=tools%2Fgbs.git Used --repo build options instead of GBS_BUILD_REPOAUTH hack This change uses repox.* config options from 'build' section to prepare proper repository urls, including auth info. Note that auth info is also allowed in repo urls, so repo1.url = 'https://user:password@host.com/' is valid and repo1.user/repo1.password are not required in this case. Change-Id: I464f86f4fffd05a60a134af7a838255263a23fae --- diff --git a/gitbuildsys/cmd_build.py b/gitbuildsys/cmd_build.py index ea3c640..4c015ae 100644 --- a/gitbuildsys/cmd_build.py +++ b/gitbuildsys/cmd_build.py @@ -20,11 +20,11 @@ """ import os -import subprocess -import urlparse import re +import subprocess import tempfile -import base64 +import urllib2 +from urlparse import urlsplit, urlunsplit import msger import utils @@ -170,60 +170,67 @@ def get_env_proxies(): return proxies -def get_reops_conf(): +def get_repos_conf(): + """ + Make list of urls using repox.url, repox.user and repox.passwd + configuration file parameters from 'build' section. + Validate configuration parameters. + """ - repos = set() + repos = {} # get repo settings form build section for opt in configmgr.options('build'): if opt.startswith('repo'): try: - (name, key) = opt.split('.') + key, name = opt.split('.') except ValueError: raise errors.ConfigError("invalid repo option: %s" % opt) - else: - if key not in ('url', 'user', 'passwdx'): - raise errors.ConfigError("invalid repo option: %s" % opt) - repos.add(name) - # get repo settings form build section - repo_urls = [] - repo_auths = set() - for repo in repos: - repo_auth = '' - # get repo url - try: - repo_url = configmgr.get(repo + '.url', 'build') - repo_urls.append(repo_url) - except: - continue + if name not in ('url', 'user', 'passwdx'): + raise errors.ConfigError("invalid repo option: %s" % opt) - try: - repo_server = re.match('(https?://.*?)/.*', repo_url).groups()[0] - except AttributeError: - if repo_url.startswith('/') and os.path.exists(repo_url): - repo_server = repo_url + if key not in repos: + repos[key] = {} + + if name in repos[key]: + raise errors.ConfigError('Duplicate entry %s' % opt) + + value = configmgr.get(opt, 'build') + if name == 'passwdx': + try: + value = value.decode('base64').decode('bz2') + except (TypeError, IOError), err: + raise errors.ConfigError('Error decoding %s: %s' % \ + (opt, err)) + repos[key]['passwd'] = urllib2.quote(value, safe='') else: - raise errors.ConfigError("Invalid repo url: %s" % repo_url) - repo_auth = 'url' + ':' + repo_server + ';' + repos[key][name] = value - valid = True - for key in ['user', 'passwdx']: - try: - value = configmgr.get(repo+'.'+ key, 'build').strip() - except: - valid = False - break - if not value: - valid = False - break - repo_auth = repo_auth + key + ':' + value + ';' - - if not valid: - continue - repo_auth = repo_auth[:-1] - repo_auths.add(repo_auth) - - return repo_urls, ' '.join(repo_auths) + result = [] + for key, item in repos.iteritems(): + if 'url' not in item: + raise errors.ConfigError("Url is not specified for %s" % key) + + splitted = urlsplit(item['url']) + if splitted.username and item['user'] or \ + splitted.password and item['passwd']: + raise errors.ConfigError("Auth info specified twice for %s" % key) + + # Get auth info from the url or urlx.user and urlx.pass + user = item.get('user') or splitted.username + passwd = item.get('passwd') or splitted.password + + if (user and not passwd) or (passwd and not user): + raise errors.ConfigError("Both user and password " + "has to be specified for %s" % key) + + splitted_list = list(splitted) + splitted_list[1] = "%s:%s@%s" % (urllib2.quote(user, safe=''), passwd, + splitted.hostname) + + result.append(urlunsplit(splitted_list)) + + return result def do(opts, args): @@ -301,53 +308,6 @@ def do(opts, args): if opts.debuginfo: cmd += ['--debug'] - repos_urls_conf, repo_auth_conf = get_reops_conf() - - repos = {} - if opts.repositories: - for repourl in opts.repositories: - (scheme, host, path, parm, query, frag) = \ - urlparse.urlparse(repourl.rstrip('/') + '/') - repos[repourl] = {} - if '@' in host: - try: - user_pass, host = host.rsplit('@', 1) - except ValueError: - raise errors.ConfigError('Bad URL: %s' % repourl) - userpwd = user_pass.split(':', 1) - repos[repourl]['user'] = userpwd[0] - if len(userpwd) == 2: - repos[repourl]['passwd'] = userpwd[1] - else: - repos[repourl]['passwd'] = None - else: - repos[repourl]['user'] = None - repos[repourl]['passwd'] = None - - elif repos_urls_conf: - for url in repos_urls_conf: - repos[url] = {} - if repo_auth_conf: - repo_auth = {} - for item in repo_auth_conf.split(';'): - key, val = item.split(':', 1) - if key == 'passwdx': - key = 'passwd' - try: - val = base64.b64decode(val).decode('bz2') - except (TypeError, IOError), err: - raise errors.ConfigError('passwdx: %s' % err) - repo_auth[key] = val - if 'user' in repo_auth: - repos[url]['user'] = repo_auth['user'] - if 'passwd' in repo_auth: - repos[url]['passwd'] = repo_auth['passwd'] - else: - repos[url]['passwd'] = None - repos[url]['user'] = None - else: - msger.error('No package repository specified.') - if opts.noinit: cmd += ['--no-init'] else: @@ -356,13 +316,26 @@ def do(opts, args): if not os.path.exists(cachedir): os.makedirs(cachedir) msger.info('generate repositories ...') + + repos_urls_conf = get_repos_conf() + if opts.repositories: + repos = opts.repositories + elif repos_urls_conf: + repos = repos_urls_conf + else: + msger.error('No package repository specified.') + repoparser = utils.RepoParser(repos, cachedir) repourls = repoparser.get_repos_by_arch(buildarch) if not repourls: - msger.error('no repositories found for arch: %s under the ' \ - 'following repos:\n %s' % (buildarch, \ - '\n'.join(repos.keys()))) + msger.error('no repositories found for arch: %s under the '\ + 'following repos:\n %s' % \ + (buildarch, '\n'.join(repos))) for url in repourls: + if not re.match('https?://.*?/.*', url) and \ + not (url.startswith('/') and os.path.exists(url)): + msger.error("Invalid repo url: %s" % url) + cmd += ['--repository=%s' % url] if opts.dist: @@ -373,8 +346,9 @@ def do(opts, args): msger.info('failed to get build conf, use default build conf') distconf = configmgr.get('distconf', 'build') else: - msger.info('build conf has been downloaded at:\n %s' - % distconf) + msger.info('build conf has been downloaded at:\n %s' \ + % distconf) + if distconf is None: msger.error('No build config file specified, please specify in '\ '~/.gbs.conf or command line using -D') @@ -443,11 +417,8 @@ def do(opts, args): (spec.name, spec.version)] # if current user is root, don't run with sucmd - if os.getuid() == 0: - os.environ['GBS_BUILD_REPOAUTH'] = repo_auth_conf - else: - cmd = ['sudo'] + proxies + ['GBS_BUILD_REPOAUTH=%s' % \ - repo_auth_conf ] + cmd + if os.getuid() != 0: + cmd = ['sudo'] + proxies + cmd # runner.show() can't support interactive mode, so use subprocess insterad. msger.debug("running command %s" % cmd) diff --git a/gitbuildsys/utils.py b/gitbuildsys/utils.py index e2b9aa8..a28c13a 100644 --- a/gitbuildsys/utils.py +++ b/gitbuildsys/utils.py @@ -210,9 +210,7 @@ class RepoParser(object): repomd_url = os.path.join(repo, 'repodata/repomd.xml') repomd_file = os.path.join(self.cachedir, 'repomd.xml') try: - urlgrab(repomd_url, repomd_file, - self.repos[baseurl]['user'],\ - self.repos[baseurl]['passwd']) + urlgrab(repomd_url, repomd_file) validrepos.append(repo) except errors.UrlError: pass @@ -221,20 +219,20 @@ class RepoParser(object): def parse(self): for repo in self.repos: + # Check if repo is standard repo with repodata/repomd.xml exist repomd_url = os.path.join(repo, 'repodata/repomd.xml') repomd_file = os.path.join(self.cachedir, 'repomd.xml') + try: - urlgrab(repomd_url, repomd_file, self.repos[repo]['user'], \ - self.repos[repo]['passwd']) + urlgrab(repomd_url, repomd_file) # Try to download build.xml buildxml_url = urlparse.urljoin(repo.rstrip('/') + '/', \ '../../../../builddata/build.xml') self.buildmeta = os.path.join(self.cachedir, \ os.path.basename(buildxml_url)) - urlgrab(buildxml_url, self.buildmeta, \ - self.repos[repo]['user'], \ - self.repos[repo]['passwd']) + urlgrab(buildxml_url, self.buildmeta) + # Try to download build conf if self.buildconf is None: build_conf = self.get_buildconf() @@ -242,9 +240,7 @@ class RepoParser(object): (buildxml_url), build_conf) self.buildconf = os.path.join(self.cachedir, \ os.path.basename(buildconf_url)) - urlgrab(buildconf_url, self.buildconf, \ - self.repos[repo]['user'],\ - self.repos[repo]['passwd']) + urlgrab(buildconf_url, self.buildconf) # buildconf downloaded succeed, break! break except errors.UrlError: @@ -259,8 +255,7 @@ class RepoParser(object): buildxml_url = os.path.join(repo, 'builddata/build.xml') self.buildmeta = os.path.join(self.cachedir, 'build.xml') try: - urlgrab(buildxml_url, self.buildmeta, self.repos[repo]['user'],\ - self.repos[repo]['passwd']) + urlgrab(buildxml_url, self.buildmeta) except errors.UrlError: self.buildmeta = None continue @@ -275,9 +270,8 @@ class RepoParser(object): 'builddata/%s' % build_conf) self.buildconf = os.path.join(self.cachedir, \ os.path.basename(buildconf_url)) - urlgrab(buildconf_url, self.buildconf, \ - self.repos[repo]['user'], \ - self.repos[repo]['passwd']) + urlgrab(buildconf_url, self.buildconf) + except errors.UrlError: self.buildconf = None @@ -289,7 +283,7 @@ class RepoParser(object): def get_repos_by_arch(self, arch): # return directly for standard repos if not self.repourls: - return self.repos.keys() + self.localrepos + return self.repos + self.localrepos if arch in ['ia32', 'i686', 'i586']: arch = 'ia32'