From: Zhang Qiang Date: Thu, 18 Apr 2013 02:36:57 +0000 (+0800) Subject: download ks files from remote repo and update it with local repo X-Git-Tag: 0.16~21 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=7ccd24227d88a06551032ae1edc996690a9f7b49;p=tools%2Fgbs.git download ks files from remote repo and update it with local repo KS files are downloaded from remote repo, and local repo will be added to involve building image. The authorization info will be added to exist repo Change-Id: I7d7af07426cbd3893df44c74e45efa69ee5d880e --- diff --git a/debian/gbs-api.install b/debian/gbs-api.install index 99b060c..c886293 100644 --- a/debian/gbs-api.install +++ b/debian/gbs-api.install @@ -4,4 +4,5 @@ usr/lib/python*/*packages/gitbuildsys/errors.py usr/lib/python*/*packages/gitbuildsys/log.py usr/lib/python*/*packages/gitbuildsys/safe_url.py usr/lib/python*/*packages/gitbuildsys/utils.py +usr/lib/python*/*packages/gitbuildsys/ks_utils.py usr/lib/python*/*packages/gbs-*.egg-info diff --git a/gitbuildsys/cmd_build.py b/gitbuildsys/cmd_build.py index adf5be1..7702a22 100644 --- a/gitbuildsys/cmd_build.py +++ b/gitbuildsys/cmd_build.py @@ -23,11 +23,14 @@ import os import shutil import pwd import re +import glob +import urlparse from gitbuildsys.utils import Temp, RepoParser from gitbuildsys.errors import GbsError, Usage from gitbuildsys.conf import configmgr from gitbuildsys.safe_url import SafeURL +from gitbuildsys.ks_utils import KSRepoUpdater from gitbuildsys.cmd_export import get_packaging_dir from gitbuildsys.log import LOGGER as log @@ -74,15 +77,56 @@ QEMU_CAN_BUILD = ['armv4l', 'armv5el', 'armv5l', 'armv6l', 'armv7l', USERID = pwd.getpwuid(os.getuid())[0] TMPDIR = os.path.join(configmgr.get('tmpdir', 'general'), '%s-gbs' % USERID) -def prepare_repos_and_build_conf(args, arch, profile): +def update_ks_files(args, repoparser, cachedir): + '''Update ks files: Add local repo and add user/pass if needed''' + if args.arch: + buildarch = args.arch + else: + buildarch = os.uname()[4] + + repourls = repoparser.get_repos_by_arch(buildarch) + localrepo_dir = os.path.join(os.environ['TIZEN_BUILD_ROOT'], 'local/repos', + repoparser.tizen_version, buildarch) + for ks_file in repoparser.ks_files: + ks_updater = KSRepoUpdater(ks_file) + ks_updater.add_repo('local', localrepo_dir, priority=1) + #ks_updater.build_ID() + for url in repourls: + #import pdb;pdb.set_trace() + hostname = urlparse.urlsplit(url).hostname + ks_updater.add_authinfo(hostname, url.user, url.passwd) + ks_updater.sync() + +def prepare_meta_files(args, repoparser): + '''prepare meta files for gbs build and image creation, including + - group patterns files for local repo + - KS files for image creation + ''' + meta_dir = os.path.join(os.environ['TIZEN_BUILD_ROOT'], 'meta') + if not os.path.exists(meta_dir): + os.makedirs(meta_dir) + if repoparser.group_file['name']: + shutil.copy(repoparser.group_file['name'], meta_dir) + if repoparser.pattern_file['name']: + shutil.copy(repoparser.pattern_file['name'], meta_dir) + + profile = get_profile(args) + + if profile.ks_dir: + ks_dir = profile.ks_dir + else: + ks_dir = configmgr.get('ks_dir', 'general') + ks_dir = os.path.expanduser(ks_dir) + if not os.path.exists(ks_dir): + os.makedirs(ks_dir) + + for ks_file in repoparser.ks_files: + shutil.copy(ks_file, ks_dir) + +def prepare_repos_and_build_conf(args, arch, profile, cachedir): '''generate repos and build conf options for depanneur''' cmd_opts = [] - cache = Temp(prefix=os.path.join(TMPDIR, 'gbscache'), - directory=True) - cachedir = cache.path - if not os.path.exists(cachedir): - os.makedirs(cachedir) log.info('generate repositories ...') if args.skip_conf_repos: @@ -103,22 +147,15 @@ def prepare_repos_and_build_conf(args, arch, profile): raise GbsError('No package repository specified.') repoparser = RepoParser(repos, cachedir) - meta_dir = os.path.join(os.environ['TIZEN_BUILD_ROOT'], 'meta') - - if not os.path.exists(meta_dir): - os.makedirs(meta_dir) - - if repoparser.group_file['name']: - shutil.copy(repoparser.group_file['name'], meta_dir) - if repoparser.pattern_file['name']: - shutil.copy(repoparser.pattern_file['name'], meta_dir) - repourls = repoparser.get_repos_by_arch(arch) if not repourls: raise GbsError('no available repositories found for arch %s under the ' 'following repos:\n%s' % (arch, '\n'.join(repos))) cmd_opts += [('--repository=%s' % url.full) for url in repourls] + update_ks_files(args, repoparser, cachedir) + prepare_meta_files(args, repoparser) + if args.dist: distconf = args.dist if not os.path.exists(distconf): @@ -269,9 +306,15 @@ def main(args): if args.clean: cmd += ['--clean'] + cache = Temp(prefix=os.path.join(TMPDIR, 'gbscache'), + directory=True) + cachedir = cache.path + if not os.path.exists(cachedir): + os.makedirs(cachedir) + # check & prepare repos and build conf if not args.noinit: - cmd += prepare_repos_and_build_conf(args, buildarch, profile) + cmd += prepare_repos_and_build_conf(args, buildarch, profile, cachedir) else: cmd += ['--noinit'] diff --git a/gitbuildsys/ks_utils.py b/gitbuildsys/ks_utils.py new file mode 100644 index 0000000..a436714 --- /dev/null +++ b/gitbuildsys/ks_utils.py @@ -0,0 +1,98 @@ +#!/usr/bin/python -tt +# vim: ai ts=4 sts=4 et sw=4 +# +# Copyright (c) 2013 Intel, Inc. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation; version 2 of the License +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., 59 +# Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +"""Helpers, convenience utils, common APIs for ks file""" + +import os +import urlparse + +from gitbuildsys.safe_url import SafeURL +from argparse import ArgumentParser +from gitbuildsys.log import LOGGER as log + +class KSRepoUpdater(object): + '''util class for updating repos in ks file''' + def __init__(self, ksfile): + self.ksfile = ksfile + with open(self.ksfile) as fobj: + self.kstext = fobj.read() + + @staticmethod + def _parse_repo(repo_str): + ''' parse repo lines into optparser strcuture''' + repoparser = ArgumentParser() + repoparser.add_argument('--baseurl') + return repoparser.parse_known_args(repo_str.split()) + + @staticmethod + def _build_repo(name, url, priority=None, user=None, passwd=None, + save=False, ssl_verify=None): + '''build repo str with specified repo options''' + repo_args = ['repo'] + if url.startswith('/') and os.path.exists(url): + url = 'file:///' + url.lstrip('/') + if user and passwd: + url = SafeURL(url, user, passwd).full + repo_args.append('--name=%s' % name) + repo_args.append('--baseurl=%s' % url) + if priority: + repo_args.append('--priority=%s' % priority) + if save: + repo_args.append('--save') + if ssl_verify: + repo_args.append('--ssl_verify=no') + return ' '.join(repo_args) + + def add_authinfo(self, host, user, passwd): + '''add user/passwd info for specified host related repo''' + kslist = self.kstext.splitlines() + for index, value in enumerate(kslist): + if value.startswith('repo'): + repoargs = self._parse_repo(value) + repo_host = urlparse.urlsplit(repoargs[0].baseurl).hostname + if repo_host != host: + continue + repoargs[0].baseurl = SafeURL(repoargs[0].baseurl, user, passwd).full + new_repo = ' '.join(repoargs[1]) + new_repo = '%s --baseurl=%s' % (new_repo, repoargs[0].baseurl) + kslist[index] = new_repo + # update to kstext + self.kstext = '\n'.join(kslist) + + def add_repo(self, name, url, priority=None, user=None, passwd=None, + save=False, ssl_verify=None): + '''add a new repo to ks file''' + kslist = self.kstext.splitlines() + for index, value in enumerate(kslist): + if value.startswith('repo'): + kslist.insert(index, self._build_repo(name, url, priority, + user, passwd, save, ssl_verify)) + self.kstext = '\n'.join(kslist) + break + else: + log.warning("no repo found, don't know where to insert new repo") + + def update_build_id(self, build_id): + '''replace @BUILD_ID@ in ks file with specified build_id''' + if "@BUILD_ID@" in self.kstext: + self.kstext = self.kstext.replace("@BUILD_ID@", build_id) + + def sync(self): + '''update changes back to original ks file''' + with open(self.ksfile, 'w') as fobj: + fobj.write(self.kstext) diff --git a/gitbuildsys/utils.py b/gitbuildsys/utils.py index d00d289..ec097ef 100644 --- a/gitbuildsys/utils.py +++ b/gitbuildsys/utils.py @@ -250,7 +250,6 @@ class URLGrabber(object): self.change_url(url, outfile, user, passwd) self.perform() - class RepoParser(object): """Repository parser for generate real repourl and build config.""" @@ -260,8 +259,10 @@ class RepoParser(object): self.buildconf = None self.group_file = defaultdict(str) self.pattern_file = defaultdict(str) + self.ks_files = [] self.standardrepos = [] self.urlgrabber = URLGrabber() + self.tizen_version = None self.localrepos, remotes = self.split_out_local_repo(repos) self.parse(remotes) @@ -306,6 +307,30 @@ class RepoParser(object): return meta + @staticmethod + def _parse_image_configs(image_configs_xml): + """ + Parse image-configs.xml + Returns: list of ks file + """ + if not (image_configs_xml and os.path.exists(image_configs_xml)): + return + + try: + etree = ET.parse(image_configs_xml) + except ET.ParseError: + log.warning('Not well formed xml: %s' % image_configs_xml) + return + + root = etree.getroot() + + ks_items = root.findall('config/path') + + if ks_items is not None: + return [ ks_item.text.strip() for ks_item in ks_items ] + else: + return [] + def build_repos_from_buildmeta(self, baseurl, meta): """Parse build.xml and pickup standard repos it contains.""" archs = meta.get('archs', []) @@ -345,6 +370,15 @@ class RepoParser(object): if build_xml: return self._parse_build_xml(build_xml) + def _fetch_image_configs(self, latest_repo_url): + """Fetch and parse image-config.xml.""" + image_configs_url = latest_repo_url.pathjoin('builddata/image-configs.xml') + image_configs_xml = self.fetch(image_configs_url) + if image_configs_xml: + return self._parse_image_configs(image_configs_xml) + else: + return [] + def _fetch_build_conf(self, latest_repo_url, meta): """Get build.conf file name from build.xml and fetch it.""" if self.buildconf: @@ -363,6 +397,7 @@ class RepoParser(object): if fname: release, _buildid = meta['id'].split('_') release = release.replace('-','') + self.tizen_version = release target_conf = os.path.join(os.path.dirname(fname), '%s.conf' % release) os.rename(fname, target_conf) @@ -392,6 +427,13 @@ class RepoParser(object): # Generate repos from build.xml self.build_repos_from_buildmeta(repo, meta) self._fetch_build_conf(repo, meta) + # Fetch ks files + for ks_file in self._fetch_image_configs(repo): + ksfile = self.fetch(repo.pathjoin( + os.path.join('builddata', ks_file))) + if ksfile: + self.ks_files.append(os.path.join(self.cachedir, + os.path.basename(ks_file))) for repo in remotes: deal_with_one_repo(repo) diff --git a/packaging/gbs.spec b/packaging/gbs.spec index d453150..fd482ab 100644 --- a/packaging/gbs.spec +++ b/packaging/gbs.spec @@ -79,4 +79,5 @@ rm -rf %{buildroot} %{python_sitelib}/gitbuildsys/log.py* %{python_sitelib}/gitbuildsys/safe_url.py* %{python_sitelib}/gitbuildsys/utils.py* +%{python_sitelib}/gitbuildsys/ks_utils.py* %{python_sitelib}/gbs-*-py*.egg-info