usr/lib/python*/*packages/gitbuildsys/log.py
usr/lib/python*/*packages/gitbuildsys/safe_url.py
usr/lib/python*/*packages/gitbuildsys/utils.py
-usr/lib/python*/*packages/gitbuildsys/ks_utils.py
usr/lib/python*/*packages/gbs-*.egg-info
import shutil
import pwd
import re
-import urlparse
from gitbuildsys.utils import Temp, RepoParser
from gitbuildsys.errors import GbsError, Usage
from gitbuildsys.conf import configmgr
from gitbuildsys.safe_url import SafeURL
-from gitbuildsys.ks_utils import KSRepoUpdater
from gitbuildsys.cmd_export import get_packaging_dir
from gitbuildsys.log import LOGGER as log
USERID = pwd.getpwuid(os.getuid())[0]
TMPDIR = os.path.join(configmgr.get('tmpdir', 'general'), '%s-gbs' % USERID)
-def update_ks_files(args, repoparser):
- '''Update ks files: Add local repo and add user/pass if needed'''
- if args.arch:
- buildarch = args.arch
- else:
- buildarch = os.uname()[4]
-
- if args.dist:
- tizen_version = os.path.basename(args.dist)[:-len('.conf')]
- elif repoparser.tizen_version:
- tizen_version = repoparser.tizen_version
- else:
- log.debug('no tizen version detected')
- return
-
- localrepo_dir = os.path.join(os.environ['TIZEN_BUILD_ROOT'], 'local/repos',
- tizen_version, buildarch)
- repourls = repoparser.get_repos_by_arch(buildarch)
- for ks_file in repoparser.ks_files:
- ks_updater = KSRepoUpdater(ks_file)
- ks_updater.add_repo('local', localrepo_dir, priority=1)
- #ks_updater.build_ID()
- for url in repourls:
- hostname = urlparse.urlsplit(url).hostname
- ks_updater.add_authinfo(hostname, url.user, url.passwd)
- ks_updater.sync()
-
-def prepare_meta_files(args, repoparser):
- '''prepare meta files for gbs build and image creation, including
- - group patterns files for local repo
- - KS files for image creation
- '''
- meta_dir = os.path.join(os.environ['TIZEN_BUILD_ROOT'], 'meta')
- if not os.path.exists(meta_dir):
- os.makedirs(meta_dir)
- if repoparser.group_file['name']:
- shutil.copy(repoparser.group_file['name'], meta_dir)
- if repoparser.pattern_file['name']:
- shutil.copy(repoparser.pattern_file['name'], meta_dir)
-
- profile = get_profile(args)
-
- if profile.ks_dir:
- ks_dir = profile.ks_dir
- else:
- ks_dir = configmgr.get('ks_dir', 'general')
- ks_dir = os.path.expanduser(ks_dir)
- if not os.path.exists(ks_dir):
- os.makedirs(ks_dir)
-
- for ks_file in repoparser.ks_files:
- shutil.copy(ks_file, ks_dir)
-
-def prepare_repos_and_build_conf(args, arch, profile, cachedir):
+def prepare_repos_and_build_conf(args, arch, profile):
'''generate repos and build conf options for depanneur'''
cmd_opts = []
+ cache = Temp(prefix=os.path.join(TMPDIR, 'gbscache'),
+ directory=True)
+ cachedir = cache.path
+ if not os.path.exists(cachedir):
+ os.makedirs(cachedir)
log.info('generate repositories ...')
if args.skip_conf_repos:
'following repos:\n%s' % (arch, '\n'.join(repos)))
cmd_opts += [('--repository=%s' % url.full) for url in repourls]
- update_ks_files(args, repoparser)
- prepare_meta_files(args, repoparser)
-
if args.dist:
distconf = args.dist
if not os.path.exists(distconf):
if args.clean:
cmd += ['--clean']
- cache = Temp(prefix=os.path.join(TMPDIR, 'gbscache'),
- directory=True)
- cachedir = cache.path
- if not os.path.exists(cachedir):
- os.makedirs(cachedir)
-
# check & prepare repos and build conf
if not args.noinit:
- cmd += prepare_repos_and_build_conf(args, buildarch, profile, cachedir)
+ cmd += prepare_repos_and_build_conf(args, buildarch, profile)
else:
cmd += ['--noinit']
"""
import os
-import glob
from gitbuildsys.errors import GbsError
-from gitbuildsys.cmd_build import get_profile
-from gitbuildsys.conf import configmgr
from gitbuildsys.log import LOGGER as log
-def createimage(args, ks_file, outdir):
+def createimage(args, ks_file):
'''create image using mic'''
- extra_mic_opts = ['--outdir=%s' % outdir]
+ extra_mic_opts = []
+ if args.outdir:
+ extra_mic_opts = ['--outdir=%s' % outdir]
if args.tmpfs:
extra_mic_opts += ['--tmpfs']
extra_mic_opts += ['--record-pkgs=name']
def main(args):
'''main entrance for createimage'''
- profile = get_profile(args)
- if profile.image_dir:
- image_dir = profile.image_dir
- else:
- image_dir = configmgr.get('image_dir', 'general')
- image_dir = os.path.expanduser(image_dir)
if args.ks_file:
if not os.path.exists(args.ks_file):
- raise GbsError('specified ks file: not exists' % args.ks_file)
+ raise GbsError('specified ks file %s does not exists' % args.ks_file)
log.info('creating image for ks file: %s' % args.ks_file)
- createimage(args, args.ks_file, image_dir)
- else:
- if profile.ks_dir:
- ks_dir = profile.ks_dir
- else:
- ks_dir = configmgr.get('ks_dir', 'general')
-
- ks_dir = os.path.expanduser(ks_dir)
- ks_list = glob.glob(os.path.join(ks_dir, '*.ks'))
- if not ks_list:
- raise GbsError('no avaliable ks file found in ks dir:%s' % ks_dir)
-
- log.debug('avaliable ks files are:\n %s '% ' '.join(ks_list))
- for ks_file in ks_list:
- log.info('creating image for ks file: %s' % ks_file)
- createimage(args, ks_file, image_dir)
+ createimage(args, args.ks_file)
'upstream_branch': 'upstream',
'upstream_tag': 'upstream/${upstreamversion}',
'squash_patches_until': '',
- 'buildroot': '~/GBS-ROOT',
+ 'buildroot': '~/GBS-ROOT/',
'packaging_dir': 'packaging',
- 'image_dir': '%(buildroot)s/images',
- 'ks_dir': '%(buildroot)s/meta/ks-dir',
},
}
self.repos = []
self.obs = None
self.buildroot = None
- self.image_dir = None
- self.ks_dir = None
def add_repo(self, repoconf):
'''add a repo to repo list of the profile'''
profile.add_repo(repoconf)
profile.buildroot = self.get_optional_item(name, 'buildroot')
- profile.ks_dir = self.get_optional_item(name, 'ks_dir')
- profile.image_dir = self.get_optional_item(name, 'image_dir')
return profile
+++ /dev/null
-#!/usr/bin/python -tt
-# vim: ai ts=4 sts=4 et sw=4
-#
-# Copyright (c) 2013 Intel, Inc.
-#
-# This program is free software; you can redistribute it and/or modify it
-# under the terms of the GNU General Public License as published by the Free
-# Software Foundation; version 2 of the License
-#
-# This program is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
-# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
-# for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc., 59
-# Temple Place - Suite 330, Boston, MA 02111-1307, USA.
-
-"""Helpers, convenience utils, common APIs for ks file"""
-
-import urlparse
-
-from gitbuildsys.safe_url import SafeURL
-from argparse import ArgumentParser
-from gitbuildsys.log import LOGGER as log
-
-class KSRepoUpdater(object):
- '''util class for updating repos in ks file'''
- def __init__(self, ksfile):
- self.ksfile = ksfile
- with open(self.ksfile) as fobj:
- self.kstext = fobj.read()
-
- @staticmethod
- def _parse_repo(repo_str):
- ''' parse repo lines into optparser strcuture'''
- repoparser = ArgumentParser()
- repoparser.add_argument('--baseurl')
- return repoparser.parse_known_args(repo_str.split())
-
- @staticmethod
- def _build_repo(name, url, priority=None, user=None, passwd=None,
- save=False, ssl_verify=None):
- '''build repo str with specified repo options'''
- repo_args = ['repo']
- if url.startswith('/'):
- url = 'file:///' + url.lstrip('/')
- if user and passwd:
- url = SafeURL(url, user, passwd).full
- repo_args.append('--name=%s' % name)
- repo_args.append('--baseurl=%s' % url)
- if priority:
- repo_args.append('--priority=%s' % priority)
- if save:
- repo_args.append('--save')
- if ssl_verify:
- repo_args.append('--ssl_verify=no')
- return ' '.join(repo_args)
-
- def add_authinfo(self, host, user, passwd):
- '''add user/passwd info for specified host related repo'''
- kslist = self.kstext.splitlines()
- for index, value in enumerate(kslist):
- if value.startswith('repo'):
- repoargs = self._parse_repo(value)
- repo_host = urlparse.urlsplit(repoargs[0].baseurl).hostname
- if repo_host != host:
- continue
- repoargs[0].baseurl = SafeURL(repoargs[0].baseurl, user,
- passwd).full
- new_repo = ' '.join(repoargs[1])
- new_repo = '%s --baseurl=%s' % (new_repo, repoargs[0].baseurl)
- kslist[index] = new_repo
- # update to kstext
- self.kstext = '\n'.join(kslist)
-
- def add_repo(self, name, url, priority=None, user=None, passwd=None,
- save=False, ssl_verify=None):
- '''add a new repo to ks file'''
- kslist = self.kstext.splitlines()
- for index, value in enumerate(kslist):
- if value.startswith('repo'):
- kslist.insert(index, self._build_repo(name, url, priority,
- user, passwd, save, ssl_verify))
- self.kstext = '\n'.join(kslist)
- break
- else:
- log.warning("no repo found, don't know where to insert new repo")
-
- def update_build_id(self, build_id):
- '''replace @BUILD_ID@ in ks file with specified build_id'''
- if "@BUILD_ID@" in self.kstext:
- self.kstext = self.kstext.replace("@BUILD_ID@", build_id)
-
- def sync(self):
- '''update changes back to original ks file'''
- with open(self.ksfile, 'w') as fobj:
- fobj.write(self.kstext)
self.change_url(url, outfile, user, passwd)
self.perform()
+
class RepoParser(object):
"""Repository parser for generate real repourl and build config."""
self.cachedir = cachedir
self.repourls = defaultdict(list)
self.buildconf = None
- self.group_file = defaultdict(str)
- self.pattern_file = defaultdict(str)
- self.ks_files = []
self.standardrepos = []
self.urlgrabber = URLGrabber()
- self.tizen_version = None
self.localrepos, remotes = self.split_out_local_repo(repos)
self.parse(remotes)
return meta
- @staticmethod
- def _parse_image_configs(image_configs_xml):
- """
- Parse image-configs.xml
- Returns: list of ks file
- """
- if not (image_configs_xml and os.path.exists(image_configs_xml)):
- return
-
- try:
- etree = ET.parse(image_configs_xml)
- except ET.ParseError:
- log.warning('Not well formed xml: %s' % image_configs_xml)
- return
-
- root = etree.getroot()
-
- ks_items = root.findall('config/path')
-
- if ks_items is not None:
- return [ ks_item.text.strip() for ks_item in ks_items ]
- else:
- return []
-
def build_repos_from_buildmeta(self, baseurl, meta):
"""Parse build.xml and pickup standard repos it contains."""
archs = meta.get('archs', [])
if build_xml:
return self._parse_build_xml(build_xml)
- def _fetch_image_configs(self, latest_repo_url):
- """Fetch and parse image-config.xml."""
- image_configs_url = latest_repo_url.pathjoin('builddata/'\
- 'image-configs.xml')
- image_configs_xml = self.fetch(image_configs_url)
- if image_configs_xml:
- return self._parse_image_configs(image_configs_xml)
- else:
- return []
-
def _fetch_build_conf(self, latest_repo_url, meta):
"""Get build.conf file name from build.xml and fetch it."""
if self.buildconf:
if fname:
release, _buildid = meta['id'].split('_')
release = release.replace('-','')
- self.tizen_version = release
target_conf = os.path.join(os.path.dirname(fname),
'%s.conf' % release)
os.rename(fname, target_conf)
# Generate repos from build.xml
self.build_repos_from_buildmeta(repo, meta)
self._fetch_build_conf(repo, meta)
- # Fetch ks files
- for ks_file in self._fetch_image_configs(repo):
- ksfile = self.fetch(repo.pathjoin(
- os.path.join('builddata', ks_file)))
- if ksfile:
- self.ks_files.append(os.path.join(self.cachedir,
- os.path.basename(ks_file)))
for repo in remotes:
deal_with_one_repo(repo)
-
- # find group/pattern files from all standard repos
- all_repos = self.standardrepos[:]
- for arch in self.repourls:
- all_repos.extend(self.repourls[arch])
- for repo in all_repos:
- group_url = repo.pathjoin('repodata/group.xml')
- group_file = self.fetch(group_url)
- if not group_file:
- continue
- with open(group_file, 'rb') as fobj:
- md5sum = hexdigest(fobj)
- if md5sum != self.group_file['md5sum']:
- log.warning('multiple differnent group files found')
- self.group_file['name'] = group_file
- self.group_file['md5sum'] = md5sum
-
- pattern_url = repo.pathjoin('repodata/patterns.xml')
- pattern_file = self.fetch(pattern_url)
- if not pattern_file:
- log.warning('pattern/group files do not exist in the same repo')
- continue
- with open(pattern_file, 'rb') as fobj:
- md5sum = hexdigest(fobj)
- if md5sum != self.pattern_file['md5sum']:
- log.warning('multiple differnent pattern files found')
- self.pattern_file['name'] = pattern_file
- self.pattern_file['md5sum'] = md5sum
-
@staticmethod
def split_out_local_repo(repos):
"""Divide repos to local and remote parts."""
%{python_sitelib}/gitbuildsys/log.py*
%{python_sitelib}/gitbuildsys/safe_url.py*
%{python_sitelib}/gitbuildsys/utils.py*
-%{python_sitelib}/gitbuildsys/ks_utils.py*
%{python_sitelib}/gbs-*-py*.egg-info
self.assertEquals(conf.getvalue(), '''[general]
profile = profile.current
-image_dir = ~/GBS-ROOT/images
-ks_dir = ~/GBS-ROOT/meta/ks-dir
[obs.remotebuild]
url = https://api/build/server
parser.add_argument('-K', '--ks-file',
help='ks file to be used')
- parser.add_argument('-P', '--profile',
- help='profile to be used for building, can be given '
- 'without the "profile." prefix')
+ parser.add_argument('-O', '--outdir',
+ help='image output directory')
parser.add_argument('--tmpfs', action='store_true',
help='use tmpfs to accelerate creating image, but '\
'plesae make sure you have enough free memory '\
parser.set_defaults(alias="cr")
return parser
-
@subparser
def remotebuild_parser(parser):
"""remote build package
projects / tools / gbs.git / commitdiff