From d098dae40bc81d709297882395d8f0bd101e0b00 Mon Sep 17 00:00:00 2001 From: JF Ding Date: Mon, 16 Jun 2014 16:09:41 +0800 Subject: [PATCH] Enabled images filtering by specifying build target Current problem is the kickstart definitions have no target identification information in meta-* packages, which cause the generated images in product for each snapshot will be duplicated among the different targets. To resolve it, we need three modifications together: 1. Added the definition of relationship with images and targets, in meta-* packages. Sample as below: /profile/ivi/meta-ivi:/ivi-targets.yaml -> """ Targets: - Name: atom Images: - ivi-efi-i586.yaml - ivi-mbr-i586.yaml - Name: emulator Images: - ivi-mbr-i586-emul.yaml """ 2. In the building of image-configurations: add extra command line options for kickstarter command: kickstarter ... --targetdef .../%{_profile}-targets.yaml \ --target %{_repository} 3. This patch: to provide the new --targetdef and --target options. 3.1 --targetdef to specify the YAML file of targets definition 3.2 --target to specify the dedicated build target to filter images The final result will be: nothing changed to Jenkins jobs and BOSS, and the image-configuration*.rpm will not their name but will only contain the build target corresponding ks files. To keep the compatibility of all the changes, rule-1st is: The order of change integration to product server will NOT break current work as most as possible. So the integration order can be: {1, 3} -> {2}, change-2 should be the last step. Change-Id: Ie4fb251c8e6442880cb3b5386bd0dd9f4bd3adbe Signed-off-by: JF Ding --- kswriter/KSWriter.py | 77 ++++++++++++++++++++++++++++++++++++++++++++++------ kswriter/__init__.py | 1 - tools/kickstarter | 20 ++++++++++---- 3 files changed, 82 insertions(+), 16 deletions(-) diff --git a/kswriter/KSWriter.py b/kswriter/KSWriter.py index 20c78cd..12ba899 100644 --- a/kswriter/KSWriter.py +++ b/kswriter/KSWriter.py @@ -9,6 +9,15 @@ from urlparse import urlparse from kickstart import kickstart +class KSMetaError(Exception): + """Exception to catch malformated meta YAML files """ + def __init__(self, msg): + Exception.__init__(self) + self.msg = msg + + def __str__(self): + return self.msg + def mkdir_p(path): try: os.makedirs(path) @@ -17,8 +26,21 @@ def mkdir_p(path): pass else: raise +def yamlload_safe(fname): + """ Safer wrapper for yaml.load() """ + try: + with file(fname) as f: + return yaml.load(f) + + except IOError: + raise KSMetaError('cannot read meta file: %s' % fname) + + except: + # should be YAML format issue + raise KSMetaError('format error of meta file: %s' % fname) + class KSWriter(): - def __init__(self, configs=None, repos=None, outdir=".", config=None, packages=False, external=[]): + def __init__(self, configs=None, repos=None, outdir=".", config=None, packages=False, external=None, targetdefs=None, target=None): self.dist = None self.arch = None self.image_filename = os.path.abspath(os.path.expanduser(configs)) @@ -26,27 +48,44 @@ class KSWriter(): self.external = external self.packages = packages self.config = config - self.image_stream = file(self.image_filename, 'r') - self.image_meta = yaml.load(self.image_stream) + self.image_meta = yamlload_safe(self.image_filename) self.extra = {} self.repos = self.parse_repos(repos) + # parse and record target definitions + self.target = target + self.targetdefs = self.parse_targetdef(targetdefs) + def parse_repos(self, repos): prepos = [] for repo in repos: - repo_stream = file(repo, 'r') - repo_meta = yaml.load(repo_stream) + repo_meta = yamlload_safe(repo) prepos = prepos + repo_meta['Repositories'] return prepos - + + def parse_targetdef(self, targetdefs): + """ Parse the targets defs in YAML to list of dict. + Will join them if multiple files provided. + Once one wrong file found, raise exception to abort + """ + if not targetdefs: + return [] + + all_defs = [] + for tdef in targetdefs: + tmeta = yamlload_safe(tdef) + + all_defs.extend(tmeta['Targets']) + + return all_defs def merge(*input): return list(reduce(set.union, input, set())) def dump(self): - print yaml.dump(yaml.load(self.stream)) + print yaml.dump(yamlload_safe(self.stream)) def parse(self, img): conf = copy.copy(self.image_meta['Default']) @@ -137,6 +176,22 @@ class KSWriter(): f.write(a) f.close() + def _image_for_current_target(self, fname): + """ To check whether this image is belong to current + target, which is specifed by user + """ + + if not self.target or not self.targetdefs: + # if user doesn't specify them, skip the checking + return True + + for tdef in self.targetdefs: + if self.target == tdef['Name'] and fname in tdef['Images']: + # found + return True + + return False + def generate(self): out = {} if self.image_meta.has_key('Configurations'): @@ -167,8 +222,12 @@ class KSWriter(): for f in os.listdir(external_config_dir): if f.endswith('.yaml'): - fp = file('%s/%s' %(external_config_dir, f), 'r') - local = yaml.load(fp) + + if not self._image_for_current_target(f): + # skip + continue + + local = yamlload_safe('%s/%s' %(external_config_dir, f)) conf = self.parse(local) if self.config: if self.config == conf['FileName']: diff --git a/kswriter/__init__.py b/kswriter/__init__.py index 82212cb..e69de29 100644 --- a/kswriter/__init__.py +++ b/kswriter/__init__.py @@ -1 +0,0 @@ -from KSWriter import KSWriter diff --git a/tools/kickstarter b/tools/kickstarter index e699670..7acc485 100755 --- a/tools/kickstarter +++ b/tools/kickstarter @@ -1,8 +1,9 @@ #!/usr/bin/python # Anas Nashif -import yaml, sys + +import yaml, sys import re, os -from kswriter import KSWriter +from kswriter.KSWriter import KSWriter, KSMetaError, yamlload_safe import copy import time @@ -70,8 +71,7 @@ def create_xml(image_meta, external_configs=[]): for path in external: for f in os.listdir(path): if f.endswith('.yaml'): - fp = file('%s/%s' %(path, f), 'r') - local = yaml.load(fp) + local = yamlload_safe('%s/%s' %(path, f)) conf = ks.parse(local) if conf.has_key('Active') and conf['Active']: image_xml(root,conf) @@ -96,6 +96,10 @@ if __name__ == '__main__': help="Limit to this configuration file") parser.add_option("-p", "--packages", action="store_true", dest="packages", default=False, help="return list of packages to be installed for this configuration") + parser.add_option("-T", "--targetdef", action="append", type="string", dest="targetdefs", + help="Targets meta file to specify images for corresponding build target") + parser.add_option("-t", "--target", type="string", dest="target", default=None, + help="Specify build target to filter ks generation") (options, args) = parser.parse_args() @@ -103,8 +107,12 @@ if __name__ == '__main__': print "you need to provide meta files with --configs and --repos" sys.exit(1) - ks = KSWriter(options.configsfile, options.repofile, options.outdir, options.config, options.packages, options.external) - ks.generate() + try: + ks = KSWriter(options.configsfile, options.repofile, options.outdir, options.config, options.packages, options.external, options.targetdefs, options.target) + ks.generate() + except KSMetaError as err: + print 'ERROR:', str(err) + sys.exit(1) if options.indexfile: str = create_xml(ks.image_meta, options.external) -- 2.7.4