Enabled images filtering by specifying build target

Current problem is the kickstart definitions have no target
identification information in meta-* packages, which cause the generated
images in product for each snapshot will be duplicated among the
different targets. To resolve it, we need three modifications together:

1. Added the definition of relationship with images and targets, in
   meta-* packages. Sample as below:

   /profile/ivi/meta-ivi:/ivi-targets.yaml ->
   """
        Targets:
            -   Name: atom
                Images:
                    - ivi-efi-i586.yaml
                    - ivi-mbr-i586.yaml
            -   Name: emulator
                Images:
                    - ivi-mbr-i586-emul.yaml
   """

2. In the building of image-configurations: add extra command line
   options for kickstarter command:

    kickstarter ... --targetdef .../%{_profile}-targets.yaml \
                    --target %{_repository}

3. This patch: to provide the new --targetdef and --target options.
  3.1 --targetdef to specify the YAML file of targets definition
  3.2 --target to specify the dedicated build target to filter images

The final result will be: nothing changed to Jenkins jobs and BOSS, and
the image-configuration*.rpm will not their name but will only contain
the build target corresponding ks files.

To keep the compatibility of all the changes, rule-1st is:

 The order of change integration to product server will NOT break
 current work as most as possible.

So the integration order can be: {1, 3} -> {2}, change-2 should be the
last step.

Change-Id: Ie4fb251c8e6442880cb3b5386bd0dd9f4bd3adbe
Signed-off-by: JF Ding <jian-feng.ding@intel.com>

diff --git a/kswriter/KSWriter.py b/kswriter/KSWriter.py
index 20c78cd..12ba899 100644 (file)
--- a/kswriter/KSWriter.py
+++ b/kswriter/KSWriter.py
@@ -9,6 +9,15 @@ from urlparse import urlparse
 
 from kickstart import kickstart
 
+class KSMetaError(Exception):
+    """Exception to catch malformated meta YAML files """
+    def __init__(self, msg):
+        Exception.__init__(self)
+        self.msg = msg
+
+    def __str__(self):
+        return self.msg
+
 def mkdir_p(path):
     try:
         os.makedirs(path)
@@ -17,8 +26,21 @@ def mkdir_p(path):
             pass
         else: raise
 
+def yamlload_safe(fname):
+    """ Safer wrapper for yaml.load() """
+    try:
+        with file(fname) as f:
+            return yaml.load(f)
+
+    except IOError:
+        raise KSMetaError('cannot read meta file: %s' % fname)
+
+    except:
+        # should be YAML format issue
+        raise KSMetaError('format error of meta file: %s' % fname)
+
 class KSWriter():
-    def __init__(self, configs=None, repos=None, outdir=".", config=None, packages=False, external=[]):
+    def __init__(self, configs=None, repos=None, outdir=".", config=None, packages=False, external=None, targetdefs=None, target=None):
         self.dist = None
         self.arch = None
         self.image_filename = os.path.abspath(os.path.expanduser(configs))
@@ -26,27 +48,44 @@ class KSWriter():
         self.external = external
         self.packages = packages
         self.config = config
-        self.image_stream = file(self.image_filename, 'r')
-        self.image_meta = yaml.load(self.image_stream)
+        self.image_meta = yamlload_safe(self.image_filename)
         self.extra = {}
 
         self.repos = self.parse_repos(repos)
 
+        # parse and record target definitions
+        self.target = target
+        self.targetdefs = self.parse_targetdef(targetdefs)
+
     def parse_repos(self, repos):
         prepos = []
         for repo in repos:
-            repo_stream = file(repo, 'r')
-            repo_meta = yaml.load(repo_stream)
+            repo_meta = yamlload_safe(repo)
             prepos = prepos + repo_meta['Repositories']
 
         return prepos
-       
+
+    def parse_targetdef(self, targetdefs):
+        """ Parse the targets defs in YAML to list of dict.
+            Will join them if multiple files provided.
+            Once one wrong file found, raise exception to abort
+        """
+        if not targetdefs:
+            return []
+
+        all_defs = []
+        for tdef in targetdefs:
+            tmeta = yamlload_safe(tdef)
+
+            all_defs.extend(tmeta['Targets'])
+
+        return all_defs
 
     def merge(*input):
         return list(reduce(set.union, input, set()))
 
     def dump(self):
-        print yaml.dump(yaml.load(self.stream))
+        print yaml.dump(yamlload_safe(self.stream))
 
     def parse(self, img):
         conf = copy.copy(self.image_meta['Default'])
@@ -137,6 +176,22 @@ class KSWriter():
             f.write(a)
             f.close()
 
+    def _image_for_current_target(self, fname):
+        """ To check whether this image is belong to current
+            target, which is specifed by user
+        """
+
+        if not self.target or not self.targetdefs:
+            # if user doesn't specify them, skip the checking
+            return True
+
+        for tdef in self.targetdefs:
+            if self.target == tdef['Name'] and fname in tdef['Images']:
+                # found
+                return True
+
+        return False
+
     def generate(self):
         out = {}
         if self.image_meta.has_key('Configurations'):
@@ -167,8 +222,12 @@ class KSWriter():
 
             for f in os.listdir(external_config_dir):
                 if f.endswith('.yaml'):
-                    fp = file('%s/%s' %(external_config_dir, f), 'r')
-                    local = yaml.load(fp)
+
+                    if not self._image_for_current_target(f):
+                        # skip
+                        continue
+
+                    local = yamlload_safe('%s/%s' %(external_config_dir, f))
                     conf = self.parse(local)
                     if self.config:
                         if self.config == conf['FileName']:

diff --git a/kswriter/__init__.py b/kswriter/__init__.py
index 82212cb..e69de29 100644 (file)
--- a/kswriter/__init__.py
+++ b/kswriter/__init__.py
@@ -1 +0,0 @@
-from KSWriter import KSWriter

diff --git a/tools/kickstarter b/tools/kickstarter
index e699670..7acc485 100755 (executable)
--- a/tools/kickstarter
+++ b/tools/kickstarter
@@ -1,8 +1,9 @@
 #!/usr/bin/python
 # Anas Nashif <anas.nashif@intel.com>
-import yaml,  sys
+
+import yaml, sys
 import re, os
-from kswriter import KSWriter
+from kswriter.KSWriter import KSWriter, KSMetaError, yamlload_safe
 
 import copy
 import time
@@ -70,8 +71,7 @@ def create_xml(image_meta, external_configs=[]):
     for path in external:
         for f in os.listdir(path):
             if f.endswith('.yaml'):
-                fp = file('%s/%s' %(path, f), 'r')
-                local = yaml.load(fp)
+                local = yamlload_safe('%s/%s' %(path, f))
                 conf = ks.parse(local)
                 if conf.has_key('Active') and conf['Active']:
                     image_xml(root,conf)
@@ -96,6 +96,10 @@ if __name__ == '__main__':
                     help="Limit to this configuration file")
     parser.add_option("-p", "--packages", action="store_true", dest="packages", default=False,
                     help="return list of packages to be installed for this configuration")
+    parser.add_option("-T", "--targetdef", action="append", type="string", dest="targetdefs",
+                    help="Targets meta file to specify images for corresponding build target")
+    parser.add_option("-t", "--target", type="string", dest="target", default=None,
+                    help="Specify build target to filter ks generation")
 
     (options, args) = parser.parse_args()
 
@@ -103,8 +107,12 @@ if __name__ == '__main__':
         print "you need to provide meta files with --configs and --repos"
         sys.exit(1)
 
-    ks = KSWriter(options.configsfile, options.repofile, options.outdir, options.config, options.packages, options.external)
-    ks.generate()
+    try:
+        ks = KSWriter(options.configsfile, options.repofile, options.outdir, options.config, options.packages, options.external, options.targetdefs, options.target)
+        ks.generate()
+    except KSMetaError as err:
+        print 'ERROR:', str(err)
+        sys.exit(1)
 
     if options.indexfile:
         str = create_xml(ks.image_meta, options.external)