Draft of refactored create_images

I'm sending it for review just to show the approach. This code is not
ready for merging yet.

Change-Id: Ibda3f0ac3e95901ea4a972b891860ad3f6cc1b22

diff --git a/common/repomaker.py b/common/repomaker.py
index fc6763e..7979e5d 100644 (file)
--- a/common/repomaker.py
+++ b/common/repomaker.py
@@ -30,21 +30,21 @@ class RepoMakerError(Exception):
     pass
 
 # Helper functions
-def find(topdir, suffix='.rpm'):
+def find_files(topdir, prefix='', suffix='.rpm'):
     """Find files in the tree."""
     for item in os.listdir(topdir):
         path = os.path.join(topdir, item)
         if os.path.isdir(path):
             for fpath in find(path):
                 yield fpath
-        elif item.endswith(suffix):
+        elif item.startswith(prefix) and item.endswith(suffix):
             yield path
 
 def collect(in_dir):
     """Collect files, binary archs and destinations."""
     files = []
     archs = set()
-    for fname in find(in_dir):
+    for fname in find_files(in_dir):
         ftype = fname.split('.')[-2]
         if ftype not in ("src", "noarch"):
             archs.add(ftype)
@@ -92,6 +92,25 @@ def move_or_hardlink(fpath, target_dirs, move=False):
         else:
             os.link(fpath, tpath)
 
+def extract_ks(rpm):
+    """Extract .ks file from rpm."""
+
+    try:
+        output = check_output("rpm2cpio %s |cpio -t" % rpm, shell=True)
+    except CalledProcessError, err:
+        raise RepoMakerError("Can't get content of %s: %s" % (rpm, err))
+
+    for path in output.split():
+        if path.endswith('.ks'):
+            try:
+                content = check_output('rpm2cpio %s |cpio -i --quiet '\
+                                       '--to-stdout %s' % (rpm, path),
+                                       shell=True)
+            except CalledProcessError, err:
+                raise RepoMakerError("Can't extract %s from %s: %s" % \
+                                     (path, rpm, err))
+
+            yield os.path.basename(path), content
 
 class RepoMaker(object):
     """Makes rpm repositories."""
@@ -108,6 +127,7 @@ class RepoMaker(object):
         self.build_id = build_id
         self.outdir = os.path.join(outdir, build_id)
         self.repos = {}
+        self.images = []
 
     def add_repo(self, in_dir, name, buildconf=None, move=False, gpg_key=None,
                  signer='/usr/bin/sign'):
@@ -154,10 +174,10 @@ class RepoMaker(object):
                               (fpath, filename, os.path.join(repo_dir, ftype,
                                                              "repodata",
                                                              filename)))
-            # extract .ks file
+            # get names and content of .ks files from rpm
             if is_imageconf:
-                # TODO: save content of .ks file and other image parameters
-                pass
+                for ksname, kscontent in extract_ks(fpath):
+                    self.images.append(ksname, kscontent)
 
         # Save build configuration file if provided
         if buildconf:
@@ -168,7 +188,9 @@ class RepoMaker(object):
         # Generate or update build.xml
         self.update_builddata(name, dirs, buildconf)
 
-        # TODO: Generate images.xml from previously saved data (see above)
+        # Generate/Update images.xml
+        if self.images:
+            self.update_imgdata()
 
         # Run createrepo
         for _rtype, _rarch, rpath in dirs:
@@ -219,3 +241,9 @@ class RepoMaker(object):
             bdata.save(outf)
         except BuildDataError, err:
             raise RepoMakerError("Unable to generate build.xml: %s" % err)
+
+    def add_images(self, images):
+        """Add image configurations."""
+        for name, content in images:
+            self.images.append((name, content))
+

diff --git a/job_pre_release_obs.py b/job_pre_release_obs.py
index e4c5518..1016537 100755 (executable)
--- a/job_pre_release_obs.py
+++ b/job_pre_release_obs.py
@@ -7,29 +7,37 @@ This code is called by jenkins jobs triggered by OBS events.
 
 import os
 import sys
-import re
 
-from common.tempbuildpkg import ProjectBuildService, ProjectRepository
-from common.tempbuildpkg import RepoConf
-from common.buildtrigger import trigger_info, trigger_next
-from testprojects.prerelease import PreRelease2
+from common.buildtrigger import trigger_info
 from common.buildservice import BuildService
-from common.repomaker import RepoMaker, RepoMakerError
+from common.repomaker import find_files, extract_ks, RepoMaker, RepoMakerError
 from common.backenddb import BackendDb, BackendDBError
 
 class LocalError(Exception):
     """Local error exception."""
     pass
 
-def make_repo(project, repo, redis_host, redis_port):
-    """Create download repo from live repo."""
+def create_images(project, repo, redis_host, redis_port):
+    """
+    Create images.
+
+    Args:
+        project (str): OBS pre-release project name
+        repo (str): name of the OBS live repository
+    Raises:
+        LocalError if can't create repos or can't find image configurations
+    """
 
+    # Make build id from latest snapshot + project suffix
     try:
         data = BackendDb(redis_host, redis_port).get_repos()[project]
     except BackendDBError, err:
         raise LocalError("Can't get information about project '%s': %s" \
                          % (project, str(err)))
+    latest_snapshot = data['latest_snapshot']
     build_id = '%s.%s' % (data['latest_snapshot'], project.split(':')[-1])
+
+    # Convert live repo to download structure
     repomaker = RepoMaker(build_id, data['prerelease_dir'])
     live_repo_path = os.path.join('/srv/obs/repos/', project.replace(':', ':/'))
     # TODO: get buldconf from OBS
@@ -38,62 +46,39 @@ def make_repo(project, repo, redis_host, redis_port):
     except RepoMakerError, err:
         raise LocalError("Unable to create download repo: %s" % err)
 
-def create_images(prerelease):
-    """Create images using live prerelease repository."""
-    linked, depended = prerelease.get_parent_project()
-
-    # TODO, update when boss-repomaker updated
-    repo_config = RepoConf('/etc/repos/repos.yaml')
-
-    # SERVER configuration
-    server_conf = {}
-    server_conf['REPO_PUB_BASE_URL'] = os.getenv('REPO_PUB_BASE_URL')
-    server_conf['LIVEREPO_PUB_BASE_URL'] = os.getenv('LIVEREPO_PUB_BASE_URL')
-
-    repo = ProjectRepository(prerelease.obs_project, repo_conf = repo_config,
-                             server_conf = server_conf)
-
-    for link_prj in linked:
-        # TODO: get_url from repo.yaml configuration
-        link_prj_path = repo.get_latest_repo_path(link_prj)
-        print link_prj_path
-        repo.extract_image_ks(repo.get_live_repo_path(),
-                              link_prj_path)
-
-    ks_info = repo.get_valid_image_ks(os.path.join(repo.get_live_repo_path(),
-                                                   'builddata'))
-
-    # KS modification, add one more source, and replace the old URLS
-    url_dict = {}
-    for obs_project in linked + depended:
-        url_dict[repo_config.get_name(obs_project)] = \
-            repo.get_latest_repo_pub_url(obs_project)
-
-    new_repo_line = "repo --name=prerelease --baseurl=%s --save  "\
-        "--ssl_verify=no --priority 1" % repo.get_live_repo_pub_url()
-
-    for ksi in ks_info.keys():
-        new_ks_lines = []
-        with open(os.path.join(repo.get_live_repo_path(), 'builddata',
-                               ks_info[ksi]['path'])) as ks_fh:
-            for line in ks_fh.readlines():
-                if line.startswith('repo') and \
-                        'baseurl=' in line and \
-                        '@BUILD_ID@' in line:
-                    for name in url_dict.keys():
-                        if line.find('--name=%s' %name) > 0:
-                            line = re.sub(r'--baseurl=http.*/repos/([a-zA-z\-]*)',
-                                          r'--baseurl=%s' %url_dict[name], line)
-                new_ks_lines.append(line.strip())
-        print new_ks_lines
-        new_ks_lines.insert(new_ks_lines.index('%packages')-1,
-                            new_repo_line)
-
-        ks_info[ksi]['kickstart'] = '\n'.join(new_ks_lines)
-
-        for index, image_info in enumerate(ks_info.values()):
-            trigger_next('%s/image_trigger_%s' %(os.getenv('WORKSPACE'), index),
-                         image_info)
+    # TODO: remove output_dir
+
+    # Assuming that there can be just one image-configurations- rpm in the repo
+    if not repomaker.images:
+        # repomaker did not found image-configurations in pre_release repo,
+        # let's take it from snapshot repo
+        snapshot_dir = os.path.join(data['snapshot_dir'], latest_snapshot, repo)
+        # Add image configuration to pre-release repo
+        repomaker.add_images(get_ks_files(snapshot_dir))
+
+    if not repomaker.images:
+        raise LocalError("Image configuration not found")
+
+    for name, content in repomaker.images:
+        # TODO: Update urls in .ks file and add .ks files to repomaker again
+        # start image creation
+        pass
+
+def get_ks_files(repo_dir, prefix="image-configurations-"):
+    """
+    Find rpm packages started with prefix in the repo.
+    Extract .ks files from rpm packages
+
+    Args:
+        repo_dir (str): top level directory of rpm repo
+        prefix (str): prefix for rpm packages to find
+    Generates:
+        tuple (name, content) for found .ks files
+    """
+    for rpm in find_files(repo_dir, prefix=prefix, suffix='.rpm'):
+        for name, content in extract_ks(rpm):
+            yield name, content
+
 
 def main(name, action):
     """Script entry point.
@@ -119,13 +104,12 @@ def main(name, action):
         repo = content.get("repo")
         # TODO: sync_repo(output_dir)
         make_repo(project, repo, redis_host, redis_port)
-        projectbuild = ProjectBuildService(project, obs_api, obs_user, obs_passwd)
-        prerelease = PreRelease2(project, projectbuild)
-        create_images(prerelease)
+        create_images(project, repo)
     elif action == 'cleanup':
         build.cleanup(project, "Cleaned up by %s" % name)
     else:
-        print "Not supported method of Pre-release jobs:", action
+        raise LocalError("Not supported method of pre_release_obs job: %s" \
+                          % action)
 
 if __name__ == '__main__':
     try: