From: Ed Bartosh Date: Wed, 15 May 2013 17:31:40 +0000 (+0300) Subject: Draft of refactored create_images X-Git-Tag: 0.14~174 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=bac76926d113d81bc8f1b5acbab94e473478b285;p=services%2Fjenkins-scripts.git Draft of refactored create_images I'm sending it for review just to show the approach. This code is not ready for merging yet. Change-Id: Ibda3f0ac3e95901ea4a972b891860ad3f6cc1b22 --- diff --git a/common/repomaker.py b/common/repomaker.py index fc6763e..7979e5d 100644 --- a/common/repomaker.py +++ b/common/repomaker.py @@ -30,21 +30,21 @@ class RepoMakerError(Exception): pass # Helper functions -def find(topdir, suffix='.rpm'): +def find_files(topdir, prefix='', suffix='.rpm'): """Find files in the tree.""" for item in os.listdir(topdir): path = os.path.join(topdir, item) if os.path.isdir(path): for fpath in find(path): yield fpath - elif item.endswith(suffix): + elif item.startswith(prefix) and item.endswith(suffix): yield path def collect(in_dir): """Collect files, binary archs and destinations.""" files = [] archs = set() - for fname in find(in_dir): + for fname in find_files(in_dir): ftype = fname.split('.')[-2] if ftype not in ("src", "noarch"): archs.add(ftype) @@ -92,6 +92,25 @@ def move_or_hardlink(fpath, target_dirs, move=False): else: os.link(fpath, tpath) +def extract_ks(rpm): + """Extract .ks file from rpm.""" + + try: + output = check_output("rpm2cpio %s |cpio -t" % rpm, shell=True) + except CalledProcessError, err: + raise RepoMakerError("Can't get content of %s: %s" % (rpm, err)) + + for path in output.split(): + if path.endswith('.ks'): + try: + content = check_output('rpm2cpio %s |cpio -i --quiet '\ + '--to-stdout %s' % (rpm, path), + shell=True) + except CalledProcessError, err: + raise RepoMakerError("Can't extract %s from %s: %s" % \ + (path, rpm, err)) + + yield os.path.basename(path), content class RepoMaker(object): """Makes rpm repositories.""" @@ -108,6 +127,7 @@ class RepoMaker(object): self.build_id = build_id self.outdir = os.path.join(outdir, build_id) self.repos = {} + self.images = [] def add_repo(self, in_dir, name, buildconf=None, move=False, gpg_key=None, signer='/usr/bin/sign'): @@ -154,10 +174,10 @@ class RepoMaker(object): (fpath, filename, os.path.join(repo_dir, ftype, "repodata", filename))) - # extract .ks file + # get names and content of .ks files from rpm if is_imageconf: - # TODO: save content of .ks file and other image parameters - pass + for ksname, kscontent in extract_ks(fpath): + self.images.append(ksname, kscontent) # Save build configuration file if provided if buildconf: @@ -168,7 +188,9 @@ class RepoMaker(object): # Generate or update build.xml self.update_builddata(name, dirs, buildconf) - # TODO: Generate images.xml from previously saved data (see above) + # Generate/Update images.xml + if self.images: + self.update_imgdata() # Run createrepo for _rtype, _rarch, rpath in dirs: @@ -219,3 +241,9 @@ class RepoMaker(object): bdata.save(outf) except BuildDataError, err: raise RepoMakerError("Unable to generate build.xml: %s" % err) + + def add_images(self, images): + """Add image configurations.""" + for name, content in images: + self.images.append((name, content)) + diff --git a/job_pre_release_obs.py b/job_pre_release_obs.py index e4c5518..1016537 100755 --- a/job_pre_release_obs.py +++ b/job_pre_release_obs.py @@ -7,29 +7,37 @@ This code is called by jenkins jobs triggered by OBS events. import os import sys -import re -from common.tempbuildpkg import ProjectBuildService, ProjectRepository -from common.tempbuildpkg import RepoConf -from common.buildtrigger import trigger_info, trigger_next -from testprojects.prerelease import PreRelease2 +from common.buildtrigger import trigger_info from common.buildservice import BuildService -from common.repomaker import RepoMaker, RepoMakerError +from common.repomaker import find_files, extract_ks, RepoMaker, RepoMakerError from common.backenddb import BackendDb, BackendDBError class LocalError(Exception): """Local error exception.""" pass -def make_repo(project, repo, redis_host, redis_port): - """Create download repo from live repo.""" +def create_images(project, repo, redis_host, redis_port): + """ + Create images. + + Args: + project (str): OBS pre-release project name + repo (str): name of the OBS live repository + Raises: + LocalError if can't create repos or can't find image configurations + """ + # Make build id from latest snapshot + project suffix try: data = BackendDb(redis_host, redis_port).get_repos()[project] except BackendDBError, err: raise LocalError("Can't get information about project '%s': %s" \ % (project, str(err))) + latest_snapshot = data['latest_snapshot'] build_id = '%s.%s' % (data['latest_snapshot'], project.split(':')[-1]) + + # Convert live repo to download structure repomaker = RepoMaker(build_id, data['prerelease_dir']) live_repo_path = os.path.join('/srv/obs/repos/', project.replace(':', ':/')) # TODO: get buldconf from OBS @@ -38,62 +46,39 @@ def make_repo(project, repo, redis_host, redis_port): except RepoMakerError, err: raise LocalError("Unable to create download repo: %s" % err) -def create_images(prerelease): - """Create images using live prerelease repository.""" - linked, depended = prerelease.get_parent_project() - - # TODO, update when boss-repomaker updated - repo_config = RepoConf('/etc/repos/repos.yaml') - - # SERVER configuration - server_conf = {} - server_conf['REPO_PUB_BASE_URL'] = os.getenv('REPO_PUB_BASE_URL') - server_conf['LIVEREPO_PUB_BASE_URL'] = os.getenv('LIVEREPO_PUB_BASE_URL') - - repo = ProjectRepository(prerelease.obs_project, repo_conf = repo_config, - server_conf = server_conf) - - for link_prj in linked: - # TODO: get_url from repo.yaml configuration - link_prj_path = repo.get_latest_repo_path(link_prj) - print link_prj_path - repo.extract_image_ks(repo.get_live_repo_path(), - link_prj_path) - - ks_info = repo.get_valid_image_ks(os.path.join(repo.get_live_repo_path(), - 'builddata')) - - # KS modification, add one more source, and replace the old URLS - url_dict = {} - for obs_project in linked + depended: - url_dict[repo_config.get_name(obs_project)] = \ - repo.get_latest_repo_pub_url(obs_project) - - new_repo_line = "repo --name=prerelease --baseurl=%s --save "\ - "--ssl_verify=no --priority 1" % repo.get_live_repo_pub_url() - - for ksi in ks_info.keys(): - new_ks_lines = [] - with open(os.path.join(repo.get_live_repo_path(), 'builddata', - ks_info[ksi]['path'])) as ks_fh: - for line in ks_fh.readlines(): - if line.startswith('repo') and \ - 'baseurl=' in line and \ - '@BUILD_ID@' in line: - for name in url_dict.keys(): - if line.find('--name=%s' %name) > 0: - line = re.sub(r'--baseurl=http.*/repos/([a-zA-z\-]*)', - r'--baseurl=%s' %url_dict[name], line) - new_ks_lines.append(line.strip()) - print new_ks_lines - new_ks_lines.insert(new_ks_lines.index('%packages')-1, - new_repo_line) - - ks_info[ksi]['kickstart'] = '\n'.join(new_ks_lines) - - for index, image_info in enumerate(ks_info.values()): - trigger_next('%s/image_trigger_%s' %(os.getenv('WORKSPACE'), index), - image_info) + # TODO: remove output_dir + + # Assuming that there can be just one image-configurations- rpm in the repo + if not repomaker.images: + # repomaker did not found image-configurations in pre_release repo, + # let's take it from snapshot repo + snapshot_dir = os.path.join(data['snapshot_dir'], latest_snapshot, repo) + # Add image configuration to pre-release repo + repomaker.add_images(get_ks_files(snapshot_dir)) + + if not repomaker.images: + raise LocalError("Image configuration not found") + + for name, content in repomaker.images: + # TODO: Update urls in .ks file and add .ks files to repomaker again + # start image creation + pass + +def get_ks_files(repo_dir, prefix="image-configurations-"): + """ + Find rpm packages started with prefix in the repo. + Extract .ks files from rpm packages + + Args: + repo_dir (str): top level directory of rpm repo + prefix (str): prefix for rpm packages to find + Generates: + tuple (name, content) for found .ks files + """ + for rpm in find_files(repo_dir, prefix=prefix, suffix='.rpm'): + for name, content in extract_ks(rpm): + yield name, content + def main(name, action): """Script entry point. @@ -119,13 +104,12 @@ def main(name, action): repo = content.get("repo") # TODO: sync_repo(output_dir) make_repo(project, repo, redis_host, redis_port) - projectbuild = ProjectBuildService(project, obs_api, obs_user, obs_passwd) - prerelease = PreRelease2(project, projectbuild) - create_images(prerelease) + create_images(project, repo) elif action == 'cleanup': build.cleanup(project, "Cleaned up by %s" % name) else: - print "Not supported method of Pre-release jobs:", action + raise LocalError("Not supported method of pre_release_obs job: %s" \ + % action) if __name__ == '__main__': try: