From 8ea77adc9c8e767b6210f59d48873dfb225f6a29 Mon Sep 17 00:00:00 2001 From: Ed Bartosh Date: Sat, 8 Jun 2013 23:54:31 +0300 Subject: [PATCH] Implemented get_prerelease_data Collected all pre-release paths and url making in one place: application level pre-release API get_prerelease_data. This function returns all data, required by prerelease applications. This function can be moved under common/ when there will be more than one consumers of prerelease data. Left only read_ and get_ methods in BackendDB as the rest of functionality is implemented in get_prerelease_data. Change-Id: I40a311f98d32002cdd10931baa80b75f9b2955de Signed-off-by: Ed Bartosh --- common/backenddb.py | 66 ----------------------- job_pre_release_obs.py | 141 ++++++++++++++++++++++++++++++++++--------------- 2 files changed, 97 insertions(+), 110 deletions(-) diff --git a/common/backenddb.py b/common/backenddb.py index ad883e0..79c9f01 100644 --- a/common/backenddb.py +++ b/common/backenddb.py @@ -145,69 +145,3 @@ class BackendDB(): """Return release id entity object.""" return Entity(self._redis, "releaseid:") - def get_release_id(self, key): - """Return latest release id.""" - return self.get_release_ids()[key] - - def get_repo(self, name): - """Return all info of one repo conf. - - Args: - name (str): the obs project name, like "Tizen:Main" - - """ - db_repos = self.get_repos() - - for repo_name in db_repos: - repo = db_repos[repo_name] - if repo['Project'] == name: - return repo - - def get_repo_by_name(self, name): - """Return all info of one repo conf. - - Args: - name (str): the repo name in repos.yaml, like "Tizen-main" - - """ - return self.get_repos()[name] - - def get_repo_data(self, name): - """Return all need info of repo conf. - - Args: - name (str): the obs project name, like "Tizen:Main" - - """ - data = {} - - repo = self.get_repo(name) - release_id = self.get_release_id('%s-%s' % (repo['PartOf'], - repo['Release'])) - - data['latest_snapshot'] = '%s_%s' % (repo['Release'], release_id) - snapshot_dir = repo['SnapshotDir'] - data['snapshot_dir'] = snapshot_dir[snapshot_dir.find('snapshots/'):] - data['prerelease_dir'] = data['snapshot_dir'].replace('snapshots/', 'prerelease/', 1) - - return data - - def get_repo_data_by_name(self, name): - """Return all need info of repo conf. - - Args: - name (str): the repo name in repos.yaml, like "Tizen-main" - - """ - data = {} - - repo = self.get_repo_by_name(name) - release_id = self.get_release_id('%s-%s' % (repo['PartOf'], - repo['Release'])) - - data['latest_snapshot'] = '%s_%s' % (repo['Release'], release_id) - snapshot_dir = repo['SnapshotDir'] - data['snapshot_dir'] = snapshot_dir[snapshot_dir.find('snapshots/'):] - data['prerelease_dir'] = data['snapshot_dir'].replace('snapshots/', 'prerelease/', 1) - - return data diff --git a/job_pre_release_obs.py b/job_pre_release_obs.py index ae4c007..a7ad5a6 100755 --- a/job_pre_release_obs.py +++ b/job_pre_release_obs.py @@ -8,6 +8,7 @@ This code is called by jenkins jobs triggered by OBS events. import os import sys import re +import shutil from common.buildtrigger import trigger_info, trigger_next from common.buildservice import BuildService @@ -18,14 +19,79 @@ class LocalError(Exception): """Local error exception.""" pass -def update_ks(imagedata, backenddb): + +def get_prerelease_data(backenddb, base_url, base_path='/srv/download', + repo_name=None, obs_project=None, tstamp=None): + """ + Return all info about repos, required by pre-release from backend database. + + Args: + backenddb: BackendDB object + base_path: base path to the repo on download server + base_url: base url of the repo + repo_name: repository name (tizen, tizen:ivi, etc) + obs_project: OBS project name: (Tizen:Main, Tizen:IVI, etc) + tstamp: pre-release time stamp + """ + + # Hardcoded architecture mappings + archmap = {'ia32': 'i586'} + + try: + repo_name = repo_name or \ + backenddb.get_obs_project_maps()[obs_project]['name'] + except BackendDBError, err: + raise LocalError("Can't get OBS project for repo '%s' : %s" \ + % (obs_project, str(err))) + + try: + repo = backenddb.get_repos()[repo_name] + except BackendDBError, err: + raise LocalError("Can't get info for repo '%s' : %s" \ + % (repo_name, str(err))) + + release_key = '%s-%s' % (repo['PartOf'], repo['Release']) + try: + release_id = backenddb.get_releases()[release_key] + except BackendDBError, err: + raise LocalError("Can't get release info for '%s' : %s" \ + % (release_key, str(err))) + + latest_snapshot = '%s_%s' % (repo['Release'], release_id) + snapshot_dir = os.path.join(repo['SnapshotDir'], latest_snapshot) + prerelease_dir = os.path.join(repo['PrereleaseDir'], latest_snapshot) + build_id = '%s.%s' % (latest_snapshot, tstamp or 'xxx') + + pkg_urls = {} + for arch in repo['Architectures']: + pkg_urls[arch] = os.path.join(base_url, prerelease_dir, build_id, + 'repos', repo, + archmap.get(arch, arch), 'packages') + + return {'snapshot_path': os.path.join(base_path, snapshot_dir), + 'snapshot_url': os.path.join(base_url, snapshot_dir), + 'prerelease_dir': prerelease_dir, + 'prerelease_path': os.path.join(base_path, prerelease_dir), + 'prerelease_build_id': build_id, + 'pkg_urls': pkg_urls + } + + +def update_ks(imagedata, backenddb, info, base_url): """ update the repo url point to right URL and add prerelease repo url with highest priority Args: imagedata (ImageData): [(ks_file_name, ks_file_content),] backenddb (BackendDB): backend database instance + info: data from backend db """ + + # Figure out repo line + repo_lines = {} + for arch, url in info['pkg_urls'].iteritems(): + repo_lines[arch] = "repo --name=prerelease --baseurl=%s --save "\ + "--ssl_verify=no --priority 1" % url images_ks = {} # update ULRs in ks file for name, content in imagedata.ks.items(): @@ -37,17 +103,15 @@ def update_ks(imagedata, backenddb): match = re.match("repo --name=([\w\-\.]*) --baseurl=.*", line) if match: - release_name = match.group(1) - # Get repo info from backend DB - repo = backenddb.get_repo_data_by_name(release_name) - repo_url = os.path.join(os.getenv("URL_PUBLIC_REPO_BASE", - "/"), - repo['snapshot_dir'], - repo['latest_snapshot']) + data = get_prerelease_data(backenddb, + base_url=base_url, + repo_name=match.group(1)) line = re.sub(r'--baseurl=http.*@BUILD_ID@', - r'--baseurl=%s' %repo_url, line) + r'--baseurl=%s' % data['snapshot_url'], line) new_ks_lines.append(line.strip()) + new_ks_lines.insert(new_ks_lines.index('%packages')-1, + repo_lines[imagedata.images[name]['arch']]) # Update the ks files in imagedata new_ks_content = '\n'.join(new_ks_lines) images_ks[name] = new_ks_content @@ -73,7 +137,7 @@ def trigger_image_creation(images_ks, build_id, path_repo): trigger_next('%s/image_trigger_%s' % (os.getenv('WORKSPACE'), index), data) -def make_repo(project, repo, backenddb): +def make_repo(project, repo, backenddb, base_url, base_path): """ make repo. @@ -87,25 +151,18 @@ def make_repo(project, repo, backenddb): # Make build id from latest snapshot + project suffix target_project = ':'.join(project.split(':')[:-1][3:]) - try: - data = backenddb.get_repo_data(target_project) - except BackendDBError, err: - raise LocalError("Can't get information about project '%s': %s" \ - % (project, str(err))) - latest_snapshot = data['latest_snapshot'] - # Prerelease repo id: ${target_repo_id}.${time_stampe} - build_id = '%s.%s' % (data['latest_snapshot'], project.split(':')[-1]) + data = get_prerelease_data(backenddb, base_url, base_path, + obs_project=target_project, + tstamp=project.split(':')[-1]) # The repo local path on the running target server - path_to_repo = os.path.join(os.getenv('PATH_REPO_BASE', '/srv/download'), - data['prerelease_dir']) + prerelease_path = data['prerelease_path'] # Convert live repo to download structure - repomaker = RepoMaker(build_id, path_to_repo) - live_repo_path = os.path.join(os.getenv('PATH_LIVE_REPO_BASE', - '/srv/obs/repos/'), - project.replace(':', ':/')) + build_id = data['prerelease_build_id'] + repomaker = RepoMaker(build_id, prerelease_path) + live_repo_path = os.path.join(prerelease_path, project.replace(':', ':/')) # TODO: get buldconf from OBS try: repomaker.add_repo(live_repo_path, repo, move=False) #,buildconf @@ -116,12 +173,9 @@ def make_repo(project, repo, backenddb): if not repomaker.imagedata: # repomaker did not found image-configurations in pre_release repo, # let's take it from target repo, only one package repo is enough - path_target_repo = os.path.join(os.getenv('PATH_REPO_BASE', - '/srv/download'), - data['snapshot_dir'], - latest_snapshot) + # Add image configuration to pre-release repo - for rpm in find_files(path_target_repo, + for rpm in find_files(data['snapshot_path'], prefix="image-configurations-", suffix='noarch.rpm'): repomaker.load_imagedata(rpm) @@ -130,7 +184,7 @@ def make_repo(project, repo, backenddb): raise LocalError("Image configuration not found") # Update ks files - images_ks = update_ks(repomaker.imagedata, backenddb) + images_ks = update_ks(repomaker.imagedata, backenddb, data, base_url) # Generate image info to builddata/ dir repomaker.gen_image_info(images_ks) @@ -165,11 +219,15 @@ def main(name, action): obs_api = os.getenv("OBS_API_URL") obs_user = os.getenv("OBS_API_USERNAME") obs_passwd = os.getenv("OBS_API_PASSWD") + base_url = os.getenv("URL_PUBLIC_REPO_BASE") + base_path = os.getenv('PATH_REPO_BASE') + content = trigger_info(os.getenv("TRIGGER_INFO")) project = content.get("project") build = BuildService(obs_api, obs_user, obs_passwd) + # Init backend database redis_host = os.getenv("REDIS_HOST") redis_port = int(os.getenv("REDIS_PORT")) @@ -184,33 +242,28 @@ def main(name, action): for target_arch, status in repo_status.items(): if target_arch.split('/')[0] == repo and status != 'published': print "The project: %s target: %s repo doesn't published yet" \ - %(project, target_arch) + % (project, target_arch) return True - make_repo(project, repo, backenddb) + make_repo(project, repo, backenddb, base_url, base_path) elif action == 'cleanup': build.cleanup(project, "Cleaned up by %s" % name) # Get target project from prerelease project name target_project = ':'.join(project.split(':')[:-1][3:]) + # Get pre-release data from db + data = get_prerelease_data(backenddb, base_url, + obs_project=target_project) + # Get tag time stamp from prerelease project name time_stamp = project.split(':')[-1] - try: - data = backenddb.get_repo_data(target_project) - except BackendDBError, err: - raise LocalError("Can't get information about project '%s': %s" \ - % (project, str(err))) - - path_to_prelease_repo = os.path.join(os.getenv('PATH_REPO_BASE', '/srv/download'), - data['prerelease_dir']) - - published_repos = get_published_repo(path_to_prelease_repo, time_stamp) - + published_repos = get_published_repo(data['prerelease_path'], + time_stamp) for repo in published_repos: - print 'Removing the repo: %s' %(repo) + print 'Removing the repo: %s' % repo shutil.rmtree(repo) else: -- 2.7.4