From 276441507278bd120aaa9deaf7df96ca06d6c2c4 Mon Sep 17 00:00:00 2001 From: Hasan Wan Date: Mon, 11 Nov 2013 06:26:17 +0000 Subject: [PATCH] Generate repo manifest file Fixes: #1392 Change-Id: I1f6b055ec59ad8bc5caaef681ff09bc08bd5c9d3 Signed-off-by: Hasan Wan --- common/manifest.py | 86 ++++++++++++++++++++++++++++++++++++++++++ common/repomaker.py | 46 ++++++++++++++++++++++ job_create_snapshot.py | 5 +++ packaging/jenkins-scripts.spec | 1 + 4 files changed, 138 insertions(+) create mode 100644 common/manifest.py diff --git a/common/manifest.py b/common/manifest.py new file mode 100644 index 0000000..6158b33 --- /dev/null +++ b/common/manifest.py @@ -0,0 +1,86 @@ +# vim: ai ts=4 sts=4 et sw=4 +# +# Copyright (c) 2012 Intel, Inc. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation; version 2 of the License +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., 59 +# Temple Place - Suite 330, Boston, MA 02111-1307, USA. + + +import os +import gzip + +from common.utils import xml2obj + +HEADER=""" + + + +""" +XML_ITEM="""\n""" +FOOTER="""""" + +def gen_repo_manifest(data, fetch_url, review_url): + """ Generate manifest file for android repo tool + """ + manifest_head = HEADER %(fetch_url, review_url) + manifest_body = "" + manifest_tail = FOOTER + + manifest_info= {} + for prj_name, prj_path, revision in data: + manifest_body +=XML_ITEM %(prj_name, prj_path, revision) + + return manifest_head + manifest_body + manifest_tail + +def get_package_vcs_tag(primary_md): + """ Get package vcs tag from repo primary md + """ + + ret_data = {} + xml_file = gzip.open(primary_md) + + primary = xml2obj(xml_file) + + for package in primary.package: + vcs_tag = package.version.vcs + if vcs_tag: + ret_data[package.name] = vcs_tag + else: + ret_data[package.name] = "" + + return ret_data + +def get_repo_primary_md(path_snapshot_base, repo): + """ Get primary md from build.xml + """ + + # return as dict in {'package_name': "VCS Tag"} + ret_data= {} + + build_info = xml2obj(open(os.path.join(path_snapshot_base, + 'build.xml'))) + + for buildtarget in build_info.buildtargets.buildtarget: + if buildtarget.name == repo: + for repo in buildtarget.repo: + if repo.type == 'binary' and repo.data.endswith('packages'): + for md_file in os.listdir(os.path.join(path_snapshot_base, + repo.data, + 'repodata')): + if md_file.endswith('primary.xml.gz'): + ret_data[repo.arch] = \ + os.path.join(path_snapshot_base, + repo.data, + 'repodata', + md_file) + return ret_data diff --git a/common/repomaker.py b/common/repomaker.py index 51377de..7d7338b 100644 --- a/common/repomaker.py +++ b/common/repomaker.py @@ -25,6 +25,7 @@ from hashlib import sha256 # pylint: disable-msg=E0611 from common.builddata import BuildData, BuildDataError from common.imagedata import ImageData +from common import manifest # arch map table: # key->val: map the arch from 'key' to 'val' to make the url unify @@ -269,3 +270,48 @@ class RepoMaker(object): bdata.save(outf) except BuildDataError, err: raise RepoMakerError("Unable to generate build.xml: %s" % err) + + def gen_manifest_info(self, name, gerrit_fetch_url, gerrit_review_url): + """ + Generate manifest for repo + + Args: + name (str): name of the target repository + gerrit_fetch_url (str): url base to fetch gerrit project + gerrit_review_url (str): url base for changes review + Raises: + RepoMakerError + """ + + manifest_dir = os.path.join(self.outdir, 'builddata','manifest') + + if not os.path.exists(manifest_dir): + os.makedirs(manifest_dir) + else: + raise RepoMakerError("The directory %s supposed must be clean" + % sync_out_dir ) + + repo_primary = manifest.get_repo_primary_md(self.outdir, name) + + # Generate manifest for every repo arch + for arch in repo_primary.keys(): + # package and vcs tag dict + package_vcs_tag = manifest.get_package_vcs_tag(repo_primary[arch]) + # tuple set in (gitprj, git_path, revision) + data = set([]) + + for pkg in package_vcs_tag.keys(): + try: + git_prj, commit_id = package_vcs_tag[pkg].split('#') + data.add((git_prj, git_prj, commit_id)) + except ValueError: + # No vcs tag found + data.add((pkg, '', '')) + + manifest_string = manifest.gen_repo_manifest(sorted(data), + gerrit_fetch_url, + gerrit_review_url) + with open(os.path.join(manifest_dir, + "%s_%s.xml" %(self.build_id, arch)), + 'w') as manifest_fh: + manifest_fh.write(manifest_string) diff --git a/job_create_snapshot.py b/job_create_snapshot.py index addbcba..a3b47fc 100755 --- a/job_create_snapshot.py +++ b/job_create_snapshot.py @@ -90,6 +90,11 @@ def make_repo(project, repo, backenddb, base_path, # Generate image info to builddata/ dir repomaker.gen_image_info(repomaker.imagedata.ksi) + # Generate repo manifest + repomaker.gen_manifest_info(repo, + os.getenv('GERRIT_FETCH_URL'), + os.getenv('GERRIT_REVIEW_URL')) + return {'project': project, 'repo': repo, 'repo_path': os.path.join(snapshot.dir, snapshot.build_id), diff --git a/packaging/jenkins-scripts.spec b/packaging/jenkins-scripts.spec index 8341082..c43fdd6 100644 --- a/packaging/jenkins-scripts.spec +++ b/packaging/jenkins-scripts.spec @@ -117,6 +117,7 @@ fi %{destdir}/common/send_mail.py %{destdir}/common/snapshot.py %{destdir}/common/utils.py +%{destdir}/common/manifest.py %{destdir}/job_create_snapshot.py %{destdir}/job_buildlogs.py %{destdir}/job_jobs_dispatcher.py -- 2.7.4