From: Ed Bartosh Date: Mon, 13 May 2013 11:38:03 +0000 (+0300) Subject: Implemented repomaker API X-Git-Tag: 0.14~181 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=b6a2fd6d6b0e92a7edac8ceedb61d77f532096d1;p=services%2Fjenkins-scripts.git Implemented repomaker API This API takes care of generating rpm repos in appropriate directory structure. It also generates build.xml and images.xml Change-Id: Ib2b31ea0173581d8a0d150ec12474d8b68548967 Signed-off-by: Ed Bartosh --- diff --git a/common/repomaker.py b/common/repomaker.py new file mode 100644 index 0000000..4cbe8a9 --- /dev/null +++ b/common/repomaker.py @@ -0,0 +1,217 @@ +# vim: ai ts=4 sts=4 et sw=4 +# +# Copyright (c) 2012 Intel, Inc. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation; version 2 of the License +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., 59 +# Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +""" +RepoMaker - creates download repos +""" + +import os +import shutil +from hashlib import sha256 + +from common.builddata import BuildData, BuildDataError + +class RepoMakerError(Exception): + """Custom RepoMaker exception.""" + pass + +# Helper functions +def find(topdir, suffix='.rpm'): + """Find files in the tree.""" + for item in os.listdir(topdir): + path = os.path.join(topdir, item) + if os.path.isdir(path): + for fpath in find(path): + yield fpath + elif item.endswith(suffix): + yield path + +def collect(in_dir): + """Collect files, binary archs and destinations.""" + files = [] + archs = set() + for fname in find(in_dir): + ftype = fname.split('.')[-2] + if ftype not in ("src", "noarch"): + archs.add(ftype) + + is_debug = "-debugsource-" in fname or "-debuginfo-" in fname + is_group = fname.startswith("package-groups-") + is_imageconf = fname.startswith("image-configurations-") + files.append((fname, ftype, is_debug, is_group, is_imageconf)) + return files, archs + +def create_dirs(repo_dir, archs): + """Create directory structure of the repos.""" + dirs = [os.path.join(repo_dir, arch, subdir) for arch in archs \ + for subdir in ('packages', 'debug')] + dirs.append(os.path.join(repo_dir, "sources")) + dirs.append(os.path.join(repo_dir, "repodata")) + for dirname in dirs: + if not os.path.exists(dirname): + os.makedirs(dirname) + return dirs + +def gen_target_dirs(repo_dir, archs, ftype, is_debug): + """"Prepare list of target dirs depending on type of package.""" + if ftype == "src": + return [os.path.join(repo_dir, "sources")] + else: + subdir = ["packages", "debug"][is_debug] + if ftype == "noarch": + return [os.path.join(repo_dir, binarch, subdir) \ + for binarch in archs] + else: + return [os.path.join(repo_dir, ftype, subdir)] + +def move_or_hardlink(fpath, target_dirs, move=False): + """Move or hardlink file to target directories.""" + fname = os.path.basename(fpath) + for tdir in target_dirs: + tpath = os.path.join(tdir, fname) + if move: + if os.path.exists(fpath): + shutil.move(fpath, tpath) + else: + # already moved noarch package. symlink it. + os.symlink(os.path.join(target_dirs[0], fname), tpath) + else: + os.link(fpath, tpath) + + +class RepoMaker(object): + """Makes rpm repositories.""" + + def __init__(self, build_id, outdir): + """ + RepoMaker init + + Args: + build_id (str): Build identifier + out_dir (str): Top output directory. + """ + + self.build_id = build_id + self.outdir = os.path.join(outdir, build_id) + self.repos = {} + + def add_repo(self, in_dir, name, buildconf=None, move=False, gpg_key=None, + signer='/usr/bin/sign'): + """ + Convert repository to download structure. + Create or update repository using packages from repo_dir + + Args: + in_dir (str): path to repository to convert + name (str): name of the repository to create + buildconf(str): content of build configuration + move (bool): move files instead of hardlinking + gpg_key (str): path to file with gpg key + signer (str): command to sign the repo + + Raises: RepoMakerError + + """ + if not os.path.exists(in_dir): + raise RepoMakerError("Directory %s doesn't exist" % in_dir) + + repo_dir = os.path.join(self.outdir, "repos", name) + if name not in self.repos: + self.repos[name] = {'archs': set()} + + files, archs = collect(in_dir) + self.repos[name]['archs'].update(archs) + + # Create directory structure + dirs = create_dirs(repo_dir, self.repos[name]['archs']) + + for fpath, ftype, is_debug, is_group, is_imageconf in files: + # Prepare list of target directories + target_dirs = gen_target_dirs(repo_dir, archs, ftype, is_debug) + + # Move or hardlink .rpm to target dirs + move_or_hardlink(fpath, target_dirs, move) + + # For package-groups package update package groups and patterns + if is_group: + for filename in ("group.xml", "patterns.xml"): + os.system("rpm2cpio %s | cpio -i --to-stdout "\ + "./usr/share/package-groups/%s > %s" % \ + (fpath, filename, os.path.join(repo_dir, + "repodata", + filename))) + # extract .ks file + if is_imageconf: + # TODO: save content of .ks file and other image parameters + pass + + # Save build configuration file if provided + if buildconf: + confpath = os.path.join(repo_dir, "repodata", + "%s-build.conf" % name) + with open(confpath, 'w') as conf: + conf.write(buildconf) + # Generate or update build.xml + self.update_builddata(name, buildconf) + + # TODO: Generate images.xml from previously saved data (see above) + + # Run createrepo + for repo in dirs: + # run createrepo + os.system('createrepo --quiet %s' % repo) + + # sign if gpg_key is provided + if gpg_key and os.path.exists(signer) and os.access(signer, os.X_OK): + for repo in dirs: + repomd_path = os.path.join(repo_dir, repo, "repodata", + "repomd.xml") + os.system('%s -d % s' % (signer, repomd_path)) + + + def update_builddata(self, name, buildconf=None): + """ + Update or create build.xml + + Args: + name (str): name of the target repository + buildconf(str): content of build configuration + + Raises: RepoMakerError + + """ + try: + bdata = BuildData(self.build_id) + target = {"name": name, "archs": list(self.repos[name]['archs'])} + if buildconf: + target["buildconf"] = { + "location": os.path.join("repos", name), + "checksum": { + "type": "sh256", + "value": sha256(buildconf).hexdigest() + } + } + + # Create or update build.xml + outf = os.path.join(self.outdir, 'build.xml') + if os.path.exists(outf): + bdata.load(open(outf)) + + bdata.add_target(target) + bdata.save(outf) + except BuildDataError, err: + raise RepoMakerError("Unable to generate build.xml: %s" % err) diff --git a/packaging/jenkins-scripts.spec b/packaging/jenkins-scripts.spec index 49ceb53..db598e6 100644 --- a/packaging/jenkins-scripts.spec +++ b/packaging/jenkins-scripts.spec @@ -9,6 +9,7 @@ Source: %{name}-%{version}.tar.gz Requires: git-core Requires: python-mysql Requires: python-yaml +Requires: createrepo Requires: rpmlint >= 1.3 Requires: gbs-api BuildRoot: %{_tmppath}/%{name}-%{version}-build diff --git a/tests/test_repomaker.py b/tests/test_repomaker.py new file mode 100644 index 0000000..6894aa3 --- /dev/null +++ b/tests/test_repomaker.py @@ -0,0 +1,148 @@ +#!/usr/bin/python -tt +# vim: ai ts=4 sts=4 et sw=4 +# +# Copyright (c) 2012 Intel, Inc. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation; version 2 of the License +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., 59 +# Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +"""Unit tests for RepoMaker""" + +import os +import unittest +import tempfile +import shutil +import contextlib +import base64 +import glob +import gzip + +from xml.dom import minidom + +from common.repomaker import RepoMaker, RepoMakerError + +# rpm content +RPM = base64.b64decode("""7avu2wMAAAEAAXBrZzEtMS4wLTEAAAAAAAAAAAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAUAAAAAAAAAAAAAAAAAAAAA +jq3oAQAAAAAAAAAFAAAAVAAAAD4AAAAHAAAARAAAABAAAAPoAAAABAAAAAAAAAABAAAD7AAAAAcA +AAAEAAAAEAAAAQ0AAAAGAAAAFAAAAAEAAAPvAAAABAAAAEAAAAABAAAGAZifaNg82W1fAFmktnky +lPk5ZjRlY2QzYjdlNmY2MjRjZDQyMzEwYTdiNzAwYjllZWQyM2E0NzMwAAAAAAAAAywAAAA+AAAA +B////7AAAAAQAAAAAI6t6AEAAAAAAAAAKgAAAXkAAAA/AAAABwAAAWkAAAAQAAAAZAAAAAgAAAAA +AAAAAQAAA+gAAAAGAAAAAgAAAAEAAAPpAAAABgAAAAcAAAABAAAD6gAAAAYAAAALAAAAAQAAA+wA +AAAJAAAADQAAAAEAAAPtAAAA +CQAAACUAAAABAAAD7gAAAAQAAABQAAAAAQAAA+8AAAAGAAAAVAAAAAEAAAPxAAAABAAAAFgAAAAB +AAAD9gAAAAYAAABcAAAAAQAAA/gAAAAJAAAAYgAAAAEAAAP8AAAABgAAAHoAAAABAAAD/QAAAAYA +AACOAAAAAQAAA/4AAAAGAAAAlAAAAAEAAAQEAAAABAAAAJwAAAABAAAEBgAAAAMAAACgAAAAAQAA +BAkAAAADAAAAogAAAAEAAAQKAAAABAAAAKQAAAABAAAECwAAAAgAAACoAAAAAQAABAwAAAAIAAAA +yQAAAAEAAAQNAAAABAAAAMwAAAABAAAEDwAAAAgAAADQAAAAAQAABBAAAAAIAAAA1QAAAAEAAAQV +AAAABAAAANwAAAABAAAEGAAAAAQAAADgAAAAAQAABBkAAAAIAAAA5AAAAAEAAAQaAAAACAAAAQAA +AAABAAAEKAAAAAYAAAEIAAAAAQAABDgAAAAEAAABEAAAAAEAAAQ5AAAACAAAARQAAAABAAAEOgAA +AAgAAAEtAAAAAQAABEcAAAAEAAABNAAAAAEAAARIAAAABAAAATgAAAABAAAESQAAAAgAAAE8AAAA +AQAABFwAAAAEAAABQAAAAAEAAARdAAAACAAAAUQAAAABAAAEXgAAAAgAAAFOAAAAAQAABGQAAAAG +AAABTwAAAAEAAARlAAAABgAAAVQAAAABAAAEZgAAAAYAAAFZAAAAAQAABEYAAAAGAAABWwAAAAFD +AHBrZzEAMS4wADEAVGVzdGluZyBtaW5pbWFsIHBhY2thZ2UATWluaW1hbCBycG0gcGFja2FnZSBm +b3IgdGVzdGluZyBwdXJwb3Nlcy4AAFGNBQtlZAAAAAACN0dQTHYyAERldmVsb3BtZW50L1Rvb2xz +L090aGVyAGh0dHA6Ly9mYWtlLmNvbS9ibGEAbGludXgAaTU4NgAAAAAAAAI3gaQAAFGNBQg0M2Rm +ZDI1ZmZmNWVjNTU0MDVmOTBhNjlhNmI4NjI1ZQAAAAAAAAAgcm9vdAByb290AAAA/////wEAAApy +cG1saWIoQ29tcHJlc3NlZEZpbGVOYW1lcykAMy4wLjQtMQA0LjkuMS4yAFGM4UBlZHVhcmQuYmFy +dG9zaEBpbnRlbC5jb20ALSBJbml0AAAACAEAEUgfAAAAAAAAAABwa2cxLnNwZWMAAGNwaW8AZ3pp +cAA5AGVkIDEzNjgxOTYzNjMAAAAAPwAAAAf///1gAAAAEB+LCAAAAAAAAgONUmFv2jAQ5Wv9K45u +SDA1iU2LQBRV20TbIcGoMrqvyDhOsHBsy3YqVYj/PsOStps2qe+Dfb737s56Nh7iISYYE3I1IjkO +GBF6hf8NMiCjDA/w6HTqXw7r/Kjh/1NHm8DsChI7w1nrOy35GF5xZNBPbp3Q6iVPYoxSLjl1r1qC +flRlSe3zS2bFnReqgFIoUVIJhrIdLTiaC8bVm8r7h/lTH91bXZnx2dmUP3GpTcmVT1ZaS5cs/ZZb +9JjOA7v13oyTJKc7HjNdJhtJEepk3DErjA93RIt6mjVlMxFybcHXtzGVNdpxF4c6Y7lBXUY9TCa3 +yzv0QSgmq4zDxPlM6Hh7g4TyUFKhuj3Yo5MhNqTy7vk3LqW+gM5eBccO7fPe9Ym23FdWAb5GB3Rs +2YObRhOzMHJTCZmhgjGIdEP8IRDKeSolqneIphCVeDgYnJ4CPu7Th8X66+NsPl2ny+XqkFTOJhuh +krpJaJELyd3RlZx6b7vRhdXa/16iHurs10GeCXt4U8K2VBXB9wKhT3BnBSzoMxAMfUwuIQKeVdRm +8YZar932c3CAy6P/KIKZEh5QC9f/9V14r+5vbJpglX6ZzW/TdrvdCvgF1lylJCwDAAA=""") + +@contextlib.contextmanager +def tempdir(suffix='', prefix='tmp'): + """ + A context manager for creating and then + deleting a temporary directory. + """ + tmpdir = tempfile.mkdtemp(prefix=prefix, suffix=suffix) + try: + yield tmpdir + finally: + shutil.rmtree(tmpdir) + + +class RepoMakerTest(unittest.TestCase): + '''Tests for RepoMaker functionality.''' + + def test_non_existing_dir(self): + """Test exception raised when output dir doesn't exist.""" + maker = RepoMaker('test-id', 'outrepo') + self.assertRaises(RepoMakerError, maker.add_repo, + 'aj45sdjfoqwiq4334w', 'test-repo') + + def test_add_repo(self): + """Test converting repository to the download structure.""" + with tempdir(prefix='repomaker.', suffix='.inrepo') as in_repo: + # prepare directory with rpms + arches = ('src', 'noarch', 'i586', 'x86_64') + files = ["%s-%s.%s.rpm" % (name, version, arch) \ + for name in ("pkg1", "pkg2", "pkg3") \ + for version in ("0.1-2.3", "1.2-3.4") \ + for arch in arches] + for fname in files: + subdir = ('', fname)[files.index(fname)%2] + dir_path = os.path.join(in_repo, subdir) + if not os.path.exists(dir_path): + os.makedirs(dir_path) + with open(os.path.join(dir_path, fname), 'w') as frpm: + frpm.write(RPM) + + with tempdir(prefix='repomaker.', suffix='.outrepo') as out_repo: + maker = RepoMaker('test-id', out_repo) + maker.add_repo(in_repo, 'testrepo', move=True, gpg_key='key', + signer='/bin/true', buildconf='buildconf') + # Check repo structure + results = { + 'sources': + ['pkg1-0.1-2.3.src.rpm', 'pkg1-1.2-3.4.src.rpm', + 'pkg2-0.1-2.3.src.rpm', 'pkg2-1.2-3.4.src.rpm', + 'pkg3-0.1-2.3.src.rpm', 'pkg3-1.2-3.4.src.rpm'], + 'i586/packages': + ['pkg1-0.1-2.3.i586.rpm', 'pkg1-0.1-2.3.noarch.rpm', + 'pkg1-1.2-3.4.i586.rpm', 'pkg1-1.2-3.4.noarch.rpm', + 'pkg2-0.1-2.3.i586.rpm', 'pkg2-0.1-2.3.noarch.rpm', + 'pkg2-1.2-3.4.i586.rpm', 'pkg2-1.2-3.4.noarch.rpm', + 'pkg3-0.1-2.3.i586.rpm', 'pkg3-0.1-2.3.noarch.rpm', + 'pkg3-1.2-3.4.i586.rpm', 'pkg3-1.2-3.4.noarch.rpm'], + 'i586/debug': [], + 'x86_64/packages': + ['pkg1-0.1-2.3.noarch.rpm', 'pkg1-0.1-2.3.x86_64.rpm', + 'pkg1-1.2-3.4.noarch.rpm', 'pkg1-1.2-3.4.x86_64.rpm', + 'pkg2-0.1-2.3.noarch.rpm', 'pkg2-0.1-2.3.x86_64.rpm', + 'pkg2-1.2-3.4.noarch.rpm', 'pkg2-1.2-3.4.x86_64.rpm', + 'pkg3-0.1-2.3.noarch.rpm', 'pkg3-0.1-2.3.x86_64.rpm', + 'pkg3-1.2-3.4.noarch.rpm', 'pkg3-1.2-3.4.x86_64.rpm'], + 'x86_64/debug': [] + } + + for arch in ('sources', 'i586/packages', 'i586/debug', + 'x86_64/packages', 'x86_64/debug'): + primary_path = glob.glob(os.path.join(out_repo, + 'test-id', + 'repos', + 'testrepo', + arch, + 'repodata', + '*primary.xml.gz'))[0] + dom = minidom.parse(gzip.open(primary_path)) + packages = sorted(elem.getAttribute('href') for elem in \ + dom.getElementsByTagName('location')) + self.assertEqual(packages, results[arch])