--- /dev/null
+# vim: ai ts=4 sts=4 et sw=4
+#
+# Copyright (c) 2012 Intel, Inc.
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation; version 2 of the License
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+# for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc., 59
+# Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+
+"""
+RepoMaker - creates download repos
+"""
+
+import os
+import shutil
+from hashlib import sha256
+
+from common.builddata import BuildData, BuildDataError
+
+class RepoMakerError(Exception):
+ """Custom RepoMaker exception."""
+ pass
+
+# Helper functions
+def find(topdir, suffix='.rpm'):
+ """Find files in the tree."""
+ for item in os.listdir(topdir):
+ path = os.path.join(topdir, item)
+ if os.path.isdir(path):
+ for fpath in find(path):
+ yield fpath
+ elif item.endswith(suffix):
+ yield path
+
+def collect(in_dir):
+ """Collect files, binary archs and destinations."""
+ files = []
+ archs = set()
+ for fname in find(in_dir):
+ ftype = fname.split('.')[-2]
+ if ftype not in ("src", "noarch"):
+ archs.add(ftype)
+
+ is_debug = "-debugsource-" in fname or "-debuginfo-" in fname
+ is_group = fname.startswith("package-groups-")
+ is_imageconf = fname.startswith("image-configurations-")
+ files.append((fname, ftype, is_debug, is_group, is_imageconf))
+ return files, archs
+
+def create_dirs(repo_dir, archs):
+ """Create directory structure of the repos."""
+ dirs = [os.path.join(repo_dir, arch, subdir) for arch in archs \
+ for subdir in ('packages', 'debug')]
+ dirs.append(os.path.join(repo_dir, "sources"))
+ dirs.append(os.path.join(repo_dir, "repodata"))
+ for dirname in dirs:
+ if not os.path.exists(dirname):
+ os.makedirs(dirname)
+ return dirs
+
+def gen_target_dirs(repo_dir, archs, ftype, is_debug):
+ """"Prepare list of target dirs depending on type of package."""
+ if ftype == "src":
+ return [os.path.join(repo_dir, "sources")]
+ else:
+ subdir = ["packages", "debug"][is_debug]
+ if ftype == "noarch":
+ return [os.path.join(repo_dir, binarch, subdir) \
+ for binarch in archs]
+ else:
+ return [os.path.join(repo_dir, ftype, subdir)]
+
+def move_or_hardlink(fpath, target_dirs, move=False):
+ """Move or hardlink file to target directories."""
+ fname = os.path.basename(fpath)
+ for tdir in target_dirs:
+ tpath = os.path.join(tdir, fname)
+ if move:
+ if os.path.exists(fpath):
+ shutil.move(fpath, tpath)
+ else:
+ # already moved noarch package. symlink it.
+ os.symlink(os.path.join(target_dirs[0], fname), tpath)
+ else:
+ os.link(fpath, tpath)
+
+
+class RepoMaker(object):
+ """Makes rpm repositories."""
+
+ def __init__(self, build_id, outdir):
+ """
+ RepoMaker init
+
+ Args:
+ build_id (str): Build identifier
+ out_dir (str): Top output directory.
+ """
+
+ self.build_id = build_id
+ self.outdir = os.path.join(outdir, build_id)
+ self.repos = {}
+
+ def add_repo(self, in_dir, name, buildconf=None, move=False, gpg_key=None,
+ signer='/usr/bin/sign'):
+ """
+ Convert repository to download structure.
+ Create or update repository using packages from repo_dir
+
+ Args:
+ in_dir (str): path to repository to convert
+ name (str): name of the repository to create
+ buildconf(str): content of build configuration
+ move (bool): move files instead of hardlinking
+ gpg_key (str): path to file with gpg key
+ signer (str): command to sign the repo
+
+ Raises: RepoMakerError
+
+ """
+ if not os.path.exists(in_dir):
+ raise RepoMakerError("Directory %s doesn't exist" % in_dir)
+
+ repo_dir = os.path.join(self.outdir, "repos", name)
+ if name not in self.repos:
+ self.repos[name] = {'archs': set()}
+
+ files, archs = collect(in_dir)
+ self.repos[name]['archs'].update(archs)
+
+ # Create directory structure
+ dirs = create_dirs(repo_dir, self.repos[name]['archs'])
+
+ for fpath, ftype, is_debug, is_group, is_imageconf in files:
+ # Prepare list of target directories
+ target_dirs = gen_target_dirs(repo_dir, archs, ftype, is_debug)
+
+ # Move or hardlink .rpm to target dirs
+ move_or_hardlink(fpath, target_dirs, move)
+
+ # For package-groups package update package groups and patterns
+ if is_group:
+ for filename in ("group.xml", "patterns.xml"):
+ os.system("rpm2cpio %s | cpio -i --to-stdout "\
+ "./usr/share/package-groups/%s > %s" % \
+ (fpath, filename, os.path.join(repo_dir,
+ "repodata",
+ filename)))
+ # extract .ks file
+ if is_imageconf:
+ # TODO: save content of .ks file and other image parameters
+ pass
+
+ # Save build configuration file if provided
+ if buildconf:
+ confpath = os.path.join(repo_dir, "repodata",
+ "%s-build.conf" % name)
+ with open(confpath, 'w') as conf:
+ conf.write(buildconf)
+ # Generate or update build.xml
+ self.update_builddata(name, buildconf)
+
+ # TODO: Generate images.xml from previously saved data (see above)
+
+ # Run createrepo
+ for repo in dirs:
+ # run createrepo
+ os.system('createrepo --quiet %s' % repo)
+
+ # sign if gpg_key is provided
+ if gpg_key and os.path.exists(signer) and os.access(signer, os.X_OK):
+ for repo in dirs:
+ repomd_path = os.path.join(repo_dir, repo, "repodata",
+ "repomd.xml")
+ os.system('%s -d % s' % (signer, repomd_path))
+
+
+ def update_builddata(self, name, buildconf=None):
+ """
+ Update or create build.xml
+
+ Args:
+ name (str): name of the target repository
+ buildconf(str): content of build configuration
+
+ Raises: RepoMakerError
+
+ """
+ try:
+ bdata = BuildData(self.build_id)
+ target = {"name": name, "archs": list(self.repos[name]['archs'])}
+ if buildconf:
+ target["buildconf"] = {
+ "location": os.path.join("repos", name),
+ "checksum": {
+ "type": "sh256",
+ "value": sha256(buildconf).hexdigest()
+ }
+ }
+
+ # Create or update build.xml
+ outf = os.path.join(self.outdir, 'build.xml')
+ if os.path.exists(outf):
+ bdata.load(open(outf))
+
+ bdata.add_target(target)
+ bdata.save(outf)
+ except BuildDataError, err:
+ raise RepoMakerError("Unable to generate build.xml: %s" % err)
--- /dev/null
+#!/usr/bin/python -tt
+# vim: ai ts=4 sts=4 et sw=4
+#
+# Copyright (c) 2012 Intel, Inc.
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation; version 2 of the License
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+# for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc., 59
+# Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+
+"""Unit tests for RepoMaker"""
+
+import os
+import unittest
+import tempfile
+import shutil
+import contextlib
+import base64
+import glob
+import gzip
+
+from xml.dom import minidom
+
+from common.repomaker import RepoMaker, RepoMakerError
+
+# rpm content
+RPM = base64.b64decode("""7avu2wMAAAEAAXBrZzEtMS4wLTEAAAAAAAAAAAAAAAAAAAAAAAAA
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAUAAAAAAAAAAAAAAAAAAAAA
+jq3oAQAAAAAAAAAFAAAAVAAAAD4AAAAHAAAARAAAABAAAAPoAAAABAAAAAAAAAABAAAD7AAAAAcA
+AAAEAAAAEAAAAQ0AAAAGAAAAFAAAAAEAAAPvAAAABAAAAEAAAAABAAAGAZifaNg82W1fAFmktnky
+lPk5ZjRlY2QzYjdlNmY2MjRjZDQyMzEwYTdiNzAwYjllZWQyM2E0NzMwAAAAAAAAAywAAAA+AAAA
+B////7AAAAAQAAAAAI6t6AEAAAAAAAAAKgAAAXkAAAA/AAAABwAAAWkAAAAQAAAAZAAAAAgAAAAA
+AAAAAQAAA+gAAAAGAAAAAgAAAAEAAAPpAAAABgAAAAcAAAABAAAD6gAAAAYAAAALAAAAAQAAA+wA
+AAAJAAAADQAAAAEAAAPtAAAA
+CQAAACUAAAABAAAD7gAAAAQAAABQAAAAAQAAA+8AAAAGAAAAVAAAAAEAAAPxAAAABAAAAFgAAAAB
+AAAD9gAAAAYAAABcAAAAAQAAA/gAAAAJAAAAYgAAAAEAAAP8AAAABgAAAHoAAAABAAAD/QAAAAYA
+AACOAAAAAQAAA/4AAAAGAAAAlAAAAAEAAAQEAAAABAAAAJwAAAABAAAEBgAAAAMAAACgAAAAAQAA
+BAkAAAADAAAAogAAAAEAAAQKAAAABAAAAKQAAAABAAAECwAAAAgAAACoAAAAAQAABAwAAAAIAAAA
+yQAAAAEAAAQNAAAABAAAAMwAAAABAAAEDwAAAAgAAADQAAAAAQAABBAAAAAIAAAA1QAAAAEAAAQV
+AAAABAAAANwAAAABAAAEGAAAAAQAAADgAAAAAQAABBkAAAAIAAAA5AAAAAEAAAQaAAAACAAAAQAA
+AAABAAAEKAAAAAYAAAEIAAAAAQAABDgAAAAEAAABEAAAAAEAAAQ5AAAACAAAARQAAAABAAAEOgAA
+AAgAAAEtAAAAAQAABEcAAAAEAAABNAAAAAEAAARIAAAABAAAATgAAAABAAAESQAAAAgAAAE8AAAA
+AQAABFwAAAAEAAABQAAAAAEAAARdAAAACAAAAUQAAAABAAAEXgAAAAgAAAFOAAAAAQAABGQAAAAG
+AAABTwAAAAEAAARlAAAABgAAAVQAAAABAAAEZgAAAAYAAAFZAAAAAQAABEYAAAAGAAABWwAAAAFD
+AHBrZzEAMS4wADEAVGVzdGluZyBtaW5pbWFsIHBhY2thZ2UATWluaW1hbCBycG0gcGFja2FnZSBm
+b3IgdGVzdGluZyBwdXJwb3Nlcy4AAFGNBQtlZAAAAAACN0dQTHYyAERldmVsb3BtZW50L1Rvb2xz
+L090aGVyAGh0dHA6Ly9mYWtlLmNvbS9ibGEAbGludXgAaTU4NgAAAAAAAAI3gaQAAFGNBQg0M2Rm
+ZDI1ZmZmNWVjNTU0MDVmOTBhNjlhNmI4NjI1ZQAAAAAAAAAgcm9vdAByb290AAAA/////wEAAApy
+cG1saWIoQ29tcHJlc3NlZEZpbGVOYW1lcykAMy4wLjQtMQA0LjkuMS4yAFGM4UBlZHVhcmQuYmFy
+dG9zaEBpbnRlbC5jb20ALSBJbml0AAAACAEAEUgfAAAAAAAAAABwa2cxLnNwZWMAAGNwaW8AZ3pp
+cAA5AGVkIDEzNjgxOTYzNjMAAAAAPwAAAAf///1gAAAAEB+LCAAAAAAAAgONUmFv2jAQ5Wv9K45u
+SDA1iU2LQBRV20TbIcGoMrqvyDhOsHBsy3YqVYj/PsOStps2qe+Dfb737s56Nh7iISYYE3I1IjkO
+GBF6hf8NMiCjDA/w6HTqXw7r/Kjh/1NHm8DsChI7w1nrOy35GF5xZNBPbp3Q6iVPYoxSLjl1r1qC
+flRlSe3zS2bFnReqgFIoUVIJhrIdLTiaC8bVm8r7h/lTH91bXZnx2dmUP3GpTcmVT1ZaS5cs/ZZb
+9JjOA7v13oyTJKc7HjNdJhtJEepk3DErjA93RIt6mjVlMxFybcHXtzGVNdpxF4c6Y7lBXUY9TCa3
+yzv0QSgmq4zDxPlM6Hh7g4TyUFKhuj3Yo5MhNqTy7vk3LqW+gM5eBccO7fPe9Ym23FdWAb5GB3Rs
+2YObRhOzMHJTCZmhgjGIdEP8IRDKeSolqneIphCVeDgYnJ4CPu7Th8X66+NsPl2ny+XqkFTOJhuh
+krpJaJELyd3RlZx6b7vRhdXa/16iHurs10GeCXt4U8K2VBXB9wKhT3BnBSzoMxAMfUwuIQKeVdRm
+8YZar932c3CAy6P/KIKZEh5QC9f/9V14r+5vbJpglX6ZzW/TdrvdCvgF1lylJCwDAAA=""")
+
+@contextlib.contextmanager
+def tempdir(suffix='', prefix='tmp'):
+ """
+ A context manager for creating and then
+ deleting a temporary directory.
+ """
+ tmpdir = tempfile.mkdtemp(prefix=prefix, suffix=suffix)
+ try:
+ yield tmpdir
+ finally:
+ shutil.rmtree(tmpdir)
+
+
+class RepoMakerTest(unittest.TestCase):
+ '''Tests for RepoMaker functionality.'''
+
+ def test_non_existing_dir(self):
+ """Test exception raised when output dir doesn't exist."""
+ maker = RepoMaker('test-id', 'outrepo')
+ self.assertRaises(RepoMakerError, maker.add_repo,
+ 'aj45sdjfoqwiq4334w', 'test-repo')
+
+ def test_add_repo(self):
+ """Test converting repository to the download structure."""
+ with tempdir(prefix='repomaker.', suffix='.inrepo') as in_repo:
+ # prepare directory with rpms
+ arches = ('src', 'noarch', 'i586', 'x86_64')
+ files = ["%s-%s.%s.rpm" % (name, version, arch) \
+ for name in ("pkg1", "pkg2", "pkg3") \
+ for version in ("0.1-2.3", "1.2-3.4") \
+ for arch in arches]
+ for fname in files:
+ subdir = ('', fname)[files.index(fname)%2]
+ dir_path = os.path.join(in_repo, subdir)
+ if not os.path.exists(dir_path):
+ os.makedirs(dir_path)
+ with open(os.path.join(dir_path, fname), 'w') as frpm:
+ frpm.write(RPM)
+
+ with tempdir(prefix='repomaker.', suffix='.outrepo') as out_repo:
+ maker = RepoMaker('test-id', out_repo)
+ maker.add_repo(in_repo, 'testrepo', move=True, gpg_key='key',
+ signer='/bin/true', buildconf='buildconf')
+ # Check repo structure
+ results = {
+ 'sources':
+ ['pkg1-0.1-2.3.src.rpm', 'pkg1-1.2-3.4.src.rpm',
+ 'pkg2-0.1-2.3.src.rpm', 'pkg2-1.2-3.4.src.rpm',
+ 'pkg3-0.1-2.3.src.rpm', 'pkg3-1.2-3.4.src.rpm'],
+ 'i586/packages':
+ ['pkg1-0.1-2.3.i586.rpm', 'pkg1-0.1-2.3.noarch.rpm',
+ 'pkg1-1.2-3.4.i586.rpm', 'pkg1-1.2-3.4.noarch.rpm',
+ 'pkg2-0.1-2.3.i586.rpm', 'pkg2-0.1-2.3.noarch.rpm',
+ 'pkg2-1.2-3.4.i586.rpm', 'pkg2-1.2-3.4.noarch.rpm',
+ 'pkg3-0.1-2.3.i586.rpm', 'pkg3-0.1-2.3.noarch.rpm',
+ 'pkg3-1.2-3.4.i586.rpm', 'pkg3-1.2-3.4.noarch.rpm'],
+ 'i586/debug': [],
+ 'x86_64/packages':
+ ['pkg1-0.1-2.3.noarch.rpm', 'pkg1-0.1-2.3.x86_64.rpm',
+ 'pkg1-1.2-3.4.noarch.rpm', 'pkg1-1.2-3.4.x86_64.rpm',
+ 'pkg2-0.1-2.3.noarch.rpm', 'pkg2-0.1-2.3.x86_64.rpm',
+ 'pkg2-1.2-3.4.noarch.rpm', 'pkg2-1.2-3.4.x86_64.rpm',
+ 'pkg3-0.1-2.3.noarch.rpm', 'pkg3-0.1-2.3.x86_64.rpm',
+ 'pkg3-1.2-3.4.noarch.rpm', 'pkg3-1.2-3.4.x86_64.rpm'],
+ 'x86_64/debug': []
+ }
+
+ for arch in ('sources', 'i586/packages', 'i586/debug',
+ 'x86_64/packages', 'x86_64/debug'):
+ primary_path = glob.glob(os.path.join(out_repo,
+ 'test-id',
+ 'repos',
+ 'testrepo',
+ arch,
+ 'repodata',
+ '*primary.xml.gz'))[0]
+ dom = minidom.parse(gzip.open(primary_path))
+ packages = sorted(elem.getAttribute('href') for elem in \
+ dom.getElementsByTagName('location'))
+ self.assertEqual(packages, results[arch])
projects / services / jenkins-scripts.git / commitdiff