From d8b16ec74cbf37d948eca9e091ef9554cf1bf596 Mon Sep 17 00:00:00 2001 From: Zhang Qiang Date: Thu, 15 Mar 2012 23:26:03 +0800 Subject: [PATCH] first version of gbs import This version of gbs import support the following features: 1. import source rpm to git repository, with two commits, one is upstream tar ball, another is packaging files, including patches and spec file; 2. import sources, including tarball, patches and specfile. spec file need to provide. --- gitbuildsys/cmd_import.py | 118 ++++++++++++++++++++++++++++++++++++++++++++++ gitbuildsys/conf.py | 8 ++++ gitbuildsys/git.py | 103 +++++++++++++++++++++++++++++++++++++++- gitbuildsys/utils.py | 93 ++++++++++++++++++++++++++++++++++++ tools/gbs | 28 +++++++++++ 5 files changed, 349 insertions(+), 1 deletion(-) create mode 100644 gitbuildsys/cmd_import.py diff --git a/gitbuildsys/cmd_import.py b/gitbuildsys/cmd_import.py new file mode 100644 index 0000000..f60873d --- /dev/null +++ b/gitbuildsys/cmd_import.py @@ -0,0 +1,118 @@ +#!/usr/bin/python -tt +# vim: ai ts=4 sts=4 et sw=4 +# +# Copyright (c) 2012 Intel, Inc. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation; version 2 of the License +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., 59 +# Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +"""Implementation of subcmd: import +""" + +import os +import time +import tempfile +import glob +import shutil + +import msger +import runner +import utils +from conf import configmgr +import git +import errors + +USER = configmgr.get('user', 'build') +TMPDIR = configmgr.get('tmpdir') +COMM_NAME = configmgr.get('commit_name', 'import') +COMM_EMAIL = configmgr.get('commit_email', 'import') + +def do(opts, args): + + workdir = os.getcwd() + tmpdir = '%s/%s' % (TMPDIR, USER) + specfile = None + + #import pdb;pdb.set_trace() + if len(args) != 1: + msger.error('missning argument, please reference gbs import --help.') + if args[0].endswith('.src.rpm'): + srcrpmdir = tempfile.mkdtemp(prefix='%s/%s' % (tmpdir, 'src.rpm')) + + msger.info('unpack source rpm package: %s' % args[0]) + ret = runner.quiet("rpm -i --define '_topdir %s' %s" % (srcrpmdir, args[0])) + if ret != 0: + msger.error('source rpm %s unpack fails' % args[0]) + specfile = glob.glob("%s/SPECS/*" % srcrpmdir)[0] + for f in glob.glob("%s/SOURCES/*" % srcrpmdir): + shutil.move(f, "%s/SPECS/" % srcrpmdir) + elif args[0].endswith('.spec'): + specfile = args[0] + else: + msger.error('tar ball import support have not been implemented') + + if not os.path.exists(tmpdir): + os.makedirs(tmpdir) + + basedir = os.path.abspath(os.path.dirname(specfile)) + tarball = os.path.join(basedir, utils.parse_spec(specfile, 'SOURCE0')) + if not os.path.exists(tarball): + msger.error('tarball %s not exist, please check that' % tarball) + pkgname = utils.parse_spec(specfile, 'name') + pkgversion = utils.parse_spec(specfile, 'version') + + try: + repo = git.Git('.') + except errors.GitInvalid: + is_empty = True + msger.info("No git repository found, creating one.") + repo = git.Git.create(pkgname) + + tardir = tempfile.mkdtemp(prefix='%s/%s' % (tmpdir, pkgname)) + + msger.info('unpack upstream tar ball ...') + upstream = utils.UpstreamTarball(tarball) + upstream.unpack(tardir) + tag = repo.version_to_tag("%(version)s", pkgversion) + msg = "Upstream version %s" % (pkgversion) + + os.chdir(repo.path) + msger.info('submit the upstream data as first commit') + commit = repo.commit_dir(upstream.unpacked, msg, + author = {'name':COMM_NAME, + 'email':COMM_EMAIL + } + ) + msger.info('create tag named: %s' % tag) + repo.create_tag(tag, msg, commit) + + packagingdir = '%s/packaging' % upstream.unpacked + if not os.path.exists(packagingdir): + os.makedirs(packagingdir) + + packagingfiles = glob.glob('%s/*' % basedir) + for f in packagingfiles: + if f.endswith(os.path.basename(tarball)): + continue + shutil.copy(f, packagingdir) + + msger.info('submit packaging files as second commit') + commit = repo.commit_dir(upstream.unpacked, 'packaging files for tizen', + author = {'name':COMM_NAME, + 'email':COMM_EMAIL + } + ) + shutil.rmtree(tardir) + if args[0].endswith('.src.rpm'): + shutil.rmtree(srcrpmdir) + msger.info('done.') diff --git a/gitbuildsys/conf.py b/gitbuildsys/conf.py index 3f09ec5..d8ce408 100644 --- a/gitbuildsys/conf.py +++ b/gitbuildsys/conf.py @@ -230,6 +230,11 @@ class ConfigMgr(object): 'su-wrapper': 'su -c', 'distconf': None, }, + 'import': { + 'commit_name': None, + 'commit_email': None, + }, + } DEFAULT_CONF_TEMPLATE="""[general] @@ -246,6 +251,9 @@ build_cmd = /usr/bin/build build_root= /var/tmp/build-root-gbs su-wrapper= su -c distconf= /usr/share/gbs/tizen-1.0.conf +[import] +commit_name= +commit_email= """ # make the manager class as singleton diff --git a/gitbuildsys/git.py b/gitbuildsys/git.py index acfe893..d89194d 100644 --- a/gitbuildsys/git.py +++ b/gitbuildsys/git.py @@ -30,11 +30,16 @@ class Git: raise errors.GitInvalid(path) self.path = os.path.abspath(path) + self._git_dir = os.path.join(path, '.git') # as cache self.cur_branch = None self.branches = None + def _is_sha1(self, val): + sha1_re = re.compile(r'[0-9a-f]{40}$') + return True if sha1_re.match(val) else False + def _exec_git(self, command, args=[]): """Exec a git command and return the output """ @@ -70,11 +75,29 @@ class Git: """ return filter(None, self._exec_git('ls-files').splitlines()) + def rev_parse(self, name): + """ Find the SHA1 of a given name commit id""" + options = [ "--quiet", "--verify", name ] + cmd = ['git', 'rev-parse'] + ret, commit = runner.runtool(' '.join(cmd + options)) + if ret == 0: + return commit.strip() + else: + return None + + def create_branch(self, branch, rev=None): + if rev and not self._is_sha1(rev): + rev = self.rev_parse(rev) + if not branch: + raise errors.GitError('Branch name should not be None') + + options = [branch, rev, '-f'] + self._exec_git('branch', options) + def _get_branches(self): """Return the branches list, current working branch is the first element. """ - branches = [] for line in self._exec_git('branch', ['--no-color']).splitlines(): br = line.strip().split()[-1] @@ -120,6 +143,44 @@ class Git: else: return (br in self.get_branches()[1]) + def commit_dir(self, unpack_dir, msg, branch = 'master', other_parents=None, + author={}, committer={}, create_missing_branch=False): + + for key, val in author.items(): + if val: + os.environ['GIT_AUTHOR_%s' % key.upper()] = val + for key, val in committer.items(): + if val: + os.environ['GIT_COMMITTER_%s' % key.upper()] = val + + os.environ['GIT_WORK_TREE'] = unpack_dir + options = ['.', '-f'] + self._exec_git("add", options) + + options = ['--quiet','-a', '-m %s' % msg,] + self._exec_git("commit", options) + + commit_id = self._exec_git('log', ['--oneline', '-1']).split()[0] + + del os.environ['GIT_WORK_TREE'] + for key, val in author.items(): + if val: + del os.environ['GIT_AUTHOR_%s' % key.upper()] + for key, val in committer.items(): + if val: + del os.environ['GIT_COMMITTER_%s' % key.upper()] + + self._exec_git('reset', ['--hard', commit_id]) + + return commit_id + + def create_tag(self, name, msg, commit): + """Creat a tag with name at commit""" + if self.rev_parse(commit) is None: + raise errors.GitError('%s is invalid commit ID' % commit) + options = [name, '-m %s' % msg, commit] + self._exec_git('tag', options) + def archive(self, prefix, tarfname, treeish='HEAD'): """Archive git tree from 'treeish', detect archive type from the extname of output filename. @@ -175,3 +236,43 @@ class Git: if finalname != tarfname: os.rename(finalname, tarfname) + + @staticmethod + def _formatlize(version): + return version.replace('~', '_').replace(':', '%') + + @staticmethod + def version_to_tag(format, version): + return format % dict(version=Git._formatlize(version)) + + @classmethod + def create(klass, path, description=None, bare=False): + """ + Create a repository at path + @path: where to create the repository + """ + abspath = os.path.abspath(path) + options = [] + if bare: + options = [ '--bare' ] + git_dir = '' + else: + options = [] + git_dir = '.git' + + try: + if not os.path.exists(abspath): + os.makedirs(abspath) + + with Workdir(abspath): + cmd = ['git', 'init'] + options; + runner.quiet(' '.join(cmd)) + if description: + with file(os.path.join(abspath, git_dir, "description"), 'w') as f: + description += '\n' if description[-1] != '\n' else '' + f.write(description) + return klass(abspath) + except OSError, err: + raise errors.GitError("Cannot create Git repository at '%s': %s" + % (abspath, err[1])) + return None diff --git a/gitbuildsys/utils.py b/gitbuildsys/utils.py index 3b0b01a..78154f5 100644 --- a/gitbuildsys/utils.py +++ b/gitbuildsys/utils.py @@ -18,10 +18,20 @@ from __future__ import with_statement import os +import glob import msger import runner +compressor_opts = { 'gzip' : [ '-n', 'gz' ], + 'bzip2' : [ '', 'bz2' ], + 'lzma' : [ '', 'lzma' ], + 'xz' : [ '', 'xz' ] } + +# Map frequently used names of compression types to the internal ones: +compressor_aliases = { 'bz2' : 'bzip2', + 'gz' : 'gzip', } + class Workdir(object): def __init__(self, path): self._newdir = path @@ -123,3 +133,86 @@ def get_hostarch(): if hostarch == 'i686': hostarch = 'i586' return hostarch + +class UnpackTarArchive(object): + """Wrap tar to unpack a compressed tar archive""" + def __init__(self, archive, dir, filters=[], compression=None): + self.archive = archive + self.dir = dir + exclude = [("--exclude=%s" % filter) for filter in filters] + + if not compression: + compression = '-a' + + cmd = ' '.join(['tar']+ exclude + ['-C', dir, compression, '-xf', archive ]) + runner.quiet(cmd) + +class UnpackZipArchive(object): + """Wrap zip to Unpack a zip file""" + def __init__(self, archive, dir): + self.archive = archive + self.dir = dir + + cmd = ' '.join(['unzip'] + [ "-q", archive, '-d', dir ]) + msger.info(cmd) + runner.quiet(cmd) + +class UpstreamTarball(object): + def __init__(self, name, unpacked=None): + self._orig = False + self._path = name + self.unpacked = unpacked + + @property + def path(self): + return self._path.rstrip('/') + + def unpack(self, dir, filters=[]): + """ + Unpack packed upstream sources into a given directory + and determine the toplevel of the source tree. + """ + if not filters: + filters = [] + + if type(filters) != type([]): + raise GbpError, "Filters must be a list" + + self._unpack_archive(dir, filters) + self.unpacked = self._unpacked_toplevel(dir) + + def _unpack_archive(self, dir, filters): + """ + Unpack packed upstream sources into a given directory. + """ + ext = os.path.splitext(self.path)[1] + if ext in [ ".zip", ".xpi" ]: + self._unpack_zip(dir) + else: + self._unpack_tar(dir, filters) + + def _unpack_zip(self, dir): + try: + UnpackZipArchive(self.path, dir) + except CmdError: + raise CmdError, "Unpacking of %s failed" % self.path + + def _unpacked_toplevel(self, dir): + """unpacked archives can contain a leading directory or not""" + unpacked = glob.glob('%s/*' % dir) + unpacked.extend(glob.glob("%s/.*" % dir)) + + # Check that dir contains nothing but a single folder: + if len(unpacked) == 1 and os.path.isdir(unpacked[0]): + return unpacked[0] + else: + return dir + + def _unpack_tar(self, dir, filters): + """ + Unpack a tarball to dir applying a list of filters. + """ + try: + unpackArchive = UnpackTarArchive(self.path, dir, filters) + except gbpc.CommandExecFailed: + raise GbpError diff --git a/tools/gbs b/tools/gbs index e756656..efc105e 100755 --- a/tools/gbs +++ b/tools/gbs @@ -176,6 +176,34 @@ class TizenPkg(cmdln.Cmdln): from gitbuildsys import cmd_build as cmd cmd.do(opts, args) + + @cmdln.alias("im") + @cmdln.option('--author-name', + default=None, + dest='author_name', + help='author name of git commit') + @cmdln.option('--author-email', + default=None, + dest='author_email', + help='author email of git commit') + def do_import(self, subcmd, opts, *args): + """${cmd_name}: Import spec file/source rpm/tar ball to git repository + + Usage: + gbs import [options] specfile | source rpm | tar ball + + + Examples: + $ gbs import /path/to/specfile/ + $ gbs import /path/to/src.rpm + $ gbs import /path/to/tarball + ${cmd_option_list} + """ + + from gitbuildsys import cmd_import as cmd + cmd.do(opts, args) + + @cmdln.alias("cfg") @cmdln.option('-s', '--section', metavar='SECTION', -- 2.7.4