From d4b556048e5ad7a644c56b681930c88ed7c94e30 Mon Sep 17 00:00:00 2001 From: wanchao-xu Date: Fri, 26 Apr 2024 18:33:55 +0800 Subject: [PATCH] import_orig_rpm: introduce gbp import-orig-rpm tool. Change-Id: Iadc9525271b2caa91ba115c8a779e5fe3366afba Signed-off-by: wanchao-xu --- debian/git-buildpackage-rpm.install | 1 + debian/git-buildpackage-rpm.manpages | 1 + docs/Makefile | 1 + docs/common.ent | 1 + docs/man.gbp.xml | 1 + docs/manpages/gbp-buildpackage-rpm.xml | 1 + docs/manpages/gbp-import-orig-rpm.xml | 348 +++++++++++++++++ docs/manpages/gbp-pq-rpm.xml | 1 + docs/manpages/gbp-rpm-ch.xml | 1 + docs/manpages/manpages.ent | 1 + docs/manual.xml | 1 + docs/po/po4a.cfg | 1 + gbp/command_wrappers.py | 13 +- gbp/config.py | 8 + gbp/pkg/upstreamsource.py | 135 ++++++- gbp/scripts/import_orig.py | 2 - gbp/scripts/import_orig_rpm.py | 500 +++++++++++++++++++++++++ packaging/git-buildpackage.spec | 1 + 18 files changed, 1007 insertions(+), 11 deletions(-) create mode 100644 docs/manpages/gbp-import-orig-rpm.xml create mode 100644 gbp/scripts/import_orig_rpm.py diff --git a/debian/git-buildpackage-rpm.install b/debian/git-buildpackage-rpm.install index f79d4f17..37163902 100644 --- a/debian/git-buildpackage-rpm.install +++ b/debian/git-buildpackage-rpm.install @@ -1,5 +1,6 @@ usr/bin/gbp-builder-mock /usr/share/git-buildpackage/ usr/lib/python3.*/dist-packages/gbp/rpm usr/lib/python3/dist-packages/gbp/ +usr/lib/python3.*//dist-packages/gbp/scripts/import_orig_rpm.py usr/lib/python3/dist-packages/gbp/scripts/ usr/lib/python3.*//dist-packages/gbp/scripts/import_srpm.py usr/lib/python3/dist-packages/gbp/scripts/ usr/lib/python3.*/dist-packages/gbp/scripts/pq_rpm.py usr/lib/python3/dist-packages/gbp/scripts/ usr/lib/python3.*/dist-packages/gbp/scripts/buildpackage_rpm.py usr/lib/python3/dist-packages/gbp/scripts/ diff --git a/debian/git-buildpackage-rpm.manpages b/debian/git-buildpackage-rpm.manpages index 183ab2fd..16f43a86 100644 --- a/debian/git-buildpackage-rpm.manpages +++ b/debian/git-buildpackage-rpm.manpages @@ -1,3 +1,4 @@ +docs/gbp-import-orig-rpm.1 docs/gbp-import-srpm.1 docs/gbp-pq-rpm.1 docs/gbp-buildpackage-rpm.1 diff --git a/docs/Makefile b/docs/Makefile index 30be4a78..17aeba1e 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -19,6 +19,7 @@ MAN1S = \ gbp-setup-gitattributes \ gbp-tag \ gbp-buildpackage-rpm \ + gbp-import-orig-rpm \ gbp-import-srpm \ gbp-pq-rpm \ gbp-rpm-ch \ diff --git a/docs/common.ent b/docs/common.ent index 794d47ce..b149b122 100644 --- a/docs/common.ent +++ b/docs/common.ent @@ -25,6 +25,7 @@ gbp import-dsc"> gbp import-dscs"> gbp import-orig"> + gbp import-orig-rpm"> gbp import-ref"> gbp import-srpm"> gbp pq"> diff --git a/docs/man.gbp.xml b/docs/man.gbp.xml index d74e906e..648c3619 100644 --- a/docs/man.gbp.xml +++ b/docs/man.gbp.xml @@ -23,6 +23,7 @@ &man.gbp.importdscs; &man.gbp.importorig; &man.gbp.importref; +&man.gbp.import.orig.rpm; &man.gbp.import.srpm; &man.gbp.pq; &man.gbp.pq.rpm; diff --git a/docs/manpages/gbp-buildpackage-rpm.xml b/docs/manpages/gbp-buildpackage-rpm.xml index aaebe548..82cd4600 100644 --- a/docs/manpages/gbp-buildpackage-rpm.xml +++ b/docs/manpages/gbp-buildpackage-rpm.xml @@ -794,6 +794,7 @@ SEE ALSO + , , , , diff --git a/docs/manpages/gbp-import-orig-rpm.xml b/docs/manpages/gbp-import-orig-rpm.xml new file mode 100644 index 00000000..6e3eeed9 --- /dev/null +++ b/docs/manpages/gbp-import-orig-rpm.xml @@ -0,0 +1,348 @@ + + +
+ &rpm-email; +
+ + &rpm-firstname; + &rpm-surname; + + + + gbp-import-orig-rpm + &rpm-mansection; + + + gbp-import-orig-rpm + Import an upstream source into a git repository + + + + &gbp-import-orig-rpm; + + &man.common.options.synopsis; + VERSION + + BRANCH-NAME + BRANCH-NAME + DIRECTORY + + TAG-NAME + + GPG-KEYID + TAG-FORMAT + PATTERN + + + + + + + + filename + url + + + + + DESCRIPTION + + &gbp-import-orig-rpm; is an basically identical to the &gbp-import-orig; + tool, with only some rpm-specific functionality added and some + Debian-specific functionality removed. + + + + + filename: A file in the local + file system. Gzip, bzip2, lzma and xz compressed tar + archives, zip archives and already unpacked source trees are + supported. + + + + + url: The tarball is downloaded + from a http + or https url. + This needs the python3-request package installed. + + + + + &gbp-import-orig-rpm; tries to download the archive from a remote server if + a remote URL is given. In addition, if no + UPSTREAM-SOURCE is given &gbp-import-orig-rpm; + takes the archive URI from the spec file - this makes it possible to import + a new upstream version just by bumping the version number in the spec file + and running &gbp-import-orig-rpm; (assuming that the spec file contains + a full URL for the archive and its filename automatically follows the + package version e.g. by using the %{version} macro, of course). + + + The sources are placed on the upstream branch (default: + upstream) and tagged. + + + + OPTIONS + + &man.common.options.description; + + + =VERSION + VERSION + + + The upstream version number. + + + + + + + + Merge the upstream branch to the packaging branch after import. + + + + + =BRANCH-NAME + + + + The branch in the &git; repository the upstream sources are put + onto. Default is upstream. + + + + + =BRANCH-NAME + + + + The branch in the &git; repository the package is being developed on, + default is master. After importing the new + sources on the upstream branch, &gbp-import-orig-rpm; will try to + merge the new version onto this branch. + + + + + DIRECTORY + + + + Subdirectory that contains the RPM packaging files. + &gbp-import-orig-rpm; uses this to try to find a spec file which, in + turn, is used to get the upstream source archive URI if one is not + specified on the command line. + + + + + + + + + Create missing upstream branch if it does not exist. + + + + + =TAG-NAME + + + + Add TAG-NAME as additional parent to the + commit of the upstream tarball. Useful when upstream uses git and you + want to link to it's revision history. + + + + + + + + + GPG sign all created tags. + + + + + GPG-KEYID + + + + Use this + for gpg signing tags. + + + + + TAG-FORMAT + + + + Use this tag format when tagging upstream versions, + default is upstream/%(version)s. + + + + + MSG-FORMAT + + + + Use this format string for the commit message when importing upstream + versions, default is + Imported Upstream version %(version)s. + + + + + PATTERN + + + + Filter out files glob-matching + . This + option can be given multiple times. + + + + + + + + + Generate pristine-tar delta file. + + + + + + + + + If using a filter, also filter the files out of the tarball + passed to pristine-tar. + + + + + FILENAME + + + + Filename to record to pristine-tar. This does not alter the tarball + content, just the filename with which the tarball can be checked out + with pristine-tar. + + + + + PREFIX + + + + Prefix (directory) to be used when importing sources into + pristine-tar. Only takes effect when + is used. Special value auto causes &gbp-import-orig-rpm; to guess + the prefix when importing unpacked sources, or, not to change the + prefix when importing source archives. + + + + Using this option will alter the source archive that is imported to + pristine-tar! That is, pristine-tar does not produce and identical + copy of the original tarball (but the mangled tarball, instead). + + + + + + + + + Run CMD after the import. The + hook gets the following environment variables passed: + + + GBP_BRANCH + + The name of the packaging branch + + + + GBP_TAG + + The name of the just created upstream tag + + + + GBP_UPSTREAM_VERSION + + The just imported upstream version + + + + + + + + + + + Run command interactively, i.e. ask package name and version if + needed. + + + + + + + EXAMPLES + + Download and import a new upstream version using the informantion from the + spec file + + + &gbp-import-orig-rpm; + + + After manually downloading an upstream import it + + + &gbp-import-orig-rpm; ../upstream-tarball-0.1.tar.gz + + + Import unpacked sources + + + &gbp-import-orig-rpm; --orig-prefix=upstream-0.1 ../upstream/ + + + + &man.gbp.config-files; + + + SEE ALSO + + , + , + , + , + , + &man.seealso.common; + + + + AUTHOR + + &rpm-username; &rpm-email; + + + + diff --git a/docs/manpages/gbp-pq-rpm.xml b/docs/manpages/gbp-pq-rpm.xml index 48a96344..6d6f2aa1 100644 --- a/docs/manpages/gbp-pq-rpm.xml +++ b/docs/manpages/gbp-pq-rpm.xml @@ -387,6 +387,7 @@ SEE ALSO , + , , diff --git a/docs/manpages/gbp-rpm-ch.xml b/docs/manpages/gbp-rpm-ch.xml index ec23ee7c..69f7edf7 100644 --- a/docs/manpages/gbp-rpm-ch.xml +++ b/docs/manpages/gbp-rpm-ch.xml @@ -364,6 +364,7 @@ SEE ALSO , + , , , &man.seealso.common; diff --git a/docs/manpages/manpages.ent b/docs/manpages/manpages.ent index f07f6c2f..489f94a0 100644 --- a/docs/manpages/manpages.ent +++ b/docs/manpages/manpages.ent @@ -8,6 +8,7 @@ + diff --git a/docs/manual.xml b/docs/manual.xml index 09a7a1fb..68d6cbef 100644 --- a/docs/manual.xml +++ b/docs/manual.xml @@ -47,6 +47,7 @@ &man.gbp.tag; &man.gbp.conf; &man.gbp.buildpackage.rpm; + &man.gbp.import.orig.rpm; &man.gbp.import.srpm; &man.gbp.pq.rpm; &man.gbp.rpm.ch; diff --git a/docs/po/po4a.cfg b/docs/po/po4a.cfg index fd6b6d71..adcc443a 100644 --- a/docs/po/po4a.cfg +++ b/docs/po/po4a.cfg @@ -25,6 +25,7 @@ [type: docbook] manpages/gbp-import-dsc.xml $lang:_gen/$lang/manpages/gbp-import-dsc.xml [type: docbook] manpages/gbp-import-orig.xml $lang:_gen/$lang/manpages/gbp-import-orig.xml [type: docbook] manpages/gbp-import-ref.xml $lang:_gen/$lang/manpages/gbp-import-ref.xml +[type: docbook] manpages/gbp-import-orig-rpm.xml $lang:_gen/$lang/manpages/gbp-import-orig-rpm.xml [type: docbook] manpages/gbp-import-srpm.xml $lang:_gen/$lang/manpages/gbp-import-srpm.xml [type: docbook] manpages/gbp-pq-rpm.xml $lang:_gen/$lang/manpages/gbp-pq-rpm.xml [type: docbook] manpages/gbp-pq.xml $lang:_gen/$lang/manpages/gbp-pq.xml diff --git a/gbp/command_wrappers.py b/gbp/command_wrappers.py index 609f3b44..f7cb3964 100644 --- a/gbp/command_wrappers.py +++ b/gbp/command_wrappers.py @@ -290,7 +290,8 @@ class UnpackTarArchive(Command): class PackTarArchive(Command): """Wrap tar to pack a compressed tar archive""" - def __init__(self, archive, dir, dest, filters=[], compression=None): + def __init__(self, archive, dir, dest, filters=[], compression=None, + transform=None): self.archive = archive self.dir = dir exclude = [("--exclude=%s" % _filter) for _filter in filters] @@ -298,8 +299,14 @@ class PackTarArchive(Command): if not compression: compression = '-a' - Command.__init__(self, 'tar', exclude + - ['-C', dir, compression, '-cf', archive, dest]) + args = exclude + ['-C', dir, compression, '-cf', archive] + + if transform is not None: + args.append('--transform=%s' % transform) + + args.append(dest) + + Command.__init__(self, 'tar', args) self.run_error = self._f("Couldn't repack '%s': {err_reason}", self.archive) diff --git a/gbp/config.py b/gbp/config.py index a082bc64..eb75090e 100644 --- a/gbp/config.py +++ b/gbp/config.py @@ -828,6 +828,8 @@ class GbpOptionParserRpm(GbpOptionParser): 'patch-export-ignore-path': '', 'patch-export-compress': '0', 'patch-export-squash-until': '', + 'pristine-tarball-name': 'auto', + 'orig-prefix': 'auto', 'patch-import': 'True', 'export-sourcedir': 'SOURCES', 'export-specdir': 'SPECS', @@ -887,6 +889,12 @@ class GbpOptionParserRpm(GbpOptionParser): "Squash commits (from upstream) until given tree-ish into one " "big diff, format is '[:]'. " "Default is '%(patch-export-squash-until)s'", + 'pristine-tarball-name': + "Filename to record to pristine-tar, set to 'auto' to not " + "mangle the file name, default is '%(pristine-tarball-name)s'", + 'orig-prefix': + "Prefix (dir) to be used when generating/importing tarballs, " + "default is '%(orig-prefix)s'", 'patch-import': "Import patches to the packaging branch, default is " "'%(patch-import)s'", diff --git a/gbp/pkg/upstreamsource.py b/gbp/pkg/upstreamsource.py index 2d9ca245..b0141608 100644 --- a/gbp/pkg/upstreamsource.py +++ b/gbp/pkg/upstreamsource.py @@ -17,9 +17,14 @@ import glob import os +import subprocess +import stat +import re +import zipfile import gbp.command_wrappers as gbpc +from gbp.pkg.archive import Archive from gbp.pkg.compressor import Compressor from gbp.pkg.pkgpolicy import PkgPolicy @@ -39,12 +44,22 @@ class UpstreamSource(object): @cvar _unpacked: path to the unpacked source tree @type _unpacked: string """ - def __init__(self, name, unpacked=None, pkg_policy=PkgPolicy, sig=None): + def __init__(self, name, unpacked=None, pkg_policy=PkgPolicy, sig=None, + prefix=None): self._orig = False + self._tarball = False self._pkg_policy = pkg_policy self._path = name + if not os.path.exists(self._path): + raise GbpError('UpstreamSource: unable to find %s' % self._path) self.unpacked = unpacked self._sig = sig + (self._filename_base, + self._archive_fmt, + self._compression) = Archive.parse_filename(os.path.basename(self.path)) + self._prefix = prefix + if self._prefix is None: + self._determine_prefix() self._check_orig() if self.is_dir(): @@ -67,8 +82,11 @@ class UpstreamSource(object): """ if self.is_dir(): self._orig = False + self._tarball = False return + self._tarball = True if self.archive_fmt == 'tar' else False + parts = self._path.split('.') try: if parts[-1] == 'tgz': @@ -88,6 +106,13 @@ class UpstreamSource(object): """ return self._orig + def is_tarball(self): + """ + @return: C{True} if source is a tarball, C{False} otherwise + @rtype: C{bool} + """ + return self._tarball + def is_dir(self): """ @return: C{True} if if upstream sources are an unpacked directory, @@ -100,6 +125,93 @@ class UpstreamSource(object): def path(self): return self._path.rstrip('/') + @staticmethod + def _get_topdir_files(file_list): + """Parse content of the top directory from a file list + + >>> UpstreamSource._get_topdir_files([]) + set() + >>> UpstreamSource._get_topdir_files([('-', 'foo/bar')]) + {('d', 'foo')} + >>> UpstreamSource._get_topdir_files([('d', 'foo/'), ('-', 'foo/bar')]) + {('d', 'foo')} + >>> UpstreamSource._get_topdir_files([('d', 'foo'), ('-', 'foo/bar')]) + {('d', 'foo')} + >>> UpstreamSource._get_topdir_files([('-', './foo/bar')]) + {('d', 'foo')} + """ + topdir_files = set() + for typ, path in file_list: + split = re.sub('^(?:./|../)*', '', path).split('/') + if len(split) == 1: + topdir_files.add((typ, path)) + else: + topdir_files.add(('d', split[0])) + return topdir_files + + def _determine_prefix(self): + """Determine the prefix, i.e. the "leading directory name""" + self._prefix = '' + if self.is_dir(): + # For directories we presume that the prefix is just the dirname + self._prefix = os.path.basename(self.path.rstrip('/')) + else: + files = [] + if self._archive_fmt == 'zip': + archive = zipfile.ZipFile(self.path) + for info in archive.infolist(): + typ = 'd' if stat.S_ISDIR(info.external_attr >> 16) else '?' + files.append((typ, info.filename)) + elif self._archive_fmt == 'tar': + popen = subprocess.Popen(['tar', '-t', '-v', '-f', self.path], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + out, _err = popen.communicate() + if popen.returncode: + raise GbpError("Listing tar archive content failed") + for line in out.splitlines(): + fields = line.split(None, 5) + files.append((fields[0][0], fields[-1].decode())) + else: + raise GbpError("Unsupported archive format %s, unable to " + "determine prefix for '%s'" % + (self._archive_fmt, self.path)) + # Determine prefix from the archive content + topdir_files = self._get_topdir_files(files) + if len(topdir_files) == 1: + typ, name = topdir_files.pop() + if typ == 'd': + self._prefix = name + + @property + def archive_fmt(self): + """Archive format of the sources, e.g. 'tar'""" + """ + >>> UpstreamSource('foo/bar.tar.gz').archive_fmt + 'tar' + >>> UpstreamSource('foo.bar.zip').archive_fmt + 'zip' + >>> UpstreamSource('foo.bar.baz').archive_fmt + """ + return self._archive_fmt + + @property + def compression(self): + """Compression format of the sources, e.g. 'gzip'""" + """ + >>> UpstreamSource('foo/bar.tar.gz').compression + 'gzip' + >>> UpstreamSource('foo.bar.zip').compression + >>> UpstreamSource('foo.bz2').compression + 'bzip2' + """ + return self._compression + + @property + def prefix(self): + """Prefix, i.e. the 'leading directory name' of the sources""" + return self._prefix + @property def signaturefile(self): return self._sig @@ -167,7 +279,7 @@ class UpstreamSource(object): # unpackArchive already printed an error message raise GbpError - def pack(self, newarchive, filters=None): + def pack(self, newarchive, filters=None, newprefix=None): """ Recreate a new archive from the current one @@ -175,6 +287,8 @@ class UpstreamSource(object): @type newarchive: string @param filters: tar filters to apply @type filters: array of strings + @param newprefix: new prefix, None implies that prefix is not mangled + @type newprefix: string or None @return: the new upstream source @rtype: UpstreamSource """ @@ -187,12 +301,21 @@ class UpstreamSource(object): if not isinstance(filters, list): raise GbpError("Filters must be a list") + run_dir = os.path.dirname(self.unpacked.rstrip('/')) + pack_this = os.path.basename(self.unpacked.rstrip('/')) + transform = None + if newprefix is not None: + newprefix = newprefix.strip('/.') + if newprefix: + transform = 's!%s!%s!' % (pack_this, newprefix) + else: + transform = 's!%s!%s!' % (pack_this, '.') try: - unpacked = self.unpacked.rstrip('/') repackArchive = gbpc.PackTarArchive(newarchive, - os.path.dirname(unpacked), - os.path.basename(unpacked), - filters) + run_dir, + pack_this, + filters, + transform=transform) repackArchive() except gbpc.CommandExecFailed: # repackArchive already printed an error diff --git a/gbp/scripts/import_orig.py b/gbp/scripts/import_orig.py index db7f1181..776bfa3e 100644 --- a/gbp/scripts/import_orig.py +++ b/gbp/scripts/import_orig.py @@ -163,8 +163,6 @@ def find_upstream(use_uscan, args, version=None): Traceback (most recent call last): ... gbp.errors.GbpError: you can't pass both --uscan and a filename. - >>> find_upstream(False, ['tarball']).path - 'tarball' """ if use_uscan: if args: diff --git a/gbp/scripts/import_orig_rpm.py b/gbp/scripts/import_orig_rpm.py new file mode 100644 index 00000000..c6bf4827 --- /dev/null +++ b/gbp/scripts/import_orig_rpm.py @@ -0,0 +1,500 @@ +# vim: set fileencoding=utf-8 : +# +# (C) 2006, 2007, 2009, 2011, 2015, 2016, 2019 Guido Günther +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, please see +# +# +"""Import a new upstream version into a Git repository""" + +import os +import re +import sys +import tempfile + +import gbp.command_wrappers as gbpc +from gbp.rpm import (RpmUpstreamSource, NoSpecError, guess_spec, guess_spec_repo) +from gbp.rpm.policy import RpmPkgPolicy +from gbp.rpm.git import (GitRepositoryError, RpmGitRepository) +from gbp.config import GbpOptionParserRpm, GbpOptionGroup, no_upstream_branch_msg +from gbp.errors import GbpError +from gbp.format import format_str +from gbp.pkg.archive import Archive +import gbp.log +from gbp.scripts.common import ExitCodes +from gbp.scripts.common.import_orig import (cleanup_tmp_tree, ask_package_name, + ask_package_version) +from gbp.scripts.common.hook import Hook +from gbp.scripts.import_srpm import download_file + + +def prepare_sources(source, pkg_name, pkg_version, pristine_commit_name, + filters, filter_pristine, prefix, tmpdir): + """ + Prepare upstream sources for importing + + Unpack, filter and repack sources for importing to git and to pristine-tar. + + @param source: original upstream sources + @type source: C{UpstreamSource} + @param pkg_name: package name + @type pkg_name: C{str} + @param pkg_version: upstream version of the package + @type pkg_version: C{str} + @param pristine_commit_name: archive filename to commit to pristine-tar + @type pristine_commit_name: C{str} or C{None} + @param filters: filter to exclude files + @type filters: C{list} of C{str} + @param filter_pristine: filter pristine-tar, too + @type filter_pristine: C{bool} + @param prefix: prefix (i.e. leading directory of files) to use in + pristine-tar, set to C{None} to not mangle orig archive + @type prefix: C{str} or C{None} + @param tmpdir: temporary working dir (cleanup left to caller) + @type tmpdir: C{str} + @return: path to prepared source tree and tarball to commit to pristine-tar + @rtype: C{tuple} of C{str} + """ + pristine = None + # Determine parameters for pristine tar + pristine_filters = filters if filters and filter_pristine else None + pristine_prefix = None + filtered = source + if prefix is not None and prefix != 'auto': + prefix_subst = {'name': pkg_name, + 'version': pkg_version, + 'upstreamversion': pkg_version} + pristine_prefix = prefix % prefix_subst + # Handle unpacked sources, i.e. importing a directory + if source.is_dir(): + if pristine_commit_name: + gbp.log.warn('Preparing unpacked sources for pristine-tar') + pristine = prepare_pristine_tar(source, pkg_name, pkg_version, + pristine_commit_name, + pristine_filters, pristine_prefix, + tmpdir) + if filters: + # Re-use sources packed for pristine-tar, if available + if pristine: + packed = pristine + else: + packed_fn = tempfile.mkstemp(prefix="packed_", dir=tmpdir, + suffix='.tar')[1] + gbp.log.debug("Packing '%s' to '%s'" % (source.path, packed_fn)) + packed = source.pack(packed_fn) + unpack_dir = tempfile.mkdtemp(prefix='filtered_', dir=tmpdir) + packed.unpack(unpack_dir, filters) + filtered = packed + # Handle source archives + else: + unpack_dir = tempfile.mkdtemp(prefix='filtered_', dir=tmpdir) + gbp.log.debug("Unpacking '%s' to '%s'" % (source.path, unpack_dir)) + source.unpack(unpack_dir, filters) + if pristine_commit_name: + pristine = prepare_pristine_tar(source, pkg_name, pkg_version, + pristine_commit_name, + pristine_filters, pristine_prefix, + tmpdir) + pristine_path = pristine.path if pristine else '' + return (filtered.unpacked, pristine_path) + + +def prepare_pristine_tar(source, pkg_name, pkg_version, pristine_commit_name, + filters=None, prefix=None, tmpdir=None): + """ + Prepare the upstream sources for pristine-tar import + + @param source: original upstream sources + @type source: C{UpstreamSource} + @param pkg_name: package name + @type pkg_name: C{str} + @param pkg_version: upstream version of the package + @type pkg_version: C{str} + @param pristine_commit_name: archive filename to commit to pristine-tar + @type pristine_commit_name: C{str} or C{None} + @param filters: filter to exclude files + @type filters: C{list} of C{str} or C{None} + @param prefix: prefix (i.e. leading directory of files) to use in + pristine-tar, set to C{None} to not mangle orig archive + @type prefix: C{str} or C{None} + @param tmpdir: temporary working dir (cleanup left to caller) + @type tmpdir: C{str} + @return: prepared source archive + @rtype: C{UpstreamSource} + """ + need_repack = False + if source.is_dir(): + if prefix is None: + prefix = '%s-%s' % (pkg_name, pkg_version) + gbp.log.info("Using guessed prefix '%s/' for pristine-tar" % prefix) + need_repack = True + else: + if prefix is not None and prefix == source.prefix: + prefix = None + comp = Archive.parse_filename(pristine_commit_name)[2] + if filters or prefix is not None or source.compression != comp: + if not source.unpacked: + unpack_dir = tempfile.mkdtemp(prefix='pristine_unpack_', + dir=tmpdir) + source.unpack(unpack_dir) + need_repack = True + pristine_path = os.path.join(tmpdir, pristine_commit_name) + if need_repack: + gbp.log.debug("Packing '%s' from '%s' for pristine-tar" % + (pristine_path, source.unpacked)) + pristine = source.pack(pristine_path, filters, prefix) + else: + # Just create symlink for mangling the pristine tarball name + os.symlink(source.path, pristine_path) + pristine = source.__class__(pristine_path) + + return pristine + + +def upstream_import_commit_msg(options, version): + return options.import_msg % dict(version=version) + + +def find_spec(repo, options): + """Find spec in the working tree or repository""" + try: + preferred_fn = os.path.basename(repo.path) + '.spec' + spec = guess_spec(os.path.join(repo.path, options.packaging_dir), True, + preferred_fn) + except NoSpecError: + try: + # Check the spec file from the repository, in case we're not on the + # packaging-branch (but upstream, for example). + spec = guess_spec_repo(repo, options.packaging_branch, + options.packaging_dir, True, preferred_fn) + except NoSpecError: + spec = None + return spec + + +def detect_name_and_version(repo, source, spec, options): + # Guess defaults for the package name and version from the + # original tarball. + guessed_package, guessed_version = source.guess_version() or ('', '') + + # Try to find the source package name + if spec: + sourcepackage = spec.name + else: + if options.interactive: + sourcepackage = ask_package_name(guessed_package, + RpmPkgPolicy.is_valid_packagename, + RpmPkgPolicy.packagename_msg) + else: + if guessed_package: + sourcepackage = guessed_package + else: + raise GbpError("Couldn't determine upstream package name. Use --interactive.") + + # Try to find the version. + if options.version: + version = options.version + else: + if options.interactive: + version = ask_package_version(guessed_version, + RpmPkgPolicy.is_valid_upstreamversion, + RpmPkgPolicy.upstreamversion_msg) + else: + if guessed_version: + version = guessed_version + else: + raise GbpError("Couldn't determine upstream version. Use '-u' or --interactive.") + + return (sourcepackage, version) + + +def find_upstream(spec, args): + """Find the main tarball to import + @return: upstream source filename or None if nothing to import + @rtype: string + @raise GbpError: raised on all detected errors + """ + if len(args) > 1: # source specified + raise GbpError("More than one archive specified. Try --help.") + elif len(args) == 0: + if spec and spec.orig_src: + path = spec.orig_src['uri'] + gbp.log.info("Archive file path from spec is used ('%s')" % path) + elif spec: + raise GbpError("No archive to import specified and unable to " + "determine source from spec. Try --help.") + else: + raise GbpError("No archive to import specified and no spec file " + "found. Try --help.") + else: + path = args[0] + if re.match(r'[a-z]{1,5}://', path): + path = download_file('..', path) + return RpmUpstreamSource(path) + + +def postimport_hook(repo, tag, version, options): + if options.postimport: + info = {'upstreamversion': version} + env = {'GBP_BRANCH': options.packaging_branch, + 'GBP_TAG': tag, + 'GBP_UPSTREAM_VERSION': version, + } + Hook('Postimport', + format_str(options.postimport, info), + extra_env=env)() + + +def pristine_tarball_name(source, pkg_name, pkg_version, pristine_name): + old_filename = os.path.basename(source.path) + base_name, _fmt, _comp = Archive.parse_filename(old_filename) + if pristine_name != 'auto': + ext = old_filename.replace(base_name, '', 1) + return pristine_name % {'name': pkg_name, + 'version': pkg_version, + 'upstreamversion': pkg_version, + 'filename_base': base_name, + 'filename_ext': ext} + # Need to repack and mangle filename if the archive is not + # pristine-tar-compatible -> we decide to create gz compressed tarball + elif not source.is_tarball(): + return "%s.tar.gz" % base_name + return old_filename + + +def set_bare_repo_options(options): + """Modify options for import into a bare repository""" + if options.pristine_tar or options.merge: + gbp.log.info("Bare repository: setting %s%s options" + % (["", " '--no-pristine-tar'"][options.pristine_tar], + ["", " '--no-merge'"][options.merge])) + options.pristine_tar = False + options.merge = False + + +def build_parser(name): + try: + parser = GbpOptionParserRpm(command=os.path.basename(name), prefix='', + usage='%prog [options] /path/to/upstream-version.tar.gz') + except GbpError as err: + gbp.log.err(err) + return None + + import_group = GbpOptionGroup(parser, "import options", + "pristine-tar and filtering") + tag_group = GbpOptionGroup(parser, "tag options", + "options related to git tag creation") + branch_group = GbpOptionGroup(parser, "version and branch naming options", + "version number and branch layout options") + cmd_group = GbpOptionGroup(parser, "external command options", + "how and when to invoke external commands and hooks") + for group in [import_group, branch_group, tag_group, cmd_group]: + parser.add_option_group(group) + + branch_group.add_option("-u", "--upstream-version", dest="version", + help="Upstream Version") + branch_group.add_config_file_option(option_name="packaging-branch", + dest="packaging_branch") + branch_group.add_config_file_option(option_name="upstream-branch", + dest="upstream_branch") + branch_group.add_config_file_option(option_name="upstream-vcs-tag", dest="vcs_tag") + branch_group.add_boolean_config_file_option(option_name="merge", dest="merge") + branch_group.add_config_file_option(option_name="packaging-dir", dest="packaging_dir") + branch_group.add_boolean_config_file_option(option_name="create-missing-branches", + dest="create_missing_branches") + + tag_group.add_boolean_config_file_option(option_name="sign-tags", + dest="sign_tags") + tag_group.add_config_file_option(option_name="keyid", + dest="keyid") + tag_group.add_config_file_option(option_name="upstream-tag", + dest="upstream_tag") + import_group.add_config_file_option(option_name="filter", + dest="filters", action="append") + import_group.add_boolean_config_file_option(option_name="pristine-tar", + dest="pristine_tar") + import_group.add_boolean_config_file_option(option_name="filter-pristine-tar", + dest="filter_pristine_tar") + import_group.add_config_file_option(option_name="pristine-tarball-name", + dest="pristine_tarball_name") + import_group.add_config_file_option(option_name="orig-prefix", + dest="orig_prefix") + import_group.add_config_file_option(option_name="import-msg", + dest="import_msg") + cmd_group.add_config_file_option(option_name="postimport", dest="postimport") + + parser.add_boolean_config_file_option(option_name="interactive", + dest='interactive') + parser.add_option("-v", "--verbose", action="store_true", dest="verbose", default=False, + help="verbose command execution") + parser.add_config_file_option(option_name="color", dest="color", type='tristate') + parser.add_config_file_option(option_name="color-scheme", + dest="color_scheme") + parser.add_config_file_option(option_name="tmp-dir", dest="tmp_dir") + + return parser + + +def parse_args(argv): + """Parse the command line arguments + @return: options and arguments + """ + + parser = build_parser(argv[0]) + if not parser: + return None, None + + (options, args) = parser.parse_args(argv[1:]) + gbp.log.setup(options.color, options.verbose, options.color_scheme) + + return options, args + + +def main(argv): + ret = 0 + + (options, args) = parse_args(argv) + if not options: + return ExitCodes.parse_error + + tmpdir = tempfile.mkdtemp(dir=options.tmp_dir, prefix='import-orig-rpm_') + try: + try: + repo = RpmGitRepository('.') + except GitRepositoryError: + raise GbpError("%s is not a git repository" % (os.path.abspath('.'))) + + spec = find_spec(repo, options) + source = find_upstream(spec, args) + + is_empty = repo.is_empty() + + if not repo.has_branch(options.upstream_branch): + if options.create_missing_branches: + gbp.log.info("Will create missing branch '%s'" % + options.upstream_branch) + elif is_empty: + options.create_missing_branches = True + else: + raise GbpError(no_upstream_branch_msg % options.upstream_branch) + + (clean, out) = repo.is_clean() + if not clean and not is_empty: + gbp.log.err("Repository has uncommitted changes, commit these first: ") + raise GbpError(out) + + # The main tarball + (name, version) = detect_name_and_version(repo, source, spec, options) + + tag_str_fields = dict(upstreamversion=version, vendor="Upstream") + tag = repo.version_to_tag(options.upstream_tag, tag_str_fields) + if repo.has_tag(tag): + raise GbpError("Upstream tag '%s' already exists" % tag) + + if repo.bare: + set_bare_repo_options(options) + + # Prepare sources for importing + if options.pristine_tar: + prepare_pristine = pristine_tarball_name(source, name, version, + options.pristine_tarball_name) + else: + prepare_pristine = None + unpacked_orig, pristine_orig = prepare_sources(source, name, version, + prepare_pristine, + options.filters, + options.filter_pristine_tar, + options.orig_prefix, tmpdir) + + # Don't mess up our repo with git metadata from an upstream tarball + try: + if os.path.isdir(os.path.join(unpacked_orig, '.git/')): + raise GbpError("The orig tarball contains .git metadata - giving up.") + except OSError: + pass + + try: + import_branch = options.upstream_branch + filter_msg = ["", " (filtering out %s)" + % options.filters][len(options.filters) > 0] + gbp.log.info("Importing '%s' to branch '%s'%s..." % (source.path, + import_branch, + filter_msg)) + gbp.log.info("Source package is %s" % name) + gbp.log.info("Upstream version is %s" % version) + + msg = upstream_import_commit_msg(options, version) + + if options.vcs_tag: + parents = [repo.rev_parse("%s^{}" % options.vcs_tag)] + else: + parents = None + + commit = repo.commit_dir(unpacked_orig, + msg=msg, + branch=import_branch, + other_parents=parents, + create_missing_branch=options.create_missing_branches) + + if options.pristine_tar and pristine_orig: + gbp.log.info("Pristine-tar: commiting %s" % pristine_orig) + repo.pristine_tar.commit(pristine_orig, import_branch) + + repo.create_tag(name=tag, + msg="Upstream version %s" % version, + commit=commit, + sign=options.sign_tags, + keyid=options.keyid) + + if options.merge: + gbp.log.info("Merging to '%s'" % options.packaging_branch) + if repo.has_branch(options.packaging_branch): + repo.set_branch(options.packaging_branch) + try: + repo.merge(tag) + except GitRepositoryError: + raise GbpError("""Merge failed, please resolve.""") + else: + repo.create_branch(options.packaging_branch, rev=options.upstream_branch) + if repo.get_branch() == options.packaging_branch: + repo.force_head(options.packaging_branch, hard=True) + # Update working copy and index if we've possibly updated the + # checked out branch + current_branch = repo.get_branch() + if current_branch in [options.upstream_branch, + repo.pristine_tar_branch]: + repo.force_head(current_branch, hard=True) + + postimport_hook(repo, tag, version, options) + except (gbpc.CommandExecFailed, GitRepositoryError) as err: + msg = str(err) or 'Unknown error, please report a bug' + raise GbpError("Import of %s failed: %s" % (source.path, msg)) + except KeyboardInterrupt: + raise GbpError("Import of %s failed: aborted by user" % (source.path)) + except GbpError as err: + if str(err): + gbp.log.err(err) + ret = 1 + + if tmpdir: + cleanup_tmp_tree(tmpdir) + + if not ret: + gbp.log.info("Successfully imported version %s of %s" % (version, source.path)) + return ret + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) + +# vim:et:ts=4:sw=4:et:sts=4:ai:set list listchars=tab\:»·,trail\:·: diff --git a/packaging/git-buildpackage.spec b/packaging/git-buildpackage.spec index a4bccdb3..81ac8d18 100644 --- a/packaging/git-buildpackage.spec +++ b/packaging/git-buildpackage.spec @@ -293,6 +293,7 @@ done %if %{with docs} %{_mandir}/man1/gbp-buildpackage-rpm.1* %{_mandir}/man1/gbp-pq-rpm.1* +%{_mandir}/man1/gbp-import-orig-rpm.1* %{_mandir}/man1/gbp-import-srpm.1* %{_mandir}/man1/gbp-rpm-ch.1* %endif -- 2.34.1