# Author: Chenxiong Qi
import argparse
+import glob
import logging
+import mimetypes
import os
import re
import subprocess
import sys
from collections import namedtuple
-from itertools import groupby
+from itertools import chain
import xdg.BaseDirectory
# /path/to/package1.rpm \
# /path/to/package2.rpm
#
-# PkgInfo is a three-elements tuple in format
+# ComparisonHalf is a three-elements tuple in format
#
# (/path/to/package1.rpm, /path/to/package1-debuginfo.rpm /path/to/package1-devel.rpm)
#
+# - the first element is the subject representing the package to compare.
+# - the rest are ancillary packages used for the comparison. So, the second one
+# is the debuginfo package, and the last one is the package containing API of
+# the ELF shared libraries carried by subject.
+#
# So, before calling abipkgdiff, fedabipkgdiff must prepare and pass
# the following information
#
# (/path/to/package1.rpm, /path/to/package1-debuginfo.rpm /path/to/package1-devel.rpm)
# (/path/to/package2.rpm, /path/to/package2-debuginfo.rpm /path/to/package1-devel.rpm)
#
-PkgInfo = namedtuple('PkgInfo', 'package debuginfo_package devel_package')
+ComparisonHalf = namedtuple('ComparisonHalf',
+ ['subject', 'ancillary_debug', 'ancillary_devel'])
global_config = None
# There is no way to configure the log format so far. I hope I would have time
# to make it available so that if fedabipkgdiff is scheduled and run by some
# service, the logs logged into log file is muc usable.
-logging.basicConfig(format='[%(levelname)s] %(message)s',
- level=logging.CRITICAL)
+logging.basicConfig(format='[%(levelname)s] %(message)s', level=logging.CRITICAL)
logger = logging.getLogger(os.path.basename(__file__))
return re.match(r'^(fc|el)\d{1,2}$', distro) is not None
+def match_nvr(s):
+ """Determine if a string is a N-V-R"""
+ return re.match(r'^([^/]+)-(.+)-(.+)$', s) is not None
+
+
+def match_nvra(s):
+ """Determine if a string is a N-V-R.A"""
+ return re.match(r'^([^/]+)-(.+)-(.+)\.(.+)$', s) is not None
+
+
+def is_rpm_file(filename):
+ """Return if a file is a RPM"""
+ return os.path.isfile(filename) and \
+ mimetypes.guess_type(filename)[0] == 'application/x-rpm'
+
+
def cmp_nvr(left, right):
"""Compare function for sorting a sequence of NVRs
else:
raise AttributeError('No attribute name {0}'.format(name))
+ def is_peer(self, another_rpm):
+ """Determine if this is the peer of a given rpm"""
+ return self.name == another_rpm.name and \
+ self.arch == another_rpm.arch and \
+ self.release != another_rpm.release
+
@property
def nvra(self):
"""Return a RPM's N-V-R-A representation
An example: libabigail-1.0-0.8.rc4.1.fc23.x86_64
"""
- return '%(name)s-%(version)s-%(release)s.%(arch)s' % self.rpm_info
+ nvra, _ = os.path.splitext(self.filename)
+ return nvra
@property
def filename(self):
An example: libabigail-1.0-0.8.rc4.1.fc23.x86_64.rpm
"""
- return '{0}.rpm'.format(self.nvra)
+ return os.path.basename(pathinfo.rpm(self.rpm_info))
@property
def is_debuginfo(self):
return self._find_rpm(filename)
+class RPMCollection(object):
+ """Collection of RPMs
+
+ This is a simple collection containing RPMs collected from a
+ directory on the local filesystem or retrieved from Koji.
+
+ A collection can contain one or more sets of RPMs. Each set of
+ RPMs being for a particular architecture.
+
+ For a given architecture, a set of RPMs is made of one RPM and its
+ ancillary RPMs. An ancillary RPM is either a debuginfo RPM or a
+ devel RPM.
+
+ So a given RPMCollection would (informally) look like:
+
+ {
+ i686 => {foo.i686.rpm, foo-debuginfo.i686.rpm, foo-devel.i686.rpm}
+ x86_64 => {foo.x86_64.rpm, foo-debuginfo.x86_64.rpm, foo-devel.x86_64.rpm,}
+ }
+
+ """
+
+ def __init__(self, rpms=None):
+ # Mapping from arch to a list of rpm_infos.
+ # Note that *all* RPMs of the collections are present in this
+ # map; that is the RPM to consider and its ancillary RPMs.
+ self.rpms = {}
+
+ # Mapping from arch to another mapping containing index of debuginfo
+ # and development package
+ # e.g.
+ # self.ancillary_rpms = {'i686', {'debuginfo': foo-debuginfo.rpm,
+ # 'devel': foo-devel.rpm}}
+ self.ancillary_rpms = {}
+
+ if rpms:
+ map(self.add, rpms)
+
+ @classmethod
+ def gather_from_dir(cls, rpm_file, all_rpms=None):
+ """Gather RPM collection from local directory"""
+ dir_name = os.path.dirname(os.path.abspath(rpm_file))
+ filename = os.path.basename(rpm_file)
+
+ nvra = koji.parse_NVRA(filename)
+ rpm_files = glob.glob(os.path.join(
+ dir_name, '*-%(version)s-%(release)s.%(arch)s.rpm' % nvra))
+ rpm_col = cls()
+
+ if all_rpms:
+ selector = lambda rpm: True
+ else:
+ selector = lambda rpm: local_rpm.is_devel or \
+ local_rpm.is_debuginfo or local_rpm.filename == filename
+
+ found_debuginfo = 1
+
+ for rpm_file in rpm_files:
+ local_rpm = LocalRPM(rpm_file)
+
+ if local_rpm.is_debuginfo:
+ found_debuginfo <<= 1
+ if found_debuginfo == 4:
+ raise RuntimeError(
+ 'Found more than one debuginfo package in '
+ 'this directory. At the moment, fedabipkgdiff '
+ 'is not able to deal with this case. '
+ 'Please create two separate directories and '
+ 'put an RPM and its ancillary debuginfo and '
+ 'devel RPMs in each directory.')
+
+ if selector(local_rpm):
+ rpm_col.add(local_rpm)
+
+ return rpm_col
+
+ def add(self, rpm):
+ """Add a RPM into this collection"""
+ self.rpms.setdefault(rpm.arch, []).append(rpm)
+
+ devel_debuginfo_default = {'debuginfo': None, 'devel': None}
+
+ if rpm.is_debuginfo:
+ self.ancillary_rpms.setdefault(
+ rpm.arch, devel_debuginfo_default)['debuginfo'] = rpm
+
+ if rpm.is_devel:
+ self.ancillary_rpms.setdefault(
+ rpm.arch, devel_debuginfo_default)['devel'] = rpm
+
+ def rpms_iter(self, arches=None, default_behavior=True):
+ """Iterator of RPMs to go through RPMs with specific arches"""
+ arches = self.rpms.keys()
+ arches.sort()
+
+ for arch in arches:
+ for _rpm in self.rpms[arch]:
+ yield _rpm
+
+ def get_sibling_debuginfo(self, rpm):
+ """Get sibling debuginfo package of given rpm"""
+ if rpm.arch not in self.ancillary_rpms:
+ return None
+ return self.ancillary_rpms[rpm.arch].get('debuginfo')
+
+ def get_sibling_devel(self, rpm):
+ """Get sibling devel package of given rpm"""
+ if rpm.arch not in self.ancillary_rpms:
+ return None
+ return self.ancillary_rpms[rpm.arch].get('devel')
+
+ def get_peer_rpm(self, rpm):
+ """Get peer rpm of rpm from this collection"""
+ for _rpm in self.rpms[rpm.arch]:
+ if _rpm.is_peer(rpm):
+ return _rpm
+ return None
+
+
+def generate_comparison_halves(rpm_col1, rpm_col2):
+ """Iterate RPM collection and peer's to generate comparison halves"""
+ for _rpm in rpm_col1.rpms_iter():
+ if _rpm.is_debuginfo:
+ continue
+ if _rpm.is_devel and not global_config.check_all_subpackages:
+ continue
+
+ rpm2 = rpm_col2.get_peer_rpm(_rpm)
+ if rpm2 is None:
+ logger.warning('Peer RPM of {0} is not found.'.format(_rpm.filename))
+ continue
+
+ debuginfo1 = rpm_col1.get_sibling_debuginfo(_rpm)
+ devel1 = rpm_col1.get_sibling_devel(_rpm)
+
+ debuginfo2 = rpm_col2.get_sibling_debuginfo(rpm2)
+ devel2 = rpm_col2.get_sibling_devel(rpm2)
+
+ yield (ComparisonHalf(subject=_rpm,
+ ancillary_debug=debuginfo1,
+ ancillary_devel=devel1),
+ ComparisonHalf(subject=rpm2,
+ ancillary_debug=debuginfo2,
+ ancillary_devel=devel2))
+
+
class Brew(object):
"""Interface to Koji XMLRPC API with enhancements specific to fedabipkgdiff
"""
rpm = self.session.getRPM(rpminfo)
if rpm is None:
- raise RpmNotFound('Cannot find RPM {0}'.format(args[0]))
+ raise RpmNotFound('Cannot find RPM {0}'.format(rpminfo))
return rpm
@log_call
By default, fedabipkgdiff requires the RPM package, as well as
its associated debuginfo and devel packages. These three
packages are selected, and noarch and src are excluded.
-
+
:param int build_id: from which build to select rpms.
:param str package_name: which rpm to select that matches this name.
:param arches: which arches to select. If arches omits, rpms with all
rpm_infos = self.listRPMs(buildID=build_id,
arches=arches,
selector=selector)
- return [RPM(rpm_info) for rpm_info in rpm_infos]
+ return RPMCollection((RPM(rpm_info) for rpm_info in rpm_infos))
@log_call
def get_latest_built_rpms(self, package_name, distro, arches=None):
@log_call
-def abipkgdiff(pkg_info1, pkg_info2):
+def abipkgdiff(cmp_half1, cmp_half2):
"""Run abipkgdiff against found two RPM packages
Construct and execute abipkgdiff to get ABI diff
called synchronously. fedabipkgdiff does not return until underlying
abipkgdiff finishes.
- :param PkgInfo pkg_info1: the first package information provided for
- abipkgdiff package1 paramter.
- :param PkgInfo pkg_info2: the second package information provided for
- abipkgdiff package2 paramter.
+ :param ComparisonHalf cmp_half1: the first comparison half.
+ :param ComparisonHalf cmp_half2: the second comparison half.
:return: return code of underlying abipkgdiff execution.
:rtype: int
"""
abipkgdiff_tool = build_path_to_abipkgdiff()
- devel_pkg1 = '' if global_config.no_devel_pkg else \
- '--devel-pkg1 {0}'.format(pkg_info1.devel_package.downloaded_file)
- devel_pkg2 = '' if global_config.no_devel_pkg else \
- '--devel-pkg2 {0}'.format(pkg_info2.devel_package.downloaded_file)
+
+ if global_config.no_devel_pkg:
+ devel_pkg1 = ''
+ devel_pkg2 = ''
+ else:
+ if cmp_half1.ancillary_devel is None:
+ msg = 'Development package for {0} does not exist.'.format(cmp_half1.subject.filename)
+ if global_config.error_on_warning:
+ raise RuntimeError(msg)
+ else:
+ devel_pkg1 = ''
+ logger.warning('{0} Ignored.'.format(msg))
+ else:
+ devel_pkg1 = '--devel-pkg1 {0}'.format(cmp_half1.ancillary_devel.downloaded_file)
+
+ if cmp_half2.ancillary_devel is None:
+ msg = 'Development package for {0} does not exist.'.format(cmp_half2.subject.filename)
+ if global_config.error_on_warning:
+ raise RuntimeError(msg)
+ else:
+ devel_pkg2 = ''
+ logger.warning('{0} Ignored.'.format(msg))
+ else:
+ devel_pkg2 = '--devel-pkg2 {0}'.format(cmp_half2.ancillary_devel.downloaded_file)
+
+ if cmp_half1.ancillary_debug is None:
+ msg = 'Debuginfo package for {0} does not exist.'.format(cmp_half1.subject.filename)
+ if global_config.error_on_warning:
+ raise RuntimeError(msg)
+ else:
+ debuginfo_pkg1 = ''
+ logger.warning('{0} Ignored.'.format(msg))
+ else:
+ debuginfo_pkg1 = '--d1 {0}'.format(cmp_half1.ancillary_debug.downloaded_file)
+
+ if cmp_half2.ancillary_debug is None:
+ msg = 'Debuginfo package for {0} does not exist.'.format(cmp_half2.subject.filename)
+ if global_config.error_on_warning:
+ raise RuntimeError(msg)
+ else:
+ debuginfo_pkg2 = ''
+ logger.warning('{0} Ignored.'.format(msg))
+ else:
+ debuginfo_pkg2 = '--d2 {0}'.format(cmp_half2.ancillary_debug.downloaded_file)
cmd = [
abipkgdiff_tool,
'--show-identical-binaries' if global_config.show_identical_binaries else '',
'--no-default-suppression' if global_config.no_default_suppr else '',
'--dso-only' if global_config.dso_only else '',
- '--d1', pkg_info1.debuginfo_package.downloaded_file,
- '--d2', pkg_info2.debuginfo_package.downloaded_file,
+ debuginfo_pkg1,
+ debuginfo_pkg2,
devel_pkg1,
devel_pkg2,
- pkg_info1.package.downloaded_file,
- pkg_info2.package.downloaded_file,
+ cmp_half1.subject.downloaded_file,
+ cmp_half2.subject.downloaded_file,
]
cmd = filter(lambda s: s != '', cmd)
logger.debug('Run: %s', ' '.join(cmd))
print 'Comparing the ABI of binaries between {0} and {1}:'.format(
- pkg_info1.package.filename, pkg_info2.package.filename)
+ cmp_half1.subject.filename, cmp_half2.subject.filename)
print
proc = subprocess.Popen(' '.join(cmd), shell=True,
stdout, stderr = proc.communicate()
is_ok = proc.returncode == ABIDIFF_OK
- is_internal_error = proc.returncode & ABIDIFF_ERROR or \
- proc.returncode & ABIDIFF_USAGE_ERROR
+ is_internal_error = proc.returncode & ABIDIFF_ERROR or proc.returncode & ABIDIFF_USAGE_ERROR
has_abi_change = proc.returncode & ABIDIFF_ABI_CHANGE
if is_internal_error:
return proc.returncode
-def magic_construct(rpms):
- """Construct RPMs into a magic structure
-
- Convert list of
-
- foo-1.0-1.fc22.i686
- foo-debuginfo-1.0-1.fc22.i686
- foo-devel-1.0-1.fc22.i686
-
- to list of
-
- (foo-1.0-1.fc22.i686, foo-debuginfo-1.0-1.fc22.i686,
- foo-devel-1.0-1.fc22.i686)
-
- and if to check all subpackages
-
- (foo-devel-1.0-1.fc22.i686, foo-debuginfo-1.0-1.fc22.i686,
- foo-devel-1.0-1.fc22.i686)
-
- :param rpms: a sequence of RPM packages.
- :type rpms: list or tuple
- :return: list of two-element tuple where the first element is a RPM package
- and the second one is the debuginfo package.
- :rtype: list
- """
- debuginfo = None
- devel_package = None
- packages = []
- for rpm in rpms:
- if rpm.is_debuginfo:
- debuginfo = rpm
- elif rpm.is_devel:
- devel_package = rpm
- if global_config.check_all_subpackages:
- packages.append(rpm)
- else:
- packages.append(rpm)
- return [PkgInfo(package, debuginfo, devel_package) for package in packages]
-
-
@log_call
-def run_abipkgdiff(pkg1_infos, pkg2_infos):
+def run_abipkgdiff(rpm_col1, rpm_col2):
"""Run abipkgdiff
If one of the executions finds ABI differences, the return code is the
return code from abipkgdiff.
- :param dict pkg1_infos: a mapping from arch to list of RPMs
+ :param RPMCollection rpm_col1: a collection of RPMs
+ :param RPMCollection rpm_col2: same as rpm_col1
:return: exit code of the last non-zero returned from underlying abipkgdiff
- :rtype: number
+ :rtype: int
"""
- arches = pkg1_infos.keys()
- arches.sort()
-
- return_code = 0
-
- for arch in arches:
- pkg_infos = magic_construct(pkg1_infos[arch])
-
- for pkg_info in pkg_infos:
- rpms = pkg2_infos[arch]
-
- package = [rpm for rpm in rpms
- if rpm.name == pkg_info.package.name][0]
- debuginfo = [rpm for rpm in rpms
- if rpm.name == pkg_info.debuginfo_package.name][0]
- devel_package = [rpm for rpm in rpms
- if rpm.name == pkg_info.devel_package.name][0]
-
- ret = abipkgdiff(pkg_info,
- PkgInfo(package=package,
- debuginfo_package=debuginfo,
- devel_package=devel_package))
- if ret > 0:
- return_code = ret
-
- return return_code
+ return_codes = [
+ abipkgdiff(cmp_half1, cmp_half2) for cmp_half1, cmp_half2
+ in generate_comparison_halves(rpm_col1, rpm_col2)]
+ return max(return_codes)
@log_call
raise ValueError('{0} does not exist.'.format(local_rpm_file))
local_rpm = LocalRPM(local_rpm_file)
- local_debuginfo = local_rpm.find_debuginfo()
- local_devel = local_rpm.find_devel()
- if local_debuginfo is None:
- raise ValueError(
- 'debuginfo rpm {0} does not exist.'.format(local_debuginfo))
- if local_devel is None and global_config.no_devel_pkg is not None:
- raise ValueError(
- 'development package {0} does not exist.'.format(local_devel))
-
- rpms = session.get_latest_built_rpms(local_rpm.name,
- from_distro,
- arches=local_rpm.arch)
- download_rpms(rpms)
- pkg_infos = make_rpms_usable_for_abipkgdiff(rpms)
-
- rpms = pkg_infos.values()[0]
- package, debuginfo, devel_package = sorted(rpms, key=lambda rpm: rpm.name)
- return abipkgdiff(PkgInfo(package=package,
- debuginfo_package=debuginfo,
- devel_package=devel_package),
- PkgInfo(package=local_rpm,
- debuginfo_package=local_debuginfo,
- devel_package=local_devel))
+ rpm_col1 = session.get_latest_built_rpms(local_rpm.name,
+ from_distro,
+ arches=local_rpm.arch)
+ rpm_col2 = RPMCollection.gather_from_dir(local_rpm_file)
-
-@log_call
-def make_rpms_usable_for_abipkgdiff(rpms):
- """Prepare package information structure for running abipkgdiff
-
- So far, RPMs input to this method are queried from Koji and abipkgdiff will
- run against these RPMs. For convenience, these RPMs should be restructured
- into a mapping so that subsequent operations could easily find RPMs from
- arch.
-
- For example, input RPMs are
-
- [RPM(arch='x86_64', name='httpd'),
- RPM(arch='i686', name='httpd'),
- RPM(arch='x86_64', name='httpd-devel'),
- RPM(arch='i686', name='http-debuginfo'),
- RPM(arch='x86_64', name='httpd-debuginfo'),
- ]
-
- it is converted into mapping
-
- {
- 'x86_64': [RPM(arch='x86_64', name='httpd'),
- RPM(arch='x86_64', name='httpd-devel'),
- RPM(arch='x86_64', name='httpd-debuginfo')],
- 'i686': [RPM(arch='i686', name='httpd'),
- RPM(arch='i686', name='http-debuginfo')],
- }
-
- The order RPMs in the mapping is unpredictable. So, if they must be in a
- particular order, caller is responsible for this.
-
- :param list rpms: a list of RPMs
- :return: a mapping from an arch to corresponding list of RPMs
- :rtype: dict
- """
- result = {}
- rpms_iter = groupby(sorted(rpms, key=lambda rpm: rpm.arch),
- key=lambda item: item.arch)
- for arch, rpms in rpms_iter:
- result[arch] = list(rpms)
- return result
+ download_rpms(rpm_col1.rpms_iter())
+ return run_abipkgdiff(rpm_col1, rpm_col2)
@log_call
package_name = global_config.NVR[0]
- rpms = session.get_latest_built_rpms(package_name,
- distro=global_config.from_distro)
- download_rpms(rpms)
- pkg1_infos = make_rpms_usable_for_abipkgdiff(rpms)
+ rpm_col1 = session.get_latest_built_rpms(package_name,
+ distro=global_config.from_distro)
+ rpm_col2 = session.get_latest_built_rpms(package_name,
+ distro=global_config.to_distro)
- rpms = session.get_latest_built_rpms(package_name,
- distro=global_config.to_distro)
- download_rpms(rpms)
- pkg2_infos = make_rpms_usable_for_abipkgdiff(rpms)
+ download_rpms(chain(rpm_col1.rpms_iter(), rpm_col2.rpms_iter()))
- return run_abipkgdiff(pkg1_infos, pkg2_infos)
+ return run_abipkgdiff(rpm_col1, rpm_col2)
@log_call
right_rpm['arch'])
build_id = session.get_rpm_build_id(*params1)
- rpms = session.select_rpms_from_a_build(
+ rpm_col1 = session.select_rpms_from_a_build(
build_id, params1[0], arches=params1[3],
select_subpackages=global_config.check_all_subpackages)
- download_rpms(rpms)
- pkg1_infos = make_rpms_usable_for_abipkgdiff(rpms)
build_id = session.get_rpm_build_id(*params2)
- rpms = session.select_rpms_from_a_build(
+ rpm_col2 = session.select_rpms_from_a_build(
build_id, params2[0], arches=params2[3],
select_subpackages=global_config.check_all_subpackages)
- download_rpms(rpms)
- pkg2_infos = make_rpms_usable_for_abipkgdiff(rpms)
- return run_abipkgdiff(pkg1_infos, pkg2_infos)
+ download_rpms(chain(rpm_col1.rpms_iter(), rpm_col2.rpms_iter()))
+
+ return run_abipkgdiff(rpm_col1, rpm_col2)
+
+
+@log_call
+def diff_from_two_rpm_files(from_rpm_file, to_rpm_file):
+ """Diff two RPM files"""
+ rpm_col1 = RPMCollection.gather_from_dir(from_rpm_file)
+ rpm_col2 = RPMCollection.gather_from_dir(to_rpm_file)
+ download_rpms(chain(rpm_col1.rpms_iter(), rpm_col2.rpms_iter()))
+ return run_abipkgdiff(rpm_col1, rpm_col2)
def build_commandline_args_parser():
parser.add_argument(
'NVR',
nargs='*',
- help='RPM package N-V-R, N-V-R-A, N, or a local RPM '
- 'file name with relative or absolute path.')
+ help='RPM package N-V-R, N-V-R-A, N, or local RPM '
+ 'file names with relative or absolute path.')
parser.add_argument(
'--dry-run',
required=False,
action='store_true',
dest='show_identical_binaries',
help='Show information about binaries whose ABI are identical')
+ parser.add_argument(
+ '--error-on-warning',
+ required=False,
+ action='store_true',
+ dest='error_on_warning',
+ help='Raise error instead of warning')
return parser
if global_config.from_distro and global_config.to_distro is None and \
global_config.NVR:
- returncode = diff_local_rpm_with_latest_rpm_from_koji()
+ return diff_local_rpm_with_latest_rpm_from_koji()
- elif global_config.from_distro and global_config.to_distro and \
+ if global_config.from_distro and global_config.to_distro and \
global_config.NVR:
- returncode = diff_latest_rpms_based_on_distros()
+ return diff_latest_rpms_based_on_distros()
- elif global_config.from_distro is None and \
- global_config.to_distro is None and len(global_config.NVR) > 1:
- returncode = diff_two_nvras_from_koji()
+ if global_config.from_distro is None and global_config.to_distro is None:
+ if len(global_config.NVR) > 1:
+ left_one = global_config.NVR[0]
+ right_one = global_config.NVR[1]
- else:
- print >>sys.stderr, 'Unknown arguments. Please refer to --help.'
- returncode = 1
+ if is_rpm_file(left_one) and is_rpm_file(right_one):
+ return diff_from_two_rpm_files(left_one, right_one)
+
+ both_nvr = match_nvr(left_one) and match_nvr(right_one)
+ both_nvra = match_nvra(left_one) and match_nvra(right_one)
+
+ if both_nvr or both_nvra:
+ return diff_two_nvras_from_koji()
- return returncode
+ print >>sys.stderr, 'Unknown arguments. Please refer to --help.'
+ return 1
if __name__ == '__main__':
try:
- main()
+ sys.exit(main())
except KeyboardInterrupt:
if global_config.debug:
logger.debug('Terminate by user')
projects / platform / upstream / libabigail.git / commitdiff