-#!/usr/bin/python -tt
+#!/usr/bin/python3 -tt
#
# Copyright (c) 2010, 2011 Intel, Inc.
#
import os
import shutil
-import urlparse
+import urllib.parse
import rpm
+import glob
-import zypp
+import zypp #pylint: disable=import-error
if not hasattr(zypp, 'PoolQuery') or \
not hasattr(zypp.RepoManager, 'loadSolvFile'):
raise ImportError("python-zypp in host system cannot support PoolQuery or "
from mic.utils.grabber import myurlgrab, TextProgress
from mic.utils.proxy import get_proxy_for
from mic.utils.errors import CreatorError, RepoError, RpmError
-from mic.imager.baseimager import BaseImageCreator
+from mic.conf import configmgr
class RepositoryStub:
def __init__(self):
class Zypp(BackendPlugin):
name = 'zypp'
- def __init__(self, target_arch, instroot, cachedir):
+ def __init__(self, target_arch, instroot, cachedir, strict_mode = False):
self.cachedir = cachedir
self.instroot = instroot
self.target_arch = target_arch
+ self.strict_mode = strict_mode
self.__pkgs_license = {}
self.__pkgs_content = {}
self.incpkgs = {}
self.excpkgs = {}
self.pre_pkgs = []
+ self.check_pkgs = []
self.probFilterFlags = [ rpm.RPMPROB_FILTER_OLDPACKAGE,
rpm.RPMPROB_FILTER_REPLACEPKG ]
self.has_prov_query = True
self.install_debuginfo = False
+ # this can't be changed, it is used by zypp
+ self.tmp_file_path = '/var/tmp'
def doFileLogSetup(self, uid, logfile):
# don't do the file log for the livecd as it can lead to open fds
self.closeRpmDB()
- if not os.path.exists("/etc/fedora-release") and \
- not os.path.exists("/etc/meego-release"):
- for i in range(3, os.sysconf("SC_OPEN_MAX")):
- try:
- os.close(i)
- except:
- pass
-
def __del__(self):
self.close()
def _cleanupRpmdbLocks(self, installroot):
# cleans up temporary files left by bdb so that differing
# versions of rpm don't cause problems
- import glob
for f in glob.glob(installroot + "/var/lib/rpm/__db*"):
os.unlink(f)
def setup(self):
self._cleanupRpmdbLocks(self.instroot)
+ # '/var/tmp' is used by zypp to build cache, so make sure
+ # if it exists
+ if not os.path.exists(self.tmp_file_path ):
+ os.makedirs(self.tmp_file_path)
def whatObsolete(self, pkg):
query = zypp.PoolQuery()
query.addKind(zypp.ResKind.package)
- query.addAttribute(zypp.SolvAttr.obsoletes, pkg)
+ query.addDependency(zypp.SolvAttr.obsoletes, pkg.name(), pkg.edition())
query.setMatchExact()
for pi in query.queryResults(self.Z.pool()):
return pi
def _zyppQueryPackage(self, pkg):
query = zypp.PoolQuery()
query.addKind(zypp.ResKind.package)
- query.addAttribute(zypp.SolvAttr.name,pkg)
+ query.addAttribute(zypp.SolvAttr.name, pkg)
query.setMatchExact()
for pi in query.queryResults(self.Z.pool()):
return pi
return None
def _splitPkgString(self, pkg):
- sp = pkg.rsplit(".",1)
+ sp = pkg.rsplit(".", 1)
name = sp[0]
arch = None
if len(sp) == 2:
obs.status().setToBeInstalled (zypp.ResStatus.USER)
def cmpEVR(p1, p2):
+ # compare criterion: arch compatibility first, then repo
+ # priority, and version last
a1 = p1.arch()
a2 = p2.arch()
if str(a1) != str(a2):
return -1
else:
return 1
+ # Priority of a repository is an integer value between 0 (the
+ # highest priority) and 99 (the lowest priority)
+ pr1 = int(p1.repoInfo().priority())
+ pr2 = int(p2.repoInfo().priority())
+ if pr1 > pr2:
+ return -1
+ elif pr1 < pr2:
+ return 1
+
ed1 = p1.edition()
ed2 = p2.edition()
- (e1, v1, r1) = map(str, [ed1.epoch(), ed1.version(), ed1.release()])
- (e2, v2, r2) = map(str, [ed2.epoch(), ed2.version(), ed2.release()])
+ (e1, v1, r1) = list(map(str, [ed1.epoch(), ed1.version(), ed1.release()]))
+ (e2, v2, r2) = list(map(str, [ed2.epoch(), ed2.version(), ed2.release()]))
return rpm.labelCompare((e1, v1, r1), (e2, v2, r2))
found = False
if endx and startx:
pattern = '%s' % (pkg[1:-1])
q.setMatchRegex()
- q.addAttribute(zypp.SolvAttr.name,pattern)
+ q.addAttribute(zypp.SolvAttr.name, pattern)
elif arch:
q.setMatchExact()
- q.addAttribute(zypp.SolvAttr.name,name)
+ q.addAttribute(zypp.SolvAttr.name, name)
else:
q.setMatchExact()
- q.addAttribute(zypp.SolvAttr.name,pkg)
+ q.addAttribute(zypp.SolvAttr.name, pkg)
- for item in sorted(
+ for pitem in sorted(
q.queryResults(self.Z.pool()),
- cmp=lambda x,y: cmpEVR(x, y),
+ cmp=lambda x,y: cmpEVR(zypp.asKindPackage(x), zypp.asKindPackage(y)),
reverse=True):
-
- if item.name() in self.excpkgs.keys() and \
+ item = zypp.asKindPackage(pitem)
+ if item.name() in list(self.excpkgs.keys()) and \
self.excpkgs[item.name()] == item.repoInfo().name():
continue
- if item.name() in self.incpkgs.keys() and \
+ if item.name() in list(self.incpkgs.keys()) and \
self.incpkgs[item.name()] != item.repoInfo().name():
continue
found = True
- obspkg = self.whatObsolete(item.name())
+ obspkg = self.whatObsolete(item)
if arch:
if arch == str(item.arch()):
- item.status().setToBeInstalled (zypp.ResStatus.USER)
+ pitem.status().setToBeInstalled (zypp.ResStatus.USER)
else:
- markPoolItem(obspkg, item)
+ markPoolItem(obspkg, pitem)
if not ispattern:
break
q.addAttribute(zypp.SolvAttr.provides, pkg)
q.addAttribute(zypp.SolvAttr.name,'')
- for item in sorted(
+ for pitem in sorted(
q.queryResults(self.Z.pool()),
- cmp=lambda x,y: cmpEVR(x.edition(), y.edition()),
+ cmp=lambda x,y: cmpEVR(zypp.asKindPackage(x), zypp.asKindPackage(y)),
reverse=True):
- if item.name() in self.excpkgs.keys() and \
+ item = zypp.asKindPackage(pitem)
+ if item.name() in list(self.excpkgs.keys()) and \
self.excpkgs[item.name()] == item.repoInfo().name():
continue
- if item.name() in self.incpkgs.keys() and \
+ if item.name() in list(self.incpkgs.keys()) and \
self.incpkgs[item.name()] != item.repoInfo().name():
continue
found = True
- obspkg = self.whatObsolete(item.name())
- markPoolItem(obspkg, item)
+ obspkg = self.whatObsolete(item)
+ markPoolItem(obspkg, pitem)
break
if found:
else:
raise CreatorError("Unable to find package: %s" % (pkg,))
- def inDeselectPackages(self, item):
+ def inDeselectPackages(self, pitem):
"""check if specified pacakges are in the list of inDeselectPackages
"""
-
+ item = zypp.asKindPackage(pitem)
name = item.name()
for pkg in self.to_deselect:
startx = pkg.startswith("*")
if not ispattern:
if pkgarch:
if name == pkgname and str(item.arch()) == pkgarch:
- return True;
+ return True
else:
if name == pkgname:
- return True;
+ return True
else:
if startx and name.endswith(pkg[1:]):
- return True;
+ return True
if endx and name.startswith(pkg[:-1]):
- return True;
+ return True
- return False;
+ return False
def deselectPackage(self, pkg):
"""collect packages should not be installed"""
self.to_deselect.append(pkg)
def selectGroup(self, grp, include = ksparser.GROUP_DEFAULT):
+ def compareGroup(pitem):
+ item = zypp.asKindPattern(pitem)
+ return item.repoInfo().priority()
if not self.Z:
self.__initialize_zypp()
found = False
- q=zypp.PoolQuery()
+ q = zypp.PoolQuery()
q.addKind(zypp.ResKind.pattern)
- for item in q.queryResults(self.Z.pool()):
+ for pitem in sorted(q.queryResults(self.Z.pool()), key=compareGroup):
+ item = zypp.asKindPattern(pitem)
summary = "%s" % item.summary()
name = "%s" % item.name()
if name == grp or summary == grp:
found = True
- item.status().setToBeInstalled (zypp.ResStatus.USER)
+ pitem.status().setToBeInstalled (zypp.ResStatus.USER)
break
if found:
if include == ksparser.GROUP_REQUIRED:
- map(
- lambda p: self.deselectPackage(p),
- grp.default_packages.keys())
+ list([self.deselectPackage(p) for p in list(grp.default_packages.keys())])
return None
else:
for pkg in exc:
self.excpkgs[pkg] = name
- # check LICENSE files
- if not rpmmisc.checkRepositoryEULA(name, repo):
- msger.warning('skip repo:%s for failed EULA confirmation' % name)
- return None
-
if mirrorlist:
repo.mirrorlist = mirrorlist
# Enable gpg check for verifying corrupt packages
repo.gpgcheck = 1
- if priority:
- repo.priority = priority
+ if priority is not None:
+ # priority 0 has issue in RepoInfo.setPriority
+ repo.priority = priority + 1
try:
repo_info = zypp.RepoInfo()
repo_info.setEnabled(repo.enabled)
repo_info.setAutorefresh(repo.autorefresh)
repo_info.setKeepPackages(repo.keeppackages)
- baseurl = zypp.Url(repo.baseurl[0])
+ baseurl = zypp.Url(repo.baseurl[0].full)
if not ssl_verify:
baseurl.setQueryParam("ssl_verify", "no")
if proxy:
- scheme, host, path, parm, query, frag = urlparse.urlparse(proxy)
+ host = urllib.parse.urlparse(proxy)[1]
+ # scheme, host, path, parm, query, frag = urlparse.urlparse(proxy)
- proxyinfo = host.split(":")
+ proxyinfo = host.rsplit(":", 1)
host = proxyinfo[0]
port = "80"
host = proxy.rsplit(':', 1)[0]
port = proxy.rsplit(':', 1)[1]
+ # parse user/pass from proxy host
+ proxyinfo = host.rsplit("@", 1)
+ if len(proxyinfo) == 2:
+ host = proxyinfo[1]
+ # Known Issue: If password contains ":", which should be
+ # quoted, for example, use '123%3Aabc' instead of 123:abc
+ userpassinfo = proxyinfo[0].rsplit(":", 1)
+ if len(userpassinfo) == 2:
+ proxy_username = userpassinfo[0]
+ proxy_password = userpassinfo[1]
+ elif len(userpassinfo) == 1:
+ proxy_username = userpassinfo[0]
+
baseurl.setQueryParam ("proxy", host)
baseurl.setQueryParam ("proxyport", port)
+ if proxy_username:
+ baseurl.setQueryParam ("proxyuser", proxy_username)
+ if proxy_password:
+ baseurl.setQueryParam ("proxypass", proxy_password)
+ else:
+ baseurl.setQueryParam ("proxy", "_none_")
- repo.baseurl[0] = baseurl.asCompleteString()
self.repos.append(repo)
repo_info.addBaseUrl(baseurl)
- if repo.priority:
+ if repo.priority is not None:
repo_info.setPriority(repo.priority)
# this hack is used to change zypp credential file location
self.__build_repo_cache(name)
- except RuntimeError, e:
+ except RuntimeError as e:
raise CreatorError(str(e))
msger.verbose('repo: %s was added' % name)
def preInstall(self, pkg):
self.pre_pkgs.append(pkg)
+ def checkPackage(self, pkg):
+ self.check_pkgs.append(pkg)
+
+ def _get_local_packages(self):
+ """Return a list of rpm path to be local installed.
+ This is the hook where subclasses may specify a set of rpms which
+ it requires to be installed locally.
+ This returns an empty list by default.
+ Note, subclasses should usually chain up to the base class
+ implementation of this hook.
+ """
+ cropts = configmgr.create
+ if cropts['local_pkgs_path']:
+ if os.path.isdir(cropts['local_pkgs_path']):
+ return glob.glob(
+ os.path.join(cropts['local_pkgs_path'], '*.rpm'))
+ elif os.path.splitext(cropts['local_pkgs_path'])[-1] == '.rpm':
+ return [cropts['local_pkgs_path']]
+ return []
+ def __localinst_packages(self):
+ for rpm_path in self._get_local_packages():
+ self.installLocal(rpm_path)
def runInstall(self, checksize = 0):
os.environ["HOME"] = "/"
os.environ["LD_PRELOAD"] = ""
self.buildTransaction()
+ self.__localinst_packages()
todo = zypp.GetResolvablesToInsDel(self.Z.pool())
installed_pkgs = todo._toInstall
dlpkgs = []
- for item in installed_pkgs:
- if not zypp.isKindPattern(item) and \
- not self.inDeselectPackages(item):
+
+ for pitem in installed_pkgs:
+ if not zypp.isKindPattern(pitem) and \
+ not self.inDeselectPackages(pitem):
+ item = zypp.asKindPackage(pitem)
dlpkgs.append(item)
+ if item.name() in self.check_pkgs:
+ self.check_pkgs.remove(item.name())
+
if not self.install_debuginfo or str(item.arch()) == "noarch":
continue
dipkg = self._zyppQueryPackage("%s-debuginfo" % item.name())
if dipkg:
- dlpkgs.append(dipkg)
+ ditem = zypp.asKindPackage(dipkg)
+ dlpkgs.append(ditem)
else:
msger.warning("No debuginfo rpm found for: %s" \
% item.name())
+ if self.check_pkgs:
+ raise CreatorError('Packages absent in image: %s' % ','.join(self.check_pkgs))
+
# record all pkg and the content
- localpkgs = self.localpkgs.keys()
+ localpkgs = list(self.localpkgs.keys())
for pkg in dlpkgs:
license = ''
if pkg.name() in localpkgs:
'release': pkg.edition().release()
}
- package = zypp.asKindPackage(pkg)
- license = package.license()
+ license = pkg.license()
- if license in self.__pkgs_license.keys():
+ if license in list(self.__pkgs_license.keys()):
self.__pkgs_license[license].append(pkg_long_name)
else:
self.__pkgs_license[license] = [pkg_long_name]
total_count = len(dlpkgs)
cached_count = 0
- download_total_size = sum(map(lambda x: int(x.downloadSize()), dlpkgs))
- localpkgs = self.localpkgs.keys()
+ download_total_size = sum([int(x.downloadSize()) for x in dlpkgs])
+ localpkgs = list(self.localpkgs.keys())
msger.info("Checking packages cached ...")
for po in dlpkgs:
raise CreatorError("No enough space used for downloading.")
# record the total size of installed pkgs
- install_total_size = sum(map(lambda x: int(x.installSize()), dlpkgs))
+ install_total_size = sum([int(x.installSize()) for x in dlpkgs])
# check needed size before actually download and install
# FIXME: for multiple partitions for loop type, check fails
if download_count > 0:
msger.info("Downloading packages ...")
self.downloadPkgs(dlpkgs, download_count)
+ except CreatorError as e:
+ raise CreatorError("Package download failed: %s" %(e,))
+ try:
self.installPkgs(dlpkgs)
-
except (RepoError, RpmError):
raise
- except Exception, e:
+ except Exception as e:
raise CreatorError("Package installation failed: %s" % (e,))
def getVcsInfo(self):
'version': hdr['version'],
'release': hdr['release']
}
- self.__pkgs_vcsinfo[lname] = hdr['VCS']
+ try:
+ self.__pkgs_vcsinfo[lname] = hdr['VCS']
+ except ValueError:
+ # if rpm not support VCS, set to None
+ self.__pkgs_vcsinfo[lname] = None
return self.__pkgs_vcsinfo
self.repo_manager.loadFromCache(repo)
self.Z = zypp.ZYppFactory_instance().getZYpp()
+ if configmgr.create['block_recommends']:
+ msger.info("zypp not install recommend packages")
+ self.Z.resolver().setOnlyRequires(True)
self.Z.initializeTarget(zypp.Pathname(self.instroot))
self.Z.target().load()
def getLocalPkgPath(self, po):
repoinfo = po.repoInfo()
cacheroot = repoinfo.packagesPath()
- location= zypp.asKindPackage(po).location()
+ location = po.location()
rpmpath = str(location.filename())
pkgpath = "%s/%s" % (cacheroot, os.path.basename(rpmpath))
return pkgpath
% (pkg, hdr['arch']))
def downloadPkgs(self, package_objects, count):
- localpkgs = self.localpkgs.keys()
+ localpkgs = list(self.localpkgs.keys())
progress_obj = TextProgress(count)
for po in package_objects:
proxies = self.get_proxies(po)
try:
- filename = myurlgrab(url, filename, proxies, progress_obj)
+ filename = myurlgrab(url.full, filename, proxies, progress_obj)
except CreatorError:
self.close()
raise
for e in errors:
msger.warning(e[0])
raise RepoError('Could not run transaction.')
+ def show_unresolved_dependencies_msg(self, unresolved_dependencies):
+ for pkg, need, needflags, sense, key in unresolved_dependencies:
+
+ package = '-'.join(pkg)
+
+ if needflags == rpm.RPMSENSE_LESS:
+ deppkg = ' < '.join(need)
+ elif needflags == rpm.RPMSENSE_EQUAL:
+ deppkg = ' = '.join(need)
+ elif needflags == rpm.RPMSENSE_GREATER:
+ deppkg = ' > '.join(need)
+ else:
+ deppkg = '-'.join(need)
+
+ if sense == rpm.RPMDEP_SENSE_REQUIRES:
+ msger.warning("[%s] Requires [%s], which is not provided" \
+ % (package, deppkg))
+
+ elif sense == rpm.RPMDEP_SENSE_CONFLICTS:
+ msger.warning("[%s] Conflicts with [%s]" % (package, deppkg))
def installPkgs(self, package_objects):
if not self.ts:
self.ts.setProbFilter(probfilter)
self.ts_pre.setProbFilter(probfilter)
- localpkgs = self.localpkgs.keys()
+ localpkgs = list(self.localpkgs.keys())
for po in package_objects:
pkgname = po.name()
if len(errors) == 0:
msger.warning('scriptlet or other non-fatal errors occurred '
'during transaction.')
-
+ if self.strict_mode:
+ raise CreatorError("mic failes to install some packages")
else:
for e in errors:
msger.warning(e[0])
raise RepoError('Could not run transaction.')
else:
- for pkg, need, needflags, sense, key in unresolved_dependencies:
- package = '-'.join(pkg)
-
- if needflags == rpm.RPMSENSE_LESS:
- deppkg = ' < '.join(need)
- elif needflags == rpm.RPMSENSE_EQUAL:
- deppkg = ' = '.join(need)
- elif needflags == rpm.RPMSENSE_GREATER:
- deppkg = ' > '.join(need)
- else:
- deppkg = '-'.join(need)
-
- if sense == rpm.RPMDEP_SENSE_REQUIRES:
- msger.warning("[%s] Requires [%s], which is not provided" \
- % (package, deppkg))
-
- elif sense == rpm.RPMDEP_SENSE_CONFLICTS:
- msger.warning("[%s] Conflicts with [%s]" %(package,deppkg))
-
+ self.show_unresolved_dependencies_msg(unresolved_dependencies)
raise RepoError("Unresolved dependencies, transaction failed.")
def __initialize_transaction(self):
proxies = None
repoinfo = pobj.repoInfo()
reponame = "%s" % repoinfo.name()
- repos = filter(lambda r: r.name == reponame, self.repos)
+ repos = [r for r in self.repos if r.name == reponame]
repourl = str(repoinfo.baseUrls()[0])
if repos:
except IndexError:
return None
- baseurl = repo.baseurl[0]
-
- index = baseurl.find("?")
- if index > -1:
- baseurl = baseurl[:index]
-
- location = zypp.asKindPackage(pobj).location()
+ location = pobj.location()
location = str(location.filename())
if location.startswith("./"):
location = location[2:]
- return os.path.join(baseurl, location)
+ return repo.baseurl[0].join(location)
def package_url(self, pkgname):
- def cmpEVR(ed1, ed2):
- (e1, v1, r1) = map(str, [ed1.epoch(), ed1.version(), ed1.release()])
- (e2, v2, r2) = map(str, [ed2.epoch(), ed2.version(), ed2.release()])
+ def cmpEVR(p1, p2):
+ ed1 = p1.edition()
+ ed2 = p2.edition()
+ (e1, v1, r1) = list(map(str, [ed1.epoch(), ed1.version(), ed1.release()]))
+ (e2, v2, r2) = list(map(str, [ed2.epoch(), ed2.version(), ed2.release()]))
return rpm.labelCompare((e1, v1, r1), (e2, v2, r2))
if not self.Z:
q.setMatchExact()
q.addAttribute(zypp.SolvAttr.name, pkgname)
items = sorted(q.queryResults(self.Z.pool()),
- cmp=lambda x,y: cmpEVR(x.edition(), y.edition()),
+ cmp=lambda x,y: cmpEVR(zypp.asKindPackage(x), zypp.asKindPackage(y)),
reverse=True)
if items:
- url = self.get_url(items[0])
- proxies = self.get_proxies(items[0])
+ item = zypp.asKindPackage(items[0])
+ url = self.get_url(item)
+ proxies = self.get_proxies(item)
return (url, proxies)
return (None, None)