3 # Copyright 2010, 2011 Intel, Inc.
5 # This copyrighted material is made available to anyone wishing to use, modify,
6 # copy, or redistribute it subject to the terms and conditions of the GNU
7 # General Public License v.2. This program is distributed in the hope that it
8 # will be useful, but WITHOUT ANY WARRANTY expressed or implied, including the
9 # implied warranties of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
10 # See the GNU General Public License for more details.
12 # You should have received a copy of the GNU General Public License along with
13 # this program; if not, write to the Free Software Foundation, Inc., 51
14 # Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. Any Red Hat
15 # trademarks that are incorporated in the source code or documentation are not
16 # subject to the GNU General Public License and may only be used or replicated
17 # with the express permission of Red Hat, Inc.
23 import rpmUtils.miscutils
24 import rpmUtils.transaction
29 from mic.kickstart import ksparser
30 from mic.utils import rpmmisc, fs_related as fs
31 from mic.utils.proxy import get_proxy_for
32 from mic.utils.errors import CreatorError
33 from mic.imager.baseimager import BaseImageCreator
35 if not hasattr(zypp, 'PoolQuery'):
36 raise ImportError("python-zypp in host system cannot support PoolQuery interface, please "
37 "update it to enhanced version which can be found in repo.meego.com/tools")
43 self.mirrorlist = None
45 self.proxy_username = None
46 self.proxy_password = None
47 self.includepkgs = None
48 self.includepkgs = None
52 self.autorefresh = True
53 self.keeppackages = True
55 class RepoError(CreatorError):
58 class RpmError(CreatorError):
61 from mic.pluginbase import BackendPlugin
62 class Zypp(BackendPlugin):
65 def __init__(self, creator = None, recording_pkgs=None):
66 if not isinstance(creator, BaseImageCreator):
67 raise CreatorError("Invalid argument: creator")
69 self.__recording_pkgs = recording_pkgs
70 self.__pkgs_content = {}
71 self.creator = creator
75 self.repo_manager = None
76 self.repo_manager_options = None
79 self.probFilterFlags = []
83 self.has_prov_query = True
86 def doFileLogSetup(self, uid, logfile):
87 # don't do the file log for the livecd as it can lead to open fds
88 # being left and an inability to clean up after ourself
96 if not os.path.exists("/etc/fedora-release") and not os.path.exists("/etc/meego-release"):
97 for i in range(3, os.sysconf("SC_OPEN_MAX")):
109 def _cleanupRpmdbLocks(self, installroot):
110 # cleans up temporary files left by bdb so that differing
111 # versions of rpm don't cause problems
113 for f in glob.glob(installroot + "/var/lib/rpm/__db*"):
116 def setup(self, confpath, installroot):
117 self._cleanupRpmdbLocks(installroot)
118 self.installroot = installroot
120 def whatObsolete(self, pkg):
121 query = zypp.PoolQuery()
122 query.addKind(zypp.ResKind.package)
123 query.addAttribute(zypp.SolvAttr.obsoletes, pkg)
124 query.setMatchExact()
125 for pi in query.queryResults(self.Z.pool()):
129 def selectPackage(self, pkg):
130 """ Select a given package or package pattern, can be specified with name.arch or name* or *name """
132 self.__initialize_zypp()
134 def markPoolItem(obs, pi):
136 pi.status().setToBeInstalled (zypp.ResStatus.USER)
138 obs.status().setToBeInstalled (zypp.ResStatus.USER)
141 startx = pkg.startswith("*")
142 endx = pkg.endswith("*")
143 ispattern = startx or endx
144 sp = pkg.rsplit(".", 1)
147 q.addKind(zypp.ResKind.package)
149 if startx and not endx:
150 pattern = '%s$' % (pkg[1:])
151 if endx and not startx:
152 pattern = '^%s' % (pkg[0:-1])
154 pattern = '%s' % (pkg[1:-1])
156 q.addAttribute(zypp.SolvAttr.name,pattern)
159 q.addAttribute(zypp.SolvAttr.name,sp[0])
162 q.addAttribute(zypp.SolvAttr.name,pkg)
164 for item in q.queryResults(self.Z.pool()):
166 obspkg = self.whatObsolete(item.name())
168 if item.arch() == sp[1]:
169 item.status().setToBeInstalled (zypp.ResStatus.USER)
171 markPoolItem(obspkg, item)
172 # Can't match using package name, then search from packge provides infomation
173 if found == False and not ispattern:
174 q.addAttribute(zypp.SolvAttr.provides, pkg)
175 q.addAttribute(zypp.SolvAttr.name,'')
176 for item in q.queryResults(self.Z.pool()):
178 obspkg = self.whatObsolete(item.name())
179 markPoolItem(obspkg, item)
184 raise CreatorError("Unable to find package: %s" % (pkg,))
185 def inDeselectPackages(self, name):
186 """check if specified pacakges are in the list of inDeselectPackages"""
187 for pkg in self.to_deselect:
188 startx = pkg.startswith("*")
189 endx = pkg.endswith("*")
190 ispattern = startx or endx
191 sp = pkg.rsplit(".", 2)
194 arch = "%s" % item.arch()
195 if name == sp[0] and arch == sp[1]:
201 if startx and name.endswith(sp[0][1:]):
203 if endx and name.startswith(sp[0][:-1]):
207 def deselectPackage(self, pkg):
208 """collect packages should not be installed"""
209 self.to_deselect.append(pkg)
211 def __selectIncpkgs(self):
213 for pkg in self.incpkgs:
214 for item in self.Z.pool():
215 kind = "%s" % item.kind()
216 if kind == "package":
217 name = "%s" % item.name()
218 repoalias = "%s" % item.repoInfo().alias()
219 if name == pkg and repoalias.endswith("include"):
221 item.status().setToBeInstalled (zypp.ResStatus.USER)
224 raise CreatorError("Unable to find package: %s" % (pkg,))
226 def __selectExcpkgs(self):
228 for pkg in self.excpkgs:
229 for item in self.Z.pool():
230 kind = "%s" % item.kind()
231 if kind == "package":
232 name = "%s" % item.name()
233 repoalias = "%s" % item.repoInfo().alias()
234 if name == pkg and not repoalias.endswith("exclude"):
236 item.status().setToBeInstalled (zypp.ResStatus.USER)
239 raise CreatorError("Unable to find package: %s" % (pkg,))
241 def selectGroup(self, grp, include = ksparser.GROUP_DEFAULT):
243 self.__initialize_zypp()
246 q.addKind(zypp.ResKind.pattern)
247 for item in q.queryResults(self.Z.pool()):
248 summary = "%s" % item.summary()
249 name = "%s" % item.name()
250 if name == grp or summary == grp:
252 item.status().setToBeInstalled (zypp.ResStatus.USER)
256 if include == ksparser.GROUP_REQUIRED:
257 map(lambda p: self.deselectPackage(p), grp.default_packages.keys())
260 raise CreatorError("Unable to find pattern: %s" % (grp,))
262 def addRepository(self, name, url = None, mirrorlist = None, proxy = None, proxy_username = None, proxy_password = None, inc = None, exc = None):
263 if not self.repo_manager:
264 self.__initialize_repo_manager()
266 repo = RepositoryStub()
270 repo.proxy_username = proxy_username
271 repo.proxy_password = proxy_password
272 repo.baseurl.append(url)
275 repo_alias = name + "include"
278 repo_alias = name + "exclude"
281 # check LICENSE files
282 if not rpmmisc.checkRepositoryEULA(name, repo):
283 msger.warning('skip repo:%s for failed EULA confirmation' % name)
287 repo.mirrorlist = mirrorlist
289 # Enable gpg check for verifying corrupt packages
291 self.repos.append(repo)
294 repo_info = zypp.RepoInfo()
295 repo_info.setAlias(repo_alias)
296 repo_info.setName(repo.name)
297 repo_info.setEnabled(repo.enabled)
298 repo_info.setAutorefresh(repo.autorefresh)
299 repo_info.setKeepPackages(repo.keeppackages)
300 repo_info.addBaseUrl(zypp.Url(repo.baseurl[0]))
301 self.repo_manager.addRepository(repo_info)
302 self.__build_repo_cache(name)
303 except RuntimeError, e:
304 raise CreatorError(str(e))
306 msger.verbose('repo: %s was added' % name)
309 def installHasFile(self, file):
312 def runInstall(self, checksize = 0):
314 self.__selectIncpkgs()
316 self.__selectExcpkgs()
318 os.environ["HOME"] = "/"
319 self.buildTransaction()
321 todo = zypp.GetResolvablesToInsDel(self.Z.pool())
322 installed_pkgs = todo._toInstall
324 for item in installed_pkgs:
325 if not zypp.isKindPattern(item) and not self.inDeselectPackages(item.name()):
328 # record the total size of installed pkgs
329 pkgs_total_size = sum(map(lambda x: int(x.installSize()), dlpkgs))
331 # check needed size before actually download and install
332 if checksize and pkgs_total_size > checksize:
333 raise CreatorError("Size of specified root partition in kickstart file is too small to install all selected packages.")
335 if self.__recording_pkgs:
336 # record all pkg and the content
338 pkg_long_name = "%s-%s.%s.rpm" % (pkg.name(), pkg.edition(), pkg.arch())
339 self.__pkgs_content[pkg_long_name] = {} #TBD: to get file list
341 total_count = len(dlpkgs)
343 localpkgs = self.localpkgs.keys()
344 msger.info("Checking packages cache and packages integrity ...")
346 """ Check if it is cached locally """
347 if po.name() in localpkgs:
350 local = self.getLocalPkgPath(po)
351 if os.path.exists(local):
352 if self.checkPkg(local) != 0:
356 download_count = total_count - cached_count
357 msger.info("%d packages to be installed, %d packages gotten from cache, %d packages to be downloaded" % (total_count, cached_count, download_count))
359 if download_count > 0:
360 msger.info("Downloading packages ...")
361 self.downloadPkgs(dlpkgs, download_count)
362 self.installPkgs(dlpkgs)
365 raise CreatorError("Unable to download from repo : %s" % (e,))
367 raise CreatorError("Unable to install: %s" % (e,))
369 def getAllContent(self):
370 return self.__pkgs_content
372 def __initialize_repo_manager(self):
373 if self.repo_manager:
376 """ Clean up repo metadata """
377 shutil.rmtree(self.creator.cachedir + "/etc", ignore_errors = True)
379 zypp.KeyRing.setDefaultAccept( zypp.KeyRing.ACCEPT_UNSIGNED_FILE
380 | zypp.KeyRing.ACCEPT_VERIFICATION_FAILED
381 | zypp.KeyRing.ACCEPT_UNKNOWNKEY
382 | zypp.KeyRing.TRUST_KEY_TEMPORARILY
384 self.repo_manager_options = zypp.RepoManagerOptions(zypp.Pathname(self.creator._instroot))
385 self.repo_manager_options.knownReposPath = zypp.Pathname(self.creator.cachedir + "/etc/zypp/repos.d")
386 self.repo_manager_options.repoCachePath = zypp.Pathname(self.creator.cachedir)
387 self.repo_manager_options.repoRawCachePath = zypp.Pathname(self.creator.cachedir + "/raw")
388 self.repo_manager_options.repoSolvCachePath = zypp.Pathname(self.creator.cachedir + "/solv")
389 self.repo_manager_options.repoPackagesCachePath = zypp.Pathname(self.creator.cachedir + "/packages")
391 self.repo_manager = zypp.RepoManager(self.repo_manager_options)
393 def __build_repo_cache(self, name):
394 repo = self.repo_manager.getRepositoryInfo(name)
395 if self.repo_manager.isCached(repo) or not repo.enabled():
397 self.repo_manager.buildCache(repo, zypp.RepoManager.BuildIfNeeded)
399 def __initialize_zypp(self):
403 zconfig = zypp.ZConfig_instance()
405 """ Set system architecture """
406 if self.creator.target_arch and self.creator.target_arch.startswith("arm"):
407 arches = ["armv7l", "armv7nhl", "armv7hl", "armv5tel"]
408 if self.creator.target_arch not in arches:
409 raise CreatorError("Invalid architecture: %s" % self.creator.target_arch)
413 arch_map[self.creator.target_arch] = zypp.Arch(self.creator.target_arch)
414 except AttributeError:
415 msger.error('libzypp/python-zypp in host system cannot support arch %s, please'
416 ' update it to enhanced version which can be found in repo.meego.com/tools'\
417 % self.creator.target_arch)
419 zconfig.setSystemArchitecture(arch_map[self.creator.target_arch])
421 msger.info("zypp architecture is <%s>" % zconfig.systemArchitecture())
423 """ repoPackagesCachePath is corrected by this """
424 self.repo_manager = zypp.RepoManager(self.repo_manager_options)
425 repos = self.repo_manager.knownRepositories()
427 if not repo.enabled():
429 self.repo_manager.loadFromCache(repo)
431 self.Z = zypp.ZYppFactory_instance().getZYpp()
432 self.Z.initializeTarget(zypp.Pathname(self.creator._instroot))
433 self.Z.target().load()
436 def buildTransaction(self):
437 if not self.Z.resolver().resolvePool():
438 msger.warning("Problem count: %d" % len(self.Z.resolver().problems()))
439 for problem in self.Z.resolver().problems():
440 msger.warning("Problem: %s, %s" % (problem.description().decode("utf-8"), problem.details().decode("utf-8")))
442 def getLocalPkgPath(self, po):
443 repoinfo = po.repoInfo()
445 cacheroot = repoinfo.packagesPath()
447 edition = po.edition()
448 version = "%s-%s" % (edition.version(), edition.release())
449 pkgpath = "%s/%s/%s-%s.%s.rpm" % (cacheroot, arch, name, version, arch)
452 def installLocal(self, pkg, po=None, updateonly=False):
454 self.__initialize_transaction()
455 ts = rpmUtils.transaction.initReadOnlyTransaction()
457 hdr = rpmUtils.miscutils.hdrFromPackage(ts, pkg)
458 except RpmUtilsError, e:
459 raise Errors.MiscError, \
460 'Could not open local rpm file: %s: %s' % (self.localpath, e)
461 arch = zypp.Arch(hdr.arch)
462 if self.creator.target_arch == None:
463 # TODO, get the default_arch from conf or detected from global settings
464 sysarch = zypp.Arch('i686')
466 sysarch = zypp.Arch(self.creator.target_arch)
467 if arch.compatible_with (sysarch):
468 pkgname = self.__get_pkg_name(pkg)
469 self.localpkgs[pkgname] = pkg
470 self.selectPackage(pkgname)
471 msger.info("Marking %s to be installed" % (pkg))
473 msger.warning ("Cannot add package %s to transaction. Not a compatible architecture: %s" % (pkg, hdr.arch))
475 def __get_pkg_name(self, pkgpath):
476 h = rpmmisc.readRpmHeader(self.ts, pkgpath)
479 def downloadPkgs(self, package_objects, count):
480 localpkgs = self.localpkgs.keys()
481 progress_obj = fs.TextProgress(count)
482 for po in package_objects:
483 if po.name() in localpkgs:
485 filename = self.getLocalPkgPath(po)
486 if os.path.exists(filename):
487 if self.checkPkg(filename) == 0:
489 dirn = os.path.dirname(filename)
490 if not os.path.exists(dirn):
492 baseurl = po.repoInfo().baseUrls()[0].__str__()
493 proxy = self.get_proxy(po.repoInfo())
496 proxies = {str(proxy.split(":")[0]):str(proxy)}
498 location = zypp.asKindPackage(po).location()
499 location = location.filename().__str__()
500 if location.startswith("./"):
501 location = location[2:]
502 url = baseurl + "/%s" % location
504 filename = fs.myurlgrab(url, filename, proxies, progress_obj)
509 def installPkgs(self, package_objects):
511 self.__initialize_transaction()
515 for flag in self.probFilterFlags:
517 self.ts.setProbFilter(probfilter)
519 localpkgs = self.localpkgs.keys()
520 for po in package_objects:
522 if pkgname in localpkgs:
523 rpmpath = self.localpkgs[pkgname]
525 rpmpath = self.getLocalPkgPath(po)
526 if not os.path.exists(rpmpath):
527 """ Maybe it is a local repo """
528 baseurl = po.repoInfo().baseUrls()[0].__str__()
529 baseurl = baseurl.strip()
530 if baseurl.startswith("file:/"):
531 rpmpath = baseurl[5:] + "/%s/%s" % (po.arch(), os.path.basename(rpmpath))
532 if not os.path.exists(rpmpath):
533 raise RpmError("Error: %s doesn't exist" % rpmpath)
534 h = rpmmisc.readRpmHeader(self.ts, rpmpath)
535 self.ts.addInstall(h, rpmpath, 'u')
537 unresolved_dependencies = self.ts.check()
538 if not unresolved_dependencies:
540 cb = rpmmisc.RPMInstallCallback(self.ts)
541 self.ts.run(cb.callback, '')
545 for pkg, need, needflags, sense, key in unresolved_dependencies:
546 package = '-'.join(pkg)
547 if needflags == rpm.RPMSENSE_LESS:
548 deppkg = ' < '.join(need)
549 elif needflags == rpm.RPMSENSE_EQUAL:
550 deppkg = ' = '.join(need)
551 elif needflags == rpm.RPMSENSE_GREATER:
552 deppkg = ' > '.join(need)
554 deppkg = '-'.join(need)
556 if sense == rpm.RPMDEP_SENSE_REQUIRES:
557 msger.warning ("[%s] Requires [%s], which is not provided" % (package, deppkg))
558 elif sense == rpm.RPMDEP_SENSE_CONFLICTS:
559 msger.warning ("[%s] Conflicts with [%s]" % (package, deppkg))
561 raise RepoError("Unresolved dependencies, transaction failed.")
563 def __initialize_transaction(self):
565 self.ts = rpm.TransactionSet(self.creator._instroot)
566 # Set to not verify DSA signatures.
567 self.ts.setVSFlags(rpm._RPMVSF_NOSIGNATURES|rpm._RPMVSF_NODIGESTS)
569 def checkPkg(self, pkg):
571 if not os.path.exists(pkg):
573 ret = rpmmisc.checkRpmIntegrity('rpm', pkg)
575 msger.warning("package %s is damaged: %s" % (os.path.basename(pkg), pkg))
579 def _add_prob_flags(self, *flags):
581 if flag not in self.probFilterFlags:
582 self.probFilterFlags.append(flag)
584 def get_proxy(self, repoinfo):
586 reponame = "%s" % repoinfo.name()
587 for repo in self.repos:
588 if repo.name == reponame:
595 repourl = repoinfo.baseUrls()[0].__str__()
596 return get_proxy_for(repourl)