3 # Copyright (c) 2010, 2011 Intel, Inc.
5 # This program is free software; you can redistribute it and/or modify it
6 # under the terms of the GNU General Public License as published by the Free
7 # Software Foundation; version 2 of the License
9 # This program is distributed in the hope that it will be useful, but
10 # WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
11 # or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 # You should have received a copy of the GNU General Public License along
15 # with this program; if not, write to the Free Software Foundation, Inc., 59
16 # Temple Place - Suite 330, Boston, MA 02111-1307, USA.
24 if not hasattr(zypp, 'PoolQuery') or \
25 not hasattr(zypp.RepoManager, 'loadSolvFile'):
26 raise ImportError("python-zypp in host system cannot support PoolQuery or "
27 "loadSolvFile interface, please update it to enhanced "
28 "version which can be found in repo.meego.com/tools")
31 from mic.kickstart import ksparser
32 from mic.utils import misc, rpmmisc, runner, fs_related
33 from mic.utils.proxy import get_proxy_for
34 from mic.utils.errors import CreatorError
35 from mic.imager.baseimager import BaseImageCreator
41 self.mirrorlist = None
43 self.proxy_username = None
44 self.proxy_password = None
47 self.autorefresh = True
48 self.keeppackages = True
51 class RepoError(CreatorError):
54 class RpmError(CreatorError):
57 from mic.pluginbase import BackendPlugin
58 class Zypp(BackendPlugin):
61 def __init__(self, target_arch, instroot, cachedir):
62 self.cachedir = cachedir
63 self.instroot = instroot
64 self.target_arch = target_arch
66 self.__pkgs_license = {}
67 self.__pkgs_content = {}
71 self.repo_manager = None
72 self.repo_manager_options = None
75 self.probFilterFlags = []
79 self.has_prov_query = True
81 def doFileLogSetup(self, uid, logfile):
82 # don't do the file log for the livecd as it can lead to open fds
83 # being left and an inability to clean up after ourself
96 if not os.path.exists("/etc/fedora-release") and \
97 not os.path.exists("/etc/meego-release"):
98 for i in range(3, os.sysconf("SC_OPEN_MAX")):
107 def _cleanupRpmdbLocks(self, installroot):
108 # cleans up temporary files left by bdb so that differing
109 # versions of rpm don't cause problems
111 for f in glob.glob(installroot + "/var/lib/rpm/__db*"):
115 self._cleanupRpmdbLocks(self.instroot)
117 def whatObsolete(self, pkg):
118 query = zypp.PoolQuery()
119 query.addKind(zypp.ResKind.package)
120 query.addAttribute(zypp.SolvAttr.obsoletes, pkg)
121 query.setMatchExact()
122 for pi in query.queryResults(self.Z.pool()):
126 def _splitPkgString(self, pkg):
127 sp = pkg.rsplit(".",1)
132 sysarch = zypp.Arch(self.target_arch)
133 if not zypp.Arch(arch).compatible_with (sysarch):
138 def selectPackage(self, pkg):
139 """Select a given package or package pattern, can be specified
140 with name.arch or name* or *name
144 self.__initialize_zypp()
146 def markPoolItem(obs, pi):
148 pi.status().setToBeInstalled (zypp.ResStatus.USER)
150 obs.status().setToBeInstalled (zypp.ResStatus.USER)
153 startx = pkg.startswith("*")
154 endx = pkg.endswith("*")
155 ispattern = startx or endx
156 name, arch = self._splitPkgString(pkg)
159 q.addKind(zypp.ResKind.package)
162 if startx and not endx:
163 pattern = '%s$' % (pkg[1:])
164 if endx and not startx:
165 pattern = '^%s' % (pkg[0:-1])
167 pattern = '%s' % (pkg[1:-1])
169 q.addAttribute(zypp.SolvAttr.name,pattern)
173 q.addAttribute(zypp.SolvAttr.name,name)
177 q.addAttribute(zypp.SolvAttr.name,pkg)
180 q.queryResults(self.Z.pool()),
181 key=lambda item: str(item.edition()),
184 if item.name() in self.excpkgs.keys() and \
185 self.excpkgs[item.name()] == item.repoInfo().name():
187 if item.name() in self.incpkgs.keys() and \
188 self.incpkgs[item.name()] != item.repoInfo().name():
192 obspkg = self.whatObsolete(item.name())
194 if arch == str(item.arch()):
195 item.status().setToBeInstalled (zypp.ResStatus.USER)
197 markPoolItem(obspkg, item)
201 # Can't match using package name, then search from packge
202 # provides infomation
203 if found == False and not ispattern:
204 q.addAttribute(zypp.SolvAttr.provides, pkg)
205 q.addAttribute(zypp.SolvAttr.name,'')
208 q.queryResults(self.Z.pool()),
209 key=lambda item: str(item.edition()),
211 if item.name() in self.excpkgs.keys() and \
212 self.excpkgs[item.name()] == item.repoInfo().name():
214 if item.name() in self.incpkgs.keys() and \
215 self.incpkgs[item.name()] != item.repoInfo().name():
219 obspkg = self.whatObsolete(item.name())
220 markPoolItem(obspkg, item)
226 raise CreatorError("Unable to find package: %s" % (pkg,))
228 def inDeselectPackages(self, item):
229 """check if specified pacakges are in the list of inDeselectPackages
233 for pkg in self.to_deselect:
234 startx = pkg.startswith("*")
235 endx = pkg.endswith("*")
236 ispattern = startx or endx
237 pkgname, pkgarch = self._splitPkgString(pkg)
240 if name == pkgname and str(item.arch()) == pkgarch:
246 if startx and name.endswith(pkg[1:]):
248 if endx and name.startswith(pkg[:-1]):
253 def deselectPackage(self, pkg):
254 """collect packages should not be installed"""
255 self.to_deselect.append(pkg)
257 def selectGroup(self, grp, include = ksparser.GROUP_DEFAULT):
259 self.__initialize_zypp()
262 q.addKind(zypp.ResKind.pattern)
263 for item in q.queryResults(self.Z.pool()):
264 summary = "%s" % item.summary()
265 name = "%s" % item.name()
266 if name == grp or summary == grp:
268 item.status().setToBeInstalled (zypp.ResStatus.USER)
272 if include == ksparser.GROUP_REQUIRED:
274 lambda p: self.deselectPackage(p),
275 grp.default_packages.keys())
279 raise CreatorError("Unable to find pattern: %s" % (grp,))
281 def addRepository(self, name,
285 proxy_username = None,
286 proxy_password = None,
292 # TODO: Handle cost attribute for repos
294 if not self.repo_manager:
295 self.__initialize_repo_manager()
297 repo = RepositoryStub()
301 repo.proxy_username = proxy_username
302 repo.proxy_password = proxy_password
303 repo.ssl_verify = ssl_verify
304 repo.baseurl.append(url)
307 self.incpkgs[pkg] = name
310 self.excpkgs[pkg] = name
312 # check LICENSE files
313 if not rpmmisc.checkRepositoryEULA(name, repo):
314 msger.warning('skip repo:%s for failed EULA confirmation' % name)
318 repo.mirrorlist = mirrorlist
320 # Enable gpg check for verifying corrupt packages
323 repo.priority = priority
324 self.repos.append(repo)
327 repo_info = zypp.RepoInfo()
328 repo_info.setAlias(repo.name)
329 repo_info.setName(repo.name)
330 repo_info.setEnabled(repo.enabled)
331 repo_info.setAutorefresh(repo.autorefresh)
332 repo_info.setKeepPackages(repo.keeppackages)
333 baseurl = zypp.Url(repo.baseurl[0])
335 baseurl.setQueryParam("ssl_verify", "no")
337 scheme, host, path, parm, query, frag = urlparse.urlparse(proxy)
339 proxyinfo = host.split(":")
343 if len(proxyinfo) > 1:
346 if proxy.startswith("socks") and len(proxy.rsplit(':', 1)) == 2:
347 host = proxy.rsplit(':', 1)[0]
348 port = proxy.rsplit(':', 1)[1]
350 baseurl.setQueryParam ("proxy", host)
351 baseurl.setQueryParam ("proxyport", port)
353 repo_info.addBaseUrl(baseurl)
356 repo_info.setPriority(repo.priority)
358 self.repo_manager.addRepository(repo_info)
360 self.__build_repo_cache(name)
362 except RuntimeError, e:
363 raise CreatorError(str(e))
365 msger.verbose('repo: %s was added' % name)
368 def installHasFile(self, file):
371 def runInstall(self, checksize = 0):
372 os.environ["HOME"] = "/"
373 self.buildTransaction()
375 todo = zypp.GetResolvablesToInsDel(self.Z.pool())
376 installed_pkgs = todo._toInstall
378 for item in installed_pkgs:
379 if not zypp.isKindPattern(item) and \
380 not self.inDeselectPackages(item):
383 # record all pkg and the content
384 localpkgs = self.localpkgs.keys()
387 if pkg.name() in localpkgs:
388 hdr = rpmmisc.readRpmHeader(self.ts, self.localpkgs[pkg.name()])
389 pkg_long_name = misc.RPM_FMT % {
392 'ver_rel': '%s-%s' % (hdr['version'],
395 license = hdr['license']
398 pkg_long_name = misc.RPM_FMT % {
401 'ver_rel': pkg.edition(),
404 package = zypp.asKindPackage(pkg)
405 license = package.license()
407 self.__pkgs_content[pkg_long_name] = {} #TBD: to get file list
409 if license in self.__pkgs_license.keys():
410 self.__pkgs_license[license].append(pkg_long_name)
412 self.__pkgs_license[license] = [pkg_long_name]
414 total_count = len(dlpkgs)
416 download_total_size = sum(map(lambda x: int(x.downloadSize()), dlpkgs))
417 localpkgs = self.localpkgs.keys()
419 msger.info("Checking packages cache and packages integrity ...")
421 # Check if it is cached locally
422 if po.name() in localpkgs:
425 local = self.getLocalPkgPath(po)
426 if os.path.exists(local):
427 if self.checkPkg(local) != 0:
430 download_total_size -= int(po.downloadSize())
432 cache_avail_size = misc.get_filesystem_avail(self.cachedir)
433 if cache_avail_size < download_total_size:
434 raise CreatorError("No enough space used for downloading.")
436 # record the total size of installed pkgs
437 install_total_size = sum(map(lambda x: int(x.installSize()), dlpkgs))
438 # check needed size before actually download and install
439 if checksize and install_total_size > checksize:
440 raise CreatorError("No enough space used for installing, "
441 "please resize partition size in ks file")
443 download_count = total_count - cached_count
444 msger.info("%d packages to be installed, "
445 "%d packages gotten from cache, "
446 "%d packages to be downloaded" \
447 % (total_count, cached_count, download_count))
450 if download_count > 0:
451 msger.info("Downloading packages ...")
452 self.downloadPkgs(dlpkgs, download_count)
454 self.installPkgs(dlpkgs)
457 raise CreatorError("Unable to download from repo : %s" % (e,))
459 raise CreatorError("Unable to install: %s" % (e,))
461 def getAllContent(self):
462 return self.__pkgs_content
464 def getPkgsLicense(self):
465 return self.__pkgs_license
467 def __initialize_repo_manager(self):
468 if self.repo_manager:
471 # Clean up repo metadata
472 shutil.rmtree(self.cachedir + "/etc", ignore_errors = True)
473 shutil.rmtree(self.cachedir + "/solv", ignore_errors = True)
474 shutil.rmtree(self.cachedir + "/raw", ignore_errors = True)
476 zypp.KeyRing.setDefaultAccept( zypp.KeyRing.ACCEPT_UNSIGNED_FILE
477 | zypp.KeyRing.ACCEPT_VERIFICATION_FAILED
478 | zypp.KeyRing.ACCEPT_UNKNOWNKEY
479 | zypp.KeyRing.TRUST_KEY_TEMPORARILY
482 self.repo_manager_options = \
483 zypp.RepoManagerOptions(zypp.Pathname(self.instroot))
485 self.repo_manager_options.knownReposPath = \
486 zypp.Pathname(self.cachedir + "/etc/zypp/repos.d")
488 self.repo_manager_options.repoCachePath = \
489 zypp.Pathname(self.cachedir)
491 self.repo_manager_options.repoRawCachePath = \
492 zypp.Pathname(self.cachedir + "/raw")
494 self.repo_manager_options.repoSolvCachePath = \
495 zypp.Pathname(self.cachedir + "/solv")
497 self.repo_manager_options.repoPackagesCachePath = \
498 zypp.Pathname(self.cachedir + "/packages")
500 self.repo_manager = zypp.RepoManager(self.repo_manager_options)
502 def __build_repo_cache(self, name):
503 repo = self.repo_manager.getRepositoryInfo(name)
504 if self.repo_manager.isCached(repo) or not repo.enabled():
507 self.repo_manager.buildCache(repo, zypp.RepoManager.BuildIfNeeded)
509 def __initialize_zypp(self):
513 zconfig = zypp.ZConfig_instance()
515 # Set system architecture
517 zconfig.setSystemArchitecture(zypp.Arch(self.target_arch))
519 msger.info("zypp architecture is <%s>" % zconfig.systemArchitecture())
521 # repoPackagesCachePath is corrected by this
522 self.repo_manager = zypp.RepoManager(self.repo_manager_options)
523 repos = self.repo_manager.knownRepositories()
525 if not repo.enabled():
527 self.repo_manager.loadFromCache(repo)
529 self.Z = zypp.ZYppFactory_instance().getZYpp()
530 self.Z.initializeTarget(zypp.Pathname(self.instroot))
531 self.Z.target().load()
533 def buildTransaction(self):
534 if not self.Z.resolver().resolvePool():
535 msger.warning("Problem count: %d" \
536 % len(self.Z.resolver().problems()))
538 for problem in self.Z.resolver().problems():
539 msger.warning("Problem: %s, %s" \
540 % (problem.description().decode("utf-8"),
541 problem.details().decode("utf-8")))
543 def getLocalPkgPath(self, po):
544 repoinfo = po.repoInfo()
545 cacheroot = repoinfo.packagesPath()
546 location= zypp.asKindPackage(po).location()
547 rpmpath = str(location.filename())
548 pkgpath = "%s/%s" % (cacheroot, rpmpath)
551 def installLocal(self, pkg, po=None, updateonly=False):
553 self.__initialize_transaction()
555 solvfile = "%s/.solv" % (self.cachedir)
557 rc, out = runner.runtool([fs_related.find_binary_path("rpms2solv"),
560 f = open(solvfile, "w+")
564 warnmsg = self.repo_manager.loadSolvFile(solvfile,
565 os.path.basename(pkg))
567 msger.warning(warnmsg)
571 msger.warning('Can not get %s solv data.' % pkg)
573 hdr = rpmmisc.readRpmHeader(self.ts, pkg)
574 arch = zypp.Arch(hdr['arch'])
575 sysarch = zypp.Arch(self.target_arch)
577 if arch.compatible_with (sysarch):
578 pkgname = hdr['name']
579 self.localpkgs[pkgname] = pkg
580 self.selectPackage(pkgname)
581 msger.info("Marking %s to be installed" % (pkg))
584 msger.warning("Cannot add package %s to transaction. "
585 "Not a compatible architecture: %s" \
586 % (pkg, hdr['arch']))
588 def downloadPkgs(self, package_objects, count):
589 localpkgs = self.localpkgs.keys()
590 progress_obj = rpmmisc.TextProgress(count)
592 for po in package_objects:
593 if po.name() in localpkgs:
596 filename = self.getLocalPkgPath(po)
597 if os.path.exists(filename):
598 if self.checkPkg(filename) == 0:
601 dirn = os.path.dirname(filename)
602 if not os.path.exists(dirn):
605 baseurl = str(po.repoInfo().baseUrls()[0])
606 index = baseurl.find("?")
608 baseurl = baseurl[:index]
610 proxy = self.get_proxy(po.repoInfo())
613 proxies = {str(proxy.split(":")[0]):str(proxy)}
615 location = zypp.asKindPackage(po).location()
616 location = str(location.filename())
617 if location.startswith("./"):
618 location = location[2:]
620 url = baseurl + "/%s" % location
623 filename = rpmmisc.myurlgrab(url,
631 def installPkgs(self, package_objects):
633 self.__initialize_transaction()
637 for flag in self.probFilterFlags:
639 self.ts.setProbFilter(probfilter)
641 localpkgs = self.localpkgs.keys()
643 for po in package_objects:
645 if pkgname in localpkgs:
646 rpmpath = self.localpkgs[pkgname]
648 rpmpath = self.getLocalPkgPath(po)
650 if not os.path.exists(rpmpath):
651 # Maybe it is a local repo
652 baseurl = str(po.repoInfo().baseUrls()[0])
653 baseurl = baseurl.strip()
655 location = zypp.asKindPackage(po).location()
656 location = str(location.filename())
658 if baseurl.startswith("file:/"):
659 rpmpath = baseurl[5:] + "/%s" % (location)
661 if not os.path.exists(rpmpath):
662 raise RpmError("Error: %s doesn't exist" % rpmpath)
664 h = rpmmisc.readRpmHeader(self.ts, rpmpath)
665 self.ts.addInstall(h, rpmpath, 'u')
667 unresolved_dependencies = self.ts.check()
668 if not unresolved_dependencies:
670 cb = rpmmisc.RPMInstallCallback(self.ts)
671 installlogfile = "%s/__catched_stderr.buf" % (self.instroot)
673 # start to catch stderr output from librpm
674 msger.enable_logstderr(installlogfile)
676 errors = self.ts.run(cb.callback, '')
680 elif len(errors) == 0:
681 msger.warning('scriptlet or other non-fatal errors occurred '
682 'during transaction.')
687 msger.error('Could not run transaction.')
690 msger.disable_logstderr()
696 for pkg, need, needflags, sense, key in unresolved_dependencies:
697 package = '-'.join(pkg)
699 if needflags == rpm.RPMSENSE_LESS:
700 deppkg = ' < '.join(need)
701 elif needflags == rpm.RPMSENSE_EQUAL:
702 deppkg = ' = '.join(need)
703 elif needflags == rpm.RPMSENSE_GREATER:
704 deppkg = ' > '.join(need)
706 deppkg = '-'.join(need)
708 if sense == rpm.RPMDEP_SENSE_REQUIRES:
709 msger.warning("[%s] Requires [%s], which is not provided" \
712 elif sense == rpm.RPMDEP_SENSE_CONFLICTS:
713 msger.warning("[%s] Conflicts with [%s]" %(package,deppkg))
715 raise RepoError("Unresolved dependencies, transaction failed.")
717 def __initialize_transaction(self):
719 self.ts = rpm.TransactionSet(self.instroot)
720 # Set to not verify DSA signatures.
721 self.ts.setVSFlags(rpm._RPMVSF_NOSIGNATURES|rpm._RPMVSF_NODIGESTS)
723 def checkPkg(self, pkg):
725 if not os.path.exists(pkg):
727 ret = rpmmisc.checkRpmIntegrity('rpm', pkg)
729 msger.warning("package %s is damaged: %s" \
730 % (os.path.basename(pkg), pkg))
734 def _add_prob_flags(self, *flags):
736 if flag not in self.probFilterFlags:
737 self.probFilterFlags.append(flag)
739 def get_proxy(self, repoinfo):
741 reponame = "%s" % repoinfo.name()
742 for repo in self.repos:
743 if repo.name == reponame:
750 repourl = str(repoinfo.baseUrls()[0])
751 return get_proxy_for(repourl)