3 # Copyright (c) 2010, 2011 Intel, Inc.
5 # This program is free software; you can redistribute it and/or modify it
6 # under the terms of the GNU General Public License as published by the Free
7 # Software Foundation; version 2 of the License
9 # This program is distributed in the hope that it will be useful, but
10 # WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
11 # or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 # You should have received a copy of the GNU General Public License along
15 # with this program; if not, write to the Free Software Foundation, Inc., 59
16 # Temple Place - Suite 330, Boston, MA 02111-1307, USA.
24 if not hasattr(zypp, 'PoolQuery') or \
25 not hasattr(zypp.RepoManager, 'loadSolvFile'):
26 raise ImportError("python-zypp in host system cannot support PoolQuery or "
27 "loadSolvFile interface, please update it to enhanced "
28 "version which can be found in repo.meego.com/tools")
31 from mic.kickstart import ksparser
32 from mic.utils import misc, rpmmisc, runner, fs_related
33 from mic.utils.proxy import get_proxy_for
34 from mic.utils.errors import CreatorError, RepoError, RpmError
35 from mic.imager.baseimager import BaseImageCreator
41 self.mirrorlist = None
43 self.proxy_username = None
44 self.proxy_password = None
47 self.autorefresh = True
48 self.keeppackages = True
51 from mic.pluginbase import BackendPlugin
52 class Zypp(BackendPlugin):
55 def __init__(self, target_arch, instroot, cachedir):
56 self.cachedir = cachedir
57 self.instroot = instroot
58 self.target_arch = target_arch
60 self.__pkgs_license = {}
61 self.__pkgs_content = {}
65 self.repo_manager = None
66 self.repo_manager_options = None
73 self.probFilterFlags = [ rpm.RPMPROB_FILTER_OLDPACKAGE,
74 rpm.RPMPROB_FILTER_REPLACEPKG ]
76 self.has_prov_query = True
78 def doFileLogSetup(self, uid, logfile):
79 # don't do the file log for the livecd as it can lead to open fds
80 # being left and an inability to clean up after ourself
97 if not os.path.exists("/etc/fedora-release") and \
98 not os.path.exists("/etc/meego-release"):
99 for i in range(3, os.sysconf("SC_OPEN_MAX")):
108 def _cleanupRpmdbLocks(self, installroot):
109 # cleans up temporary files left by bdb so that differing
110 # versions of rpm don't cause problems
112 for f in glob.glob(installroot + "/var/lib/rpm/__db*"):
116 self._cleanupRpmdbLocks(self.instroot)
118 def whatObsolete(self, pkg):
119 query = zypp.PoolQuery()
120 query.addKind(zypp.ResKind.package)
121 query.addAttribute(zypp.SolvAttr.obsoletes, pkg)
122 query.setMatchExact()
123 for pi in query.queryResults(self.Z.pool()):
127 def _splitPkgString(self, pkg):
128 sp = pkg.rsplit(".",1)
133 sysarch = zypp.Arch(self.target_arch)
134 if not zypp.Arch(arch).compatible_with (sysarch):
139 def selectPackage(self, pkg):
140 """Select a given package or package pattern, can be specified
141 with name.arch or name* or *name
145 self.__initialize_zypp()
147 def markPoolItem(obs, pi):
149 pi.status().setToBeInstalled (zypp.ResStatus.USER)
151 obs.status().setToBeInstalled (zypp.ResStatus.USER)
153 def cmpEVR(ed1, ed2):
154 (e1, v1, r1) = map(str, [ed1.epoch(), ed1.version(), ed1.release()])
155 (e2, v2, r2) = map(str, [ed2.epoch(), ed2.version(), ed2.release()])
156 return rpm.labelCompare((e1, v1, r1), (e2, v2, r2))
159 startx = pkg.startswith("*")
160 endx = pkg.endswith("*")
161 ispattern = startx or endx
162 name, arch = self._splitPkgString(pkg)
165 q.addKind(zypp.ResKind.package)
168 if startx and not endx:
169 pattern = '%s$' % (pkg[1:])
170 if endx and not startx:
171 pattern = '^%s' % (pkg[0:-1])
173 pattern = '%s' % (pkg[1:-1])
175 q.addAttribute(zypp.SolvAttr.name,pattern)
179 q.addAttribute(zypp.SolvAttr.name,name)
183 q.addAttribute(zypp.SolvAttr.name,pkg)
186 q.queryResults(self.Z.pool()),
187 cmp=lambda x,y: cmpEVR(x.edition(), y.edition()),
190 if item.name() in self.excpkgs.keys() and \
191 self.excpkgs[item.name()] == item.repoInfo().name():
193 if item.name() in self.incpkgs.keys() and \
194 self.incpkgs[item.name()] != item.repoInfo().name():
198 obspkg = self.whatObsolete(item.name())
200 if arch == str(item.arch()):
201 item.status().setToBeInstalled (zypp.ResStatus.USER)
203 markPoolItem(obspkg, item)
207 # Can't match using package name, then search from packge
208 # provides infomation
209 if found == False and not ispattern:
210 q.addAttribute(zypp.SolvAttr.provides, pkg)
211 q.addAttribute(zypp.SolvAttr.name,'')
214 q.queryResults(self.Z.pool()),
215 cmp=lambda x,y: cmpEVR(x.edition(), y.edition()),
217 if item.name() in self.excpkgs.keys() and \
218 self.excpkgs[item.name()] == item.repoInfo().name():
220 if item.name() in self.incpkgs.keys() and \
221 self.incpkgs[item.name()] != item.repoInfo().name():
225 obspkg = self.whatObsolete(item.name())
226 markPoolItem(obspkg, item)
232 raise CreatorError("Unable to find package: %s" % (pkg,))
234 def inDeselectPackages(self, item):
235 """check if specified pacakges are in the list of inDeselectPackages
239 for pkg in self.to_deselect:
240 startx = pkg.startswith("*")
241 endx = pkg.endswith("*")
242 ispattern = startx or endx
243 pkgname, pkgarch = self._splitPkgString(pkg)
246 if name == pkgname and str(item.arch()) == pkgarch:
252 if startx and name.endswith(pkg[1:]):
254 if endx and name.startswith(pkg[:-1]):
259 def deselectPackage(self, pkg):
260 """collect packages should not be installed"""
261 self.to_deselect.append(pkg)
263 def selectGroup(self, grp, include = ksparser.GROUP_DEFAULT):
265 self.__initialize_zypp()
268 q.addKind(zypp.ResKind.pattern)
269 for item in q.queryResults(self.Z.pool()):
270 summary = "%s" % item.summary()
271 name = "%s" % item.name()
272 if name == grp or summary == grp:
274 item.status().setToBeInstalled (zypp.ResStatus.USER)
278 if include == ksparser.GROUP_REQUIRED:
280 lambda p: self.deselectPackage(p),
281 grp.default_packages.keys())
285 raise CreatorError("Unable to find pattern: %s" % (grp,))
287 def addRepository(self, name,
291 proxy_username = None,
292 proxy_password = None,
298 # TODO: Handle cost attribute for repos
300 if not self.repo_manager:
301 self.__initialize_repo_manager()
303 if not proxy and url:
304 proxy = get_proxy_for(url)
306 repo = RepositoryStub()
310 repo.proxy_username = proxy_username
311 repo.proxy_password = proxy_password
312 repo.ssl_verify = ssl_verify
313 repo.baseurl.append(url)
316 self.incpkgs[pkg] = name
319 self.excpkgs[pkg] = name
321 # check LICENSE files
322 if not rpmmisc.checkRepositoryEULA(name, repo):
323 msger.warning('skip repo:%s for failed EULA confirmation' % name)
327 repo.mirrorlist = mirrorlist
329 # Enable gpg check for verifying corrupt packages
332 repo.priority = priority
335 repo_info = zypp.RepoInfo()
336 repo_info.setAlias(repo.name)
337 repo_info.setName(repo.name)
338 repo_info.setEnabled(repo.enabled)
339 repo_info.setAutorefresh(repo.autorefresh)
340 repo_info.setKeepPackages(repo.keeppackages)
341 baseurl = zypp.Url(repo.baseurl[0])
343 baseurl.setQueryParam("ssl_verify", "no")
345 scheme, host, path, parm, query, frag = urlparse.urlparse(proxy)
347 proxyinfo = host.split(":")
351 if len(proxyinfo) > 1:
354 if proxy.startswith("socks") and len(proxy.rsplit(':', 1)) == 2:
355 host = proxy.rsplit(':', 1)[0]
356 port = proxy.rsplit(':', 1)[1]
358 baseurl.setQueryParam ("proxy", host)
359 baseurl.setQueryParam ("proxyport", port)
361 repo.baseurl[0] = baseurl.asCompleteString()
362 self.repos.append(repo)
364 repo_info.addBaseUrl(baseurl)
367 repo_info.setPriority(repo.priority)
369 self.repo_manager.addRepository(repo_info)
371 self.__build_repo_cache(name)
373 except RuntimeError, e:
374 raise CreatorError(str(e))
376 msger.verbose('repo: %s was added' % name)
379 def installHasFile(self, file):
382 def preInstall(self, pkg):
383 self.pre_pkgs.append(pkg)
385 def runInstall(self, checksize = 0):
386 os.environ["HOME"] = "/"
387 self.buildTransaction()
389 todo = zypp.GetResolvablesToInsDel(self.Z.pool())
390 installed_pkgs = todo._toInstall
392 for item in installed_pkgs:
393 if not zypp.isKindPattern(item) and \
394 not self.inDeselectPackages(item):
397 # record all pkg and the content
398 localpkgs = self.localpkgs.keys()
401 if pkg.name() in localpkgs:
402 hdr = rpmmisc.readRpmHeader(self.ts, self.localpkgs[pkg.name()])
403 pkg_long_name = misc.RPM_FMT % {
406 'ver_rel': '%s-%s' % (hdr['version'],
409 license = hdr['license']
412 pkg_long_name = misc.RPM_FMT % {
415 'ver_rel': pkg.edition(),
418 package = zypp.asKindPackage(pkg)
419 license = package.license()
421 self.__pkgs_content[pkg_long_name] = {} #TBD: to get file list
423 if license in self.__pkgs_license.keys():
424 self.__pkgs_license[license].append(pkg_long_name)
426 self.__pkgs_license[license] = [pkg_long_name]
428 total_count = len(dlpkgs)
430 download_total_size = sum(map(lambda x: int(x.downloadSize()), dlpkgs))
431 localpkgs = self.localpkgs.keys()
433 msger.info("Checking packages cache and packages integrity ...")
435 # Check if it is cached locally
436 if po.name() in localpkgs:
439 local = self.getLocalPkgPath(po)
440 if os.path.exists(local):
441 if self.checkPkg(local) != 0:
444 download_total_size -= int(po.downloadSize())
446 cache_avail_size = misc.get_filesystem_avail(self.cachedir)
447 if cache_avail_size < download_total_size:
448 raise CreatorError("No enough space used for downloading.")
450 # record the total size of installed pkgs
451 install_total_size = sum(map(lambda x: int(x.installSize()), dlpkgs))
452 # check needed size before actually download and install
454 # FIXME: for multiple partitions for loop type, check fails
455 # skip the check temporarily
456 #if checksize and install_total_size > checksize:
457 # raise CreatorError("No enough space used for installing, "
458 # "please resize partition size in ks file")
460 download_count = total_count - cached_count
461 msger.info("%d packages to be installed, "
462 "%d packages gotten from cache, "
463 "%d packages to be downloaded" \
464 % (total_count, cached_count, download_count))
467 if download_count > 0:
468 msger.info("Downloading packages ...")
469 self.downloadPkgs(dlpkgs, download_count)
471 self.installPkgs(dlpkgs)
473 except (RepoError, RpmError):
476 raise CreatorError("Package installation failed: %s" % (e,))
478 def getAllContent(self):
479 return self.__pkgs_content
481 def getPkgsLicense(self):
482 return self.__pkgs_license
484 def getFilelist(self, pkgname):
489 self.__initialize_transaction()
491 mi = self.ts.dbMatch('name', pkgname)
493 return header['FILENAMES']
495 def __initialize_repo_manager(self):
496 if self.repo_manager:
499 # Clean up repo metadata
500 shutil.rmtree(self.cachedir + "/etc", ignore_errors = True)
501 shutil.rmtree(self.cachedir + "/solv", ignore_errors = True)
502 shutil.rmtree(self.cachedir + "/raw", ignore_errors = True)
504 zypp.KeyRing.setDefaultAccept( zypp.KeyRing.ACCEPT_UNSIGNED_FILE
505 | zypp.KeyRing.ACCEPT_VERIFICATION_FAILED
506 | zypp.KeyRing.ACCEPT_UNKNOWNKEY
507 | zypp.KeyRing.TRUST_KEY_TEMPORARILY
510 self.repo_manager_options = \
511 zypp.RepoManagerOptions(zypp.Pathname(self.instroot))
513 self.repo_manager_options.knownReposPath = \
514 zypp.Pathname(self.cachedir + "/etc/zypp/repos.d")
516 self.repo_manager_options.repoCachePath = \
517 zypp.Pathname(self.cachedir)
519 self.repo_manager_options.repoRawCachePath = \
520 zypp.Pathname(self.cachedir + "/raw")
522 self.repo_manager_options.repoSolvCachePath = \
523 zypp.Pathname(self.cachedir + "/solv")
525 self.repo_manager_options.repoPackagesCachePath = \
526 zypp.Pathname(self.cachedir + "/packages")
528 self.repo_manager = zypp.RepoManager(self.repo_manager_options)
530 def __build_repo_cache(self, name):
531 repo = self.repo_manager.getRepositoryInfo(name)
532 if self.repo_manager.isCached(repo) or not repo.enabled():
535 msger.info('Refreshing repository: %s ...' % name)
536 self.repo_manager.buildCache(repo, zypp.RepoManager.BuildIfNeeded)
538 def __initialize_zypp(self):
542 zconfig = zypp.ZConfig_instance()
544 # Set system architecture
546 zconfig.setSystemArchitecture(zypp.Arch(self.target_arch))
548 msger.info("zypp architecture is <%s>" % zconfig.systemArchitecture())
550 # repoPackagesCachePath is corrected by this
551 self.repo_manager = zypp.RepoManager(self.repo_manager_options)
552 repos = self.repo_manager.knownRepositories()
554 if not repo.enabled():
556 self.repo_manager.loadFromCache(repo)
558 self.Z = zypp.ZYppFactory_instance().getZYpp()
559 self.Z.initializeTarget(zypp.Pathname(self.instroot))
560 self.Z.target().load()
562 def buildTransaction(self):
563 if not self.Z.resolver().resolvePool():
564 probs = self.Z.resolver().problems()
566 for problem in probs:
567 msger.warning("repo problem: %s, %s" \
568 % (problem.description().decode("utf-8"),
569 problem.details().decode("utf-8")))
571 raise RepoError("found %d resolver problem, abort!" \
574 def getLocalPkgPath(self, po):
575 repoinfo = po.repoInfo()
576 cacheroot = repoinfo.packagesPath()
577 location= zypp.asKindPackage(po).location()
578 rpmpath = str(location.filename())
579 pkgpath = "%s/%s" % (cacheroot, os.path.basename(rpmpath))
582 def installLocal(self, pkg, po=None, updateonly=False):
584 self.__initialize_transaction()
586 solvfile = "%s/.solv" % (self.cachedir)
588 rc, out = runner.runtool([fs_related.find_binary_path("rpms2solv"),
591 f = open(solvfile, "w+")
595 warnmsg = self.repo_manager.loadSolvFile(solvfile,
596 os.path.basename(pkg))
598 msger.warning(warnmsg)
602 msger.warning('Can not get %s solv data.' % pkg)
604 hdr = rpmmisc.readRpmHeader(self.ts, pkg)
605 arch = zypp.Arch(hdr['arch'])
606 sysarch = zypp.Arch(self.target_arch)
608 if arch.compatible_with (sysarch):
609 pkgname = hdr['name']
610 self.localpkgs[pkgname] = pkg
611 self.selectPackage(pkgname)
612 msger.info("Marking %s to be installed" % (pkg))
615 msger.warning("Cannot add package %s to transaction. "
616 "Not a compatible architecture: %s" \
617 % (pkg, hdr['arch']))
619 def downloadPkgs(self, package_objects, count):
620 localpkgs = self.localpkgs.keys()
621 progress_obj = rpmmisc.TextProgress(count)
623 for po in package_objects:
624 if po.name() in localpkgs:
627 filename = self.getLocalPkgPath(po)
628 if os.path.exists(filename):
629 if self.checkPkg(filename) == 0:
632 dirn = os.path.dirname(filename)
633 if not os.path.exists(dirn):
636 name = str(po.repoInfo().name())
637 repo = filter(lambda r: r.name == name, self.repos)[0]
638 baseurl = repo.baseurl[0]
639 index = baseurl.find("?")
641 baseurl = baseurl[:index]
643 proxy = self.get_proxy(po.repoInfo())
646 proxies = {str(proxy.split(":")[0]):str(proxy)}
648 location = zypp.asKindPackage(po).location()
649 location = str(location.filename())
650 if location.startswith("./"):
651 location = location[2:]
653 url = baseurl + "/%s" % location
656 filename = rpmmisc.myurlgrab(url,
664 def preinstallPkgs(self):
666 self.__initialize_transaction()
669 cb = rpmmisc.RPMInstallCallback(self.ts_pre)
670 cb.headmsg = "Preinstall"
671 installlogfile = "%s/__catched_stderr.buf" % (self.instroot)
673 # start to catch stderr output from librpm
674 msger.enable_logstderr(installlogfile)
676 errors = self.ts_pre.run(cb.callback, '')
678 msger.disable_logstderr()
679 self.ts_pre.closeDB()
682 if errors is not None:
684 msger.warning('scriptlet or other non-fatal errors occurred '
685 'during transaction.')
690 raise RepoError('Could not run transaction.')
692 def installPkgs(self, package_objects):
694 self.__initialize_transaction()
698 for flag in self.probFilterFlags:
700 self.ts.setProbFilter(probfilter)
701 self.ts_pre.setProbFilter(probfilter)
703 localpkgs = self.localpkgs.keys()
705 for po in package_objects:
707 if pkgname in localpkgs:
708 rpmpath = self.localpkgs[pkgname]
710 rpmpath = self.getLocalPkgPath(po)
712 if not os.path.exists(rpmpath):
713 # Maybe it is a local repo
714 baseurl = str(po.repoInfo().baseUrls()[0])
715 baseurl = baseurl.strip()
717 location = zypp.asKindPackage(po).location()
718 location = str(location.filename())
720 if baseurl.startswith("file:/"):
721 rpmpath = baseurl[5:] + "/%s" % (location)
723 if not os.path.exists(rpmpath):
724 raise RpmError("Error: %s doesn't exist" % rpmpath)
726 h = rpmmisc.readRpmHeader(self.ts, rpmpath)
728 if pkgname in self.pre_pkgs:
729 msger.verbose("pre-install package added: %s" % pkgname)
730 self.ts_pre.addInstall(h, rpmpath, 'u')
732 self.ts.addInstall(h, rpmpath, 'u')
734 unresolved_dependencies = self.ts.check()
735 if not unresolved_dependencies:
737 self.preinstallPkgs()
740 cb = rpmmisc.RPMInstallCallback(self.ts)
741 installlogfile = "%s/__catched_stderr.buf" % (self.instroot)
743 # start to catch stderr output from librpm
744 msger.enable_logstderr(installlogfile)
746 errors = self.ts.run(cb.callback, '')
748 msger.disable_logstderr()
752 if errors is not None:
754 msger.warning('scriptlet or other non-fatal errors occurred '
755 'during transaction.')
760 raise RepoError('Could not run transaction.')
763 for pkg, need, needflags, sense, key in unresolved_dependencies:
764 package = '-'.join(pkg)
766 if needflags == rpm.RPMSENSE_LESS:
767 deppkg = ' < '.join(need)
768 elif needflags == rpm.RPMSENSE_EQUAL:
769 deppkg = ' = '.join(need)
770 elif needflags == rpm.RPMSENSE_GREATER:
771 deppkg = ' > '.join(need)
773 deppkg = '-'.join(need)
775 if sense == rpm.RPMDEP_SENSE_REQUIRES:
776 msger.warning("[%s] Requires [%s], which is not provided" \
779 elif sense == rpm.RPMDEP_SENSE_CONFLICTS:
780 msger.warning("[%s] Conflicts with [%s]" %(package,deppkg))
782 raise RepoError("Unresolved dependencies, transaction failed.")
784 def __initialize_transaction(self):
786 self.ts = rpm.TransactionSet(self.instroot)
787 # Set to not verify DSA signatures.
788 self.ts.setVSFlags(rpm._RPMVSF_NOSIGNATURES|rpm._RPMVSF_NODIGESTS)
791 self.ts_pre = rpm.TransactionSet(self.instroot)
792 # Just unpack the files, don't run scripts
793 self.ts_pre.setFlags(rpm.RPMTRANS_FLAG_ALLFILES | rpm.RPMTRANS_FLAG_NOSCRIPTS)
794 # Set to not verify DSA signatures.
795 self.ts_pre.setVSFlags(rpm._RPMVSF_NOSIGNATURES|rpm._RPMVSF_NODIGESTS)
797 def checkPkg(self, pkg):
799 if not os.path.exists(pkg):
801 ret = rpmmisc.checkRpmIntegrity('rpm', pkg)
803 msger.warning("package %s is damaged: %s" \
804 % (os.path.basename(pkg), pkg))
808 def _add_prob_flags(self, *flags):
810 if flag not in self.probFilterFlags:
811 self.probFilterFlags.append(flag)
813 def get_proxy(self, repoinfo):
815 reponame = "%s" % repoinfo.name()
816 for repo in self.repos:
817 if repo.name == reponame:
824 repourl = str(repoinfo.baseUrls()[0])
825 return get_proxy_for(repourl)
827 def package_url(self, pkg):
829 def cmpEVR(ed1, ed2):
830 (e1, v1, r1) = map(str, [ed1.epoch(), ed1.version(), ed1.release()])
831 (e2, v2, r2) = map(str, [ed2.epoch(), ed2.version(), ed2.release()])
832 return rpm.labelCompare((e1, v1, r1), (e2, v2, r2))
835 self.__initialize_zypp()
838 q.addKind(zypp.ResKind.package)
840 q.addAttribute(zypp.SolvAttr.name,pkg)
841 items = sorted(q.queryResults(self.Z.pool()),
842 cmp=lambda x,y: cmpEVR(x.edition(), y.edition()),
846 baseurl = items[0].repoInfo().baseUrls()[0]
847 location = zypp.asKindPackage(items[0]).location()
848 return os.path.join(str(baseurl), str(location.filename()))