3 # Copyright (c) 2010, 2011 Intel, Inc.
5 # This program is free software; you can redistribute it and/or modify it
6 # under the terms of the GNU General Public License as published by the Free
7 # Software Foundation; version 2 of the License
9 # This program is distributed in the hope that it will be useful, but
10 # WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
11 # or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 # You should have received a copy of the GNU General Public License along
15 # with this program; if not, write to the Free Software Foundation, Inc., 59
16 # Temple Place - Suite 330, Boston, MA 02111-1307, USA.
24 if not hasattr(zypp, 'PoolQuery') or \
25 not hasattr(zypp.RepoManager, 'loadSolvFile'):
26 raise ImportError("python-zypp in host system cannot support PoolQuery or "
27 "loadSolvFile interface, please update it to enhanced "
28 "version which can be found in download.tizen.org/tools")
31 from mic.kickstart import ksparser
32 from mic.utils import misc, rpmmisc, runner, fs_related
33 from mic.utils.grabber import myurlgrab, TextProgress
34 from mic.utils.proxy import get_proxy_for
35 from mic.utils.errors import CreatorError, RepoError, RpmError
36 from mic.imager.baseimager import BaseImageCreator
42 self.mirrorlist = None
44 self.proxy_username = None
45 self.proxy_password = None
49 self.autorefresh = True
50 self.keeppackages = True
53 from mic.pluginbase import BackendPlugin
54 class Zypp(BackendPlugin):
57 def __init__(self, target_arch, instroot, cachedir):
58 self.cachedir = cachedir
59 self.instroot = instroot
60 self.target_arch = target_arch
62 self.__pkgs_license = {}
63 self.__pkgs_content = {}
64 self.__pkgs_vcsinfo = {}
68 self.repo_manager = None
69 self.repo_manager_options = None
76 self.probFilterFlags = [ rpm.RPMPROB_FILTER_OLDPACKAGE,
77 rpm.RPMPROB_FILTER_REPLACEPKG ]
79 self.has_prov_query = True
80 self.install_debuginfo = False
82 def doFileLogSetup(self, uid, logfile):
83 # don't do the file log for the livecd as it can lead to open fds
84 # being left and an inability to clean up after ourself
101 if not os.path.exists("/etc/fedora-release") and \
102 not os.path.exists("/etc/meego-release"):
103 for i in range(3, os.sysconf("SC_OPEN_MAX")):
112 def _cleanupRpmdbLocks(self, installroot):
113 # cleans up temporary files left by bdb so that differing
114 # versions of rpm don't cause problems
116 for f in glob.glob(installroot + "/var/lib/rpm/__db*"):
119 def _cleanupZyppJunk(self, installroot):
121 shutil.rmtree(os.path.join(installroot, '.zypp'))
126 self._cleanupRpmdbLocks(self.instroot)
128 def whatObsolete(self, pkg):
129 query = zypp.PoolQuery()
130 query.addKind(zypp.ResKind.package)
131 query.addAttribute(zypp.SolvAttr.obsoletes, pkg)
132 query.setMatchExact()
133 for pi in query.queryResults(self.Z.pool()):
137 def _zyppQueryPackage(self, pkg):
138 query = zypp.PoolQuery()
139 query.addKind(zypp.ResKind.package)
140 query.addAttribute(zypp.SolvAttr.name,pkg)
141 query.setMatchExact()
142 for pi in query.queryResults(self.Z.pool()):
146 def _splitPkgString(self, pkg):
147 sp = pkg.rsplit(".",1)
152 sysarch = zypp.Arch(self.target_arch)
153 if not zypp.Arch(arch).compatible_with (sysarch):
158 def selectPackage(self, pkg):
159 """Select a given package or package pattern, can be specified
160 with name.arch or name* or *name
164 self.__initialize_zypp()
166 def markPoolItem(obs, pi):
168 pi.status().setToBeInstalled (zypp.ResStatus.USER)
170 obs.status().setToBeInstalled (zypp.ResStatus.USER)
175 if str(a1) != str(a2):
176 if a1.compatible_with(a2):
182 (e1, v1, r1) = map(str, [ed1.epoch(), ed1.version(), ed1.release()])
183 (e2, v2, r2) = map(str, [ed2.epoch(), ed2.version(), ed2.release()])
184 return rpm.labelCompare((e1, v1, r1), (e2, v2, r2))
187 startx = pkg.startswith("*")
188 endx = pkg.endswith("*")
189 ispattern = startx or endx
190 name, arch = self._splitPkgString(pkg)
193 q.addKind(zypp.ResKind.package)
196 if startx and not endx:
197 pattern = '%s$' % (pkg[1:])
198 if endx and not startx:
199 pattern = '^%s' % (pkg[0:-1])
201 pattern = '%s' % (pkg[1:-1])
203 q.addAttribute(zypp.SolvAttr.name,pattern)
207 q.addAttribute(zypp.SolvAttr.name,name)
211 q.addAttribute(zypp.SolvAttr.name,pkg)
214 q.queryResults(self.Z.pool()),
215 cmp=lambda x,y: cmpEVR(x, y),
218 if item.name() in self.excpkgs.keys() and \
219 self.excpkgs[item.name()] == item.repoInfo().name():
221 if item.name() in self.incpkgs.keys() and \
222 self.incpkgs[item.name()] != item.repoInfo().name():
226 obspkg = self.whatObsolete(item.name())
228 if arch == str(item.arch()):
229 item.status().setToBeInstalled (zypp.ResStatus.USER)
231 markPoolItem(obspkg, item)
235 # Can't match using package name, then search from packge
236 # provides infomation
237 if found == False and not ispattern:
238 q.addAttribute(zypp.SolvAttr.provides, pkg)
239 q.addAttribute(zypp.SolvAttr.name,'')
242 q.queryResults(self.Z.pool()),
243 cmp=lambda x,y: cmpEVR(x.edition(), y.edition()),
245 if item.name() in self.excpkgs.keys() and \
246 self.excpkgs[item.name()] == item.repoInfo().name():
248 if item.name() in self.incpkgs.keys() and \
249 self.incpkgs[item.name()] != item.repoInfo().name():
253 obspkg = self.whatObsolete(item.name())
254 markPoolItem(obspkg, item)
260 raise CreatorError("Unable to find package: %s" % (pkg,))
262 def inDeselectPackages(self, item):
263 """check if specified pacakges are in the list of inDeselectPackages
267 for pkg in self.to_deselect:
268 startx = pkg.startswith("*")
269 endx = pkg.endswith("*")
270 ispattern = startx or endx
271 pkgname, pkgarch = self._splitPkgString(pkg)
274 if name == pkgname and str(item.arch()) == pkgarch:
280 if startx and name.endswith(pkg[1:]):
282 if endx and name.startswith(pkg[:-1]):
287 def deselectPackage(self, pkg):
288 """collect packages should not be installed"""
289 self.to_deselect.append(pkg)
291 def selectGroup(self, grp, include = ksparser.GROUP_DEFAULT):
293 self.__initialize_zypp()
296 q.addKind(zypp.ResKind.pattern)
297 for item in q.queryResults(self.Z.pool()):
298 summary = "%s" % item.summary()
299 name = "%s" % item.name()
300 if name == grp or summary == grp:
302 item.status().setToBeInstalled (zypp.ResStatus.USER)
306 if include == ksparser.GROUP_REQUIRED:
308 lambda p: self.deselectPackage(p),
309 grp.default_packages.keys())
313 raise CreatorError("Unable to find pattern: %s" % (grp,))
315 def addRepository(self, name,
319 proxy_username = None,
320 proxy_password = None,
327 # TODO: Handle cost attribute for repos
329 if not self.repo_manager:
330 self.__initialize_repo_manager()
332 if not proxy and url:
333 proxy = get_proxy_for(url)
335 repo = RepositoryStub()
339 repo.proxy_username = proxy_username
340 repo.proxy_password = proxy_password
341 repo.ssl_verify = ssl_verify
342 repo.nocache = nocache
343 repo.baseurl.append(url)
346 self.incpkgs[pkg] = name
349 self.excpkgs[pkg] = name
351 # check LICENSE files
352 if not rpmmisc.checkRepositoryEULA(name, repo):
353 msger.warning('skip repo:%s for failed EULA confirmation' % name)
357 repo.mirrorlist = mirrorlist
359 # Enable gpg check for verifying corrupt packages
362 repo.priority = priority
365 repo_info = zypp.RepoInfo()
366 repo_info.setAlias(repo.name)
367 repo_info.setName(repo.name)
368 repo_info.setEnabled(repo.enabled)
369 repo_info.setAutorefresh(repo.autorefresh)
370 repo_info.setKeepPackages(repo.keeppackages)
371 baseurl = zypp.Url(repo.baseurl[0])
373 baseurl.setQueryParam("ssl_verify", "no")
375 scheme, host, path, parm, query, frag = urlparse.urlparse(proxy)
377 proxyinfo = host.split(":")
381 if len(proxyinfo) > 1:
384 if proxy.startswith("socks") and len(proxy.rsplit(':', 1)) == 2:
385 host = proxy.rsplit(':', 1)[0]
386 port = proxy.rsplit(':', 1)[1]
388 baseurl.setQueryParam ("proxy", host)
389 baseurl.setQueryParam ("proxyport", port)
391 repo.baseurl[0] = baseurl.asCompleteString()
392 self.repos.append(repo)
394 repo_info.addBaseUrl(baseurl)
397 repo_info.setPriority(repo.priority)
399 # this hack is used to change zypp credential file location
400 # the default one is $HOME/.zypp, which cause conflicts when
401 # installing some basic packages, and the location doesn't
402 # have any interface actually, so use a tricky way anyway
404 if 'HOME' in os.environ:
405 homedir = os.environ['HOME']
406 os.environ['HOME'] = '/'
408 os.environ['HOME'] = '/'
410 self.repo_manager.addRepository(repo_info)
412 # save back the $HOME env
414 os.environ['HOME'] = homedir
416 del os.environ['HOME']
418 self.__build_repo_cache(name)
420 except RuntimeError, e:
421 raise CreatorError(str(e))
423 msger.verbose('repo: %s was added' % name)
426 def installHasFile(self, file):
429 def preInstall(self, pkg):
430 self.pre_pkgs.append(pkg)
432 def runInstall(self, checksize = 0):
433 os.environ["HOME"] = "/"
434 os.environ["LD_PRELOAD"] = ""
435 self.buildTransaction()
437 todo = zypp.GetResolvablesToInsDel(self.Z.pool())
438 installed_pkgs = todo._toInstall
440 for item in installed_pkgs:
441 if not zypp.isKindPattern(item) and \
442 not self.inDeselectPackages(item):
445 if not self.install_debuginfo or str(item.arch()) == "noarch":
448 dipkg = self._zyppQueryPackage("%s-debuginfo" % item.name())
452 msger.warning("No debuginfo rpm found for: %s" \
455 # record all pkg and the content
456 localpkgs = self.localpkgs.keys()
459 if pkg.name() in localpkgs:
460 hdr = rpmmisc.readRpmHeader(self.ts, self.localpkgs[pkg.name()])
461 pkg_long_name = misc.RPM_FMT % {
464 'version': hdr['version'],
465 'release': hdr['release']
467 license = hdr['license']
470 pkg_long_name = misc.RPM_FMT % {
473 'version': pkg.edition().version(),
474 'release': pkg.edition().release()
477 package = zypp.asKindPackage(pkg)
478 license = package.license()
480 if license in self.__pkgs_license.keys():
481 self.__pkgs_license[license].append(pkg_long_name)
483 self.__pkgs_license[license] = [pkg_long_name]
485 total_count = len(dlpkgs)
487 download_total_size = sum(map(lambda x: int(x.downloadSize()), dlpkgs))
488 localpkgs = self.localpkgs.keys()
490 msger.info("Checking packages cached ...")
492 # Check if it is cached locally
493 if po.name() in localpkgs:
496 local = self.getLocalPkgPath(po)
497 name = str(po.repoInfo().name())
499 repo = filter(lambda r: r.name == name, self.repos)[0]
502 nocache = repo.nocache if repo else False
504 if os.path.exists(local):
505 if nocache or self.checkPkg(local) !=0:
508 download_total_size -= int(po.downloadSize())
510 cache_avail_size = misc.get_filesystem_avail(self.cachedir)
511 if cache_avail_size < download_total_size:
512 raise CreatorError("No enough space used for downloading.")
514 # record the total size of installed pkgs
515 install_total_size = sum(map(lambda x: int(x.installSize()), dlpkgs))
516 # check needed size before actually download and install
518 # FIXME: for multiple partitions for loop type, check fails
519 # skip the check temporarily
520 #if checksize and install_total_size > checksize:
521 # raise CreatorError("No enough space used for installing, "
522 # "please resize partition size in ks file")
524 download_count = total_count - cached_count
525 msger.info("Packages: %d Total, %d Cached, %d Missed" \
526 % (total_count, cached_count, download_count))
529 if download_count > 0:
530 msger.info("Downloading packages ...")
531 self.downloadPkgs(dlpkgs, download_count)
533 self.installPkgs(dlpkgs)
535 except (RepoError, RpmError):
538 raise CreatorError("Package installation failed: %s" % (e,))
540 def getVcsInfo(self):
541 if self.__pkgs_vcsinfo:
545 self.__initialize_transaction()
547 mi = self.ts.dbMatch()
549 lname = misc.RPM_FMT % {
552 'version': hdr['version'],
553 'release': hdr['release']
555 self.__pkgs_vcsinfo[lname] = hdr['VCS']
557 return self.__pkgs_vcsinfo
559 def getAllContent(self):
560 if self.__pkgs_content:
561 return self.__pkgs_content
564 self.__initialize_transaction()
566 mi = self.ts.dbMatch()
568 lname = misc.RPM_FMT % {
571 'version': hdr['version'],
572 'release': hdr['release']
574 self.__pkgs_content[lname] = hdr['FILENAMES']
576 return self.__pkgs_content
578 def getPkgsLicense(self):
579 return self.__pkgs_license
581 def getFilelist(self, pkgname):
586 self.__initialize_transaction()
588 mi = self.ts.dbMatch('name', pkgname)
590 return header['FILENAMES']
592 def __initialize_repo_manager(self):
593 if self.repo_manager:
596 # Clean up repo metadata
597 shutil.rmtree(self.cachedir + "/etc", ignore_errors = True)
598 shutil.rmtree(self.cachedir + "/solv", ignore_errors = True)
599 shutil.rmtree(self.cachedir + "/raw", ignore_errors = True)
601 zypp.KeyRing.setDefaultAccept( zypp.KeyRing.ACCEPT_UNSIGNED_FILE
602 | zypp.KeyRing.ACCEPT_VERIFICATION_FAILED
603 | zypp.KeyRing.ACCEPT_UNKNOWNKEY
604 | zypp.KeyRing.TRUST_KEY_TEMPORARILY
607 self.repo_manager_options = \
608 zypp.RepoManagerOptions(zypp.Pathname(self.instroot))
610 self.repo_manager_options.knownReposPath = \
611 zypp.Pathname(self.cachedir + "/etc/zypp/repos.d")
613 self.repo_manager_options.repoCachePath = \
614 zypp.Pathname(self.cachedir)
616 self.repo_manager_options.repoRawCachePath = \
617 zypp.Pathname(self.cachedir + "/raw")
619 self.repo_manager_options.repoSolvCachePath = \
620 zypp.Pathname(self.cachedir + "/solv")
622 self.repo_manager_options.repoPackagesCachePath = \
623 zypp.Pathname(self.cachedir + "/packages")
625 self.repo_manager = zypp.RepoManager(self.repo_manager_options)
627 def __build_repo_cache(self, name):
628 repo = self.repo_manager.getRepositoryInfo(name)
629 if self.repo_manager.isCached(repo) or not repo.enabled():
632 msger.info('Refreshing repository: %s ...' % name)
633 self.repo_manager.buildCache(repo, zypp.RepoManager.BuildIfNeeded)
635 def __initialize_zypp(self):
639 zconfig = zypp.ZConfig_instance()
641 # Set system architecture
643 zconfig.setSystemArchitecture(zypp.Arch(self.target_arch))
645 msger.info("zypp architecture is <%s>" % zconfig.systemArchitecture())
647 # repoPackagesCachePath is corrected by this
648 self.repo_manager = zypp.RepoManager(self.repo_manager_options)
649 repos = self.repo_manager.knownRepositories()
651 if not repo.enabled():
653 self.repo_manager.loadFromCache(repo)
655 self.Z = zypp.ZYppFactory_instance().getZYpp()
656 self.Z.initializeTarget(zypp.Pathname(self.instroot))
657 self.Z.target().load()
659 def buildTransaction(self):
660 if not self.Z.resolver().resolvePool():
661 probs = self.Z.resolver().problems()
663 for problem in probs:
664 msger.warning("repo problem: %s, %s" \
665 % (problem.description().decode("utf-8"),
666 problem.details().decode("utf-8")))
668 raise RepoError("found %d resolver problem, abort!" \
671 def getLocalPkgPath(self, po):
672 repoinfo = po.repoInfo()
673 cacheroot = repoinfo.packagesPath()
674 location= zypp.asKindPackage(po).location()
675 rpmpath = str(location.filename())
676 pkgpath = "%s/%s" % (cacheroot, os.path.basename(rpmpath))
679 def installLocal(self, pkg, po=None, updateonly=False):
681 self.__initialize_transaction()
683 solvfile = "%s/.solv" % (self.cachedir)
685 rc, out = runner.runtool([fs_related.find_binary_path("rpms2solv"),
688 f = open(solvfile, "w+")
692 warnmsg = self.repo_manager.loadSolvFile(solvfile,
693 os.path.basename(pkg))
695 msger.warning(warnmsg)
699 msger.warning('Can not get %s solv data.' % pkg)
701 hdr = rpmmisc.readRpmHeader(self.ts, pkg)
702 arch = zypp.Arch(hdr['arch'])
703 sysarch = zypp.Arch(self.target_arch)
705 if arch.compatible_with (sysarch):
706 pkgname = hdr['name']
707 self.localpkgs[pkgname] = pkg
708 self.selectPackage(pkgname)
709 msger.info("Marking %s to be installed" % (pkg))
712 msger.warning("Cannot add package %s to transaction. "
713 "Not a compatible architecture: %s" \
714 % (pkg, hdr['arch']))
716 def downloadPkgs(self, package_objects, count):
717 localpkgs = self.localpkgs.keys()
718 progress_obj = TextProgress(count)
720 for po in package_objects:
721 if po.name() in localpkgs:
724 filename = self.getLocalPkgPath(po)
725 if os.path.exists(filename):
726 if self.checkPkg(filename) == 0:
729 dirn = os.path.dirname(filename)
730 if not os.path.exists(dirn):
733 url = self.get_url(po)
734 proxies = self.get_proxies(po)
737 filename = myurlgrab(url, filename, proxies, progress_obj)
742 def preinstallPkgs(self):
744 self.__initialize_transaction()
747 cb = rpmmisc.RPMInstallCallback(self.ts_pre)
748 cb.headmsg = "Preinstall"
749 installlogfile = "%s/__catched_stderr.buf" % (self.instroot)
751 # start to catch stderr output from librpm
752 msger.enable_logstderr(installlogfile)
754 errors = self.ts_pre.run(cb.callback, '')
756 msger.disable_logstderr()
757 self.ts_pre.closeDB()
760 if errors is not None:
762 msger.warning('scriptlet or other non-fatal errors occurred '
763 'during transaction.')
768 raise RepoError('Could not run transaction.')
770 def installPkgs(self, package_objects):
772 self.__initialize_transaction()
775 self._cleanupRpmdbLocks(self.instroot)
776 self._cleanupZyppJunk(self.instroot)
779 for flag in self.probFilterFlags:
781 self.ts.setProbFilter(probfilter)
782 self.ts_pre.setProbFilter(probfilter)
784 localpkgs = self.localpkgs.keys()
786 for po in package_objects:
788 if pkgname in localpkgs:
789 rpmpath = self.localpkgs[pkgname]
791 rpmpath = self.getLocalPkgPath(po)
793 if not os.path.exists(rpmpath):
794 # Maybe it is a local repo
795 rpmuri = self.get_url(po)
796 if rpmuri.startswith("file:/"):
799 if not os.path.exists(rpmpath):
800 raise RpmError("Error: %s doesn't exist" % rpmpath)
802 h = rpmmisc.readRpmHeader(self.ts, rpmpath)
804 if pkgname in self.pre_pkgs:
805 msger.verbose("pre-install package added: %s" % pkgname)
806 self.ts_pre.addInstall(h, rpmpath, 'u')
808 self.ts.addInstall(h, rpmpath, 'u')
810 unresolved_dependencies = self.ts.check()
811 if not unresolved_dependencies:
813 self.preinstallPkgs()
816 cb = rpmmisc.RPMInstallCallback(self.ts)
817 installlogfile = "%s/__catched_stderr.buf" % (self.instroot)
819 # start to catch stderr output from librpm
820 msger.enable_logstderr(installlogfile)
822 errors = self.ts.run(cb.callback, '')
824 msger.disable_logstderr()
828 if errors is not None:
830 msger.warning('scriptlet or other non-fatal errors occurred '
831 'during transaction.')
836 raise RepoError('Could not run transaction.')
839 for pkg, need, needflags, sense, key in unresolved_dependencies:
840 package = '-'.join(pkg)
842 if needflags == rpm.RPMSENSE_LESS:
843 deppkg = ' < '.join(need)
844 elif needflags == rpm.RPMSENSE_EQUAL:
845 deppkg = ' = '.join(need)
846 elif needflags == rpm.RPMSENSE_GREATER:
847 deppkg = ' > '.join(need)
849 deppkg = '-'.join(need)
851 if sense == rpm.RPMDEP_SENSE_REQUIRES:
852 msger.warning("[%s] Requires [%s], which is not provided" \
855 elif sense == rpm.RPMDEP_SENSE_CONFLICTS:
856 msger.warning("[%s] Conflicts with [%s]" %(package,deppkg))
858 raise RepoError("Unresolved dependencies, transaction failed.")
860 def __initialize_transaction(self):
862 self.ts = rpm.TransactionSet(self.instroot)
863 # Set to not verify DSA signatures.
864 self.ts.setVSFlags(rpm._RPMVSF_NOSIGNATURES|rpm._RPMVSF_NODIGESTS)
867 self.ts_pre = rpm.TransactionSet(self.instroot)
868 # Just unpack the files, don't run scripts
869 self.ts_pre.setFlags(rpm.RPMTRANS_FLAG_ALLFILES | rpm.RPMTRANS_FLAG_NOSCRIPTS)
870 # Set to not verify DSA signatures.
871 self.ts_pre.setVSFlags(rpm._RPMVSF_NOSIGNATURES|rpm._RPMVSF_NODIGESTS)
873 def checkPkg(self, pkg):
875 if not os.path.exists(pkg):
877 ret = rpmmisc.checkRpmIntegrity('rpm', pkg)
879 msger.warning("package %s is damaged: %s" \
880 % (os.path.basename(pkg), pkg))
884 def _add_prob_flags(self, *flags):
886 if flag not in self.probFilterFlags:
887 self.probFilterFlags.append(flag)
889 def get_proxies(self, pobj):
895 repoinfo = pobj.repoInfo()
896 reponame = "%s" % repoinfo.name()
897 repos = filter(lambda r: r.name == reponame, self.repos)
898 repourl = str(repoinfo.baseUrls()[0])
901 proxy = repos[0].proxy
903 proxy = get_proxy_for(repourl)
905 proxies = {str(repourl.split(':')[0]): str(proxy)}
909 def get_url(self, pobj):
913 name = str(pobj.repoInfo().name())
915 repo = filter(lambda r: r.name == name, self.repos)[0]
919 baseurl = repo.baseurl[0]
921 index = baseurl.find("?")
923 baseurl = baseurl[:index]
925 location = zypp.asKindPackage(pobj).location()
926 location = str(location.filename())
927 if location.startswith("./"):
928 location = location[2:]
930 return os.path.join(baseurl, location)
932 def package_url(self, pkgname):
934 def cmpEVR(ed1, ed2):
935 (e1, v1, r1) = map(str, [ed1.epoch(), ed1.version(), ed1.release()])
936 (e2, v2, r2) = map(str, [ed2.epoch(), ed2.version(), ed2.release()])
937 return rpm.labelCompare((e1, v1, r1), (e2, v2, r2))
940 self.__initialize_zypp()
943 q.addKind(zypp.ResKind.package)
945 q.addAttribute(zypp.SolvAttr.name, pkgname)
946 items = sorted(q.queryResults(self.Z.pool()),
947 cmp=lambda x,y: cmpEVR(x.edition(), y.edition()),
951 url = self.get_url(items[0])
952 proxies = self.get_proxies(items[0])
953 return (url, proxies)