3 # Copyright (c) 2010, 2011 Intel, Inc.
5 # This program is free software; you can redistribute it and/or modify it
6 # under the terms of the GNU General Public License as published by the Free
7 # Software Foundation; version 2 of the License
9 # This program is distributed in the hope that it will be useful, but
10 # WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
11 # or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 # You should have received a copy of the GNU General Public License along
15 # with this program; if not, write to the Free Software Foundation, Inc., 59
16 # Temple Place - Suite 330, Boston, MA 02111-1307, USA.
22 from mic.utils import runner, fs_related
25 if not hasattr(zypp, 'PoolQuery') or not hasattr(zypp.RepoManager, 'loadSolvFile'):
26 raise ImportError("python-zypp in host system cannot support PoolQuery or loadSolvFile interface,"
27 "please update it to enhanced version which can be found in repo.meego.com/tools")
30 from mic.kickstart import ksparser
31 from mic.utils import rpmmisc, fs_related as fs
32 from mic.utils.proxy import get_proxy_for
33 from mic.utils.errors import CreatorError
34 from mic.imager.baseimager import BaseImageCreator
40 self.mirrorlist = None
42 self.proxy_username = None
43 self.proxy_password = None
46 self.autorefresh = True
47 self.keeppackages = True
49 class RepoError(CreatorError):
52 class RpmError(CreatorError):
55 from mic.pluginbase import BackendPlugin
56 class Zypp(BackendPlugin):
59 def __init__(self, creator = None):
60 if not isinstance(creator, BaseImageCreator):
61 raise CreatorError("Invalid argument: creator")
63 self.__pkgs_license = {}
64 self.__pkgs_content = {}
65 self.creator = creator
69 self.repo_manager = None
70 self.repo_manager_options = None
73 self.probFilterFlags = []
77 self.has_prov_query = True
79 def doFileLogSetup(self, uid, logfile):
80 # don't do the file log for the livecd as it can lead to open fds
81 # being left and an inability to clean up after ourself
89 if not os.path.exists("/etc/fedora-release") and not os.path.exists("/etc/meego-release"):
90 for i in range(3, os.sysconf("SC_OPEN_MAX")):
102 def _cleanupRpmdbLocks(self, installroot):
103 # cleans up temporary files left by bdb so that differing
104 # versions of rpm don't cause problems
106 for f in glob.glob(installroot + "/var/lib/rpm/__db*"):
109 def setup(self, confpath, installroot):
110 self._cleanupRpmdbLocks(installroot)
111 self.installroot = installroot
113 def whatObsolete(self, pkg):
114 query = zypp.PoolQuery()
115 query.addKind(zypp.ResKind.package)
116 query.addAttribute(zypp.SolvAttr.obsoletes, pkg)
117 query.setMatchExact()
118 for pi in query.queryResults(self.Z.pool()):
122 def _splitPkgString(self, pkg):
123 sp = pkg.rsplit(".",1)
128 if self.creator.target_arch == None:
129 # TODO, get the default_arch from conf or detected from global settings
130 sysarch = zypp.Arch('i686')
132 sysarch = zypp.Arch(self.creator.target_arch)
133 if not zypp.Arch(arch).compatible_with (sysarch):
138 def selectPackage(self, pkg):
139 """ Select a given package or package pattern, can be specified with name.arch or name* or *name """
141 self.__initialize_zypp()
143 def markPoolItem(obs, pi):
145 pi.status().setToBeInstalled (zypp.ResStatus.USER)
147 obs.status().setToBeInstalled (zypp.ResStatus.USER)
150 startx = pkg.startswith("*")
151 endx = pkg.endswith("*")
152 ispattern = startx or endx
153 name, arch = self._splitPkgString(pkg)
156 q.addKind(zypp.ResKind.package)
158 if startx and not endx:
159 pattern = '%s$' % (pkg[1:])
160 if endx and not startx:
161 pattern = '^%s' % (pkg[0:-1])
163 pattern = '%s' % (pkg[1:-1])
165 q.addAttribute(zypp.SolvAttr.name,pattern)
168 q.addAttribute(zypp.SolvAttr.name,name)
171 q.addAttribute(zypp.SolvAttr.name,pkg)
173 for item in sorted(q.queryResults(self.Z.pool()), key=lambda item: str(item.edition()), reverse=True):
174 if item.name() in self.excpkgs.keys() and self.excpkgs[item.name()] == item.repoInfo().name():
176 if item.name() in self.incpkgs.keys() and self.incpkgs[item.name()] != item.repoInfo().name():
179 obspkg = self.whatObsolete(item.name())
181 if arch == str(item.arch()):
182 item.status().setToBeInstalled (zypp.ResStatus.USER)
184 markPoolItem(obspkg, item)
187 # Can't match using package name, then search from packge provides infomation
188 if found == False and not ispattern:
189 q.addAttribute(zypp.SolvAttr.provides, pkg)
190 q.addAttribute(zypp.SolvAttr.name,'')
191 for item in sorted(q.queryResults(self.Z.pool()), key=lambda item: str(item.edition()), reverse=True):
192 if item.name() in self.excpkgs.keys() and self.excpkgs[item.name()] == item.repoInfo().name():
194 if item.name() in self.incpkgs.keys() and self.incpkgs[item.name()] != item.repoInfo().name():
197 obspkg = self.whatObsolete(item.name())
198 markPoolItem(obspkg, item)
203 raise CreatorError("Unable to find package: %s" % (pkg,))
205 def inDeselectPackages(self, item):
206 """check if specified pacakges are in the list of inDeselectPackages"""
208 for pkg in self.to_deselect:
209 startx = pkg.startswith("*")
210 endx = pkg.endswith("*")
211 ispattern = startx or endx
212 pkgname, pkgarch = self._splitPkgString(pkg)
215 if name == pkgname and str(item.arch()) == pkgarch:
221 if startx and name.endswith(pkg[1:]):
223 if endx and name.startswith(pkg[:-1]):
227 def deselectPackage(self, pkg):
228 """collect packages should not be installed"""
229 self.to_deselect.append(pkg)
231 def selectGroup(self, grp, include = ksparser.GROUP_DEFAULT):
233 self.__initialize_zypp()
236 q.addKind(zypp.ResKind.pattern)
237 for item in q.queryResults(self.Z.pool()):
238 summary = "%s" % item.summary()
239 name = "%s" % item.name()
240 if name == grp or summary == grp:
242 item.status().setToBeInstalled (zypp.ResStatus.USER)
246 if include == ksparser.GROUP_REQUIRED:
247 map(lambda p: self.deselectPackage(p), grp.default_packages.keys())
250 raise CreatorError("Unable to find pattern: %s" % (grp,))
252 def addRepository(self, name, url = None, mirrorlist = None, proxy = None, proxy_username = None, proxy_password = None, inc = None, exc = None):
253 if not self.repo_manager:
254 self.__initialize_repo_manager()
256 repo = RepositoryStub()
260 repo.proxy_username = proxy_username
261 repo.proxy_password = proxy_password
262 repo.baseurl.append(url)
265 self.incpkgs[pkg] = name
268 self.excpkgs[pkg] = name
270 # check LICENSE files
271 if not rpmmisc.checkRepositoryEULA(name, repo):
272 msger.warning('skip repo:%s for failed EULA confirmation' % name)
276 repo.mirrorlist = mirrorlist
278 # Enable gpg check for verifying corrupt packages
280 self.repos.append(repo)
283 repo_info = zypp.RepoInfo()
284 repo_info.setAlias(repo.name)
285 repo_info.setName(repo.name)
286 repo_info.setEnabled(repo.enabled)
287 repo_info.setAutorefresh(repo.autorefresh)
288 repo_info.setKeepPackages(repo.keeppackages)
289 baseurl = zypp.Url(repo.baseurl[0])
291 (scheme, host, path, parm, query, frag) = urlparse.urlparse(proxy)
292 proxyinfo = host.split(":")
293 baseurl.setQueryParam ("proxy", proxyinfo[0])
295 if len(proxyinfo) > 1:
297 baseurl.setQueryParam ("proxyport", port)
298 repo_info.addBaseUrl(baseurl)
299 self.repo_manager.addRepository(repo_info)
300 self.__build_repo_cache(name)
301 except RuntimeError, e:
302 raise CreatorError(str(e))
304 msger.verbose('repo: %s was added' % name)
307 def installHasFile(self, file):
310 def runInstall(self, checksize = 0):
311 os.environ["HOME"] = "/"
312 self.buildTransaction()
314 todo = zypp.GetResolvablesToInsDel(self.Z.pool())
315 installed_pkgs = todo._toInstall
317 for item in installed_pkgs:
318 if not zypp.isKindPattern(item) and not self.inDeselectPackages(item):
321 # record the total size of installed pkgs
322 pkgs_total_size = sum(map(lambda x: int(x.installSize()), dlpkgs))
324 # check needed size before actually download and install
325 if checksize and pkgs_total_size > checksize:
326 raise CreatorError("Size of specified root partition in kickstart file is too small to install all selected packages.")
328 # record all pkg and the content
329 localpkgs = self.localpkgs.keys()
332 if pkg.name() in localpkgs:
333 hdr = rpmmisc.readRpmHeader(self.ts, self.localpkgs[pkg.name()])
334 pkg_long_name = "%s-%s-%s.%s.rpm" % (hdr['name'], hdr['version'], hdr['release'], hdr['arch'])
335 license = hdr['license']
337 pkg_long_name = "%s-%s.%s.rpm" % (pkg.name(), pkg.edition(), pkg.arch())
338 package = zypp.asKindPackage(pkg)
339 license = package.license()
340 self.__pkgs_content[pkg_long_name] = {} #TBD: to get file list
341 if license in self.__pkgs_license.keys():
342 self.__pkgs_license[license].append(pkg_long_name)
344 self.__pkgs_license[license] = [pkg_long_name]
346 total_count = len(dlpkgs)
348 localpkgs = self.localpkgs.keys()
349 msger.info("Checking packages cache and packages integrity ...")
351 """ Check if it is cached locally """
352 if po.name() in localpkgs:
355 local = self.getLocalPkgPath(po)
356 if os.path.exists(local):
357 if self.checkPkg(local) != 0:
361 download_count = total_count - cached_count
362 msger.info("%d packages to be installed, %d packages gotten from cache, %d packages to be downloaded" % (total_count, cached_count, download_count))
364 if download_count > 0:
365 msger.info("Downloading packages ...")
366 self.downloadPkgs(dlpkgs, download_count)
367 self.installPkgs(dlpkgs)
370 raise CreatorError("Unable to download from repo : %s" % (e,))
372 raise CreatorError("Unable to install: %s" % (e,))
374 def getAllContent(self):
375 return self.__pkgs_content
377 def getPkgsLicense(self):
378 return self.__pkgs_license
380 def __initialize_repo_manager(self):
381 if self.repo_manager:
384 """ Clean up repo metadata """
385 shutil.rmtree(self.creator.cachedir + "/etc", ignore_errors = True)
386 shutil.rmtree(self.creator.cachedir + "/solv", ignore_errors = True)
387 shutil.rmtree(self.creator.cachedir + "/raw", ignore_errors = True)
389 zypp.KeyRing.setDefaultAccept( zypp.KeyRing.ACCEPT_UNSIGNED_FILE
390 | zypp.KeyRing.ACCEPT_VERIFICATION_FAILED
391 | zypp.KeyRing.ACCEPT_UNKNOWNKEY
392 | zypp.KeyRing.TRUST_KEY_TEMPORARILY
394 self.repo_manager_options = zypp.RepoManagerOptions(zypp.Pathname(self.creator._instroot))
395 self.repo_manager_options.knownReposPath = zypp.Pathname(self.creator.cachedir + "/etc/zypp/repos.d")
396 self.repo_manager_options.repoCachePath = zypp.Pathname(self.creator.cachedir)
397 self.repo_manager_options.repoRawCachePath = zypp.Pathname(self.creator.cachedir + "/raw")
398 self.repo_manager_options.repoSolvCachePath = zypp.Pathname(self.creator.cachedir + "/solv")
399 self.repo_manager_options.repoPackagesCachePath = zypp.Pathname(self.creator.cachedir + "/packages")
401 self.repo_manager = zypp.RepoManager(self.repo_manager_options)
403 def __build_repo_cache(self, name):
404 repo = self.repo_manager.getRepositoryInfo(name)
405 if self.repo_manager.isCached(repo) or not repo.enabled():
407 self.repo_manager.buildCache(repo, zypp.RepoManager.BuildIfNeeded)
409 def __initialize_zypp(self):
413 zconfig = zypp.ZConfig_instance()
415 """ Set system architecture """
416 if self.creator.target_arch:
417 zconfig.setSystemArchitecture(zypp.Arch(self.creator.target_arch))
419 msger.info("zypp architecture is <%s>" % zconfig.systemArchitecture())
421 """ repoPackagesCachePath is corrected by this """
422 self.repo_manager = zypp.RepoManager(self.repo_manager_options)
423 repos = self.repo_manager.knownRepositories()
425 if not repo.enabled():
427 self.repo_manager.loadFromCache(repo)
429 self.Z = zypp.ZYppFactory_instance().getZYpp()
430 self.Z.initializeTarget(zypp.Pathname(self.creator._instroot))
431 self.Z.target().load()
434 def buildTransaction(self):
435 if not self.Z.resolver().resolvePool():
436 msger.warning("Problem count: %d" % len(self.Z.resolver().problems()))
437 for problem in self.Z.resolver().problems():
438 msger.warning("Problem: %s, %s" % (problem.description().decode("utf-8"), problem.details().decode("utf-8")))
440 def getLocalPkgPath(self, po):
441 repoinfo = po.repoInfo()
442 cacheroot = repoinfo.packagesPath()
443 location= zypp.asKindPackage(po).location()
444 rpmpath = str(location.filename())
445 pkgpath = "%s/%s" % (cacheroot, rpmpath)
448 def installLocal(self, pkg, po=None, updateonly=False):
450 self.__initialize_transaction()
451 solvfile = "%s/.solv" % (self.creator.cachedir)
452 rc, out = runner.runtool([fs_related.find_binary_path("rpms2solv"), pkg])
454 f = open(solvfile, "w+")
457 warnmsg = self.repo_manager.loadSolvFile(solvfile , os.path.basename(pkg))
459 msger.warning(warnmsg)
462 msger.warning('Can not get %s solv data.' % pkg)
463 hdr = rpmmisc.readRpmHeader(self.ts, pkg)
464 arch = zypp.Arch(hdr['arch'])
465 if self.creator.target_arch == None:
466 # TODO, get the default_arch from conf or detected from global settings
467 sysarch = zypp.Arch('i686')
469 sysarch = zypp.Arch(self.creator.target_arch)
470 if arch.compatible_with (sysarch):
471 pkgname = hdr['name']
472 self.localpkgs[pkgname] = pkg
473 self.selectPackage(pkgname)
474 msger.info("Marking %s to be installed" % (pkg))
476 msger.warning ("Cannot add package %s to transaction. Not a compatible architecture: %s" % (pkg, hdr['arch']))
478 def downloadPkgs(self, package_objects, count):
479 localpkgs = self.localpkgs.keys()
480 progress_obj = fs.TextProgress(count)
481 for po in package_objects:
482 if po.name() in localpkgs:
484 filename = self.getLocalPkgPath(po)
485 if os.path.exists(filename):
486 if self.checkPkg(filename) == 0:
488 dirn = os.path.dirname(filename)
489 if not os.path.exists(dirn):
491 baseurl = str(po.repoInfo().baseUrls()[0])
492 index = baseurl.find("?")
494 baseurl = baseurl[:index]
495 proxy = self.get_proxy(po.repoInfo())
498 proxies = {str(proxy.split(":")[0]):str(proxy)}
500 location = zypp.asKindPackage(po).location()
501 location = str(location.filename())
502 if location.startswith("./"):
503 location = location[2:]
504 url = baseurl + "/%s" % location
506 filename = fs.myurlgrab(url, filename, proxies, progress_obj)
511 def installPkgs(self, package_objects):
513 self.__initialize_transaction()
517 for flag in self.probFilterFlags:
519 self.ts.setProbFilter(probfilter)
521 localpkgs = self.localpkgs.keys()
522 for po in package_objects:
524 if pkgname in localpkgs:
525 rpmpath = self.localpkgs[pkgname]
527 rpmpath = self.getLocalPkgPath(po)
528 if not os.path.exists(rpmpath):
529 """ Maybe it is a local repo """
530 baseurl = str(po.repoInfo().baseUrls()[0])
531 baseurl = baseurl.strip()
532 location = zypp.asKindPackage(po).location()
533 location = str(location.filename())
534 if baseurl.startswith("file:/"):
535 rpmpath = baseurl[5:] + "/%s" % (location)
536 if not os.path.exists(rpmpath):
537 raise RpmError("Error: %s doesn't exist" % rpmpath)
538 h = rpmmisc.readRpmHeader(self.ts, rpmpath)
539 self.ts.addInstall(h, rpmpath, 'u')
541 unresolved_dependencies = self.ts.check()
542 if not unresolved_dependencies:
544 cb = rpmmisc.RPMInstallCallback(self.ts)
545 installlogfile = "%s/__catched_stderr.buf" % (self.creator._instroot)
546 msger.enable_logstderr(installlogfile)
547 errors = self.ts.run(cb.callback, '')
550 elif len(errors) == 0:
551 msger.warning('scriptlet or other non-fatal errors occurred during transaction.')
555 msger.error('Could not run transaction.')
556 msger.disable_logstderr()
561 for pkg, need, needflags, sense, key in unresolved_dependencies:
562 package = '-'.join(pkg)
563 if needflags == rpm.RPMSENSE_LESS:
564 deppkg = ' < '.join(need)
565 elif needflags == rpm.RPMSENSE_EQUAL:
566 deppkg = ' = '.join(need)
567 elif needflags == rpm.RPMSENSE_GREATER:
568 deppkg = ' > '.join(need)
570 deppkg = '-'.join(need)
572 if sense == rpm.RPMDEP_SENSE_REQUIRES:
573 msger.warning ("[%s] Requires [%s], which is not provided" % (package, deppkg))
574 elif sense == rpm.RPMDEP_SENSE_CONFLICTS:
575 msger.warning ("[%s] Conflicts with [%s]" % (package, deppkg))
577 raise RepoError("Unresolved dependencies, transaction failed.")
579 def __initialize_transaction(self):
581 self.ts = rpm.TransactionSet(self.creator._instroot)
582 # Set to not verify DSA signatures.
583 self.ts.setVSFlags(rpm._RPMVSF_NOSIGNATURES|rpm._RPMVSF_NODIGESTS)
585 def checkPkg(self, pkg):
587 if not os.path.exists(pkg):
589 ret = rpmmisc.checkRpmIntegrity('rpm', pkg)
591 msger.warning("package %s is damaged: %s" % (os.path.basename(pkg), pkg))
595 def _add_prob_flags(self, *flags):
597 if flag not in self.probFilterFlags:
598 self.probFilterFlags.append(flag)
600 def get_proxy(self, repoinfo):
602 reponame = "%s" % repoinfo.name()
603 for repo in self.repos:
604 if repo.name == reponame:
611 repourl = str(repoinfo.baseUrls()[0])
612 return get_proxy_for(repourl)