3 # Copyright (c) 2010, 2011 Intel, Inc.
5 # This program is free software; you can redistribute it and/or modify it
6 # under the terms of the GNU General Public License as published by the Free
7 # Software Foundation; version 2 of the License
9 # This program is distributed in the hope that it will be useful, but
10 # WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
11 # or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 # You should have received a copy of the GNU General Public License along
15 # with this program; if not, write to the Free Software Foundation, Inc., 59
16 # Temple Place - Suite 330, Boston, MA 02111-1307, USA.
22 from mic.utils import runner, fs_related
25 if not hasattr(zypp, 'PoolQuery') or not hasattr(zypp.RepoManager, 'loadSolvFile'):
26 raise ImportError("python-zypp in host system cannot support PoolQuery or loadSolvFile interface, "
27 "please update it to enhanced version which can be found in repo.meego.com/tools")
30 from mic.kickstart import ksparser
31 from mic.utils import rpmmisc
32 from mic.utils import misc
33 from mic.utils.proxy import get_proxy_for
34 from mic.utils.errors import CreatorError
35 from mic.imager.baseimager import BaseImageCreator
41 self.mirrorlist = None
43 self.proxy_username = None
44 self.proxy_password = None
47 self.autorefresh = True
48 self.keeppackages = True
51 class RepoError(CreatorError):
54 class RpmError(CreatorError):
57 from mic.pluginbase import BackendPlugin
58 class Zypp(BackendPlugin):
61 def __init__(self, target_arch, instroot, cachedir):
62 self.cachedir = cachedir
63 self.instroot = instroot
64 self.target_arch = target_arch
66 self.__pkgs_license = {}
67 self.__pkgs_content = {}
71 self.repo_manager = None
72 self.repo_manager_options = None
75 self.probFilterFlags = []
79 self.has_prov_query = True
81 def doFileLogSetup(self, uid, logfile):
82 # don't do the file log for the livecd as it can lead to open fds
83 # being left and an inability to clean up after ourself
94 if not os.path.exists("/etc/fedora-release") and not os.path.exists("/etc/meego-release"):
95 for i in range(3, os.sysconf("SC_OPEN_MAX")):
104 def _cleanupRpmdbLocks(self, installroot):
105 # cleans up temporary files left by bdb so that differing
106 # versions of rpm don't cause problems
108 for f in glob.glob(installroot + "/var/lib/rpm/__db*"):
111 def setup(self, confpath=None, instroot=None):
112 if not self.instroot:
113 self.instroot = instroot
114 self._cleanupRpmdbLocks(self.instroot)
116 def whatObsolete(self, pkg):
117 query = zypp.PoolQuery()
118 query.addKind(zypp.ResKind.package)
119 query.addAttribute(zypp.SolvAttr.obsoletes, pkg)
120 query.setMatchExact()
121 for pi in query.queryResults(self.Z.pool()):
125 def _splitPkgString(self, pkg):
126 sp = pkg.rsplit(".",1)
131 sysarch = zypp.Arch(self.target_arch)
132 if not zypp.Arch(arch).compatible_with (sysarch):
137 def selectPackage(self, pkg):
138 """ Select a given package or package pattern, can be specified with name.arch or name* or *name """
140 self.__initialize_zypp()
142 def markPoolItem(obs, pi):
144 pi.status().setToBeInstalled (zypp.ResStatus.USER)
146 obs.status().setToBeInstalled (zypp.ResStatus.USER)
149 startx = pkg.startswith("*")
150 endx = pkg.endswith("*")
151 ispattern = startx or endx
152 name, arch = self._splitPkgString(pkg)
155 q.addKind(zypp.ResKind.package)
157 if startx and not endx:
158 pattern = '%s$' % (pkg[1:])
159 if endx and not startx:
160 pattern = '^%s' % (pkg[0:-1])
162 pattern = '%s' % (pkg[1:-1])
164 q.addAttribute(zypp.SolvAttr.name,pattern)
167 q.addAttribute(zypp.SolvAttr.name,name)
170 q.addAttribute(zypp.SolvAttr.name,pkg)
172 for item in sorted(q.queryResults(self.Z.pool()), key=lambda item: str(item.edition()), reverse=True):
173 if item.name() in self.excpkgs.keys() and self.excpkgs[item.name()] == item.repoInfo().name():
175 if item.name() in self.incpkgs.keys() and self.incpkgs[item.name()] != item.repoInfo().name():
178 obspkg = self.whatObsolete(item.name())
180 if arch == str(item.arch()):
181 item.status().setToBeInstalled (zypp.ResStatus.USER)
183 markPoolItem(obspkg, item)
186 # Can't match using package name, then search from packge provides infomation
187 if found == False and not ispattern:
188 q.addAttribute(zypp.SolvAttr.provides, pkg)
189 q.addAttribute(zypp.SolvAttr.name,'')
190 for item in sorted(q.queryResults(self.Z.pool()), key=lambda item: str(item.edition()), reverse=True):
191 if item.name() in self.excpkgs.keys() and self.excpkgs[item.name()] == item.repoInfo().name():
193 if item.name() in self.incpkgs.keys() and self.incpkgs[item.name()] != item.repoInfo().name():
196 obspkg = self.whatObsolete(item.name())
197 markPoolItem(obspkg, item)
202 raise CreatorError("Unable to find package: %s" % (pkg,))
204 def inDeselectPackages(self, item):
205 """check if specified pacakges are in the list of inDeselectPackages"""
207 for pkg in self.to_deselect:
208 startx = pkg.startswith("*")
209 endx = pkg.endswith("*")
210 ispattern = startx or endx
211 pkgname, pkgarch = self._splitPkgString(pkg)
214 if name == pkgname and str(item.arch()) == pkgarch:
220 if startx and name.endswith(pkg[1:]):
222 if endx and name.startswith(pkg[:-1]):
226 def deselectPackage(self, pkg):
227 """collect packages should not be installed"""
228 self.to_deselect.append(pkg)
230 def selectGroup(self, grp, include = ksparser.GROUP_DEFAULT):
232 self.__initialize_zypp()
235 q.addKind(zypp.ResKind.pattern)
236 for item in q.queryResults(self.Z.pool()):
237 summary = "%s" % item.summary()
238 name = "%s" % item.name()
239 if name == grp or summary == grp:
241 item.status().setToBeInstalled (zypp.ResStatus.USER)
245 if include == ksparser.GROUP_REQUIRED:
246 map(lambda p: self.deselectPackage(p), grp.default_packages.keys())
249 raise CreatorError("Unable to find pattern: %s" % (grp,))
251 def addRepository(self, name, url = None, mirrorlist = None, proxy = None,
252 proxy_username = None, proxy_password = None,
253 inc = None, exc = None, ssl_verify = True, cost=None,
255 # TODO: Handle cost attribute for repos
256 if not self.repo_manager:
257 self.__initialize_repo_manager()
259 repo = RepositoryStub()
263 repo.proxy_username = proxy_username
264 repo.proxy_password = proxy_password
265 repo.ssl_verify = ssl_verify
266 repo.baseurl.append(url)
269 self.incpkgs[pkg] = name
272 self.excpkgs[pkg] = name
274 # check LICENSE files
275 if not rpmmisc.checkRepositoryEULA(name, repo):
276 msger.warning('skip repo:%s for failed EULA confirmation' % name)
280 repo.mirrorlist = mirrorlist
282 # Enable gpg check for verifying corrupt packages
285 repo.priority = priority
286 self.repos.append(repo)
289 repo_info = zypp.RepoInfo()
290 repo_info.setAlias(repo.name)
291 repo_info.setName(repo.name)
292 repo_info.setEnabled(repo.enabled)
293 repo_info.setAutorefresh(repo.autorefresh)
294 repo_info.setKeepPackages(repo.keeppackages)
295 baseurl = zypp.Url(repo.baseurl[0])
297 baseurl.setQueryParam("ssl_verify", "no")
299 (scheme, host, path, parm, query, frag) = urlparse.urlparse(proxy)
300 proxyinfo = host.split(":")
303 if len(proxyinfo) > 1:
305 if proxy.startswith("socks") and len(proxy.rsplit(':', 1)) == 2:
306 host = proxy.rsplit(':', 1)[0]
307 port = proxy.rsplit(':', 1)[1]
308 baseurl.setQueryParam ("proxy", host)
309 baseurl.setQueryParam ("proxyport", port)
310 repo_info.addBaseUrl(baseurl)
312 repo_info.setPriority(repo.priority)
313 self.repo_manager.addRepository(repo_info)
314 self.__build_repo_cache(name)
315 except RuntimeError, e:
316 raise CreatorError(str(e))
318 msger.verbose('repo: %s was added' % name)
321 def installHasFile(self, file):
324 def runInstall(self, checksize = 0):
325 os.environ["HOME"] = "/"
326 self.buildTransaction()
328 todo = zypp.GetResolvablesToInsDel(self.Z.pool())
329 installed_pkgs = todo._toInstall
331 for item in installed_pkgs:
332 if not zypp.isKindPattern(item) and not self.inDeselectPackages(item):
335 # record all pkg and the content
336 localpkgs = self.localpkgs.keys()
339 if pkg.name() in localpkgs:
340 hdr = rpmmisc.readRpmHeader(self.ts, self.localpkgs[pkg.name()])
341 pkg_long_name = "%s.%s %s-%s" % (hdr['name'], hdr['arch'], hdr['version'], hdr['release'])
342 license = hdr['license']
344 pkg_long_name = "%s.%s %s" % (pkg.name(), pkg.arch(), pkg.edition())
345 package = zypp.asKindPackage(pkg)
346 license = package.license()
347 self.__pkgs_content[pkg_long_name] = {} #TBD: to get file list
348 if license in self.__pkgs_license.keys():
349 self.__pkgs_license[license].append(pkg_long_name)
351 self.__pkgs_license[license] = [pkg_long_name]
353 total_count = len(dlpkgs)
355 download_total_size = sum(map(lambda x: int(x.downloadSize()), dlpkgs))
356 localpkgs = self.localpkgs.keys()
357 msger.info("Checking packages cache and packages integrity ...")
359 """ Check if it is cached locally """
360 if po.name() in localpkgs:
363 local = self.getLocalPkgPath(po)
364 if os.path.exists(local):
365 if self.checkPkg(local) != 0:
368 download_total_size -= int(po.downloadSize())
370 cache_avail_size = misc.get_filesystem_avail(self.cachedir)
371 if cache_avail_size < download_total_size:
372 raise CreatorError("No enough space used for downloading.")
374 # record the total size of installed pkgs
375 install_total_size = sum(map(lambda x: int(x.installSize()), dlpkgs))
376 # check needed size before actually download and install
377 if checksize and install_total_size > checksize:
378 raise CreatorError("No enough space used for installing, please resize partition size in ks file")
380 download_count = total_count - cached_count
381 msger.info("%d packages to be installed, %d packages gotten from cache, %d packages to be downloaded" % (total_count, cached_count, download_count))
383 if download_count > 0:
384 msger.info("Downloading packages ...")
385 self.downloadPkgs(dlpkgs, download_count)
386 self.installPkgs(dlpkgs)
389 raise CreatorError("Unable to download from repo : %s" % (e,))
391 raise CreatorError("Unable to install: %s" % (e,))
393 def getAllContent(self):
394 return self.__pkgs_content
396 def getPkgsLicense(self):
397 return self.__pkgs_license
399 def __initialize_repo_manager(self):
400 if self.repo_manager:
403 """ Clean up repo metadata """
404 shutil.rmtree(self.cachedir + "/etc", ignore_errors = True)
405 shutil.rmtree(self.cachedir + "/solv", ignore_errors = True)
406 shutil.rmtree(self.cachedir + "/raw", ignore_errors = True)
408 zypp.KeyRing.setDefaultAccept( zypp.KeyRing.ACCEPT_UNSIGNED_FILE
409 | zypp.KeyRing.ACCEPT_VERIFICATION_FAILED
410 | zypp.KeyRing.ACCEPT_UNKNOWNKEY
411 | zypp.KeyRing.TRUST_KEY_TEMPORARILY
413 self.repo_manager_options = zypp.RepoManagerOptions(zypp.Pathname(self.instroot))
414 self.repo_manager_options.knownReposPath = zypp.Pathname(self.cachedir + "/etc/zypp/repos.d")
415 self.repo_manager_options.repoCachePath = zypp.Pathname(self.cachedir)
416 self.repo_manager_options.repoRawCachePath = zypp.Pathname(self.cachedir + "/raw")
417 self.repo_manager_options.repoSolvCachePath = zypp.Pathname(self.cachedir + "/solv")
418 self.repo_manager_options.repoPackagesCachePath = zypp.Pathname(self.cachedir + "/packages")
420 self.repo_manager = zypp.RepoManager(self.repo_manager_options)
422 def __build_repo_cache(self, name):
423 repo = self.repo_manager.getRepositoryInfo(name)
424 if self.repo_manager.isCached(repo) or not repo.enabled():
426 self.repo_manager.buildCache(repo, zypp.RepoManager.BuildIfNeeded)
428 def __initialize_zypp(self):
432 zconfig = zypp.ZConfig_instance()
434 """ Set system architecture """
436 zconfig.setSystemArchitecture(zypp.Arch(self.target_arch))
438 msger.info("zypp architecture is <%s>" % zconfig.systemArchitecture())
440 """ repoPackagesCachePath is corrected by this """
441 self.repo_manager = zypp.RepoManager(self.repo_manager_options)
442 repos = self.repo_manager.knownRepositories()
444 if not repo.enabled():
446 self.repo_manager.loadFromCache(repo)
448 self.Z = zypp.ZYppFactory_instance().getZYpp()
449 self.Z.initializeTarget(zypp.Pathname(self.instroot))
450 self.Z.target().load()
453 def buildTransaction(self):
454 if not self.Z.resolver().resolvePool():
455 msger.warning("Problem count: %d" % len(self.Z.resolver().problems()))
456 for problem in self.Z.resolver().problems():
457 msger.warning("Problem: %s, %s" % (problem.description().decode("utf-8"), problem.details().decode("utf-8")))
459 def getLocalPkgPath(self, po):
460 repoinfo = po.repoInfo()
461 cacheroot = repoinfo.packagesPath()
462 location= zypp.asKindPackage(po).location()
463 rpmpath = str(location.filename())
464 pkgpath = "%s/%s" % (cacheroot, rpmpath)
467 def installLocal(self, pkg, po=None, updateonly=False):
469 self.__initialize_transaction()
470 solvfile = "%s/.solv" % (self.cachedir)
471 rc, out = runner.runtool([fs_related.find_binary_path("rpms2solv"), pkg])
473 f = open(solvfile, "w+")
476 warnmsg = self.repo_manager.loadSolvFile(solvfile , os.path.basename(pkg))
478 msger.warning(warnmsg)
481 msger.warning('Can not get %s solv data.' % pkg)
482 hdr = rpmmisc.readRpmHeader(self.ts, pkg)
483 arch = zypp.Arch(hdr['arch'])
484 sysarch = zypp.Arch(self.target_arch)
485 if arch.compatible_with (sysarch):
486 pkgname = hdr['name']
487 self.localpkgs[pkgname] = pkg
488 self.selectPackage(pkgname)
489 msger.info("Marking %s to be installed" % (pkg))
491 msger.warning ("Cannot add package %s to transaction. Not a compatible architecture: %s" % (pkg, hdr['arch']))
493 def downloadPkgs(self, package_objects, count):
494 localpkgs = self.localpkgs.keys()
495 progress_obj = rpmmisc.TextProgress(count)
496 for po in package_objects:
497 if po.name() in localpkgs:
499 filename = self.getLocalPkgPath(po)
500 if os.path.exists(filename):
501 if self.checkPkg(filename) == 0:
503 dirn = os.path.dirname(filename)
504 if not os.path.exists(dirn):
506 baseurl = str(po.repoInfo().baseUrls()[0])
507 index = baseurl.find("?")
509 baseurl = baseurl[:index]
510 proxy = self.get_proxy(po.repoInfo())
513 proxies = {str(proxy.split(":")[0]):str(proxy)}
515 location = zypp.asKindPackage(po).location()
516 location = str(location.filename())
517 if location.startswith("./"):
518 location = location[2:]
519 url = baseurl + "/%s" % location
521 filename = rpmmisc.myurlgrab(url, filename, proxies, progress_obj)
526 def installPkgs(self, package_objects):
528 self.__initialize_transaction()
532 for flag in self.probFilterFlags:
534 self.ts.setProbFilter(probfilter)
536 localpkgs = self.localpkgs.keys()
537 for po in package_objects:
539 if pkgname in localpkgs:
540 rpmpath = self.localpkgs[pkgname]
542 rpmpath = self.getLocalPkgPath(po)
543 if not os.path.exists(rpmpath):
544 """ Maybe it is a local repo """
545 baseurl = str(po.repoInfo().baseUrls()[0])
546 baseurl = baseurl.strip()
547 location = zypp.asKindPackage(po).location()
548 location = str(location.filename())
549 if baseurl.startswith("file:/"):
550 rpmpath = baseurl[5:] + "/%s" % (location)
551 if not os.path.exists(rpmpath):
552 raise RpmError("Error: %s doesn't exist" % rpmpath)
553 h = rpmmisc.readRpmHeader(self.ts, rpmpath)
554 self.ts.addInstall(h, rpmpath, 'u')
556 unresolved_dependencies = self.ts.check()
557 if not unresolved_dependencies:
559 cb = rpmmisc.RPMInstallCallback(self.ts)
560 installlogfile = "%s/__catched_stderr.buf" % (self.instroot)
561 msger.enable_logstderr(installlogfile)
562 errors = self.ts.run(cb.callback, '')
565 elif len(errors) == 0:
566 msger.warning('scriptlet or other non-fatal errors occurred during transaction.')
570 msger.error('Could not run transaction.')
571 msger.disable_logstderr()
576 for pkg, need, needflags, sense, key in unresolved_dependencies:
577 package = '-'.join(pkg)
578 if needflags == rpm.RPMSENSE_LESS:
579 deppkg = ' < '.join(need)
580 elif needflags == rpm.RPMSENSE_EQUAL:
581 deppkg = ' = '.join(need)
582 elif needflags == rpm.RPMSENSE_GREATER:
583 deppkg = ' > '.join(need)
585 deppkg = '-'.join(need)
587 if sense == rpm.RPMDEP_SENSE_REQUIRES:
588 msger.warning ("[%s] Requires [%s], which is not provided" % (package, deppkg))
589 elif sense == rpm.RPMDEP_SENSE_CONFLICTS:
590 msger.warning ("[%s] Conflicts with [%s]" % (package, deppkg))
592 raise RepoError("Unresolved dependencies, transaction failed.")
594 def __initialize_transaction(self):
596 self.ts = rpm.TransactionSet(self.instroot)
597 # Set to not verify DSA signatures.
598 self.ts.setVSFlags(rpm._RPMVSF_NOSIGNATURES|rpm._RPMVSF_NODIGESTS)
600 def checkPkg(self, pkg):
602 if not os.path.exists(pkg):
604 ret = rpmmisc.checkRpmIntegrity('rpm', pkg)
606 msger.warning("package %s is damaged: %s" % (os.path.basename(pkg), pkg))
610 def _add_prob_flags(self, *flags):
612 if flag not in self.probFilterFlags:
613 self.probFilterFlags.append(flag)
615 def get_proxy(self, repoinfo):
617 reponame = "%s" % repoinfo.name()
618 for repo in self.repos:
619 if repo.name == reponame:
626 repourl = str(repoinfo.baseUrls()[0])
627 return get_proxy_for(repourl)