13 import pykickstart.parser
14 from mic.utils.errors import *
15 from mic.imager.baseimager import BaseImageCreator as ImageCreator
16 from mic.utils.fs_related import *
17 from mic.utils.misc import *
18 from mic.utils.rpmmisc import *
19 from mic.pluginbase.backend_plugin import BackendPlugin
25 self.mirrorlist = None
27 self.proxy_username = None
28 self.proxy_password = None
29 self.includepkgs = None
30 self.includepkgs = None
34 self.autorefresh = True
35 self.keeppackages = True
37 class RepoError(CreatorError):
40 class RpmError(CreatorError):
43 class Zypp(BackendPlugin):
44 def __init__(self, creator = None, recording_pkgs=None):
45 if not isinstance(creator, ImageCreator):
46 raise CreatorError("Invalid argument: creator")
48 self.__recording_pkgs = recording_pkgs
49 self.__pkgs_content = {}
50 self.creator = creator
55 self.repo_manager = None
56 self.repo_manager_options = None
59 self.probFilterFlags = []
60 self.bin_rpm = find_binary_path("rpm")
64 def doFileLogSetup(self, uid, logfile):
65 # don't do the file log for the livecd as it can lead to open fds
66 # being left and an inability to clean up after ourself
74 os.unlink(self.installroot + "/yum.conf")
78 if not os.path.exists("/etc/fedora-release") and not os.path.exists("/etc/meego-release"):
79 for i in range(3, os.sysconf("SC_OPEN_MAX")):
91 def _writeConf(self, confpath, installroot):
93 conf += "installroot=%s\n" % installroot
94 conf += "cachedir=/var/cache/yum\n"
97 conf += "failovermethod=priority\n"
98 conf += "http_caching=packages\n"
100 f = file(confpath, "w+")
104 os.chmod(confpath, 0644)
106 def _cleanupRpmdbLocks(self, installroot):
107 # cleans up temporary files left by bdb so that differing
108 # versions of rpm don't cause problems
109 for f in glob.glob(installroot + "/var/lib/rpm/__db*"):
112 def setup(self, confpath, installroot):
113 self._writeConf(confpath, installroot)
114 self._cleanupRpmdbLocks(installroot)
115 self.installroot = installroot
117 def selectPackage(self, pkg):
118 """ Select a given package or package pattern, can be specified with name.arch or name* or *name """
120 self.__initialize_zypp()
123 startx = pkg.startswith("*")
124 endx = pkg.endswith("*")
125 ispattern = startx or endx
126 sp = pkg.rsplit(".", 2)
127 for item in self.Z.pool():
128 kind = "%s" % item.kind()
129 if kind == "package":
130 name = "%s" % item.name()
132 if name in self.incpkgs or self.excpkgs:
136 arch = "%s" % item.arch()
137 if name == sp[0] and arch == sp[1]:
139 if name not in self.packages:
140 self.packages.append(name)
141 item.status().setToBeInstalled (zypp.ResStatus.USER)
146 if name not in self.packages:
147 self.packages.append(name)
148 item.status().setToBeInstalled (zypp.ResStatus.USER)
151 if name in self.incpkgs or self.excpkgs:
154 if startx and name.endswith(sp[0][1:]):
156 if name not in self.packages:
157 self.packages.append(name)
158 item.status().setToBeInstalled (zypp.ResStatus.USER)
160 if endx and name.startswith(sp[0][:-1]):
162 if name not in self.packages:
163 self.packages.append(name)
164 item.status().setToBeInstalled (zypp.ResStatus.USER)
168 e = CreatorError("Unable to find package: %s" % (pkg,))
171 def deselectPackage(self, pkg):
172 """Deselect package. Can be specified as name.arch or name*"""
175 self.__initialize_zypp()
177 startx = pkg.startswith("*")
178 endx = pkg.endswith("*")
179 ispattern = startx or endx
180 sp = pkg.rsplit(".", 2)
181 for item in self.Z.pool():
182 kind = "%s" % item.kind()
183 if kind == "package":
184 name = "%s" % item.name()
187 arch = "%s" % item.arch()
188 if name == sp[0] and arch == sp[1]:
189 if item.status().isToBeInstalled():
190 item.status().resetTransact(zypp.ResStatus.USER)
191 if name in self.packages:
192 self.packages.remove(name)
196 if item.status().isToBeInstalled():
197 item.status().resetTransact(zypp.ResStatus.USER)
198 if name in self.packages:
199 self.packages.remove(name)
202 if startx and name.endswith(sp[0][1:]):
203 if item.status().isToBeInstalled():
204 item.status().resetTransact(zypp.ResStatus.USER)
205 if name in self.packages:
206 self.packages.remove(name)
208 if endx and name.startswith(sp[0][:-1]):
209 if item.status().isToBeInstalled():
210 item.status().resetTransact(zypp.ResStatus.USER)
211 if name in self.packages:
212 self.packages.remove(name)
214 def __selectIncpkgs(self):
216 for pkg in self.incpkgs:
217 for item in self.Z.pool():
218 kind = "%s" % item.kind()
219 if kind == "package":
220 name = "%s" % item.name()
221 repoalias = "%s" % item.repoInfo().alias()
222 if name == pkg and repoalias.endswith("include"):
224 if name not in self.packages:
225 self.packages.append(name)
226 item.status().setToBeInstalled (zypp.ResStatus.USER)
229 raise CreatorError("Unable to find package: %s" % (pkg,))
231 def __selectExcpkgs(self):
233 for pkg in self.excpkgs:
234 for item in self.Z.pool():
235 kind = "%s" % item.kind()
236 if kind == "package":
237 name = "%s" % item.name()
238 repoalias = "%s" % item.repoInfo().alias()
239 if name == pkg and not repoalias.endswith("exclude"):
241 if name not in self.packages:
242 self.packages.append(name)
243 item.status().setToBeInstalled (zypp.ResStatus.USER)
246 raise CreatorError("Unable to find package: %s" % (pkg,))
249 def selectGroup(self, grp, include = pykickstart.parser.GROUP_DEFAULT):
251 self.__initialize_zypp()
253 for item in self.Z.pool():
254 kind = "%s" % item.kind()
255 if kind == "pattern":
256 summary = "%s" % item.summary()
257 name = "%s" % item.name()
258 if name == grp or summary == grp:
260 if name not in self.patterns:
261 self.patterns.append(name)
262 item.status().setToBeInstalled (zypp.ResStatus.USER)
266 if include == pykickstart.parser.GROUP_REQUIRED:
267 map(lambda p: self.deselectPackage(p), grp.default_packages.keys())
268 elif include == pykickstart.parser.GROUP_ALL:
269 map(lambda p: self.selectPackage(p), grp.optional_packages.keys())
272 e = CreatorError("Unable to find pattern: %s" % (grp,))
275 def __checkAndDownloadURL(self, u2opener, url, savepath):
278 f = u2opener.open(url)
281 except u2.HTTPError, httperror:
282 if httperror.code in (404, 503):
285 raise CreatorError(httperror)
286 except OSError, oserr:
290 raise CreatorError(oserr)
291 except IOError, oserr:
292 if hasattr(oserr, "reason") and oserr.reason.errno == 2:
295 raise CreatorError(oserr)
296 except u2.URLError, err:
297 raise CreatorError(err)
300 licf = open(savepath, "w")
307 def __pagerFile(self, savepath):
308 if os.path.splitext(savepath)[1].upper() in ('.HTM', '.HTML'):
309 pagers = ('w3m', 'links', 'lynx', 'less', 'more')
311 pagers = ('less', 'more')
316 subprocess.call([pager, savepath])
326 raw_input('press <ENTER> to continue...')
328 def checkRepositoryEULA(self, name, repo):
329 """ This function is to check the LICENSE file if provided. """
331 # when proxy needed, make urllib2 follow it
333 proxy_username = repo.proxy_username
334 proxy_password = repo.proxy_password
337 auth_handler = u2.HTTPBasicAuthHandler(u2.HTTPPasswordMgrWithDefaultRealm())
341 proxy_netloc = urlparse.urlsplit(proxy).netloc
343 proxy_url = 'http://%s:%s@%s' % (proxy_username, proxy_password, proxy_netloc)
345 proxy_url = 'http://%s@%s' % (proxy_username, proxy_netloc)
349 proxy_support = u2.ProxyHandler({'http': proxy_url,
351 handlers.append(proxy_support)
353 # download all remote files to one temp dir
355 repo_lic_dir = tempfile.mkdtemp(prefix = 'repolic')
357 for url in repo.baseurl:
358 if not url.endswith('/'):
360 tmphandlers = handlers
361 (scheme, host, path, parm, query, frag) = urlparse.urlparse(url)
362 if scheme not in ("http", "https", "ftp", "ftps", "file"):
363 raise CreatorError("Error: invalid url %s" % url)
366 user_pass, host = host.split('@', 1)
368 user, password = user_pass.split(':', 1)
369 except ValueError, e:
370 raise CreatorError('Bad URL: %s' % url)
371 print "adding HTTP auth: %s, %s" %(user, password)
372 auth_handler.add_password(None, host, user, password)
373 tmphandlers.append(auth_handler)
374 url = scheme + "://" + host + path + parm + query + frag
375 if len(tmphandlers) != 0:
376 u2opener = u2.build_opener(*tmphandlers)
378 repo_eula_url = urlparse.urljoin(url, "LICENSE.txt")
379 repo_eula_path = self.__checkAndDownloadURL(
382 os.path.join(repo_lic_dir, repo.id + '_LICENSE.txt'))
391 # show the license file
392 print 'For the software packages in this yum repo:'
393 print ' %s: %s' % (name, baseurl)
394 print 'There is an "End User License Agreement" file that need to be checked.'
395 print 'Please read the terms and conditions outlined in it and answer the followed qustions.'
396 raw_input('press <ENTER> to continue...')
398 self.__pagerFile(repo_eula_path)
400 # Asking for the "Accept/Decline"
403 input_accept = raw_input('Would you agree to the terms and conditions outlined in the above End User License Agreement? (Yes/No): ')
404 if input_accept.upper() in ('YES', 'Y'):
406 elif input_accept.upper() in ('NO', 'N'):
408 print 'Will not install pkgs from this repo.'
412 shutil.rmtree(repo_lic_dir)
415 # try to find support_info.html for extra infomation
416 repo_info_url = urlparse.urljoin(baseurl, "support_info.html")
417 repo_info_path = self.__checkAndDownloadURL(
420 os.path.join(repo_lic_dir, repo.id + '_support_info.html'))
422 print 'There is one more file in the repo for additional support information, please read it'
423 raw_input('press <ENTER> to continue...')
424 self.__pagerFile(repo_info_path)
427 shutil.rmtree(repo_lic_dir)
430 def addRepository(self, name, url = None, mirrorlist = None, proxy = None, proxy_username = None, proxy_password = None, inc = None, exc = None):
431 if not self.repo_manager:
432 self.__initialize_repo_manager()
434 repo = RepositoryStub()
438 repo.proxy_username = proxy_username
439 repo.proxy_password = proxy_password
440 repo.baseurl.append(url)
443 repo_alias = name + "include"
446 repo_alias = name + "exclude"
449 # check LICENSE files
450 if not self.checkRepositoryEULA(name, repo):
454 repo.mirrorlist = mirrorlist
456 # Enable gpg check for verifying corrupt packages
458 self.repos.append(repo)
462 repo_info = zypp.RepoInfo()
463 repo_info.setAlias(repo_alias)
464 repo_info.setName(repo.name)
465 repo_info.setEnabled(repo.enabled)
466 repo_info.setAutorefresh(repo.autorefresh)
467 repo_info.setKeepPackages(repo.keeppackages)
468 repo_info.addBaseUrl(zypp.Url(repo.baseurl[0]))
469 self.repo_manager.addRepository(repo_info)
470 self.__build_repo_cache(name)
471 except RuntimeError, e:
472 raise CreatorError("%s" % (e,))
476 def installHasFile(self, file):
479 def runInstall(self, checksize = 0):
481 self.__selectIncpkgs()
483 self.__selectExcpkgs()
485 os.environ["HOME"] = "/"
486 self.buildTransaction()
488 todo = zypp.GetResolvablesToInsDel(self.Z.pool())
489 installed_pkgs = todo._toInstall
491 for item in installed_pkgs:
492 if not zypp.isKindPattern(item):
495 # record the total size of installed pkgs
496 pkgs_total_size = sum(map(lambda x: int(x.installSize()), dlpkgs))
498 # check needed size before actually download and install
499 if checksize and pkgs_total_size > checksize:
500 raise CreatorError("Size of specified root partition in kickstart file is too small to install all selected packages.")
502 if self.__recording_pkgs:
503 # record all pkg and the content
505 pkg_long_name = "%s-%s.%s.rpm" % (pkg.name(), pkg.edition(), pkg.arch())
506 self.__pkgs_content[pkg_long_name] = {} #TBD: to get file list
508 total_count = len(dlpkgs)
510 localpkgs = self.localpkgs.keys()
511 print "Checking packages cache and packages integrity..."
513 """ Check if it is cached locally """
514 if po.name() in localpkgs:
517 local = self.getLocalPkgPath(po)
518 if os.path.exists(local):
519 if self.checkPkg(local) != 0:
523 print "%d packages to be installed, %d packages gotten from cache, %d packages to be downloaded" % (total_count, cached_count, total_count - cached_count)
525 print "downloading packages..."
526 self.downloadPkgs(dlpkgs)
527 self.installPkgs(dlpkgs)
530 raise CreatorError("Unable to download from repo : %s" % (e,))
532 raise CreatorError("Unable to install: %s" % (e,))
534 def getAllContent(self):
535 return self.__pkgs_content
537 def __initialize_repo_manager(self):
538 if self.repo_manager:
541 """ Clean up repo metadata """
542 shutil.rmtree(self.creator.cachedir + "/var", ignore_errors = True)
543 shutil.rmtree(self.creator.cachedir + "/etc", ignore_errors = True)
544 shutil.rmtree(self.creator.cachedir + "/raw", ignore_errors = True)
545 shutil.rmtree(self.creator.cachedir + "/solv", ignore_errors = True)
547 zypp.KeyRing.setDefaultAccept( zypp.KeyRing.ACCEPT_UNSIGNED_FILE
548 | zypp.KeyRing.ACCEPT_VERIFICATION_FAILED
549 | zypp.KeyRing.ACCEPT_UNKNOWNKEY
550 | zypp.KeyRing.TRUST_KEY_TEMPORARILY
552 self.repo_manager_options = zypp.RepoManagerOptions(zypp.Pathname(self.creator._instroot))
553 self.repo_manager_options.knownReposPath = zypp.Pathname(self.creator.cachedir + "/etc/zypp/repos.d")
554 self.repo_manager_options.repoCachePath = zypp.Pathname(self.creator.cachedir + "/var/cache/zypp")
555 self.repo_manager_options.repoRawCachePath = zypp.Pathname(self.creator.cachedir + "/raw")
556 self.repo_manager_options.repoSolvCachePath = zypp.Pathname(self.creator.cachedir + "/solv")
557 self.repo_manager_options.repoPackagesCachePath = zypp.Pathname(self.creator.cachedir + "/packages")
559 self.repo_manager = zypp.RepoManager(self.repo_manager_options)
562 def __build_repo_cache(self, name):
563 repos = self.repo_manager.knownRepositories()
565 if not repo.enabled():
567 reponame = "%s" % repo.name()
570 if self.repo_manager.isCached( repo ):
572 #print "Retrieving repo metadata from %s ..." % repo.url()
573 self.repo_manager.buildCache( repo, zypp.RepoManager.BuildIfNeeded )
576 def __initialize_zypp(self):
580 zconfig = zypp.ZConfig_instance()
582 """ Set system architecture """
583 if self.creator.target_arch and self.creator.target_arch.startswith("arm"):
584 arches = ["armv7l", "armv7nhl", "armv7hl"]
585 if self.creator.target_arch not in arches:
586 raise CreatorError("Invalid architecture: %s" % self.creator.target_arch)
588 if self.creator.target_arch == "armv7l":
589 arch_map["armv7l"] = zypp.Arch_armv7l()
590 elif self.creator.target_arch == "armv7nhl":
591 arch_map["armv7nhl"] = zypp.Arch_armv7nhl()
592 elif self.creator.target_arch == "armv7hl":
593 arch_map["armv7hl"] = zypp.Arch_armv7hl()
594 zconfig.setSystemArchitecture(arch_map[self.creator.target_arch])
596 print "zypp architecture: %s" % zconfig.systemArchitecture()
598 """ repoPackagesCachePath is corrected by this """
599 self.repo_manager = zypp.RepoManager(self.repo_manager_options)
600 repos = self.repo_manager.knownRepositories()
602 if not repo.enabled():
604 if not self.repo_manager.isCached( repo ):
605 print "Retrieving repo metadata from %s ..." % repo.url()
606 self.repo_manager.buildCache( repo, zypp.RepoManager.BuildIfNeeded )
608 self.repo_manager.refreshMetadata(repo, zypp.RepoManager.BuildIfNeeded)
609 self.repo_manager.loadFromCache( repo );
611 self.Z = zypp.ZYppFactory_instance().getZYpp()
612 self.Z.initializeTarget( zypp.Pathname(self.creator._instroot) )
613 self.Z.target().load();
616 def buildTransaction(self):
617 if not self.Z.resolver().resolvePool():
618 print "Problem count: %d" % len(self.Z.resolver().problems())
619 for problem in self.Z.resolver().problems():
620 print "Problem: %s, %s" % (problem.description().decode("utf-8"), problem.details().decode("utf-8"))
622 def getLocalPkgPath(self, po):
623 repoinfo = po.repoInfo()
625 cacheroot = repoinfo.packagesPath()
627 edition = po.edition()
628 version = "%s-%s" % (edition.version(), edition.release())
629 pkgpath = "%s/%s/%s-%s.%s.rpm" % (cacheroot, arch, name, version, arch)
632 def installLocal(self, pkg, po=None, updateonly=False):
634 self.__initialize_transaction()
635 pkgname = self.__get_pkg_name(pkg)
636 self.localpkgs[pkgname] = pkg
637 self.selectPackage(pkgname)
639 def __get_pkg_name(self, pkgpath):
640 h = readRpmHeader(self.ts, pkgpath)
643 def downloadPkgs(self, package_objects):
644 localpkgs = self.localpkgs.keys()
645 for po in package_objects:
646 if po.name() in localpkgs:
648 filename = self.getLocalPkgPath(po)
649 if os.path.exists(filename):
650 if self.checkPkg(filename) == 0:
652 dir = os.path.dirname(filename)
653 if not os.path.exists(dir):
655 baseurl = po.repoInfo().baseUrls()[0].__str__()
656 proxy = self.get_proxy(po.repoInfo())
659 proxies = {str(proxy.split(":")[0]):str(proxy)}
661 location = zypp.asKindPackage(po).location()
662 location = location.filename().__str__()
663 if location.startswith("./"):
664 location = location[2:]
665 url = baseurl + "/%s" % location
667 filename = myurlgrab(url, filename, proxies)
668 except CreatorError, e:
670 raise CreatorError("%s" % e)
672 def installPkgs(self, package_objects):
674 self.__initialize_transaction()
678 for flag in self.probFilterFlags:
680 self.ts.setProbFilter(probfilter)
682 localpkgs = self.localpkgs.keys()
683 for po in package_objects:
685 if pkgname in localpkgs:
686 rpmpath = self.localpkgs[pkgname]
688 rpmpath = self.getLocalPkgPath(po)
689 if not os.path.exists(rpmpath):
690 """ Maybe it is a local repo """
691 baseurl = po.repoInfo().baseUrls()[0].__str__()
692 baseurl = baseurl.strip()
693 if baseurl.startswith("file:/"):
694 rpmpath = baseurl[5:] + "/%s/%s" % (po.arch(), os.path.basename(rpmpath))
695 if not os.path.exists(rpmpath):
696 raise RpmError("Error: %s doesn't exist" % rpmpath)
697 h = readRpmHeader(self.ts, rpmpath)
698 self.ts.addInstall(h, rpmpath, 'u')
700 unresolved_dependencies = self.ts.check()
701 if not unresolved_dependencies:
703 cb = RPMInstallCallback(self.ts)
704 self.ts.run(cb.callback, '')
708 print unresolved_dependencies
709 raise RepoError("Error: Unresolved dependencies, transaction failed.")
711 def __initialize_transaction(self):
713 self.ts = rpm.TransactionSet(self.creator._instroot)
714 # Set to not verify DSA signatures.
715 self.ts.setVSFlags(rpm._RPMVSF_NOSIGNATURES|rpm._RPMVSF_NODIGESTS)
717 def checkPkg(self, pkg):
719 if not os.path.exists(pkg):
721 ret = checkRpmIntegrity(self.bin_rpm, pkg)
723 print "Package %s is damaged: %s" % (os.path.basename(pkg), pkg)
726 def zypp_install(self):
727 policy = zypp.ZYppCommitPolicy()
728 policy.downloadMode(zypp.DownloadInAdvance)
729 policy.dryRun( False )
730 policy.syncPoolAfterCommit( False )
731 result = self.Z.commit( policy )
734 def _add_prob_flags(self, *flags):
736 if flag not in self.probFilterFlags:
737 self.probFilterFlags.append(flag)
739 def get_proxy(self, repoinfo):
741 reponame = "%s" % repoinfo.name()
742 for repo in self.repos:
743 if repo.name == reponame:
749 repourl = repoinfo.baseUrls()[0].__str__()
750 return get_proxy(repourl)
752 mic_plugin = ["zypp", Zypp]