13 from micng.utils.errors import *
14 from micng.imager.BaseImageCreator import ImageCreator
15 import pykickstart.parser
16 from micng.utils.fs_related import *
17 from micng.utils.misc import *
18 from micng.utils.rpmmisc import *
24 self.mirrorlist = None
26 self.proxy_username = None
27 self.proxy_password = None
28 self.includepkgs = None
29 self.includepkgs = None
33 self.autorefresh = True
34 self.keeppackages = True
36 class RepoError(CreatorError):
39 class RpmError(CreatorError):
43 def __init__(self, creator = None, recording_pkgs=None):
44 if not isinstance(creator, ImageCreator):
45 raise CreatorError("Invalid argument: creator")
47 self.__recording_pkgs = recording_pkgs
48 self.__pkgs_content = {}
49 self.creator = creator
54 self.repo_manager = None
55 self.repo_manager_options = None
58 self.probFilterFlags = []
59 self.bin_rpm = find_binary_path("rpm")
63 def doFileLogSetup(self, uid, logfile):
64 # don't do the file log for the livecd as it can lead to open fds
65 # being left and an inability to clean up after ourself
73 os.unlink(self.installroot + "/yum.conf")
77 if not os.path.exists("/etc/fedora-release") and not os.path.exists("/etc/meego-release"):
78 for i in range(3, os.sysconf("SC_OPEN_MAX")):
90 def _writeConf(self, confpath, installroot):
92 conf += "installroot=%s\n" % installroot
93 conf += "cachedir=/var/cache/yum\n"
96 conf += "failovermethod=priority\n"
97 conf += "http_caching=packages\n"
99 f = file(confpath, "w+")
103 os.chmod(confpath, 0644)
105 def _cleanupRpmdbLocks(self, installroot):
106 # cleans up temporary files left by bdb so that differing
107 # versions of rpm don't cause problems
108 for f in glob.glob(installroot + "/var/lib/rpm/__db*"):
111 def setup(self, confpath, installroot):
112 self._writeConf(confpath, installroot)
113 self._cleanupRpmdbLocks(installroot)
114 self.installroot = installroot
116 def selectPackage(self, pkg):
117 """ Select a given package or package pattern, can be specified with name.arch or name* or *name """
119 self.__initialize_zypp()
122 startx = pkg.startswith("*")
123 endx = pkg.endswith("*")
124 ispattern = startx or endx
125 sp = pkg.rsplit(".", 2)
126 for item in self.Z.pool():
127 kind = "%s" % item.kind()
128 if kind == "package":
129 name = "%s" % item.name()
131 if name in self.incpkgs or self.excpkgs:
135 arch = "%s" % item.arch()
136 if name == sp[0] and arch == sp[1]:
138 if name not in self.packages:
139 self.packages.append(name)
140 item.status().setToBeInstalled (zypp.ResStatus.USER)
145 if name not in self.packages:
146 self.packages.append(name)
147 item.status().setToBeInstalled (zypp.ResStatus.USER)
150 if name in self.incpkgs or self.excpkgs:
153 if startx and name.endswith(sp[0][1:]):
155 if name not in self.packages:
156 self.packages.append(name)
157 item.status().setToBeInstalled (zypp.ResStatus.USER)
159 if endx and name.startswith(sp[0][:-1]):
161 if name not in self.packages:
162 self.packages.append(name)
163 item.status().setToBeInstalled (zypp.ResStatus.USER)
167 e = CreatorError("Unable to find package: %s" % (pkg,))
170 def deselectPackage(self, pkg):
171 """Deselect package. Can be specified as name.arch or name*"""
174 self.__initialize_zypp()
176 startx = pkg.startswith("*")
177 endx = pkg.endswith("*")
178 ispattern = startx or endx
179 sp = pkg.rsplit(".", 2)
180 for item in self.Z.pool():
181 kind = "%s" % item.kind()
182 if kind == "package":
183 name = "%s" % item.name()
186 arch = "%s" % item.arch()
187 if name == sp[0] and arch == sp[1]:
188 if item.status().isToBeInstalled():
189 item.status().resetTransact(zypp.ResStatus.USER)
190 if name in self.packages:
191 self.packages.remove(name)
195 if item.status().isToBeInstalled():
196 item.status().resetTransact(zypp.ResStatus.USER)
197 if name in self.packages:
198 self.packages.remove(name)
201 if startx and name.endswith(sp[0][1:]):
202 if item.status().isToBeInstalled():
203 item.status().resetTransact(zypp.ResStatus.USER)
204 if name in self.packages:
205 self.packages.remove(name)
207 if endx and name.startswith(sp[0][:-1]):
208 if item.status().isToBeInstalled():
209 item.status().resetTransact(zypp.ResStatus.USER)
210 if name in self.packages:
211 self.packages.remove(name)
213 def __selectIncpkgs(self):
215 for pkg in self.incpkgs:
216 for item in self.Z.pool():
217 kind = "%s" % item.kind()
218 if kind == "package":
219 name = "%s" % item.name()
220 repoalias = "%s" % item.repoInfo().alias()
221 if name == pkg and repoalias.endswith("include"):
223 if name not in self.packages:
224 self.packages.append(name)
225 item.status().setToBeInstalled (zypp.ResStatus.USER)
228 raise CreatorError("Unable to find package: %s" % (pkg,))
230 def __selectExcpkgs(self):
232 for pkg in self.excpkgs:
233 for item in self.Z.pool():
234 kind = "%s" % item.kind()
235 if kind == "package":
236 name = "%s" % item.name()
237 repoalias = "%s" % item.repoInfo().alias()
238 if name == pkg and not repoalias.endswith("exclude"):
240 if name not in self.packages:
241 self.packages.append(name)
242 item.status().setToBeInstalled (zypp.ResStatus.USER)
245 raise CreatorError("Unable to find package: %s" % (pkg,))
248 def selectGroup(self, grp, include = pykickstart.parser.GROUP_DEFAULT):
250 self.__initialize_zypp()
252 for item in self.Z.pool():
253 kind = "%s" % item.kind()
254 if kind == "pattern":
255 summary = "%s" % item.summary()
256 name = "%s" % item.name()
257 if name == grp or summary == grp:
259 if name not in self.patterns:
260 self.patterns.append(name)
261 item.status().setToBeInstalled (zypp.ResStatus.USER)
265 if include == pykickstart.parser.GROUP_REQUIRED:
266 map(lambda p: self.deselectPackage(p), grp.default_packages.keys())
267 elif include == pykickstart.parser.GROUP_ALL:
268 map(lambda p: self.selectPackage(p), grp.optional_packages.keys())
271 e = CreatorError("Unable to find pattern: %s" % (grp,))
274 def __checkAndDownloadURL(self, u2opener, url, savepath):
277 f = u2opener.open(url)
280 except u2.HTTPError, httperror:
281 if httperror.code in (404, 503):
284 raise CreatorError(httperror)
285 except OSError, oserr:
289 raise CreatorError(oserr)
290 except IOError, oserr:
291 if hasattr(oserr, "reason") and oserr.reason.errno == 2:
294 raise CreatorError(oserr)
295 except u2.URLError, err:
296 raise CreatorError(err)
299 licf = open(savepath, "w")
306 def __pagerFile(self, savepath):
307 if os.path.splitext(savepath)[1].upper() in ('.HTM', '.HTML'):
308 pagers = ('w3m', 'links', 'lynx', 'less', 'more')
310 pagers = ('less', 'more')
315 subprocess.call([pager, savepath])
325 raw_input('press <ENTER> to continue...')
327 def checkRepositoryEULA(self, name, repo):
328 """ This function is to check the LICENSE file if provided. """
330 # when proxy needed, make urllib2 follow it
332 proxy_username = repo.proxy_username
333 proxy_password = repo.proxy_password
336 auth_handler = u2.HTTPBasicAuthHandler(u2.HTTPPasswordMgrWithDefaultRealm())
340 proxy_netloc = urlparse.urlsplit(proxy).netloc
342 proxy_url = 'http://%s:%s@%s' % (proxy_username, proxy_password, proxy_netloc)
344 proxy_url = 'http://%s@%s' % (proxy_username, proxy_netloc)
348 proxy_support = u2.ProxyHandler({'http': proxy_url,
350 handlers.append(proxy_support)
352 # download all remote files to one temp dir
354 repo_lic_dir = tempfile.mkdtemp(prefix = 'repolic')
356 for url in repo.baseurl:
357 if not url.endswith('/'):
359 tmphandlers = handlers
360 (scheme, host, path, parm, query, frag) = urlparse.urlparse(url)
361 if scheme not in ("http", "https", "ftp", "ftps", "file"):
362 raise CreatorError("Error: invalid url %s" % url)
365 user_pass, host = host.split('@', 1)
367 user, password = user_pass.split(':', 1)
368 except ValueError, e:
369 raise CreatorError('Bad URL: %s' % url)
370 print "adding HTTP auth: %s, %s" %(user, password)
371 auth_handler.add_password(None, host, user, password)
372 tmphandlers.append(auth_handler)
373 url = scheme + "://" + host + path + parm + query + frag
374 if len(tmphandlers) != 0:
375 u2opener = u2.build_opener(*tmphandlers)
377 repo_eula_url = urlparse.urljoin(url, "LICENSE.txt")
378 repo_eula_path = self.__checkAndDownloadURL(
381 os.path.join(repo_lic_dir, repo.id + '_LICENSE.txt'))
390 # show the license file
391 print 'For the software packages in this yum repo:'
392 print ' %s: %s' % (name, baseurl)
393 print 'There is an "End User License Agreement" file that need to be checked.'
394 print 'Please read the terms and conditions outlined in it and answer the followed qustions.'
395 raw_input('press <ENTER> to continue...')
397 self.__pagerFile(repo_eula_path)
399 # Asking for the "Accept/Decline"
402 input_accept = raw_input('Would you agree to the terms and conditions outlined in the above End User License Agreement? (Yes/No): ')
403 if input_accept.upper() in ('YES', 'Y'):
405 elif input_accept.upper() in ('NO', 'N'):
407 print 'Will not install pkgs from this repo.'
411 shutil.rmtree(repo_lic_dir)
414 # try to find support_info.html for extra infomation
415 repo_info_url = urlparse.urljoin(baseurl, "support_info.html")
416 repo_info_path = self.__checkAndDownloadURL(
419 os.path.join(repo_lic_dir, repo.id + '_support_info.html'))
421 print 'There is one more file in the repo for additional support information, please read it'
422 raw_input('press <ENTER> to continue...')
423 self.__pagerFile(repo_info_path)
426 shutil.rmtree(repo_lic_dir)
429 def addRepository(self, name, url = None, mirrorlist = None, proxy = None, proxy_username = None, proxy_password = None, inc = None, exc = None):
430 if not self.repo_manager:
431 self.__initialize_repo_manager()
433 repo = RepositoryStub()
437 repo.proxy_username = proxy_username
438 repo.proxy_password = proxy_password
439 repo.baseurl.append(url)
442 repo_alias = name + "include"
445 repo_alias = name + "exclude"
448 # check LICENSE files
449 if not self.checkRepositoryEULA(name, repo):
453 repo.mirrorlist = mirrorlist
455 # Enable gpg check for verifying corrupt packages
457 self.repos.append(repo)
461 repo_info = zypp.RepoInfo()
462 repo_info.setAlias(repo_alias)
463 repo_info.setName(repo.name)
464 repo_info.setEnabled(repo.enabled)
465 repo_info.setAutorefresh(repo.autorefresh)
466 repo_info.setKeepPackages(repo.keeppackages)
467 repo_info.addBaseUrl(zypp.Url(repo.baseurl[0]))
468 self.repo_manager.addRepository(repo_info)
469 self.__build_repo_cache(name)
470 except RuntimeError, e:
471 raise CreatorError("%s" % (e,))
475 def installHasFile(self, file):
478 def runInstall(self, checksize = 0):
480 self.__selectIncpkgs()
482 self.__selectExcpkgs()
484 os.environ["HOME"] = "/"
485 self.buildTransaction()
487 todo = zypp.GetResolvablesToInsDel(self.Z.pool())
488 installed_pkgs = todo._toInstall
490 for item in installed_pkgs:
491 if not zypp.isKindPattern(item):
494 # record the total size of installed pkgs
495 pkgs_total_size = sum(map(lambda x: int(x.installSize()), dlpkgs))
497 # check needed size before actually download and install
498 if checksize and pkgs_total_size > checksize:
499 raise CreatorError("Size of specified root partition in kickstart file is too small to install all selected packages.")
501 if self.__recording_pkgs:
502 # record all pkg and the content
504 pkg_long_name = "%s-%s.%s.rpm" % (pkg.name(), pkg.edition(), pkg.arch())
505 self.__pkgs_content[pkg_long_name] = {} #TBD: to get file list
507 total_count = len(dlpkgs)
509 localpkgs = self.localpkgs.keys()
510 print "Checking packages cache and packages integrity..."
512 """ Check if it is cached locally """
513 if po.name() in localpkgs:
516 local = self.getLocalPkgPath(po)
517 if os.path.exists(local):
518 if self.checkPkg(local) != 0:
522 print "%d packages to be installed, %d packages gotten from cache, %d packages to be downloaded" % (total_count, cached_count, total_count - cached_count)
524 print "downloading packages..."
525 self.downloadPkgs(dlpkgs)
526 self.installPkgs(dlpkgs)
529 raise CreatorError("Unable to download from repo : %s" % (e,))
531 raise CreatorError("Unable to install: %s" % (e,))
533 def getAllContent(self):
534 return self.__pkgs_content
536 def __initialize_repo_manager(self):
537 if self.repo_manager:
540 """ Clean up repo metadata """
541 shutil.rmtree(self.creator.cachedir + "/var", ignore_errors = True)
542 shutil.rmtree(self.creator.cachedir + "/etc", ignore_errors = True)
543 shutil.rmtree(self.creator.cachedir + "/raw", ignore_errors = True)
544 shutil.rmtree(self.creator.cachedir + "/solv", ignore_errors = True)
546 zypp.KeyRing.setDefaultAccept( zypp.KeyRing.ACCEPT_UNSIGNED_FILE
547 | zypp.KeyRing.ACCEPT_VERIFICATION_FAILED
548 | zypp.KeyRing.ACCEPT_UNKNOWNKEY
549 | zypp.KeyRing.TRUST_KEY_TEMPORARILY
551 self.repo_manager_options = zypp.RepoManagerOptions(zypp.Pathname(self.creator._instroot))
552 self.repo_manager_options.knownReposPath = zypp.Pathname(self.creator.cachedir + "/etc/zypp/repos.d")
553 self.repo_manager_options.repoCachePath = zypp.Pathname(self.creator.cachedir + "/var/cache/zypp")
554 self.repo_manager_options.repoRawCachePath = zypp.Pathname(self.creator.cachedir + "/raw")
555 self.repo_manager_options.repoSolvCachePath = zypp.Pathname(self.creator.cachedir + "/solv")
556 self.repo_manager_options.repoPackagesCachePath = zypp.Pathname(self.creator.cachedir + "/packages")
558 self.repo_manager = zypp.RepoManager(self.repo_manager_options)
561 def __build_repo_cache(self, name):
562 repos = self.repo_manager.knownRepositories()
564 if not repo.enabled():
566 reponame = "%s" % repo.name()
569 if self.repo_manager.isCached( repo ):
571 #print "Retrieving repo metadata from %s ..." % repo.url()
572 self.repo_manager.buildCache( repo, zypp.RepoManager.BuildIfNeeded )
575 def __initialize_zypp(self):
579 zconfig = zypp.ZConfig_instance()
581 """ Set system architecture """
582 if self.creator.target_arch and self.creator.target_arch.startswith("arm"):
583 arches = ["armv7l", "armv7nhl", "armv7hl"]
584 if self.creator.target_arch not in arches:
585 raise CreatorError("Invalid architecture: %s" % self.creator.target_arch)
587 if self.creator.target_arch == "armv7l":
588 arch_map["armv7l"] = zypp.Arch_armv7l()
589 elif self.creator.target_arch == "armv7nhl":
590 arch_map["armv7nhl"] = zypp.Arch_armv7nhl()
591 elif self.creator.target_arch == "armv7hl":
592 arch_map["armv7hl"] = zypp.Arch_armv7hl()
593 zconfig.setSystemArchitecture(arch_map[self.creator.target_arch])
595 print "zypp architecture: %s" % zconfig.systemArchitecture()
597 """ repoPackagesCachePath is corrected by this """
598 self.repo_manager = zypp.RepoManager(self.repo_manager_options)
599 repos = self.repo_manager.knownRepositories()
601 if not repo.enabled():
603 if not self.repo_manager.isCached( repo ):
604 print "Retrieving repo metadata from %s ..." % repo.url()
605 self.repo_manager.buildCache( repo, zypp.RepoManager.BuildIfNeeded )
607 self.repo_manager.refreshMetadata(repo, zypp.RepoManager.BuildIfNeeded)
608 self.repo_manager.loadFromCache( repo );
610 self.Z = zypp.ZYppFactory_instance().getZYpp()
611 self.Z.initializeTarget( zypp.Pathname(self.creator._instroot) )
612 self.Z.target().load();
615 def buildTransaction(self):
616 if not self.Z.resolver().resolvePool():
617 print "Problem count: %d" % len(self.Z.resolver().problems())
618 for problem in self.Z.resolver().problems():
619 print "Problem: %s, %s" % (problem.description().decode("utf-8"), problem.details().decode("utf-8"))
621 def getLocalPkgPath(self, po):
622 repoinfo = po.repoInfo()
624 cacheroot = repoinfo.packagesPath()
626 edition = po.edition()
627 version = "%s-%s" % (edition.version(), edition.release())
628 pkgpath = "%s/%s/%s-%s.%s.rpm" % (cacheroot, arch, name, version, arch)
631 def installLocal(self, pkg, po=None, updateonly=False):
633 self.__initialize_transaction()
634 pkgname = self.__get_pkg_name(pkg)
635 self.localpkgs[pkgname] = pkg
636 self.selectPackage(pkgname)
638 def __get_pkg_name(self, pkgpath):
639 h = readRpmHeader(self.ts, pkgpath)
642 def downloadPkgs(self, package_objects):
643 localpkgs = self.localpkgs.keys()
644 for po in package_objects:
645 if po.name() in localpkgs:
647 filename = self.getLocalPkgPath(po)
648 if os.path.exists(filename):
649 if self.checkPkg(filename) == 0:
651 dir = os.path.dirname(filename)
652 if not os.path.exists(dir):
654 baseurl = po.repoInfo().baseUrls()[0].__str__()
655 proxy = self.get_proxy(po.repoInfo())
658 proxies = {str(proxy.split(":")[0]):str(proxy)}
660 location = zypp.asKindPackage(po).location()
661 location = location.filename().__str__()
662 if location.startswith("./"):
663 location = location[2:]
664 url = baseurl + "/%s" % location
666 filename = myurlgrab(url, filename, proxies)
667 except CreatorError, e:
669 raise CreatorError("%s" % e)
671 def installPkgs(self, package_objects):
673 self.__initialize_transaction()
677 for flag in self.probFilterFlags:
679 self.ts.setProbFilter(probfilter)
681 localpkgs = self.localpkgs.keys()
682 for po in package_objects:
684 if pkgname in localpkgs:
685 rpmpath = self.localpkgs[pkgname]
687 rpmpath = self.getLocalPkgPath(po)
688 if not os.path.exists(rpmpath):
689 """ Maybe it is a local repo """
690 baseurl = po.repoInfo().baseUrls()[0].__str__()
691 baseurl = baseurl.strip()
692 if baseurl.startswith("file:/"):
693 rpmpath = baseurl[5:] + "/%s/%s" % (po.arch(), os.path.basename(rpmpath))
694 if not os.path.exists(rpmpath):
695 raise RpmError("Error: %s doesn't exist" % rpmpath)
696 h = readRpmHeader(self.ts, rpmpath)
697 self.ts.addInstall(h, rpmpath, 'u')
699 unresolved_dependencies = self.ts.check()
700 if not unresolved_dependencies:
702 cb = RPMInstallCallback(self.ts)
703 self.ts.run(cb.callback, '')
707 print unresolved_dependencies
708 raise RepoError("Error: Unresolved dependencies, transaction failed.")
710 def __initialize_transaction(self):
712 self.ts = rpm.TransactionSet(self.creator._instroot)
713 # Set to not verify DSA signatures.
714 self.ts.setVSFlags(rpm._RPMVSF_NOSIGNATURES|rpm._RPMVSF_NODIGESTS)
716 def checkPkg(self, pkg):
718 if not os.path.exists(pkg):
720 ret = checkRpmIntegrity(self.bin_rpm, pkg)
722 print "Package %s is damaged: %s" % (os.path.basename(pkg), pkg)
725 def zypp_install(self):
726 policy = zypp.ZYppCommitPolicy()
727 policy.downloadMode(zypp.DownloadInAdvance)
728 policy.dryRun( False )
729 policy.syncPoolAfterCommit( False )
730 result = self.Z.commit( policy )
733 def _add_prob_flags(self, *flags):
735 if flag not in self.probFilterFlags:
736 self.probFilterFlags.append(flag)
738 def get_proxy(self, repoinfo):
740 reponame = "%s" % repoinfo.name()
741 for repo in self.repos:
742 if repo.name == reponame:
748 repourl = repoinfo.baseUrls()[0].__str__()
749 return get_proxy(repourl)
751 _pkgmgr = ["zypp", Zypp]