plugins/backend/zypppkgmgr.py

   1 #!/usr/bin/python -tt
   2 #
   3 # Copyright 2010, 2011 Intel, Inc.
   4 #
   5 # This copyrighted material is made available to anyone wishing to use, modify,
   6 # copy, or redistribute it subject to the terms and conditions of the GNU
   7 # General Public License v.2.  This program is distributed in the hope that it
   8 # will be useful, but WITHOUT ANY WARRANTY expressed or implied, including the
   9 # implied warranties of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
  10 # See the GNU General Public License for more details.
  11 #
  12 # You should have received a copy of the GNU General Public License along with
  13 # this program; if not, write to the Free Software Foundation, Inc., 51
  14 # Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.  Any Red Hat
  15 # trademarks that are incorporated in the source code or documentation are not
  16 # subject to the GNU General Public License and may only be used or replicated
  17 # with the express permission of Red Hat, Inc.
  18 #
  19
  20 import os
  21 import zypp
  22 import rpm
  23 import shutil
  24
  25 from pykickstart import parser as ksparser
  26
  27 from mic import msger
  28 from mic.imager.baseimager import BaseImageCreator
  29 from mic.utils import rpmmisc, fs_related as fs
  30 from mic.utils.proxy import get_proxy_for
  31 from mic.utils.errors import CreatorError
  32
  33 class RepositoryStub:
  34     def __init__(self):
  35         self.name = None
  36         self.baseurl = []
  37         self.mirrorlist = None
  38         self.proxy = None
  39         self.proxy_username = None
  40         self.proxy_password = None
  41         self.includepkgs = None
  42         self.includepkgs = None
  43         self.exclude = None
  44
  45         self.enabled = True
  46         self.autorefresh = True
  47         self.keeppackages = True
  48
  49 class RepoError(CreatorError):
  50     pass
  51
  52 class RpmError(CreatorError):
  53     pass
  54
  55 from mic.pluginbase import BackendPlugin
  56 class Zypp(BackendPlugin):
  57     name = 'zypp'
  58
  59     def __init__(self, creator = None, recording_pkgs=None):
  60         if not isinstance(creator, BaseImageCreator):
  61             raise CreatorError("Invalid argument: creator")
  62
  63         self.__recording_pkgs = recording_pkgs
  64         self.__pkgs_content = {}
  65         self.creator = creator
  66         self.repos = []
  67         self.packages = []
  68         self.patterns = []
  69         self.localpkgs = {}
  70         self.repo_manager = None
  71         self.repo_manager_options = None
  72         self.Z = None
  73         self.ts = None
  74         self.probFilterFlags = []
  75         self.incpkgs = []
  76         self.excpkgs = []
  77
  78     def doFileLogSetup(self, uid, logfile):
  79         # don't do the file log for the livecd as it can lead to open fds
  80         # being left and an inability to clean up after ourself
  81         pass
  82
  83     def closeRpmDB(self):
  84         pass
  85
  86     def close(self):
  87         try:
  88             os.unlink(self.installroot + "/yum.conf")
  89         except:
  90             pass
  91         self.closeRpmDB()
  92         if not os.path.exists("/etc/fedora-release") and not os.path.exists("/etc/meego-release"):
  93             for i in range(3, os.sysconf("SC_OPEN_MAX")):
  94                 try:
  95                     os.close(i)
  96                 except:
  97                     pass
  98         if self.ts:
  99             self.ts.closeDB()
 100             self.ts = None
 101
 102     def __del__(self):
 103         self.close()
 104
 105     def _writeConf(self, confpath, installroot):
 106         conf  = "[main]\n"
 107         conf += "installroot=%s\n" % installroot
 108         conf += "cachedir=/var/cache/yum\n"
 109         conf += "plugins=0\n"
 110         conf += "reposdir=\n"
 111         conf += "failovermethod=priority\n"
 112         conf += "http_caching=packages\n"
 113
 114         f = file(confpath, "w+")
 115         f.write(conf)
 116         f.close()
 117
 118         os.chmod(confpath, 0644)
 119
 120     def _cleanupRpmdbLocks(self, installroot):
 121         # cleans up temporary files left by bdb so that differing
 122         # versions of rpm don't cause problems
 123         import glob
 124         for f in glob.glob(installroot + "/var/lib/rpm/__db*"):
 125             os.unlink(f)
 126
 127     def setup(self, confpath, installroot):
 128         self._writeConf(confpath, installroot)
 129         self._cleanupRpmdbLocks(installroot)
 130         self.installroot = installroot
 131
 132     def selectPackage(self, pkg):
 133         """ Select a given package or package pattern, can be specified with name.arch or name* or *name """
 134         if not self.Z:
 135             self.__initialize_zypp()
 136
 137         found = False
 138         startx = pkg.startswith("*")
 139         endx = pkg.endswith("*")
 140         ispattern = startx or endx
 141         sp = pkg.rsplit(".", 2)
 142         for item in self.Z.pool():
 143             kind = "%s" % item.kind()
 144             if kind == "package":
 145                 name = "%s" % item.name()
 146                 if not ispattern:
 147                     if name in self.incpkgs or self.excpkgs:
 148                         found = True
 149                         break
 150                     if len(sp) == 2:
 151                         arch = "%s" % item.arch()
 152                         if name == sp[0] and arch == sp[1]:
 153                             found = True
 154                             if name not in self.packages:
 155                                 self.packages.append(name)
 156                                 item.status().setToBeInstalled (zypp.ResStatus.USER)
 157                             break
 158                     else:
 159                         if name == sp[0]:
 160                             found = True
 161                             if name not in self.packages:
 162                                 self.packages.append(name)
 163                                 item.status().setToBeInstalled (zypp.ResStatus.USER)
 164                             break
 165                 else:
 166                     if name in self.incpkgs or self.excpkgs:
 167                         found =  True
 168                         continue
 169                     if startx and name.endswith(sp[0][1:]):
 170                         found = True
 171                         if name not in self.packages:
 172                             self.packages.append(name)
 173                             item.status().setToBeInstalled (zypp.ResStatus.USER)
 174
 175                     if endx and name.startswith(sp[0][:-1]):
 176                         found = True
 177                         if name not in self.packages:
 178                             self.packages.append(name)
 179                             item.status().setToBeInstalled (zypp.ResStatus.USER)
 180         if found:
 181             return None
 182         else:
 183             e = CreatorError("Unable to find package: %s" % (pkg,))
 184             return e
 185
 186     def deselectPackage(self, pkg):
 187         """Deselect package.  Can be specified as name.arch or name*"""
 188
 189         if not self.Z:
 190             self.__initialize_zypp()
 191
 192         startx = pkg.startswith("*")
 193         endx = pkg.endswith("*")
 194         ispattern = startx or endx
 195         sp = pkg.rsplit(".", 2)
 196         for item in self.Z.pool():
 197             kind = "%s" % item.kind()
 198             if kind == "package":
 199                 name = "%s" % item.name()
 200                 if not ispattern:
 201                     if len(sp) == 2:
 202                         arch = "%s" % item.arch()
 203                         if name == sp[0] and arch == sp[1]:
 204                             if item.status().isToBeInstalled():
 205                                 item.status().resetTransact(zypp.ResStatus.USER)
 206                             if name in self.packages:
 207                                 self.packages.remove(name)
 208                             break
 209                     else:
 210                         if name == sp[0]:
 211                             if item.status().isToBeInstalled():
 212                                 item.status().resetTransact(zypp.ResStatus.USER)
 213                             if name in self.packages:
 214                                 self.packages.remove(name)
 215                             break
 216                 else:
 217                     if startx and name.endswith(sp[0][1:]):
 218                         if item.status().isToBeInstalled():
 219                             item.status().resetTransact(zypp.ResStatus.USER)
 220                         if name in self.packages:
 221                             self.packages.remove(name)
 222
 223                     if endx and name.startswith(sp[0][:-1]):
 224                         if item.status().isToBeInstalled():
 225                             item.status().resetTransact(zypp.ResStatus.USER)
 226                         if name in self.packages:
 227                             self.packages.remove(name)
 228
 229     def __selectIncpkgs(self):
 230         found = False
 231         for pkg in self.incpkgs:
 232             for item in self.Z.pool():
 233                 kind = "%s" % item.kind()
 234                 if kind == "package":
 235                     name = "%s" % item.name()
 236                     repoalias = "%s" % item.repoInfo().alias()
 237                     if name == pkg and repoalias.endswith("include"):
 238                         found = True
 239                         if name not in self.packages:
 240                             self.packages.append(name)
 241                             item.status().setToBeInstalled (zypp.ResStatus.USER)
 242                         break
 243         if not found:
 244             raise CreatorError("Unable to find package: %s" % (pkg,))
 245
 246     def __selectExcpkgs(self):
 247         found = False
 248         for pkg in self.excpkgs:
 249             for item in self.Z.pool():
 250                 kind = "%s" % item.kind()
 251                 if kind == "package":
 252                     name = "%s" % item.name()
 253                     repoalias = "%s" % item.repoInfo().alias()
 254                     if name == pkg and not repoalias.endswith("exclude"):
 255                         found = True
 256                         if name not in self.packages:
 257                             self.packages.append(name)
 258                             item.status().setToBeInstalled (zypp.ResStatus.USER)
 259                         break
 260         if not found:
 261             raise CreatorError("Unable to find package: %s" % (pkg,))
 262
 263     def selectGroup(self, grp, include = ksparser.GROUP_DEFAULT):
 264         if not self.Z:
 265             self.__initialize_zypp()
 266         found = False
 267         for item in self.Z.pool():
 268             kind = "%s" % item.kind()
 269             if kind == "pattern":
 270                 summary = "%s" % item.summary()
 271                 name = "%s" % item.name()
 272                 if name == grp or summary == grp:
 273                     found = True
 274                     if name not in self.patterns:
 275                         self.patterns.append(name)
 276                         item.status().setToBeInstalled (zypp.ResStatus.USER)
 277                     break
 278
 279         if found:
 280             if include == ksparser.GROUP_REQUIRED:
 281                 map(lambda p: self.deselectPackage(p), grp.default_packages.keys())
 282             return None
 283         else:
 284             e = CreatorError("Unable to find pattern: %s" % (grp,))
 285             return e
 286
 287     def addRepository(self, name, url = None, mirrorlist = None, proxy = None, proxy_username = None, proxy_password = None, inc = None, exc = None):
 288         if not self.repo_manager:
 289             self.__initialize_repo_manager()
 290
 291         repo = RepositoryStub()
 292         repo.name = name
 293         repo.id = name
 294         repo.proxy = proxy
 295         repo.proxy_username = proxy_username
 296         repo.proxy_password = proxy_password
 297         repo.baseurl.append(url)
 298         repo_alias = repo.id
 299         if inc:
 300             repo_alias = name + "include"
 301             self.incpkgs = inc
 302         if exc:
 303             repo_alias = name + "exclude"
 304             self.excpkgs = exc
 305
 306         # check LICENSE files
 307         if not rpmmisc.checkRepositoryEULA(name, repo):
 308             msger.warning('skip repo:%s for failed EULA confirmation' % name)
 309             return None
 310
 311         if mirrorlist:
 312             repo.mirrorlist = mirrorlist
 313
 314         # Enable gpg check for verifying corrupt packages
 315         repo.gpgcheck = 1
 316         self.repos.append(repo)
 317
 318         try:
 319             repo_info = zypp.RepoInfo()
 320             repo_info.setAlias(repo_alias)
 321             repo_info.setName(repo.name)
 322             repo_info.setEnabled(repo.enabled)
 323             repo_info.setAutorefresh(repo.autorefresh)
 324             repo_info.setKeepPackages(repo.keeppackages)
 325             repo_info.addBaseUrl(zypp.Url(repo.baseurl[0]))
 326             self.repo_manager.addRepository(repo_info)
 327             self.__build_repo_cache(name)
 328         except RuntimeError, e:
 329             raise CreatorError(str(e))
 330
 331         msger.verbose('repo: %s was added' % name)
 332         return repo
 333
 334     def installHasFile(self, file):
 335         return False
 336
 337     def runInstall(self, checksize = 0):
 338         if self.incpkgs:
 339             self.__selectIncpkgs()
 340         if self.excpkgs:
 341             self.__selectExcpkgs()
 342
 343         os.environ["HOME"] = "/"
 344         self.buildTransaction()
 345
 346         todo = zypp.GetResolvablesToInsDel(self.Z.pool())
 347         installed_pkgs = todo._toInstall
 348         dlpkgs = []
 349         for item in installed_pkgs:
 350             if not zypp.isKindPattern(item):
 351                 dlpkgs.append(item)
 352
 353         # record the total size of installed pkgs
 354         pkgs_total_size = sum(map(lambda x: int(x.installSize()), dlpkgs))
 355
 356         # check needed size before actually download and install
 357         if checksize and pkgs_total_size > checksize:
 358             raise CreatorError("Size of specified root partition in kickstart file is too small to install all selected packages.")
 359
 360         if self.__recording_pkgs:
 361             # record all pkg and the content
 362             for pkg in dlpkgs:
 363                 pkg_long_name = "%s-%s.%s.rpm" % (pkg.name(), pkg.edition(), pkg.arch())
 364                 self.__pkgs_content[pkg_long_name] = {} #TBD: to get file list
 365
 366         total_count = len(dlpkgs)
 367         cached_count = 0
 368         localpkgs = self.localpkgs.keys()
 369         msger.info("Checking packages cache and packages integrity ...")
 370         for po in dlpkgs:
 371             """ Check if it is cached locally """
 372             if po.name() in localpkgs:
 373                 cached_count += 1
 374             else:
 375                 local = self.getLocalPkgPath(po)
 376                 if os.path.exists(local):
 377                     if self.checkPkg(local) != 0:
 378                         os.unlink(local)
 379                     else:
 380                         cached_count += 1
 381         download_count =  total_count - cached_count
 382         msger.info("%d packages to be installed, %d packages gotten from cache, %d packages to be downloaded" % (total_count, cached_count, download_count))
 383         try:
 384             if download_count > 0:
 385                 msger.info("Downloading packages ...")
 386             self.downloadPkgs(dlpkgs, download_count)
 387             self.installPkgs(dlpkgs)
 388
 389         except RepoError, e:
 390             raise CreatorError("Unable to download from repo : %s" % (e,))
 391         except RpmError, e:
 392             raise CreatorError("Unable to install: %s" % (e,))
 393
 394     def getAllContent(self):
 395         return self.__pkgs_content
 396
 397     def __initialize_repo_manager(self):
 398         if self.repo_manager:
 399             return
 400
 401         """ Clean up repo metadata """
 402         shutil.rmtree(self.creator.cachedir + "/etc", ignore_errors = True)
 403
 404         zypp.KeyRing.setDefaultAccept( zypp.KeyRing.ACCEPT_UNSIGNED_FILE
 405                                      | zypp.KeyRing.ACCEPT_VERIFICATION_FAILED
 406                                      | zypp.KeyRing.ACCEPT_UNKNOWNKEY
 407                                      | zypp.KeyRing.TRUST_KEY_TEMPORARILY
 408                                      )
 409         self.repo_manager_options = zypp.RepoManagerOptions(zypp.Pathname(self.creator._instroot))
 410         self.repo_manager_options.knownReposPath = zypp.Pathname(self.creator.cachedir + "/etc/zypp/repos.d")
 411         self.repo_manager_options.repoCachePath = zypp.Pathname(self.creator.cachedir)
 412         self.repo_manager_options.repoRawCachePath = zypp.Pathname(self.creator.cachedir + "/raw")
 413         self.repo_manager_options.repoSolvCachePath = zypp.Pathname(self.creator.cachedir + "/solv")
 414         self.repo_manager_options.repoPackagesCachePath = zypp.Pathname(self.creator.cachedir + "/packages")
 415
 416         self.repo_manager = zypp.RepoManager(self.repo_manager_options)
 417
 418     def __build_repo_cache(self, name):
 419         repos = self.repo_manager.knownRepositories()
 420         for repo in repos:
 421             if not repo.enabled():
 422                 continue
 423             reponame = "%s" % repo.name()
 424             if reponame != name:
 425                 continue
 426             if self.repo_manager.isCached( repo ):
 427                 return
 428
 429             self.repo_manager.buildCache(repo, zypp.RepoManager.BuildIfNeeded)
 430
 431     def __initialize_zypp(self):
 432         if self.Z:
 433             return
 434
 435         zconfig = zypp.ZConfig_instance()
 436
 437         """ Set system architecture """
 438         if self.creator.target_arch and self.creator.target_arch.startswith("arm"):
 439             arches = ["armv7l", "armv7nhl", "armv7hl", "armv5tel"]
 440             if self.creator.target_arch not in arches:
 441                 raise CreatorError("Invalid architecture: %s" % self.creator.target_arch)
 442             arch_map = {}
 443             if self.creator.target_arch == "armv7l":
 444                 arch_map["armv7l"] = zypp.Arch_armv7l()
 445             elif self.creator.target_arch == "armv7nhl":
 446                 arch_map["armv7nhl"] = zypp.Arch_armv7nhl()
 447             elif self.creator.target_arch == "armv7hl":
 448                 arch_map["armv7hl"] = zypp.Arch_armv7hl()
 449             elif self.creator.target_arch == "armv5tel":
 450                 arch_map["armv5tel"] = zypp.Arch_armv5tel()
 451             zconfig.setSystemArchitecture(arch_map[self.creator.target_arch])
 452
 453         msger.info("zypp architecture is <%s>" % zconfig.systemArchitecture())
 454
 455         """ repoPackagesCachePath is corrected by this """
 456         self.repo_manager = zypp.RepoManager(self.repo_manager_options)
 457         repos = self.repo_manager.knownRepositories()
 458         for repo in repos:
 459             if not repo.enabled():
 460                 continue
 461             if not self.repo_manager.isCached( repo ):
 462                 msger.info("Retrieving repo metadata from %s ..." % repo.url())
 463                 self.repo_manager.buildCache( repo, zypp.RepoManager.BuildIfNeeded )
 464             else:
 465                 self.repo_manager.refreshMetadata(repo, zypp.RepoManager.BuildIfNeeded)
 466             self.repo_manager.loadFromCache( repo );
 467
 468         self.Z = zypp.ZYppFactory_instance().getZYpp()
 469         self.Z.initializeTarget( zypp.Pathname(self.creator._instroot) )
 470         self.Z.target().load();
 471
 472
 473     def buildTransaction(self):
 474         if not self.Z.resolver().resolvePool():
 475             msger.warning("Problem count: %d" % len(self.Z.resolver().problems()))
 476             for problem in self.Z.resolver().problems():
 477                 msger.warning("Problem: %s, %s" % (problem.description().decode("utf-8"), problem.details().decode("utf-8")))
 478
 479     def getLocalPkgPath(self, po):
 480         repoinfo = po.repoInfo()
 481         name = po.name()
 482         cacheroot = repoinfo.packagesPath()
 483         arch =  po.arch()
 484         edition = po.edition()
 485         version = "%s-%s" % (edition.version(), edition.release())
 486         pkgpath = "%s/%s/%s-%s.%s.rpm" % (cacheroot, arch, name, version, arch)
 487         return pkgpath
 488
 489     def installLocal(self, pkg, po=None, updateonly=False):
 490         if not self.ts:
 491             self.__initialize_transaction()
 492         pkgname = self.__get_pkg_name(pkg)
 493         self.localpkgs[pkgname] = pkg
 494         self.selectPackage(pkgname)
 495
 496     def __get_pkg_name(self, pkgpath):
 497         h = rpmmisc.readRpmHeader(self.ts, pkgpath)
 498         return h["name"]
 499
 500     def downloadPkgs(self, package_objects, count):
 501         localpkgs = self.localpkgs.keys()
 502         progress_obj = fs.TextProgress(count)
 503         for po in package_objects:
 504             if po.name() in localpkgs:
 505                 continue
 506             filename = self.getLocalPkgPath(po)
 507             if os.path.exists(filename):
 508                 if self.checkPkg(filename) == 0:
 509                     continue
 510             dirn = os.path.dirname(filename)
 511             if not os.path.exists(dirn):
 512                 os.makedirs(dirn)
 513             baseurl = po.repoInfo().baseUrls()[0].__str__()
 514             proxy = self.get_proxy(po.repoInfo())
 515             proxies = {}
 516             if proxy:
 517                 proxies = {str(proxy.split(":")[0]):str(proxy)}
 518
 519             location = zypp.asKindPackage(po).location()
 520             location = location.filename().__str__()
 521             if location.startswith("./"):
 522                 location = location[2:]
 523             url = baseurl + "/%s" % location
 524             try:
 525                 filename = fs.myurlgrab(url, filename, proxies, progress_obj)
 526             except CreatorError, e:
 527                 self.close()
 528                 raise CreatorError("%s" % e)
 529
 530     def installPkgs(self, package_objects):
 531         if not self.ts:
 532             self.__initialize_transaction()
 533
 534         """ Set filters """
 535         probfilter = 0
 536         for flag in self.probFilterFlags:
 537             probfilter |= flag
 538         self.ts.setProbFilter(probfilter)
 539
 540         localpkgs = self.localpkgs.keys()
 541         for po in package_objects:
 542             pkgname = po.name()
 543             if pkgname in localpkgs:
 544                 rpmpath = self.localpkgs[pkgname]
 545             else:
 546                 rpmpath = self.getLocalPkgPath(po)
 547             if not os.path.exists(rpmpath):
 548                 """ Maybe it is a local repo """
 549                 baseurl = po.repoInfo().baseUrls()[0].__str__()
 550                 baseurl = baseurl.strip()
 551                 if baseurl.startswith("file:/"):
 552                     rpmpath = baseurl[5:] + "/%s/%s" % (po.arch(), os.path.basename(rpmpath))
 553             if not os.path.exists(rpmpath):
 554                 raise RpmError("Error: %s doesn't exist" % rpmpath)
 555             h = rpmmisc.readRpmHeader(self.ts, rpmpath)
 556             self.ts.addInstall(h, rpmpath, 'u')
 557
 558         unresolved_dependencies = self.ts.check()
 559         if not unresolved_dependencies:
 560             self.ts.order()
 561             cb = rpmmisc.RPMInstallCallback(self.ts)
 562             self.ts.run(cb.callback, '')
 563             self.ts.closeDB()
 564             self.ts = None
 565         else:
 566             msger.warning(unresolved_dependencies)
 567             raise RepoError("Unresolved dependencies, transaction failed.")
 568
 569     def __initialize_transaction(self):
 570         if not self.ts:
 571             self.ts = rpm.TransactionSet(self.creator._instroot)
 572             # Set to not verify DSA signatures.
 573             self.ts.setVSFlags(rpm._RPMVSF_NOSIGNATURES|rpm._RPMVSF_NODIGESTS)
 574
 575     def checkPkg(self, pkg):
 576         ret = 1
 577         if not os.path.exists(pkg):
 578             return ret
 579         ret = rpmmisc.checkRpmIntegrity('rpm', pkg)
 580         if ret != 0:
 581             msger.warning("package %s is damaged: %s" % (os.path.basename(pkg), pkg))
 582
 583         return ret
 584
 585     def zypp_install(self):
 586         policy = zypp.ZYppCommitPolicy()
 587         policy.downloadMode(zypp.DownloadInAdvance)
 588         policy.dryRun( False )
 589         policy.syncPoolAfterCommit( False )
 590         result = self.Z.commit( policy )
 591         msger.info(result)
 592
 593     def _add_prob_flags(self, *flags):
 594         for flag in flags:
 595            if flag not in self.probFilterFlags:
 596                self.probFilterFlags.append(flag)
 597
 598     def get_proxy(self, repoinfo):
 599         proxy = None
 600         reponame = "%s" % repoinfo.name()
 601         for repo in self.repos:
 602             if repo.name == reponame:
 603                 proxy = repo.proxy
 604                 break
 605
 606         if proxy:
 607             return proxy
 608         else:
 609             repourl = repoinfo.baseUrls()[0].__str__()
 610             return get_proxy_for(repourl)