plugins/backend/yumpkgmgr.py

   1 #!/usr/bin/python -tt
   2 #
   3 # Copyright (c) 2007 Red Hat  Inc.
   4 # Copyright (c) 2010, 2011 Intel, Inc.
   5 #
   6 # This program is free software; you can redistribute it and/or modify it
   7 # under the terms of the GNU General Public License as published by the Free
   8 # Software Foundation; version 2 of the License
   9 #
  10 # This program is distributed in the hope that it will be useful, but
  11 # WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
  12 # or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13 # for more details.
  14 #
  15 # You should have received a copy of the GNU General Public License along
  16 # with this program; if not, write to the Free Software Foundation, Inc., 59
  17 # Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  18
  19 import os, sys
  20 import re
  21 import tempfile
  22 import glob
  23 from string import Template
  24
  25 import rpmUtils
  26 import yum
  27
  28 from mic import msger
  29 from mic.kickstart import ksparser
  30 from mic.utils import misc, rpmmisc
  31 from mic.utils.errors import CreatorError
  32 from mic.imager.baseimager import BaseImageCreator
  33
  34 YUMCONF_TEMP = """[main]
  35 installroot=$installroot
  36 cachedir=/var/cache/yum
  37 persistdir=/var/lib/yum
  38 plugins=0
  39 reposdir=
  40 failovermethod=priority
  41 http_caching=packages
  42 sslverify=1
  43 """
  44
  45 class MyYumRepository(yum.yumRepo.YumRepository):
  46     def __del__(self):
  47         pass
  48
  49     def dirSetup(self):
  50         super(MyYumRepository, self).dirSetup()
  51         # relocate package dir
  52         pkgdir = os.path.join(self.basecachedir, 'packages', self.id)
  53         self.setAttribute('_dir_setup_pkgdir', pkgdir)
  54         self._dirSetupMkdir_p(self.pkgdir)
  55
  56     def _getFile(self, url=None,
  57                        relative=None,
  58                        local=None,
  59                        start=None,
  60                        end=None,
  61                        copy_local=None,
  62                        checkfunc=None,
  63                        text=None,
  64                        reget='simple',
  65                        cache=True,
  66                        size=None):
  67
  68         m2c_connection = None
  69         if not self.sslverify:
  70             import M2Crypto
  71             m2c_connection = M2Crypto.SSL.Connection.clientPostConnectionCheck
  72             M2Crypto.SSL.Connection.clientPostConnectionCheck = None
  73
  74         size = int(size) if size else None
  75         rvalue = super(MyYumRepository, self)._getFile(url,
  76                                                        relative,
  77                                                        local,
  78                                                        start,
  79                                                        end,
  80                                                        copy_local,
  81                                                        checkfunc,
  82                                                        text,
  83                                                        reget,
  84                                                        cache,
  85                                                        size)
  86
  87         if m2c_connection and \
  88            not M2Crypto.SSL.Connection.clientPostConnectionCheck:
  89             M2Crypto.SSL.Connection.clientPostConnectionCheck = m2c_connection
  90
  91         return rvalue
  92
  93 from mic.pluginbase import BackendPlugin
  94 class Yum(BackendPlugin, yum.YumBase):
  95     name = 'yum'
  96
  97     def __init__(self, target_arch, instroot, cachedir):
  98         yum.YumBase.__init__(self)
  99
 100         self.cachedir = cachedir
 101         self.instroot  = instroot
 102         self.target_arch = target_arch
 103
 104         if self.target_arch:
 105             if not rpmUtils.arch.arches.has_key(self.target_arch):
 106                 rpmUtils.arch.arches["armv7hl"] = "noarch"
 107                 rpmUtils.arch.arches["armv7tnhl"] = "armv7nhl"
 108                 rpmUtils.arch.arches["armv7tnhl"] = "armv7thl"
 109                 rpmUtils.arch.arches["armv7thl"] = "armv7hl"
 110                 rpmUtils.arch.arches["armv7nhl"] = "armv7hl"
 111             self.arch.setup_arch(self.target_arch)
 112
 113         self.__pkgs_license = {}
 114         self.__pkgs_content = {}
 115
 116     def doFileLogSetup(self, uid, logfile):
 117         # don't do the file log for the livecd as it can lead to open fds
 118         # being left and an inability to clean up after ourself
 119         pass
 120
 121     def close(self):
 122         try:
 123             os.unlink(self.confpath)
 124             os.unlink(self.conf.installroot + "/yum.conf")
 125         except:
 126             pass
 127
 128         if self.ts:
 129             self.ts.close()
 130         self._delRepos()
 131         self._delSacks()
 132         yum.YumBase.close(self)
 133         self.closeRpmDB()
 134
 135         if not os.path.exists("/etc/fedora-release") and \
 136            not os.path.exists("/etc/meego-release"):
 137             for i in range(3, os.sysconf("SC_OPEN_MAX")):
 138                 try:
 139                     os.close(i)
 140                 except:
 141                     pass
 142
 143     def __del__(self):
 144         pass
 145
 146     def _writeConf(self, confpath, installroot):
 147         conf = Template(YUMCONF_TEMP).safe_substitute(installroot=installroot)
 148
 149         f = file(confpath, "w+")
 150         f.write(conf)
 151         f.close()
 152
 153         os.chmod(confpath, 0644)
 154
 155     def _cleanupRpmdbLocks(self, installroot):
 156         # cleans up temporary files left by bdb so that differing
 157         # versions of rpm don't cause problems
 158         for f in glob.glob(installroot + "/var/lib/rpm/__db*"):
 159             os.unlink(f)
 160
 161     def setup(self):
 162         # create yum.conf
 163         (fn, self.confpath) = tempfile.mkstemp(dir=self.cachedir,
 164                                                prefix='yum.conf-')
 165         os.close(fn)
 166         self._writeConf(self.confpath, self.instroot)
 167         self._cleanupRpmdbLocks(self.instroot)
 168         # do setup
 169         self.doConfigSetup(fn = self.confpath, root = self.instroot)
 170         self.conf.cache = 0
 171         self.doTsSetup()
 172         self.doRpmDBSetup()
 173         self.doRepoSetup()
 174         self.doSackSetup()
 175
 176     def preInstall(self, pkg):
 177         # FIXME: handle pre-install package
 178         return None
 179
 180     def selectPackage(self, pkg):
 181         """Select a given package.
 182         Can be specified with name.arch or name*
 183         """
 184
 185         try:
 186             self.install(pattern = pkg)
 187             return None
 188         except yum.Errors.InstallError:
 189             return "No package(s) available to install"
 190         except yum.Errors.RepoError, e:
 191             raise CreatorError("Unable to download from repo : %s" % (e,))
 192         except yum.Errors.YumBaseError, e:
 193             raise CreatorError("Unable to install: %s" % (e,))
 194
 195     def deselectPackage(self, pkg):
 196         """Deselect package.  Can be specified as name.arch or name*
 197         """
 198
 199         sp = pkg.rsplit(".", 2)
 200         txmbrs = []
 201         if len(sp) == 2:
 202             txmbrs = self.tsInfo.matchNaevr(name=sp[0], arch=sp[1])
 203
 204         if len(txmbrs) == 0:
 205             exact, match, unmatch = yum.packages.parsePackages(
 206                                             self.pkgSack.returnPackages(),
 207                                             [pkg],
 208                                             casematch=1)
 209             for p in exact + match:
 210                 txmbrs.append(p)
 211
 212         if len(txmbrs) > 0:
 213             for x in txmbrs:
 214                 self.tsInfo.remove(x.pkgtup)
 215                 # we also need to remove from the conditionals
 216                 # dict so that things don't get pulled back in as a result
 217                 # of them.  yes, this is ugly.  conditionals should die.
 218                 for req, pkgs in self.tsInfo.conditionals.iteritems():
 219                     if x in pkgs:
 220                         pkgs.remove(x)
 221                         self.tsInfo.conditionals[req] = pkgs
 222         else:
 223             msger.warning("No such package %s to remove" %(pkg,))
 224
 225     def selectGroup(self, grp, include = ksparser.GROUP_DEFAULT):
 226         try:
 227             yum.YumBase.selectGroup(self, grp)
 228             if include == ksparser.GROUP_REQUIRED:
 229                 for p in grp.default_packages.keys():
 230                     self.deselectPackage(p)
 231
 232             elif include == ksparser.GROUP_ALL:
 233                 for p in grp.optional_packages.keys():
 234                     self.selectPackage(p)
 235
 236             return None
 237         except (yum.Errors.InstallError, yum.Errors.GroupsError), e:
 238             return e
 239         except yum.Errors.RepoError, e:
 240             raise CreatorError("Unable to download from repo : %s" % (e,))
 241         except yum.Errors.YumBaseError, e:
 242             raise CreatorError("Unable to install: %s" % (e,))
 243
 244     def addRepository(self, name, url = None, mirrorlist = None, proxy = None,
 245                       proxy_username = None, proxy_password = None,
 246                       inc = None, exc = None, ssl_verify=True, cost = None,
 247                       priority=None):
 248         # TODO: Handle priority attribute for repos
 249         def _varSubstitute(option):
 250             # takes a variable and substitutes like yum configs do
 251             option = option.replace("$basearch", rpmUtils.arch.getBaseArch())
 252             option = option.replace("$arch", rpmUtils.arch.getCanonArch())
 253             return option
 254
 255         repo = MyYumRepository(name)
 256
 257         # Set proxy
 258         repo.proxy = proxy
 259         repo.proxy_username = proxy_username
 260         repo.proxy_password = proxy_password
 261
 262         if url:
 263             repo.baseurl.append(_varSubstitute(url))
 264
 265         # check LICENSE files
 266         if not rpmmisc.checkRepositoryEULA(name, repo):
 267             msger.warning('skip repo:%s for failed EULA confirmation' % name)
 268             return None
 269
 270         if mirrorlist:
 271             repo.mirrorlist = _varSubstitute(mirrorlist)
 272
 273         conf = yum.config.RepoConf()
 274         for k, v in conf.iteritems():
 275             if v or not hasattr(repo, k):
 276                 repo.setAttribute(k, v)
 277
 278         repo.sslverify = ssl_verify
 279
 280         repo.basecachedir = self.cachedir
 281         repo.base_persistdir = self.conf.persistdir
 282         repo.failovermethod = "priority"
 283         repo.metadata_expire = 0
 284         # Enable gpg check for verifying corrupt packages
 285         repo.gpgcheck = 1
 286         repo.enable()
 287         repo.setup(0)
 288         self.repos.add(repo)
 289         if cost:
 290             repo.cost = cost
 291
 292         msger.verbose('repo: %s was added' % name)
 293         return repo
 294
 295     def installLocal(self, pkg, po=None, updateonly=False):
 296         ts = rpmUtils.transaction.initReadOnlyTransaction()
 297         try:
 298             hdr = rpmUtils.miscutils.hdrFromPackage(ts, pkg)
 299         except rpmUtils.RpmUtilsError, e:
 300             raise yum.Errors.MiscError, \
 301                   'Could not open local rpm file: %s: %s' % (pkg, e)
 302
 303         self.deselectPackage(hdr['name'])
 304         yum.YumBase.installLocal(self, pkg, po, updateonly)
 305
 306     def installHasFile(self, file):
 307         provides_pkg = self.whatProvides(file, None, None)
 308         dlpkgs = map(
 309                     lambda x: x.po,
 310                     filter(
 311                         lambda txmbr: txmbr.ts_state in ("i", "u"),
 312                         self.tsInfo.getMembers()))
 313
 314         for p in dlpkgs:
 315             for q in provides_pkg:
 316                 if (p == q):
 317                     return True
 318
 319         return False
 320
 321     def runInstall(self, checksize = 0):
 322         os.environ["HOME"] = "/"
 323         try:
 324             (res, resmsg) = self.buildTransaction()
 325         except yum.Errors.RepoError, e:
 326             raise CreatorError("Unable to download from repo : %s" %(e,))
 327
 328         if res != 2:
 329             raise CreatorError("Failed to build transaction : %s" \
 330                                % str.join("\n", resmsg))
 331
 332         dlpkgs = map(
 333                     lambda x: x.po,
 334                     filter(
 335                         lambda txmbr: txmbr.ts_state in ("i", "u"),
 336                         self.tsInfo.getMembers()))
 337
 338         # record all pkg and the content
 339         for pkg in dlpkgs:
 340             pkg_long_name = misc.RPM_FMT % {
 341                                 'name': pkg.name,
 342                                 'arch': pkg.arch,
 343                                 'ver_rel': pkg.printVer(),
 344                             }
 345             self.__pkgs_content[pkg_long_name] = pkg.files
 346             license = pkg.license
 347             if license in self.__pkgs_license.keys():
 348                 self.__pkgs_license[license].append(pkg_long_name)
 349             else:
 350                 self.__pkgs_license[license] = [pkg_long_name]
 351
 352         total_count = len(dlpkgs)
 353         cached_count = 0
 354         download_total_size = sum(map(lambda x: int(x.packagesize), dlpkgs))
 355
 356         msger.info("\nChecking packages cache and packages integrity ...")
 357         for po in dlpkgs:
 358             local = po.localPkg()
 359             if not os.path.exists(local):
 360                 continue
 361             if not self.verifyPkg(local, po, False):
 362                 msger.warning("Package %s is damaged: %s" \
 363                               % (os.path.basename(local), local))
 364             else:
 365                 download_total_size -= int(po.packagesize)
 366                 cached_count +=1
 367
 368         cache_avail_size = misc.get_filesystem_avail(self.cachedir)
 369         if cache_avail_size < download_total_size:
 370             raise CreatorError("No enough space used for downloading.")
 371
 372         # record the total size of installed pkgs
 373         pkgs_total_size = 0L
 374         for x in dlpkgs:
 375             if hasattr(x, 'installedsize'):
 376                 pkgs_total_size += int(x.installedsize)
 377             else:
 378                 pkgs_total_size += int(x.size)
 379
 380         # check needed size before actually download and install
 381         if checksize and pkgs_total_size > checksize:
 382             raise CreatorError("No enough space used for installing, "
 383                                "please resize partition size in ks file")
 384
 385         msger.info("%d packages to be installed, "
 386                    "%d packages gotten from cache, "
 387                    "%d packages to be downloaded" \
 388                    % (total_count, cached_count, total_count - cached_count))
 389
 390         try:
 391             repos = self.repos.listEnabled()
 392             for repo in repos:
 393                 repo.setCallback(
 394                             rpmmisc.TextProgress(total_count - cached_count))
 395
 396             self.downloadPkgs(dlpkgs)
 397             # FIXME: sigcheck?
 398
 399             self.initActionTs()
 400             self.populateTs(keepold=0)
 401
 402             deps = self.ts.check()
 403             if len(deps) != 0:
 404                 # This isn't fatal, Ubuntu has this issue but it is ok.
 405                 msger.debug(deps)
 406                 msger.warning("Dependency check failed!")
 407
 408             rc = self.ts.order()
 409             if rc != 0:
 410                 raise CreatorError("ordering packages for installation failed")
 411
 412             # FIXME: callback should be refactored a little in yum
 413             cb = rpmmisc.RPMInstallCallback(self.ts)
 414             cb.tsInfo = self.tsInfo
 415             cb.filelog = False
 416
 417             msger.warning('\nCaution, do NOT interrupt the installation, '
 418                           'else mic cannot finish the cleanup.')
 419
 420             installlogfile = "%s/__catched_stderr.buf" % (self.instroot)
 421             msger.enable_logstderr(installlogfile)
 422             self.runTransaction(cb)
 423             self._cleanupRpmdbLocks(self.conf.installroot)
 424
 425         except rpmUtils.RpmUtilsError, e:
 426             raise CreatorError("mic does NOT support delta rpm: %s" % e)
 427         except yum.Errors.RepoError, e:
 428             raise CreatorError("Unable to download from repo : %s" % e)
 429         except yum.Errors.YumBaseError, e:
 430             raise CreatorError("Unable to install: %s" % e)
 431         finally:
 432             msger.disable_logstderr()
 433
 434     def getAllContent(self):
 435         return self.__pkgs_content
 436
 437     def getPkgsLicense(self):
 438         return self.__pkgs_license
 439
 440     def getFilelist(self, pkgname):
 441         if not pkgname:
 442             return None
 443
 444         pkg = filter(lambda txmbr: txmbr.po.name == pkgname, self.tsInfo.getMembers())
 445         if not pkg:
 446             return None
 447         return pkg[0].po.filelist
 448