plugins/backend/yumpkgmgr.py

   1 #!/usr/bin/python -tt
   2 #
   3 # Copyright (c) 2007 Red Hat  Inc.
   4 # Copyright (c) 2010, 2011 Intel, Inc.
   5 #
   6 # This program is free software; you can redistribute it and/or modify it
   7 # under the terms of the GNU General Public License as published by the Free
   8 # Software Foundation; version 2 of the License
   9 #
  10 # This program is distributed in the hope that it will be useful, but
  11 # WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
  12 # or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13 # for more details.
  14 #
  15 # You should have received a copy of the GNU General Public License along
  16 # with this program; if not, write to the Free Software Foundation, Inc., 59
  17 # Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  18
  19 import os
  20 import tempfile
  21 import glob
  22 from string import Template
  23
  24 import rpmUtils
  25 import yum
  26
  27 from mic import msger
  28 from mic.kickstart import ksparser
  29 from mic.utils import misc, rpmmisc
  30 from mic.utils.grabber import TextProgress
  31 from mic.utils.proxy import get_proxy_for
  32 from mic.utils.errors import CreatorError
  33 from mic.utils.safeurl import SafeURL
  34
  35
  36 YUMCONF_TEMP = """[main]
  37 installroot=$installroot
  38 cachedir=/var/cache/yum
  39 persistdir=/var/lib/yum
  40 plugins=0
  41 reposdir=
  42 failovermethod=priority
  43 http_caching=packages
  44 sslverify=1
  45 """
  46
  47 class MyYumRepository(yum.yumRepo.YumRepository):
  48     def __init__(self, repoid, nocache):
  49         super(MyYumRepository, self).__init__(repoid)
  50         self.nocache = nocache
  51
  52     def __del__(self):
  53         pass
  54
  55     def dirSetup(self):
  56         super(MyYumRepository, self).dirSetup()
  57         # relocate package dir
  58         pkgdir = os.path.join(self.basecachedir, 'packages', self.id)
  59         self.setAttribute('_dir_setup_pkgdir', pkgdir)
  60         self._dirSetupMkdir_p(self.pkgdir)
  61
  62     def _getFile(self, url=None,
  63                        relative=None,
  64                        local=None,
  65                        start=None,
  66                        end=None,
  67                        copy_local=None,
  68                        checkfunc=None,
  69                        text=None,
  70                        reget='simple',
  71                        cache=True,
  72                        size=None):
  73
  74         m2c_connection = None
  75         if not self.sslverify:
  76             try:
  77                 import M2Crypto
  78                 m2c_connection = M2Crypto.SSL.Connection.clientPostConnectionCheck
  79                 M2Crypto.SSL.Connection.clientPostConnectionCheck = None
  80             except ImportError, err:
  81                 raise CreatorError("%s, please try to install python-m2crypto" % str(err))
  82
  83         proxy = None
  84         if url:
  85             proxy = get_proxy_for(url)
  86         else:
  87             proxy = get_proxy_for(self.urls[0])
  88
  89         if proxy:
  90             self.proxy = str(proxy)
  91
  92         size = int(size) if size else None
  93         rvalue = super(MyYumRepository, self)._getFile(url,
  94                                                        relative,
  95                                                        local,
  96                                                        start,
  97                                                        end,
  98                                                        copy_local,
  99                                                        checkfunc,
 100                                                        text,
 101                                                        reget,
 102                                                        cache,
 103                                                        size)
 104
 105         if m2c_connection and \
 106            not M2Crypto.SSL.Connection.clientPostConnectionCheck:
 107             M2Crypto.SSL.Connection.clientPostConnectionCheck = m2c_connection
 108
 109         return rvalue
 110
 111 from mic.pluginbase import BackendPlugin
 112 class Yum(BackendPlugin, yum.YumBase):
 113     name = 'yum'
 114
 115     def __init__(self, target_arch, instroot, cachedir, strict_mode = False):
 116         yum.YumBase.__init__(self)
 117
 118         self.cachedir = cachedir
 119         self.instroot  = instroot
 120         self.target_arch = target_arch
 121         self.strict_mode = strict_mode
 122
 123         if self.target_arch:
 124             if not rpmUtils.arch.arches.has_key(self.target_arch):
 125                 rpmUtils.arch.arches["armv7hl"] = "noarch"
 126                 rpmUtils.arch.arches["armv7tnhl"] = "armv7nhl"
 127                 rpmUtils.arch.arches["armv7tnhl"] = "armv7thl"
 128                 rpmUtils.arch.arches["armv7thl"] = "armv7hl"
 129                 rpmUtils.arch.arches["armv7nhl"] = "armv7hl"
 130             self.arch.setup_arch(self.target_arch)
 131
 132         self.__pkgs_license = {}
 133         self.__pkgs_content = {}
 134         self.__pkgs_vcsinfo = {}
 135         self.check_pkgs = []
 136
 137         self.install_debuginfo = False
 138
 139     def doFileLogSetup(self, uid, logfile):
 140         # don't do the file log for the livecd as it can lead to open fds
 141         # being left and an inability to clean up after ourself
 142         pass
 143
 144     def close(self):
 145         try:
 146             os.unlink(self.confpath)
 147             os.unlink(self.conf.installroot + "/yum.conf")
 148         except:
 149             pass
 150
 151         if self.ts:
 152             self.ts.close()
 153         self._delRepos()
 154         self._delSacks()
 155         yum.YumBase.close(self)
 156         self.closeRpmDB()
 157
 158     def __del__(self):
 159         pass
 160
 161     def _writeConf(self, confpath, installroot):
 162         conf = Template(YUMCONF_TEMP).safe_substitute(installroot=installroot)
 163
 164         f = file(confpath, "w+")
 165         f.write(conf)
 166         f.close()
 167
 168         os.chmod(confpath, 0644)
 169
 170     def _cleanupRpmdbLocks(self, installroot):
 171         # cleans up temporary files left by bdb so that differing
 172         # versions of rpm don't cause problems
 173         for f in glob.glob(installroot + "/var/lib/rpm/__db*"):
 174             os.unlink(f)
 175
 176     def setup(self):
 177         # create yum.conf
 178         (fn, self.confpath) = tempfile.mkstemp(dir=self.cachedir,
 179                                                prefix='yum.conf-')
 180         os.close(fn)
 181         self._writeConf(self.confpath, self.instroot)
 182         self._cleanupRpmdbLocks(self.instroot)
 183         # do setup
 184         self.doConfigSetup(fn = self.confpath, root = self.instroot)
 185         self.conf.cache = 0
 186         self.doTsSetup()
 187         self.doRpmDBSetup()
 188         self.doRepoSetup()
 189         self.doSackSetup()
 190
 191     def preInstall(self, pkg):
 192         # FIXME: handle pre-install package
 193         return None
 194
 195     def checkPackage(self, pkg):
 196         self.check_pkgs.append(pkg)
 197
 198     def selectPackage(self, pkg):
 199         """Select a given package.
 200         Can be specified with name.arch or name*
 201         """
 202
 203         try:
 204             self.install(pattern = pkg)
 205             return None
 206         except yum.Errors.InstallError:
 207             return "No package(s) available to install"
 208         except yum.Errors.RepoError, e:
 209             raise CreatorError("Unable to download from repo : %s" % (e,))
 210         except yum.Errors.YumBaseError, e:
 211             raise CreatorError("Unable to install: %s" % (e,))
 212
 213     def deselectPackage(self, pkg):
 214         """Deselect package.  Can be specified as name.arch or name*
 215         """
 216
 217         sp = pkg.rsplit(".", 2)
 218         txmbrs = []
 219         if len(sp) == 2:
 220             txmbrs = self.tsInfo.matchNaevr(name=sp[0], arch=sp[1])
 221
 222         if len(txmbrs) == 0:
 223             exact, match, unmatch = yum.packages.parsePackages(
 224                                             self.pkgSack.returnPackages(),
 225                                             [pkg],
 226                                             casematch=1)
 227             for p in exact + match:
 228                 txmbrs.append(p)
 229
 230         if len(txmbrs) > 0:
 231             for x in txmbrs:
 232                 self.tsInfo.remove(x.pkgtup)
 233                 # we also need to remove from the conditionals
 234                 # dict so that things don't get pulled back in as a result
 235                 # of them.  yes, this is ugly.  conditionals should die.
 236                 for req, pkgs in self.tsInfo.conditionals.iteritems():
 237                     if x in pkgs:
 238                         pkgs.remove(x)
 239                         self.tsInfo.conditionals[req] = pkgs
 240         else:
 241             msger.warning("No such package %s to remove" %(pkg,))
 242
 243     def selectGroup(self, grp, include = ksparser.GROUP_DEFAULT):
 244         try:
 245             yum.YumBase.selectGroup(self, grp)
 246             if include == ksparser.GROUP_REQUIRED:
 247                 for p in grp.default_packages.keys():
 248                     self.deselectPackage(p)
 249
 250             elif include == ksparser.GROUP_ALL:
 251                 for p in grp.optional_packages.keys():
 252                     self.selectPackage(p)
 253
 254             return None
 255         except (yum.Errors.InstallError, yum.Errors.GroupsError), e:
 256             return e
 257         except yum.Errors.RepoError, e:
 258             raise CreatorError("Unable to download from repo : %s" % (e,))
 259         except yum.Errors.YumBaseError, e:
 260             raise CreatorError("Unable to install: %s" % (e,))
 261
 262     def addRepository(self, name, url = None, mirrorlist = None, proxy = None,
 263                       proxy_username = None, proxy_password = None,
 264                       inc = None, exc = None, ssl_verify=True, nocache=False,
 265                       cost = None, priority=None):
 266         # TODO: Handle priority attribute for repos
 267         def _varSubstitute(option):
 268             # takes a variable and substitutes like yum configs do
 269             option = option.replace("$basearch", rpmUtils.arch.getBaseArch())
 270             option = option.replace("$arch", rpmUtils.arch.getCanonArch())
 271             return option
 272         repo = MyYumRepository(name, nocache)
 273
 274         # Set proxy
 275         repo.proxy = proxy
 276         repo.proxy_username = proxy_username
 277         repo.proxy_password = proxy_password
 278
 279         if url:
 280             repo.baseurl.append(_varSubstitute(url.full))
 281
 282         if mirrorlist:
 283             repo.mirrorlist = _varSubstitute(mirrorlist)
 284
 285         conf = yum.config.RepoConf()
 286         for k, v in conf.iteritems():
 287             if v or not hasattr(repo, k):
 288                 repo.setAttribute(k, v)
 289
 290         repo.sslverify = ssl_verify
 291
 292         repo.basecachedir = self.cachedir
 293         repo.base_persistdir = self.conf.persistdir
 294         repo.failovermethod = "priority"
 295         repo.metadata_expire = 0
 296         # Enable gpg check for verifying corrupt packages
 297         repo.gpgcheck = 1
 298         repo.enable()
 299         repo.setup(0)
 300         self.repos.add(repo)
 301         if cost:
 302             repo.cost = cost
 303
 304         msger.verbose('repo: %s was added' % name)
 305         return repo
 306
 307     def installLocal(self, pkg, po=None, updateonly=False):
 308         ts = rpmUtils.transaction.initReadOnlyTransaction()
 309         try:
 310             hdr = rpmUtils.miscutils.hdrFromPackage(ts, pkg)
 311         except rpmUtils.RpmUtilsError, e:
 312             raise yum.Errors.MiscError, \
 313                   'Could not open local rpm file: %s: %s' % (pkg, e)
 314
 315         self.deselectPackage(hdr['name'])
 316         yum.YumBase.installLocal(self, pkg, po, updateonly)
 317
 318     def installHasFile(self, file):
 319         provides_pkg = self.whatProvides(file, None, None)
 320         dlpkgs = map(
 321                     lambda x: x.po,
 322                     filter(
 323                         lambda txmbr: txmbr.ts_state in ("i", "u"),
 324                         self.tsInfo.getMembers()))
 325
 326         for p in dlpkgs:
 327             for q in provides_pkg:
 328                 if (p == q):
 329                     return True
 330
 331         return False
 332
 333     def runInstall(self, checksize = 0):
 334         os.environ["HOME"] = "/"
 335         os.environ["LD_PRELOAD"] = ""
 336         try:
 337             (res, resmsg) = self.buildTransaction()
 338         except yum.Errors.RepoError, e:
 339             raise CreatorError("Unable to download from repo : %s" %(e,))
 340
 341         if res != 2:
 342             raise CreatorError("Failed to build transaction : %s" \
 343                                % str.join("\n", resmsg))
 344
 345         dlpkgs = map(
 346                     lambda x: x.po,
 347                     filter(
 348                         lambda txmbr: txmbr.ts_state in ("i", "u"),
 349                         self.tsInfo.getMembers()))
 350
 351         # record all pkg and the content
 352         for pkg in dlpkgs:
 353             pkg_long_name = misc.RPM_FMT % {
 354                                 'name': pkg.name,
 355                                 'arch': pkg.arch,
 356                                 'version': pkg.version,
 357                                 'release': pkg.release
 358                             }
 359             self.__pkgs_content[pkg_long_name] = pkg.files
 360             license = pkg.license
 361             if license in self.__pkgs_license.keys():
 362                 self.__pkgs_license[license].append(pkg_long_name)
 363             else:
 364                 self.__pkgs_license[license] = [pkg_long_name]
 365
 366             if pkg.name in self.check_pkgs:
 367                 self.check_pkgs.remove(pkg.name)
 368
 369         if self.check_pkgs:
 370             raise CreatorError('Packages absent in image: %s' % ','.join(self.check_pkgs))
 371
 372         total_count = len(dlpkgs)
 373         cached_count = 0
 374         download_total_size = sum(map(lambda x: int(x.packagesize), dlpkgs))
 375
 376         msger.info("\nChecking packages cached ...")
 377         for po in dlpkgs:
 378             local = po.localPkg()
 379             repo = filter(lambda r: r.id == po.repoid, self.repos.listEnabled())[0]
 380             if repo.nocache and os.path.exists(local):
 381                 os.unlink(local)
 382             if not os.path.exists(local):
 383                 continue
 384             if not self.verifyPkg(local, po, False):
 385                 msger.warning("Package %s is damaged: %s" \
 386                               % (os.path.basename(local), local))
 387             else:
 388                 download_total_size -= int(po.packagesize)
 389                 cached_count += 1
 390
 391         cache_avail_size = misc.get_filesystem_avail(self.cachedir)
 392         if cache_avail_size < download_total_size:
 393             raise CreatorError("No enough space used for downloading.")
 394
 395         # record the total size of installed pkgs
 396         pkgs_total_size = 0L
 397         for x in dlpkgs:
 398             if hasattr(x, 'installedsize'):
 399                 pkgs_total_size += int(x.installedsize)
 400             else:
 401                 pkgs_total_size += int(x.size)
 402
 403         # check needed size before actually download and install
 404         if checksize and pkgs_total_size > checksize:
 405             raise CreatorError("No enough space used for installing, "
 406                                "please resize partition size in ks file")
 407
 408         msger.info("Packages: %d Total, %d Cached, %d Missed" \
 409                    % (total_count, cached_count, total_count - cached_count))
 410
 411         try:
 412             repos = self.repos.listEnabled()
 413             for repo in repos:
 414                 repo.setCallback(TextProgress(total_count - cached_count))
 415
 416             self.downloadPkgs(dlpkgs)
 417             # FIXME: sigcheck?
 418
 419             self.initActionTs()
 420             self.populateTs(keepold=0)
 421
 422             deps = self.ts.check()
 423             if len(deps) != 0:
 424                 # This isn't fatal, Ubuntu has this issue but it is ok.
 425                 msger.debug(deps)
 426                 msger.warning("Dependency check failed!")
 427
 428             rc = self.ts.order()
 429             if rc != 0:
 430                 raise CreatorError("ordering packages for installation failed")
 431
 432             # FIXME: callback should be refactored a little in yum
 433             cb = rpmmisc.RPMInstallCallback(self.ts)
 434             cb.tsInfo = self.tsInfo
 435             cb.filelog = False
 436
 437             msger.warning('\nCaution, do NOT interrupt the installation, '
 438                           'else mic cannot finish the cleanup.')
 439
 440             installlogfile = "%s/__catched_stderr.buf" % (self.instroot)
 441             msger.enable_logstderr(installlogfile)
 442             transactionResult = self.runTransaction(cb)
 443             if transactionResult.return_code != 0 and self.strict_mode:
 444                 raise CreatorError("mic failes to install some packages")
 445             self._cleanupRpmdbLocks(self.conf.installroot)
 446
 447         except rpmUtils.RpmUtilsError, e:
 448             raise CreatorError("mic does NOT support delta rpm: %s" % e)
 449         except yum.Errors.RepoError, e:
 450             raise CreatorError("Unable to download from repo : %s" % e)
 451         except yum.Errors.YumBaseError, e:
 452             raise CreatorError("Unable to install: %s" % e)
 453         finally:
 454             msger.disable_logstderr()
 455
 456     def getVcsInfo(self):
 457         return self.__pkgs_vcsinfo
 458
 459     def getAllContent(self):
 460         return self.__pkgs_content
 461
 462     def getPkgsLicense(self):
 463         return self.__pkgs_license
 464
 465     def getFilelist(self, pkgname):
 466         if not pkgname:
 467             return None
 468
 469         pkg = filter(lambda txmbr: txmbr.po.name == pkgname, self.tsInfo.getMembers())
 470         if not pkg:
 471             return None
 472         return pkg[0].po.filelist
 473
 474     def package_url(self, pkgname):
 475         pkgs = self.pkgSack.searchNevra(name=pkgname)
 476         if pkgs:
 477             pkg = pkgs[0]
 478
 479             repo = pkg.repo
 480             url = SafeURL(repo.baseurl[0]).join(pkg.remote_path)
 481
 482             proxy = repo.proxy
 483             if not proxy:
 484                 proxy = get_proxy_for(url)
 485             if proxy:
 486                 proxies = {str(url.split(':')[0]): str(proxy)}
 487             else:
 488                 proxies = None
 489
 490             return (url, proxies)
 491
 492         return (None, None)