plugins/backend/yumpkgmgr.py

   1 #!/usr/bin/python -tt
   2 #
   3 # Copyright (c) 2007 Red Hat  Inc.
   4 # Copyright (c) 2010, 2011 Intel, Inc.
   5 #
   6 # This program is free software; you can redistribute it and/or modify it
   7 # under the terms of the GNU General Public License as published by the Free
   8 # Software Foundation; version 2 of the License
   9 #
  10 # This program is distributed in the hope that it will be useful, but
  11 # WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
  12 # or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13 # for more details.
  14 #
  15 # You should have received a copy of the GNU General Public License along
  16 # with this program; if not, write to the Free Software Foundation, Inc., 59
  17 # Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  18
  19 import os
  20 import tempfile
  21 import glob
  22 from string import Template
  23
  24 import rpmUtils
  25 import yum
  26
  27 from mic import msger
  28 from mic.kickstart import ksparser
  29 from mic.utils import misc, rpmmisc
  30 from mic.utils.grabber import TextProgress
  31 from mic.utils.proxy import get_proxy_for
  32 from mic.utils.errors import CreatorError
  33 from mic.utils.safeurl import SafeURL
  34
  35
  36 YUMCONF_TEMP = """[main]
  37 installroot=$installroot
  38 cachedir=/var/cache/yum
  39 persistdir=/var/lib/yum
  40 plugins=0
  41 reposdir=
  42 failovermethod=priority
  43 http_caching=packages
  44 sslverify=1
  45 """
  46
  47 class MyYumRepository(yum.yumRepo.YumRepository):
  48     def __init__(self, repoid, nocache):
  49         super(MyYumRepository, self).__init__(repoid)
  50         self.nocache = nocache
  51
  52     def __del__(self):
  53         pass
  54
  55     def dirSetup(self):
  56         super(MyYumRepository, self).dirSetup()
  57         # relocate package dir
  58         pkgdir = os.path.join(self.basecachedir, 'packages', self.id)
  59         self.setAttribute('_dir_setup_pkgdir', pkgdir)
  60         self._dirSetupMkdir_p(self.pkgdir)
  61
  62     def _getFile(self, url=None,
  63                        relative=None,
  64                        local=None,
  65                        start=None,
  66                        end=None,
  67                        copy_local=None,
  68                        checkfunc=None,
  69                        text=None,
  70                        reget='simple',
  71                        cache=True,
  72                        size=None):
  73
  74         m2c_connection = None
  75         if not self.sslverify:
  76             try:
  77                 import M2Crypto
  78                 m2c_connection = M2Crypto.SSL.Connection.clientPostConnectionCheck
  79                 M2Crypto.SSL.Connection.clientPostConnectionCheck = None
  80             except ImportError, err:
  81                 raise CreatorError("%s, please try to install python-m2crypto" % str(err))
  82
  83         proxy = None
  84         if url:
  85             proxy = get_proxy_for(url)
  86         else:
  87             proxy = get_proxy_for(self.urls[0])
  88
  89         if proxy:
  90             self.proxy = str(proxy)
  91
  92         size = int(size) if size else None
  93         rvalue = super(MyYumRepository, self)._getFile(url,
  94                                                        relative,
  95                                                        local,
  96                                                        start,
  97                                                        end,
  98                                                        copy_local,
  99                                                        checkfunc,
 100                                                        text,
 101                                                        reget,
 102                                                        cache,
 103                                                        size)
 104
 105         if m2c_connection and \
 106            not M2Crypto.SSL.Connection.clientPostConnectionCheck:
 107             M2Crypto.SSL.Connection.clientPostConnectionCheck = m2c_connection
 108
 109         return rvalue
 110
 111 from mic.pluginbase import BackendPlugin
 112 class Yum(BackendPlugin, yum.YumBase):
 113     name = 'yum'
 114
 115     def __init__(self, target_arch, instroot, cachedir):
 116         yum.YumBase.__init__(self)
 117
 118         self.cachedir = cachedir
 119         self.instroot  = instroot
 120         self.target_arch = target_arch
 121
 122         if self.target_arch:
 123             if not rpmUtils.arch.arches.has_key(self.target_arch):
 124                 rpmUtils.arch.arches["armv7hl"] = "noarch"
 125                 rpmUtils.arch.arches["armv7tnhl"] = "armv7nhl"
 126                 rpmUtils.arch.arches["armv7tnhl"] = "armv7thl"
 127                 rpmUtils.arch.arches["armv7thl"] = "armv7hl"
 128                 rpmUtils.arch.arches["armv7nhl"] = "armv7hl"
 129             self.arch.setup_arch(self.target_arch)
 130
 131         self.__pkgs_license = {}
 132         self.__pkgs_content = {}
 133         self.__pkgs_vcsinfo = {}
 134         self.check_pkgs = []
 135
 136         self.install_debuginfo = False
 137
 138     def doFileLogSetup(self, uid, logfile):
 139         # don't do the file log for the livecd as it can lead to open fds
 140         # being left and an inability to clean up after ourself
 141         pass
 142
 143     def close(self):
 144         try:
 145             os.unlink(self.confpath)
 146             os.unlink(self.conf.installroot + "/yum.conf")
 147         except:
 148             pass
 149
 150         if self.ts:
 151             self.ts.close()
 152         self._delRepos()
 153         self._delSacks()
 154         yum.YumBase.close(self)
 155         self.closeRpmDB()
 156
 157     def __del__(self):
 158         pass
 159
 160     def _writeConf(self, confpath, installroot):
 161         conf = Template(YUMCONF_TEMP).safe_substitute(installroot=installroot)
 162
 163         f = file(confpath, "w+")
 164         f.write(conf)
 165         f.close()
 166
 167         os.chmod(confpath, 0644)
 168
 169     def _cleanupRpmdbLocks(self, installroot):
 170         # cleans up temporary files left by bdb so that differing
 171         # versions of rpm don't cause problems
 172         for f in glob.glob(installroot + "/var/lib/rpm/__db*"):
 173             os.unlink(f)
 174
 175     def setup(self):
 176         # create yum.conf
 177         (fn, self.confpath) = tempfile.mkstemp(dir=self.cachedir,
 178                                                prefix='yum.conf-')
 179         os.close(fn)
 180         self._writeConf(self.confpath, self.instroot)
 181         self._cleanupRpmdbLocks(self.instroot)
 182         # do setup
 183         self.doConfigSetup(fn = self.confpath, root = self.instroot)
 184         self.conf.cache = 0
 185         self.doTsSetup()
 186         self.doRpmDBSetup()
 187         self.doRepoSetup()
 188         self.doSackSetup()
 189
 190     def preInstall(self, pkg):
 191         # FIXME: handle pre-install package
 192         return None
 193
 194     def checkPackage(self, pkg):
 195         self.check_pkgs.append(pkg)
 196
 197     def selectPackage(self, pkg):
 198         """Select a given package.
 199         Can be specified with name.arch or name*
 200         """
 201
 202         try:
 203             self.install(pattern = pkg)
 204             return None
 205         except yum.Errors.InstallError:
 206             return "No package(s) available to install"
 207         except yum.Errors.RepoError, e:
 208             raise CreatorError("Unable to download from repo : %s" % (e,))
 209         except yum.Errors.YumBaseError, e:
 210             raise CreatorError("Unable to install: %s" % (e,))
 211
 212     def deselectPackage(self, pkg):
 213         """Deselect package.  Can be specified as name.arch or name*
 214         """
 215
 216         sp = pkg.rsplit(".", 2)
 217         txmbrs = []
 218         if len(sp) == 2:
 219             txmbrs = self.tsInfo.matchNaevr(name=sp[0], arch=sp[1])
 220
 221         if len(txmbrs) == 0:
 222             exact, match, unmatch = yum.packages.parsePackages(
 223                                             self.pkgSack.returnPackages(),
 224                                             [pkg],
 225                                             casematch=1)
 226             for p in exact + match:
 227                 txmbrs.append(p)
 228
 229         if len(txmbrs) > 0:
 230             for x in txmbrs:
 231                 self.tsInfo.remove(x.pkgtup)
 232                 # we also need to remove from the conditionals
 233                 # dict so that things don't get pulled back in as a result
 234                 # of them.  yes, this is ugly.  conditionals should die.
 235                 for req, pkgs in self.tsInfo.conditionals.iteritems():
 236                     if x in pkgs:
 237                         pkgs.remove(x)
 238                         self.tsInfo.conditionals[req] = pkgs
 239         else:
 240             msger.warning("No such package %s to remove" %(pkg,))
 241
 242     def selectGroup(self, grp, include = ksparser.GROUP_DEFAULT):
 243         try:
 244             yum.YumBase.selectGroup(self, grp)
 245             if include == ksparser.GROUP_REQUIRED:
 246                 for p in grp.default_packages.keys():
 247                     self.deselectPackage(p)
 248
 249             elif include == ksparser.GROUP_ALL:
 250                 for p in grp.optional_packages.keys():
 251                     self.selectPackage(p)
 252
 253             return None
 254         except (yum.Errors.InstallError, yum.Errors.GroupsError), e:
 255             return e
 256         except yum.Errors.RepoError, e:
 257             raise CreatorError("Unable to download from repo : %s" % (e,))
 258         except yum.Errors.YumBaseError, e:
 259             raise CreatorError("Unable to install: %s" % (e,))
 260
 261     def addRepository(self, name, url = None, mirrorlist = None, proxy = None,
 262                       proxy_username = None, proxy_password = None,
 263                       inc = None, exc = None, ssl_verify=True, nocache=False,
 264                       cost = None, priority=None):
 265         # TODO: Handle priority attribute for repos
 266         def _varSubstitute(option):
 267             # takes a variable and substitutes like yum configs do
 268             option = option.replace("$basearch", rpmUtils.arch.getBaseArch())
 269             option = option.replace("$arch", rpmUtils.arch.getCanonArch())
 270             return option
 271         repo = MyYumRepository(name, nocache)
 272
 273         # Set proxy
 274         repo.proxy = proxy
 275         repo.proxy_username = proxy_username
 276         repo.proxy_password = proxy_password
 277
 278         if url:
 279             repo.baseurl.append(_varSubstitute(url.full))
 280
 281         if mirrorlist:
 282             repo.mirrorlist = _varSubstitute(mirrorlist)
 283
 284         conf = yum.config.RepoConf()
 285         for k, v in conf.iteritems():
 286             if v or not hasattr(repo, k):
 287                 repo.setAttribute(k, v)
 288
 289         repo.sslverify = ssl_verify
 290
 291         repo.basecachedir = self.cachedir
 292         repo.base_persistdir = self.conf.persistdir
 293         repo.failovermethod = "priority"
 294         repo.metadata_expire = 0
 295         # Enable gpg check for verifying corrupt packages
 296         repo.gpgcheck = 1
 297         repo.enable()
 298         repo.setup(0)
 299         self.repos.add(repo)
 300         if cost:
 301             repo.cost = cost
 302
 303         msger.verbose('repo: %s was added' % name)
 304         return repo
 305
 306     def installLocal(self, pkg, po=None, updateonly=False):
 307         ts = rpmUtils.transaction.initReadOnlyTransaction()
 308         try:
 309             hdr = rpmUtils.miscutils.hdrFromPackage(ts, pkg)
 310         except rpmUtils.RpmUtilsError, e:
 311             raise yum.Errors.MiscError, \
 312                   'Could not open local rpm file: %s: %s' % (pkg, e)
 313
 314         self.deselectPackage(hdr['name'])
 315         yum.YumBase.installLocal(self, pkg, po, updateonly)
 316
 317     def installHasFile(self, file):
 318         provides_pkg = self.whatProvides(file, None, None)
 319         dlpkgs = map(
 320                     lambda x: x.po,
 321                     filter(
 322                         lambda txmbr: txmbr.ts_state in ("i", "u"),
 323                         self.tsInfo.getMembers()))
 324
 325         for p in dlpkgs:
 326             for q in provides_pkg:
 327                 if (p == q):
 328                     return True
 329
 330         return False
 331
 332     def runInstall(self, checksize = 0):
 333         os.environ["HOME"] = "/"
 334         os.environ["LD_PRELOAD"] = ""
 335         try:
 336             (res, resmsg) = self.buildTransaction()
 337         except yum.Errors.RepoError, e:
 338             raise CreatorError("Unable to download from repo : %s" %(e,))
 339
 340         if res != 2:
 341             raise CreatorError("Failed to build transaction : %s" \
 342                                % str.join("\n", resmsg))
 343
 344         dlpkgs = map(
 345                     lambda x: x.po,
 346                     filter(
 347                         lambda txmbr: txmbr.ts_state in ("i", "u"),
 348                         self.tsInfo.getMembers()))
 349
 350         # record all pkg and the content
 351         for pkg in dlpkgs:
 352             pkg_long_name = misc.RPM_FMT % {
 353                                 'name': pkg.name,
 354                                 'arch': pkg.arch,
 355                                 'version': pkg.version,
 356                                 'release': pkg.release
 357                             }
 358             self.__pkgs_content[pkg_long_name] = pkg.files
 359             license = pkg.license
 360             if license in self.__pkgs_license.keys():
 361                 self.__pkgs_license[license].append(pkg_long_name)
 362             else:
 363                 self.__pkgs_license[license] = [pkg_long_name]
 364
 365             if pkg.name in self.check_pkgs:
 366                 self.check_pkgs.remove(pkg.name)
 367
 368         if self.check_pkgs:
 369             raise CreatorError('Packages absent in image: %s' % ','.join(self.check_pkgs))
 370
 371         total_count = len(dlpkgs)
 372         cached_count = 0
 373         download_total_size = sum(map(lambda x: int(x.packagesize), dlpkgs))
 374
 375         msger.info("\nChecking packages cached ...")
 376         for po in dlpkgs:
 377             local = po.localPkg()
 378             repo = filter(lambda r: r.id == po.repoid, self.repos.listEnabled())[0]
 379             if repo.nocache and os.path.exists(local):
 380                 os.unlink(local)
 381             if not os.path.exists(local):
 382                 continue
 383             if not self.verifyPkg(local, po, False):
 384                 msger.warning("Package %s is damaged: %s" \
 385                               % (os.path.basename(local), local))
 386             else:
 387                 download_total_size -= int(po.packagesize)
 388                 cached_count += 1
 389
 390         cache_avail_size = misc.get_filesystem_avail(self.cachedir)
 391         if cache_avail_size < download_total_size:
 392             raise CreatorError("No enough space used for downloading.")
 393
 394         # record the total size of installed pkgs
 395         pkgs_total_size = 0L
 396         for x in dlpkgs:
 397             if hasattr(x, 'installedsize'):
 398                 pkgs_total_size += int(x.installedsize)
 399             else:
 400                 pkgs_total_size += int(x.size)
 401
 402         # check needed size before actually download and install
 403         if checksize and pkgs_total_size > checksize:
 404             raise CreatorError("No enough space used for installing, "
 405                                "please resize partition size in ks file")
 406
 407         msger.info("Packages: %d Total, %d Cached, %d Missed" \
 408                    % (total_count, cached_count, total_count - cached_count))
 409
 410         try:
 411             repos = self.repos.listEnabled()
 412             for repo in repos:
 413                 repo.setCallback(TextProgress(total_count - cached_count))
 414
 415             self.downloadPkgs(dlpkgs)
 416             # FIXME: sigcheck?
 417
 418             self.initActionTs()
 419             self.populateTs(keepold=0)
 420
 421             deps = self.ts.check()
 422             if len(deps) != 0:
 423                 # This isn't fatal, Ubuntu has this issue but it is ok.
 424                 msger.debug(deps)
 425                 msger.warning("Dependency check failed!")
 426
 427             rc = self.ts.order()
 428             if rc != 0:
 429                 raise CreatorError("ordering packages for installation failed")
 430
 431             # FIXME: callback should be refactored a little in yum
 432             cb = rpmmisc.RPMInstallCallback(self.ts)
 433             cb.tsInfo = self.tsInfo
 434             cb.filelog = False
 435
 436             msger.warning('\nCaution, do NOT interrupt the installation, '
 437                           'else mic cannot finish the cleanup.')
 438
 439             installlogfile = "%s/__catched_stderr.buf" % (self.instroot)
 440             msger.enable_logstderr(installlogfile)
 441             self.runTransaction(cb)
 442             self._cleanupRpmdbLocks(self.conf.installroot)
 443
 444         except rpmUtils.RpmUtilsError, e:
 445             raise CreatorError("mic does NOT support delta rpm: %s" % e)
 446         except yum.Errors.RepoError, e:
 447             raise CreatorError("Unable to download from repo : %s" % e)
 448         except yum.Errors.YumBaseError, e:
 449             raise CreatorError("Unable to install: %s" % e)
 450         finally:
 451             msger.disable_logstderr()
 452
 453     def getVcsInfo(self):
 454         return self.__pkgs_vcsinfo
 455
 456     def getAllContent(self):
 457         return self.__pkgs_content
 458
 459     def getPkgsLicense(self):
 460         return self.__pkgs_license
 461
 462     def getFilelist(self, pkgname):
 463         if not pkgname:
 464             return None
 465
 466         pkg = filter(lambda txmbr: txmbr.po.name == pkgname, self.tsInfo.getMembers())
 467         if not pkg:
 468             return None
 469         return pkg[0].po.filelist
 470
 471     def package_url(self, pkgname):
 472         pkgs = self.pkgSack.searchNevra(name=pkgname)
 473         if pkgs:
 474             pkg = pkgs[0]
 475
 476             repo = pkg.repo
 477             url = SafeURL(repo.baseurl[0]).join(pkg.remote_path)
 478
 479             proxy = repo.proxy
 480             if not proxy:
 481                 proxy = get_proxy_for(url)
 482             if proxy:
 483                 proxies = {str(url.split(':')[0]): str(proxy)}
 484             else:
 485                 proxies = None
 486
 487             return (url, proxies)
 488
 489         return (None, None)