plugins/backend/zypppkgmgr.py

   1 #!/usr/bin/python -tt
   2 #
   3 # Copyright 2010, 2011 Intel, Inc.
   4 #
   5 # This copyrighted material is made available to anyone wishing to use, modify,
   6 # copy, or redistribute it subject to the terms and conditions of the GNU
   7 # General Public License v.2.  This program is distributed in the hope that it
   8 # will be useful, but WITHOUT ANY WARRANTY expressed or implied, including the
   9 # implied warranties of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
  10 # See the GNU General Public License for more details.
  11 #
  12 # You should have received a copy of the GNU General Public License along with
  13 # this program; if not, write to the Free Software Foundation, Inc., 51
  14 # Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.  Any Red Hat
  15 # trademarks that are incorporated in the source code or documentation are not
  16 # subject to the GNU General Public License and may only be used or replicated
  17 # with the express permission of Red Hat, Inc.
  18 #
  19
  20 import os
  21 import shutil
  22
  23 import rpmUtils.miscutils
  24 import rpmUtils.transaction
  25 import rpm
  26 import zypp
  27
  28 from mic import msger
  29 from mic.kickstart import ksparser
  30 from mic.utils import rpmmisc, fs_related as fs
  31 from mic.utils.proxy import get_proxy_for
  32 from mic.utils.errors import CreatorError
  33 from mic.imager.baseimager import BaseImageCreator
  34
  35 if not hasattr(zypp, 'PoolQuery'):
  36     raise ImportError("python-zypp in host system cannot support PoolQuery interface, please "
  37                       "update it to enhanced version which can be found in repo.meego.com/tools")
  38
  39 class RepositoryStub:
  40     def __init__(self):
  41         self.name = None
  42         self.baseurl = []
  43         self.mirrorlist = None
  44         self.proxy = None
  45         self.proxy_username = None
  46         self.proxy_password = None
  47         self.includepkgs = None
  48         self.includepkgs = None
  49         self.exclude = None
  50
  51         self.enabled = True
  52         self.autorefresh = True
  53         self.keeppackages = True
  54
  55 class RepoError(CreatorError):
  56     pass
  57
  58 class RpmError(CreatorError):
  59     pass
  60
  61 from mic.pluginbase import BackendPlugin
  62 class Zypp(BackendPlugin):
  63     name = 'zypp'
  64
  65     def __init__(self, creator = None, recording_pkgs=None):
  66         if not isinstance(creator, BaseImageCreator):
  67             raise CreatorError("Invalid argument: creator")
  68
  69         self.__recording_pkgs = recording_pkgs
  70         self.__pkgs_content = {}
  71         self.creator = creator
  72         self.repos = []
  73         self.to_deselect = []
  74         self.localpkgs = {}
  75         self.repo_manager = None
  76         self.repo_manager_options = None
  77         self.Z = None
  78         self.ts = None
  79         self.probFilterFlags = []
  80         self.incpkgs = []
  81         self.excpkgs = []
  82
  83         self.has_prov_query = True
  84
  85
  86     def doFileLogSetup(self, uid, logfile):
  87         # don't do the file log for the livecd as it can lead to open fds
  88         # being left and an inability to clean up after ourself
  89         pass
  90
  91     def closeRpmDB(self):
  92         pass
  93
  94     def close(self):
  95         self.closeRpmDB()
  96         if not os.path.exists("/etc/fedora-release") and not os.path.exists("/etc/meego-release"):
  97             for i in range(3, os.sysconf("SC_OPEN_MAX")):
  98                 try:
  99                     os.close(i)
 100                 except:
 101                     pass
 102         if self.ts:
 103             self.ts.closeDB()
 104             self.ts = None
 105
 106     def __del__(self):
 107         self.close()
 108
 109     def _cleanupRpmdbLocks(self, installroot):
 110         # cleans up temporary files left by bdb so that differing
 111         # versions of rpm don't cause problems
 112         import glob
 113         for f in glob.glob(installroot + "/var/lib/rpm/__db*"):
 114             os.unlink(f)
 115
 116     def setup(self, confpath, installroot):
 117         self._cleanupRpmdbLocks(installroot)
 118         self.installroot = installroot
 119
 120     def whatObsolete(self, pkg):
 121         query = zypp.PoolQuery()
 122         query.addKind(zypp.ResKind.package)
 123         query.addAttribute(zypp.SolvAttr.obsoletes, pkg)
 124         query.setMatchExact()
 125         for pi in query.queryResults(self.Z.pool()):
 126             return pi
 127         return None
 128
 129     def selectPackage(self, pkg):
 130         """ Select a given package or package pattern, can be specified with name.arch or name* or *name """
 131         if not self.Z:
 132             self.__initialize_zypp()
 133
 134         def markPoolItem(obs, pi):
 135             if obs == None:
 136                 pi.status().setToBeInstalled (zypp.ResStatus.USER)
 137             else:
 138                 obs.status().setToBeInstalled (zypp.ResStatus.USER)
 139
 140         found = False
 141         startx = pkg.startswith("*")
 142         endx = pkg.endswith("*")
 143         ispattern = startx or endx
 144         sp = pkg.rsplit(".", 1)
 145
 146         q = zypp.PoolQuery()
 147         q.addKind(zypp.ResKind.package)
 148         if ispattern:
 149             if startx and not endx:
 150                 pattern = '%s$' % (pkg[1:])
 151             if endx and not startx:
 152                 pattern = '^%s' % (pkg[0:-1])
 153             if endx and startx:
 154                 pattern = '%s' % (pkg[1:-1])
 155             q.setMatchRegex()
 156             q.addAttribute(zypp.SolvAttr.name,pattern)
 157         elif len(sp) == 2:
 158             q.setMatchExact()
 159             q.addAttribute(zypp.SolvAttr.name,sp[0])
 160         else:
 161             q.setMatchExact()
 162             q.addAttribute(zypp.SolvAttr.name,pkg)
 163
 164         for item in q.queryResults(self.Z.pool()):
 165             found = True
 166             obspkg = self.whatObsolete(item.name())
 167             if len(sp) == 2:
 168                 if item.arch() == sp[1]:
 169                     item.status().setToBeInstalled (zypp.ResStatus.USER)
 170             else:
 171                 markPoolItem(obspkg, item)
 172         # Can't match using package name, then search from packge provides infomation
 173         if found == False and not ispattern:
 174             q.addAttribute(zypp.SolvAttr.provides, pkg)
 175             q.addAttribute(zypp.SolvAttr.name,'')
 176             for item in q.queryResults(self.Z.pool()):
 177                 found = True
 178                 obspkg = self.whatObsolete(item.name())
 179                 markPoolItem(obspkg, item)
 180                 break
 181         if found:
 182             return None
 183         else:
 184             raise CreatorError("Unable to find package: %s" % (pkg,))
 185     def inDeselectPackages(self, name):
 186         """check if specified pacakges are in the list of inDeselectPackages"""
 187         for pkg in self.to_deselect:
 188             startx = pkg.startswith("*")
 189             endx = pkg.endswith("*")
 190             ispattern = startx or endx
 191             sp = pkg.rsplit(".", 2)
 192             if not ispattern:
 193                 if len(sp) == 2:
 194                     arch = "%s" % item.arch()
 195                     if name == sp[0] and arch == sp[1]:
 196                         return True;
 197                 else:
 198                     if name == sp[0]:
 199                         return True;
 200             else:
 201                 if startx and name.endswith(sp[0][1:]):
 202                         return True;
 203                 if endx and name.startswith(sp[0][:-1]):
 204                         return True;
 205         return False;
 206
 207     def deselectPackage(self, pkg):
 208         """collect packages should not be installed"""
 209         self.to_deselect.append(pkg)
 210
 211     def __selectIncpkgs(self):
 212         found = False
 213         for pkg in self.incpkgs:
 214             for item in self.Z.pool():
 215                 kind = "%s" % item.kind()
 216                 if kind == "package":
 217                     name = "%s" % item.name()
 218                     repoalias = "%s" % item.repoInfo().alias()
 219                     if name == pkg and repoalias.endswith("include"):
 220                         found = True
 221                         item.status().setToBeInstalled (zypp.ResStatus.USER)
 222                         break
 223         if not found:
 224             raise CreatorError("Unable to find package: %s" % (pkg,))
 225
 226     def __selectExcpkgs(self):
 227         found = False
 228         for pkg in self.excpkgs:
 229             for item in self.Z.pool():
 230                 kind = "%s" % item.kind()
 231                 if kind == "package":
 232                     name = "%s" % item.name()
 233                     repoalias = "%s" % item.repoInfo().alias()
 234                     if name == pkg and not repoalias.endswith("exclude"):
 235                         found = True
 236                         item.status().setToBeInstalled (zypp.ResStatus.USER)
 237                         break
 238         if not found:
 239             raise CreatorError("Unable to find package: %s" % (pkg,))
 240
 241     def selectGroup(self, grp, include = ksparser.GROUP_DEFAULT):
 242         if not self.Z:
 243             self.__initialize_zypp()
 244         found = False
 245         q=zypp.PoolQuery()
 246         q.addKind(zypp.ResKind.pattern)
 247         for item in q.queryResults(self.Z.pool()):
 248             summary = "%s" % item.summary()
 249             name = "%s" % item.name()
 250             if name == grp or summary == grp:
 251                 found = True
 252                 item.status().setToBeInstalled (zypp.ResStatus.USER)
 253                 break
 254
 255         if found:
 256             if include == ksparser.GROUP_REQUIRED:
 257                 map(lambda p: self.deselectPackage(p), grp.default_packages.keys())
 258             return None
 259         else:
 260             raise CreatorError("Unable to find pattern: %s" % (grp,))
 261
 262     def addRepository(self, name, url = None, mirrorlist = None, proxy = None, proxy_username = None, proxy_password = None, inc = None, exc = None):
 263         if not self.repo_manager:
 264             self.__initialize_repo_manager()
 265
 266         repo = RepositoryStub()
 267         repo.name = name
 268         repo.id = name
 269         repo.proxy = proxy
 270         repo.proxy_username = proxy_username
 271         repo.proxy_password = proxy_password
 272         repo.baseurl.append(url)
 273         repo_alias = repo.id
 274         if inc:
 275             repo_alias = name + "include"
 276             self.incpkgs = inc
 277         if exc:
 278             repo_alias = name + "exclude"
 279             self.excpkgs = exc
 280
 281         # check LICENSE files
 282         if not rpmmisc.checkRepositoryEULA(name, repo):
 283             msger.warning('skip repo:%s for failed EULA confirmation' % name)
 284             return None
 285
 286         if mirrorlist:
 287             repo.mirrorlist = mirrorlist
 288
 289         # Enable gpg check for verifying corrupt packages
 290         repo.gpgcheck = 1
 291         self.repos.append(repo)
 292
 293         try:
 294             repo_info = zypp.RepoInfo()
 295             repo_info.setAlias(repo_alias)
 296             repo_info.setName(repo.name)
 297             repo_info.setEnabled(repo.enabled)
 298             repo_info.setAutorefresh(repo.autorefresh)
 299             repo_info.setKeepPackages(repo.keeppackages)
 300             repo_info.addBaseUrl(zypp.Url(repo.baseurl[0]))
 301             self.repo_manager.addRepository(repo_info)
 302             self.__build_repo_cache(name)
 303         except RuntimeError, e:
 304             raise CreatorError(str(e))
 305
 306         msger.verbose('repo: %s was added' % name)
 307         return repo
 308
 309     def installHasFile(self, file):
 310         return False
 311
 312     def runInstall(self, checksize = 0):
 313         if self.incpkgs:
 314             self.__selectIncpkgs()
 315         if self.excpkgs:
 316             self.__selectExcpkgs()
 317
 318         os.environ["HOME"] = "/"
 319         self.buildTransaction()
 320
 321         todo = zypp.GetResolvablesToInsDel(self.Z.pool())
 322         installed_pkgs = todo._toInstall
 323         dlpkgs = []
 324         for item in installed_pkgs:
 325             if not zypp.isKindPattern(item) and not self.inDeselectPackages(item.name()):
 326                 dlpkgs.append(item)
 327
 328         # record the total size of installed pkgs
 329         pkgs_total_size = sum(map(lambda x: int(x.installSize()), dlpkgs))
 330
 331         # check needed size before actually download and install
 332         if checksize and pkgs_total_size > checksize:
 333             raise CreatorError("Size of specified root partition in kickstart file is too small to install all selected packages.")
 334
 335         if self.__recording_pkgs:
 336             # record all pkg and the content
 337             for pkg in dlpkgs:
 338                 pkg_long_name = "%s-%s.%s.rpm" % (pkg.name(), pkg.edition(), pkg.arch())
 339                 self.__pkgs_content[pkg_long_name] = {} #TBD: to get file list
 340
 341         total_count = len(dlpkgs)
 342         cached_count = 0
 343         localpkgs = self.localpkgs.keys()
 344         msger.info("Checking packages cache and packages integrity ...")
 345         for po in dlpkgs:
 346             """ Check if it is cached locally """
 347             if po.name() in localpkgs:
 348                 cached_count += 1
 349             else:
 350                 local = self.getLocalPkgPath(po)
 351                 if os.path.exists(local):
 352                     if self.checkPkg(local) != 0:
 353                         os.unlink(local)
 354                     else:
 355                         cached_count += 1
 356         download_count =  total_count - cached_count
 357         msger.info("%d packages to be installed, %d packages gotten from cache, %d packages to be downloaded" % (total_count, cached_count, download_count))
 358         try:
 359             if download_count > 0:
 360                 msger.info("Downloading packages ...")
 361             self.downloadPkgs(dlpkgs, download_count)
 362             self.installPkgs(dlpkgs)
 363
 364         except RepoError, e:
 365             raise CreatorError("Unable to download from repo : %s" % (e,))
 366         except RpmError, e:
 367             raise CreatorError("Unable to install: %s" % (e,))
 368
 369     def getAllContent(self):
 370         return self.__pkgs_content
 371
 372     def __initialize_repo_manager(self):
 373         if self.repo_manager:
 374             return
 375
 376         """ Clean up repo metadata """
 377         shutil.rmtree(self.creator.cachedir + "/etc", ignore_errors = True)
 378
 379         zypp.KeyRing.setDefaultAccept( zypp.KeyRing.ACCEPT_UNSIGNED_FILE
 380                                      | zypp.KeyRing.ACCEPT_VERIFICATION_FAILED
 381                                      | zypp.KeyRing.ACCEPT_UNKNOWNKEY
 382                                      | zypp.KeyRing.TRUST_KEY_TEMPORARILY
 383                                      )
 384         self.repo_manager_options = zypp.RepoManagerOptions(zypp.Pathname(self.creator._instroot))
 385         self.repo_manager_options.knownReposPath = zypp.Pathname(self.creator.cachedir + "/etc/zypp/repos.d")
 386         self.repo_manager_options.repoCachePath = zypp.Pathname(self.creator.cachedir)
 387         self.repo_manager_options.repoRawCachePath = zypp.Pathname(self.creator.cachedir + "/raw")
 388         self.repo_manager_options.repoSolvCachePath = zypp.Pathname(self.creator.cachedir + "/solv")
 389         self.repo_manager_options.repoPackagesCachePath = zypp.Pathname(self.creator.cachedir + "/packages")
 390
 391         self.repo_manager = zypp.RepoManager(self.repo_manager_options)
 392
 393     def __build_repo_cache(self, name):
 394         repo = self.repo_manager.getRepositoryInfo(name)
 395         if self.repo_manager.isCached(repo) or not repo.enabled():
 396             return
 397         self.repo_manager.buildCache(repo, zypp.RepoManager.BuildIfNeeded)
 398
 399     def __initialize_zypp(self):
 400         if self.Z:
 401             return
 402
 403         zconfig = zypp.ZConfig_instance()
 404
 405         """ Set system architecture """
 406         if self.creator.target_arch and self.creator.target_arch.startswith("arm"):
 407             arches = ["armv7l", "armv7nhl", "armv7hl", "armv5tel"]
 408             if self.creator.target_arch not in arches:
 409                 raise CreatorError("Invalid architecture: %s" % self.creator.target_arch)
 410
 411             arch_map = {}
 412             try:
 413                 arch_map[self.creator.target_arch] = zypp.Arch(self.creator.target_arch)
 414             except AttributeError:
 415                 msger.error('libzypp/python-zypp in host system cannot support arch %s, please'
 416                             ' update it to enhanced version which can be found in repo.meego.com/tools'\
 417                             % self.creator.target_arch)
 418
 419             zconfig.setSystemArchitecture(arch_map[self.creator.target_arch])
 420
 421         msger.info("zypp architecture is <%s>" % zconfig.systemArchitecture())
 422
 423         """ repoPackagesCachePath is corrected by this """
 424         self.repo_manager = zypp.RepoManager(self.repo_manager_options)
 425         repos = self.repo_manager.knownRepositories()
 426         for repo in repos:
 427             if not repo.enabled():
 428                 continue
 429             self.repo_manager.loadFromCache(repo)
 430
 431         self.Z = zypp.ZYppFactory_instance().getZYpp()
 432         self.Z.initializeTarget(zypp.Pathname(self.creator._instroot))
 433         self.Z.target().load()
 434
 435
 436     def buildTransaction(self):
 437         if not self.Z.resolver().resolvePool():
 438             msger.warning("Problem count: %d" % len(self.Z.resolver().problems()))
 439             for problem in self.Z.resolver().problems():
 440                 msger.warning("Problem: %s, %s" % (problem.description().decode("utf-8"), problem.details().decode("utf-8")))
 441
 442     def getLocalPkgPath(self, po):
 443         repoinfo = po.repoInfo()
 444         name = po.name()
 445         cacheroot = repoinfo.packagesPath()
 446         arch =  po.arch()
 447         edition = po.edition()
 448         version = "%s-%s" % (edition.version(), edition.release())
 449         pkgpath = "%s/%s/%s-%s.%s.rpm" % (cacheroot, arch, name, version, arch)
 450         return pkgpath
 451
 452     def installLocal(self, pkg, po=None, updateonly=False):
 453         if not self.ts:
 454             self.__initialize_transaction()
 455         ts = rpmUtils.transaction.initReadOnlyTransaction()
 456         try:
 457             hdr = rpmUtils.miscutils.hdrFromPackage(ts, pkg)
 458         except RpmUtilsError, e:
 459             raise Errors.MiscError, \
 460                 'Could not open local rpm file: %s: %s' % (self.localpath, e)
 461         arch = zypp.Arch(hdr.arch)
 462         if self.creator.target_arch == None:
 463             # TODO, get the default_arch from conf or detected from global settings
 464             sysarch = zypp.Arch('i686')
 465         else:
 466             sysarch = zypp.Arch(self.creator.target_arch)
 467         if arch.compatible_with (sysarch):
 468             pkgname = self.__get_pkg_name(pkg)
 469             self.localpkgs[pkgname] = pkg
 470             self.selectPackage(pkgname)
 471             msger.info("Marking %s to be installed" % (pkg))
 472         else:
 473             msger.warning ("Cannot add package %s to transaction. Not a compatible architecture: %s" % (pkg, hdr.arch))
 474
 475     def __get_pkg_name(self, pkgpath):
 476         h = rpmmisc.readRpmHeader(self.ts, pkgpath)
 477         return h["name"]
 478
 479     def downloadPkgs(self, package_objects, count):
 480         localpkgs = self.localpkgs.keys()
 481         progress_obj = fs.TextProgress(count)
 482         for po in package_objects:
 483             if po.name() in localpkgs:
 484                 continue
 485             filename = self.getLocalPkgPath(po)
 486             if os.path.exists(filename):
 487                 if self.checkPkg(filename) == 0:
 488                     continue
 489             dirn = os.path.dirname(filename)
 490             if not os.path.exists(dirn):
 491                 os.makedirs(dirn)
 492             baseurl = po.repoInfo().baseUrls()[0].__str__()
 493             proxy = self.get_proxy(po.repoInfo())
 494             proxies = {}
 495             if proxy:
 496                 proxies = {str(proxy.split(":")[0]):str(proxy)}
 497
 498             location = zypp.asKindPackage(po).location()
 499             location = location.filename().__str__()
 500             if location.startswith("./"):
 501                 location = location[2:]
 502             url = baseurl + "/%s" % location
 503             try:
 504                 filename = fs.myurlgrab(url, filename, proxies, progress_obj)
 505             except CreatorError:
 506                 self.close()
 507                 raise
 508
 509     def installPkgs(self, package_objects):
 510         if not self.ts:
 511             self.__initialize_transaction()
 512
 513         """ Set filters """
 514         probfilter = 0
 515         for flag in self.probFilterFlags:
 516             probfilter |= flag
 517         self.ts.setProbFilter(probfilter)
 518
 519         localpkgs = self.localpkgs.keys()
 520         for po in package_objects:
 521             pkgname = po.name()
 522             if pkgname in localpkgs:
 523                 rpmpath = self.localpkgs[pkgname]
 524             else:
 525                 rpmpath = self.getLocalPkgPath(po)
 526             if not os.path.exists(rpmpath):
 527                 """ Maybe it is a local repo """
 528                 baseurl = po.repoInfo().baseUrls()[0].__str__()
 529                 baseurl = baseurl.strip()
 530                 if baseurl.startswith("file:/"):
 531                     rpmpath = baseurl[5:] + "/%s/%s" % (po.arch(), os.path.basename(rpmpath))
 532             if not os.path.exists(rpmpath):
 533                 raise RpmError("Error: %s doesn't exist" % rpmpath)
 534             h = rpmmisc.readRpmHeader(self.ts, rpmpath)
 535             self.ts.addInstall(h, rpmpath, 'u')
 536
 537         unresolved_dependencies = self.ts.check()
 538         if not unresolved_dependencies:
 539             self.ts.order()
 540             cb = rpmmisc.RPMInstallCallback(self.ts)
 541             self.ts.run(cb.callback, '')
 542             self.ts.closeDB()
 543             self.ts = None
 544         else:
 545             for pkg, need, needflags, sense, key in unresolved_dependencies:
 546                 package = '-'.join(pkg)
 547                 if needflags == rpm.RPMSENSE_LESS:
 548                     deppkg = ' < '.join(need)
 549                 elif needflags == rpm.RPMSENSE_EQUAL:
 550                     deppkg = ' = '.join(need)
 551                 elif needflags == rpm.RPMSENSE_GREATER:
 552                     deppkg = ' > '.join(need)
 553                 else:
 554                     deppkg = '-'.join(need)
 555
 556                 if sense == rpm.RPMDEP_SENSE_REQUIRES:
 557                     msger.warning ("[%s] Requires [%s], which is not provided" % (package, deppkg))
 558                 elif sense == rpm.RPMDEP_SENSE_CONFLICTS:
 559                     msger.warning ("[%s] Conflicts with [%s]" % (package, deppkg))
 560
 561             raise RepoError("Unresolved dependencies, transaction failed.")
 562
 563     def __initialize_transaction(self):
 564         if not self.ts:
 565             self.ts = rpm.TransactionSet(self.creator._instroot)
 566             # Set to not verify DSA signatures.
 567             self.ts.setVSFlags(rpm._RPMVSF_NOSIGNATURES|rpm._RPMVSF_NODIGESTS)
 568
 569     def checkPkg(self, pkg):
 570         ret = 1
 571         if not os.path.exists(pkg):
 572             return ret
 573         ret = rpmmisc.checkRpmIntegrity('rpm', pkg)
 574         if ret != 0:
 575             msger.warning("package %s is damaged: %s" % (os.path.basename(pkg), pkg))
 576
 577         return ret
 578
 579     def _add_prob_flags(self, *flags):
 580         for flag in flags:
 581            if flag not in self.probFilterFlags:
 582                self.probFilterFlags.append(flag)
 583
 584     def get_proxy(self, repoinfo):
 585         proxy = None
 586         reponame = "%s" % repoinfo.name()
 587         for repo in self.repos:
 588             if repo.name == reponame:
 589                 proxy = repo.proxy
 590                 break
 591
 592         if proxy:
 593             return proxy
 594         else:
 595             repourl = repoinfo.baseUrls()[0].__str__()
 596             return get_proxy_for(repourl)