Merge "Add 47 system test case."
[tools/mic.git] / plugins / backend / zypppkgmgr.py
1 #!/usr/bin/python -tt
2 #
3 # Copyright (c) 2010, 2011 Intel, Inc.
4 #
5 # This program is free software; you can redistribute it and/or modify it
6 # under the terms of the GNU General Public License as published by the Free
7 # Software Foundation; version 2 of the License
8 #
9 # This program is distributed in the hope that it will be useful, but
10 # WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
11 # or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
12 # for more details.
13 #
14 # You should have received a copy of the GNU General Public License along
15 # with this program; if not, write to the Free Software Foundation, Inc., 59
16 # Temple Place - Suite 330, Boston, MA 02111-1307, USA.
17
18 import os
19 import shutil
20 import urlparse
21 import rpm
22 from mic.utils import runner, fs_related
23
24 import zypp
25 if not hasattr(zypp, 'PoolQuery') or not hasattr(zypp.RepoManager, 'loadSolvFile'):
26     raise ImportError("python-zypp in host system cannot support PoolQuery or loadSolvFile interface, "
27                       "please update it to enhanced version which can be found in repo.meego.com/tools")
28
29 from mic import msger
30 from mic.kickstart import ksparser
31 from mic.utils import rpmmisc
32 from mic.utils.proxy import get_proxy_for
33 from mic.utils.errors import CreatorError
34 from mic.imager.baseimager import BaseImageCreator
35
36 class RepositoryStub:
37     def __init__(self):
38         self.name = None
39         self.baseurl = []
40         self.mirrorlist = None
41         self.proxy = None
42         self.proxy_username = None
43         self.proxy_password = None
44
45         self.enabled = True
46         self.autorefresh = True
47         self.keeppackages = True
48         self.priority = None
49
50 class RepoError(CreatorError):
51     pass
52
53 class RpmError(CreatorError):
54     pass
55
56 from mic.pluginbase import BackendPlugin
57 class Zypp(BackendPlugin):
58     name = 'zypp'
59
60     def __init__(self, creator = None):
61         if not isinstance(creator, BaseImageCreator):
62             raise CreatorError("Invalid argument: creator")
63
64         self.__pkgs_license = {}
65         self.__pkgs_content = {}
66         self.creator = creator
67         self.repos = []
68         self.to_deselect = []
69         self.localpkgs = {}
70         self.repo_manager = None
71         self.repo_manager_options = None
72         self.Z = None
73         self.ts = None
74         self.probFilterFlags = []
75         self.incpkgs = {}
76         self.excpkgs = {}
77
78         self.has_prov_query = True
79
80     def doFileLogSetup(self, uid, logfile):
81         # don't do the file log for the livecd as it can lead to open fds
82         # being left and an inability to clean up after ourself
83         pass
84
85     def closeRpmDB(self):
86         pass
87
88     def close(self):
89         self.closeRpmDB()
90         if not os.path.exists("/etc/fedora-release") and not os.path.exists("/etc/meego-release"):
91             for i in range(3, os.sysconf("SC_OPEN_MAX")):
92                 try:
93                     os.close(i)
94                 except:
95                     pass
96         if self.ts:
97             self.ts.closeDB()
98             self.ts = None
99
100     def __del__(self):
101         self.close()
102
103     def _cleanupRpmdbLocks(self, installroot):
104         # cleans up temporary files left by bdb so that differing
105         # versions of rpm don't cause problems
106         import glob
107         for f in glob.glob(installroot + "/var/lib/rpm/__db*"):
108             os.unlink(f)
109
110     def setup(self, confpath, installroot):
111         self._cleanupRpmdbLocks(installroot)
112         self.installroot = installroot
113
114     def whatObsolete(self, pkg):
115         query = zypp.PoolQuery()
116         query.addKind(zypp.ResKind.package)
117         query.addAttribute(zypp.SolvAttr.obsoletes, pkg)
118         query.setMatchExact()
119         for pi in query.queryResults(self.Z.pool()):
120             return pi
121         return None
122
123     def _splitPkgString(self, pkg):
124         sp = pkg.rsplit(".",1)
125         name = sp[0]
126         arch = None
127         if len(sp) == 2:
128             arch = sp[1]
129             if self.creator.target_arch == None:
130                 # TODO, get the default_arch from conf or detected from global settings
131                 sysarch = zypp.Arch('i686')
132             else:
133                 sysarch = zypp.Arch(self.creator.target_arch)
134             if not zypp.Arch(arch).compatible_with (sysarch):
135                 arch = None
136                 name = ".".join(sp)
137         return name, arch
138
139     def selectPackage(self, pkg):
140         """ Select a given package or package pattern, can be specified with name.arch or name* or *name """
141         if not self.Z:
142             self.__initialize_zypp()
143
144         def markPoolItem(obs, pi):
145             if obs == None:
146                 pi.status().setToBeInstalled (zypp.ResStatus.USER)
147             else:
148                 obs.status().setToBeInstalled (zypp.ResStatus.USER)
149
150         found = False
151         startx = pkg.startswith("*")
152         endx = pkg.endswith("*")
153         ispattern = startx or endx
154         name, arch = self._splitPkgString(pkg)
155
156         q = zypp.PoolQuery()
157         q.addKind(zypp.ResKind.package)
158         if ispattern:
159             if startx and not endx:
160                 pattern = '%s$' % (pkg[1:])
161             if endx and not startx:
162                 pattern = '^%s' % (pkg[0:-1])
163             if endx and startx:
164                 pattern = '%s' % (pkg[1:-1])
165             q.setMatchRegex()
166             q.addAttribute(zypp.SolvAttr.name,pattern)
167         elif arch:
168             q.setMatchExact()
169             q.addAttribute(zypp.SolvAttr.name,name)
170         else:
171             q.setMatchExact()
172             q.addAttribute(zypp.SolvAttr.name,pkg)
173
174         for item in sorted(q.queryResults(self.Z.pool()), key=lambda item: str(item.edition()), reverse=True):
175             if item.name() in self.excpkgs.keys() and self.excpkgs[item.name()] == item.repoInfo().name():
176                 continue
177             if item.name() in self.incpkgs.keys() and self.incpkgs[item.name()] != item.repoInfo().name():
178                 continue
179             found = True
180             obspkg = self.whatObsolete(item.name())
181             if arch:
182                 if arch == str(item.arch()):
183                     item.status().setToBeInstalled (zypp.ResStatus.USER)
184             else:
185                 markPoolItem(obspkg, item)
186             if not ispattern:
187                 break
188         # Can't match using package name, then search from packge provides infomation
189         if found == False and not ispattern:
190             q.addAttribute(zypp.SolvAttr.provides, pkg)
191             q.addAttribute(zypp.SolvAttr.name,'')
192             for item in sorted(q.queryResults(self.Z.pool()), key=lambda item: str(item.edition()), reverse=True):
193                 if item.name() in self.excpkgs.keys() and self.excpkgs[item.name()] == item.repoInfo().name():
194                     continue
195                 if item.name() in self.incpkgs.keys() and self.incpkgs[item.name()] != item.repoInfo().name():
196                     continue
197                 found = True
198                 obspkg = self.whatObsolete(item.name())
199                 markPoolItem(obspkg, item)
200                 break
201         if found:
202             return None
203         else:
204             raise CreatorError("Unable to find package: %s" % (pkg,))
205
206     def inDeselectPackages(self, item):
207         """check if specified pacakges are in the list of inDeselectPackages"""
208         name = item.name()
209         for pkg in self.to_deselect:
210             startx = pkg.startswith("*")
211             endx = pkg.endswith("*")
212             ispattern = startx or endx
213             pkgname, pkgarch = self._splitPkgString(pkg)
214             if not ispattern:
215                 if pkgarch:
216                     if name == pkgname and str(item.arch()) == pkgarch:
217                         return True;
218                 else:
219                     if name == pkgname:
220                         return True;
221             else:
222                 if startx and name.endswith(pkg[1:]):
223                         return True;
224                 if endx and name.startswith(pkg[:-1]):
225                         return True;
226         return False;
227
228     def deselectPackage(self, pkg):
229         """collect packages should not be installed"""
230         self.to_deselect.append(pkg)
231
232     def selectGroup(self, grp, include = ksparser.GROUP_DEFAULT):
233         if not self.Z:
234             self.__initialize_zypp()
235         found = False
236         q=zypp.PoolQuery()
237         q.addKind(zypp.ResKind.pattern)
238         for item in q.queryResults(self.Z.pool()):
239             summary = "%s" % item.summary()
240             name = "%s" % item.name()
241             if name == grp or summary == grp:
242                 found = True
243                 item.status().setToBeInstalled (zypp.ResStatus.USER)
244                 break
245
246         if found:
247             if include == ksparser.GROUP_REQUIRED:
248                 map(lambda p: self.deselectPackage(p), grp.default_packages.keys())
249             return None
250         else:
251             raise CreatorError("Unable to find pattern: %s" % (grp,))
252
253     def addRepository(self, name, url = None, mirrorlist = None, proxy = None,
254                       proxy_username = None, proxy_password = None,
255                       inc = None, exc = None, ssl_verify = True, cost=None,
256                       priority=None):
257         # TODO: Handle cost attribute for repos
258         if not self.repo_manager:
259             self.__initialize_repo_manager()
260
261         repo = RepositoryStub()
262         repo.name = name
263         repo.id = name
264         repo.proxy = proxy
265         repo.proxy_username = proxy_username
266         repo.proxy_password = proxy_password
267         repo.ssl_verify = ssl_verify
268         repo.baseurl.append(url)
269         if inc:
270             for pkg in inc:
271                 self.incpkgs[pkg] = name
272         if exc:
273             for pkg in exc:
274                 self.excpkgs[pkg] = name
275
276         # check LICENSE files
277         if not rpmmisc.checkRepositoryEULA(name, repo):
278             msger.warning('skip repo:%s for failed EULA confirmation' % name)
279             return None
280
281         if mirrorlist:
282             repo.mirrorlist = mirrorlist
283
284         # Enable gpg check for verifying corrupt packages
285         repo.gpgcheck = 1
286         if priority:
287             repo.priority = priority
288         self.repos.append(repo)
289
290         try:
291             repo_info = zypp.RepoInfo()
292             repo_info.setAlias(repo.name)
293             repo_info.setName(repo.name)
294             repo_info.setEnabled(repo.enabled)
295             repo_info.setAutorefresh(repo.autorefresh)
296             repo_info.setKeepPackages(repo.keeppackages)
297             baseurl = zypp.Url(repo.baseurl[0])
298             if not ssl_verify:
299                 baseurl.setQueryParam("ssl_verify", "no")
300             if proxy:
301                 (scheme, host, path, parm, query, frag) = urlparse.urlparse(proxy)
302                 proxyinfo = host.split(":")
303                 baseurl.setQueryParam ("proxy", proxyinfo[0])
304                 port = "80"
305                 if len(proxyinfo) > 1:
306                     port = proxyinfo[1]
307                 baseurl.setQueryParam ("proxyport", port)
308             repo_info.addBaseUrl(baseurl)
309             if repo.priority:
310                 repo_info.setPriority(repo.priority)
311             self.repo_manager.addRepository(repo_info)
312             self.__build_repo_cache(name)
313         except RuntimeError, e:
314             raise CreatorError(str(e))
315
316         msger.verbose('repo: %s was added' % name)
317         return repo
318
319     def installHasFile(self, file):
320         return False
321
322     def runInstall(self, checksize = 0):
323         os.environ["HOME"] = "/"
324         self.buildTransaction()
325
326         todo = zypp.GetResolvablesToInsDel(self.Z.pool())
327         installed_pkgs = todo._toInstall
328         dlpkgs = []
329         for item in installed_pkgs:
330             if not zypp.isKindPattern(item) and not self.inDeselectPackages(item):
331                 dlpkgs.append(item)
332
333         # record the total size of installed pkgs
334         pkgs_total_size = sum(map(lambda x: int(x.installSize()), dlpkgs))
335
336         # check needed size before actually download and install
337         if checksize and pkgs_total_size > checksize:
338             raise CreatorError("Size of specified root partition in kickstart file is too small to install all selected packages.")
339
340         # record all pkg and the content
341         localpkgs = self.localpkgs.keys()
342         for pkg in dlpkgs:
343             license = ''
344             if pkg.name() in localpkgs:
345                 hdr = rpmmisc.readRpmHeader(self.ts, self.localpkgs[pkg.name()])
346                 pkg_long_name = "%s-%s-%s.%s.rpm" % (hdr['name'], hdr['version'], hdr['release'], hdr['arch'])
347                 license = hdr['license']
348             else:
349                 pkg_long_name = "%s-%s.%s.rpm" % (pkg.name(), pkg.edition(), pkg.arch())
350                 package = zypp.asKindPackage(pkg)
351                 license = package.license()
352             self.__pkgs_content[pkg_long_name] = {} #TBD: to get file list
353             if license in self.__pkgs_license.keys():
354                 self.__pkgs_license[license].append(pkg_long_name)
355             else:
356                 self.__pkgs_license[license] = [pkg_long_name]
357
358         total_count = len(dlpkgs)
359         cached_count = 0
360         localpkgs = self.localpkgs.keys()
361         msger.info("Checking packages cache and packages integrity ...")
362         for po in dlpkgs:
363             """ Check if it is cached locally """
364             if po.name() in localpkgs:
365                 cached_count += 1
366             else:
367                 local = self.getLocalPkgPath(po)
368                 if os.path.exists(local):
369                     if self.checkPkg(local) != 0:
370                         os.unlink(local)
371                     else:
372                         cached_count += 1
373         download_count =  total_count - cached_count
374         msger.info("%d packages to be installed, %d packages gotten from cache, %d packages to be downloaded" % (total_count, cached_count, download_count))
375         try:
376             if download_count > 0:
377                 msger.info("Downloading packages ...")
378             self.downloadPkgs(dlpkgs, download_count)
379             self.installPkgs(dlpkgs)
380
381         except RepoError, e:
382             raise CreatorError("Unable to download from repo : %s" % (e,))
383         except RpmError, e:
384             raise CreatorError("Unable to install: %s" % (e,))
385
386     def getAllContent(self):
387         return self.__pkgs_content
388
389     def getPkgsLicense(self):
390         return self.__pkgs_license
391
392     def __initialize_repo_manager(self):
393         if self.repo_manager:
394             return
395
396         """ Clean up repo metadata """
397         shutil.rmtree(self.creator.cachedir + "/etc", ignore_errors = True)
398         shutil.rmtree(self.creator.cachedir + "/solv", ignore_errors = True)
399         shutil.rmtree(self.creator.cachedir + "/raw", ignore_errors = True)
400
401         zypp.KeyRing.setDefaultAccept( zypp.KeyRing.ACCEPT_UNSIGNED_FILE
402                                      | zypp.KeyRing.ACCEPT_VERIFICATION_FAILED
403                                      | zypp.KeyRing.ACCEPT_UNKNOWNKEY
404                                      | zypp.KeyRing.TRUST_KEY_TEMPORARILY
405                                      )
406         self.repo_manager_options = zypp.RepoManagerOptions(zypp.Pathname(self.creator._instroot))
407         self.repo_manager_options.knownReposPath = zypp.Pathname(self.creator.cachedir + "/etc/zypp/repos.d")
408         self.repo_manager_options.repoCachePath = zypp.Pathname(self.creator.cachedir)
409         self.repo_manager_options.repoRawCachePath = zypp.Pathname(self.creator.cachedir + "/raw")
410         self.repo_manager_options.repoSolvCachePath = zypp.Pathname(self.creator.cachedir + "/solv")
411         self.repo_manager_options.repoPackagesCachePath = zypp.Pathname(self.creator.cachedir + "/packages")
412
413         self.repo_manager = zypp.RepoManager(self.repo_manager_options)
414
415     def __build_repo_cache(self, name):
416         repo = self.repo_manager.getRepositoryInfo(name)
417         if self.repo_manager.isCached(repo) or not repo.enabled():
418             return
419         self.repo_manager.buildCache(repo, zypp.RepoManager.BuildIfNeeded)
420
421     def __initialize_zypp(self):
422         if self.Z:
423             return
424
425         zconfig = zypp.ZConfig_instance()
426
427         """ Set system architecture """
428         if self.creator.target_arch:
429             zconfig.setSystemArchitecture(zypp.Arch(self.creator.target_arch))
430
431         msger.info("zypp architecture is <%s>" % zconfig.systemArchitecture())
432
433         """ repoPackagesCachePath is corrected by this """
434         self.repo_manager = zypp.RepoManager(self.repo_manager_options)
435         repos = self.repo_manager.knownRepositories()
436         for repo in repos:
437             if not repo.enabled():
438                 continue
439             self.repo_manager.loadFromCache(repo)
440
441         self.Z = zypp.ZYppFactory_instance().getZYpp()
442         self.Z.initializeTarget(zypp.Pathname(self.creator._instroot))
443         self.Z.target().load()
444
445
446     def buildTransaction(self):
447         if not self.Z.resolver().resolvePool():
448             msger.warning("Problem count: %d" % len(self.Z.resolver().problems()))
449             for problem in self.Z.resolver().problems():
450                 msger.warning("Problem: %s, %s" % (problem.description().decode("utf-8"), problem.details().decode("utf-8")))
451
452     def getLocalPkgPath(self, po):
453         repoinfo = po.repoInfo()
454         cacheroot = repoinfo.packagesPath()
455         location= zypp.asKindPackage(po).location()
456         rpmpath = str(location.filename())
457         pkgpath = "%s/%s" % (cacheroot, rpmpath)
458         return pkgpath
459
460     def installLocal(self, pkg, po=None, updateonly=False):
461         if not self.ts:
462             self.__initialize_transaction()
463         solvfile = "%s/.solv" % (self.creator.cachedir)
464         rc, out = runner.runtool([fs_related.find_binary_path("rpms2solv"), pkg])
465         if rc == 0:
466             f = open(solvfile, "w+")
467             f.write(out)
468             f.close()
469             warnmsg = self.repo_manager.loadSolvFile(solvfile , os.path.basename(pkg))
470             if warnmsg:
471                 msger.warning(warnmsg)
472             os.unlink(solvfile)
473         else:
474             msger.warning('Can not get %s solv data.' % pkg)
475         hdr = rpmmisc.readRpmHeader(self.ts, pkg)
476         arch = zypp.Arch(hdr['arch'])
477         if self.creator.target_arch == None:
478             # TODO, get the default_arch from conf or detected from global settings
479             sysarch = zypp.Arch('i686')
480         else:
481             sysarch = zypp.Arch(self.creator.target_arch)
482         if arch.compatible_with (sysarch):
483             pkgname = hdr['name']
484             self.localpkgs[pkgname] = pkg
485             self.selectPackage(pkgname)
486             msger.info("Marking %s to be installed" % (pkg))
487         else:
488             msger.warning ("Cannot add package %s to transaction. Not a compatible architecture: %s" % (pkg, hdr['arch']))
489
490     def downloadPkgs(self, package_objects, count):
491         localpkgs = self.localpkgs.keys()
492         progress_obj = rpmmisc.TextProgress(count)
493         for po in package_objects:
494             if po.name() in localpkgs:
495                 continue
496             filename = self.getLocalPkgPath(po)
497             if os.path.exists(filename):
498                 if self.checkPkg(filename) == 0:
499                     continue
500             dirn = os.path.dirname(filename)
501             if not os.path.exists(dirn):
502                 os.makedirs(dirn)
503             baseurl = str(po.repoInfo().baseUrls()[0])
504             index = baseurl.find("?")
505             if index > -1:
506                 baseurl = baseurl[:index]
507             proxy = self.get_proxy(po.repoInfo())
508             proxies = {}
509             if proxy:
510                 proxies = {str(proxy.split(":")[0]):str(proxy)}
511
512             location = zypp.asKindPackage(po).location()
513             location = str(location.filename())
514             if location.startswith("./"):
515                 location = location[2:]
516             url = baseurl + "/%s" % location
517             try:
518                 filename = rpmmisc.myurlgrab(url, filename, proxies, progress_obj)
519             except CreatorError:
520                 self.close()
521                 raise
522
523     def installPkgs(self, package_objects):
524         if not self.ts:
525             self.__initialize_transaction()
526
527         """ Set filters """
528         probfilter = 0
529         for flag in self.probFilterFlags:
530             probfilter |= flag
531         self.ts.setProbFilter(probfilter)
532
533         localpkgs = self.localpkgs.keys()
534         for po in package_objects:
535             pkgname = po.name()
536             if pkgname in localpkgs:
537                 rpmpath = self.localpkgs[pkgname]
538             else:
539                 rpmpath = self.getLocalPkgPath(po)
540             if not os.path.exists(rpmpath):
541                 """ Maybe it is a local repo """
542                 baseurl = str(po.repoInfo().baseUrls()[0])
543                 baseurl = baseurl.strip()
544                 location = zypp.asKindPackage(po).location()
545                 location = str(location.filename())
546                 if baseurl.startswith("file:/"):
547                     rpmpath = baseurl[5:] + "/%s" % (location)
548             if not os.path.exists(rpmpath):
549                 raise RpmError("Error: %s doesn't exist" % rpmpath)
550             h = rpmmisc.readRpmHeader(self.ts, rpmpath)
551             self.ts.addInstall(h, rpmpath, 'u')
552
553         unresolved_dependencies = self.ts.check()
554         if not unresolved_dependencies:
555             self.ts.order()
556             cb = rpmmisc.RPMInstallCallback(self.ts)
557             installlogfile = "%s/__catched_stderr.buf" % (self.creator._instroot)
558             msger.enable_logstderr(installlogfile)
559             errors = self.ts.run(cb.callback, '')
560             if errors is None:
561                 pass
562             elif len(errors) == 0:
563                 msger.warning('scriptlet or other non-fatal errors occurred during transaction.')
564             else:
565                 for e in errors:
566                     msger.warning(e[0])
567                 msger.error('Could not run transaction.')
568             msger.disable_logstderr()
569
570             self.ts.closeDB()
571             self.ts = None
572         else:
573             for pkg, need, needflags, sense, key in unresolved_dependencies:
574                 package = '-'.join(pkg)
575                 if needflags == rpm.RPMSENSE_LESS:
576                     deppkg = ' < '.join(need)
577                 elif needflags == rpm.RPMSENSE_EQUAL:
578                     deppkg = ' = '.join(need)
579                 elif needflags == rpm.RPMSENSE_GREATER:
580                     deppkg = ' > '.join(need)
581                 else:
582                     deppkg = '-'.join(need)
583
584                 if sense == rpm.RPMDEP_SENSE_REQUIRES:
585                     msger.warning ("[%s] Requires [%s], which is not provided" % (package, deppkg))
586                 elif sense == rpm.RPMDEP_SENSE_CONFLICTS:
587                     msger.warning ("[%s] Conflicts with [%s]" % (package, deppkg))
588
589             raise RepoError("Unresolved dependencies, transaction failed.")
590
591     def __initialize_transaction(self):
592         if not self.ts:
593             self.ts = rpm.TransactionSet(self.creator._instroot)
594             # Set to not verify DSA signatures.
595             self.ts.setVSFlags(rpm._RPMVSF_NOSIGNATURES|rpm._RPMVSF_NODIGESTS)
596
597     def checkPkg(self, pkg):
598         ret = 1
599         if not os.path.exists(pkg):
600             return ret
601         ret = rpmmisc.checkRpmIntegrity('rpm', pkg)
602         if ret != 0:
603             msger.warning("package %s is damaged: %s" % (os.path.basename(pkg), pkg))
604
605         return ret
606
607     def _add_prob_flags(self, *flags):
608         for flag in flags:
609            if flag not in self.probFilterFlags:
610                self.probFilterFlags.append(flag)
611
612     def get_proxy(self, repoinfo):
613         proxy = None
614         reponame = "%s" % repoinfo.name()
615         for repo in self.repos:
616             if repo.name == reponame:
617                 proxy = repo.proxy
618                 break
619
620         if proxy:
621             return proxy
622         else:
623             repourl = str(repoinfo.baseUrls()[0])
624             return get_proxy_for(repourl)