Implemented repository option --ssl_verify
[tools/mic.git] / plugins / backend / zypppkgmgr.py
1 #!/usr/bin/python -tt
2 #
3 # Copyright (c) 2010, 2011 Intel, Inc.
4 #
5 # This program is free software; you can redistribute it and/or modify it
6 # under the terms of the GNU General Public License as published by the Free
7 # Software Foundation; version 2 of the License
8 #
9 # This program is distributed in the hope that it will be useful, but
10 # WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
11 # or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
12 # for more details.
13 #
14 # You should have received a copy of the GNU General Public License along
15 # with this program; if not, write to the Free Software Foundation, Inc., 59
16 # Temple Place - Suite 330, Boston, MA 02111-1307, USA.
17
18 import os
19 import shutil
20 import urlparse
21 import rpm
22 from mic.utils import runner, fs_related
23
24 import zypp
25 if not hasattr(zypp, 'PoolQuery') or not hasattr(zypp.RepoManager, 'loadSolvFile'):
26     raise ImportError("python-zypp in host system cannot support PoolQuery or loadSolvFile interface, "
27                       "please update it to enhanced version which can be found in repo.meego.com/tools")
28
29 from mic import msger
30 from mic.kickstart import ksparser
31 from mic.utils import rpmmisc
32 from mic.utils.proxy import get_proxy_for
33 from mic.utils.errors import CreatorError
34 from mic.imager.baseimager import BaseImageCreator
35
36 class RepositoryStub:
37     def __init__(self):
38         self.name = None
39         self.baseurl = []
40         self.mirrorlist = None
41         self.proxy = None
42         self.proxy_username = None
43         self.proxy_password = None
44
45         self.enabled = True
46         self.autorefresh = True
47         self.keeppackages = True
48
49 class RepoError(CreatorError):
50     pass
51
52 class RpmError(CreatorError):
53     pass
54
55 from mic.pluginbase import BackendPlugin
56 class Zypp(BackendPlugin):
57     name = 'zypp'
58
59     def __init__(self, creator = None):
60         if not isinstance(creator, BaseImageCreator):
61             raise CreatorError("Invalid argument: creator")
62
63         self.__pkgs_license = {}
64         self.__pkgs_content = {}
65         self.creator = creator
66         self.repos = []
67         self.to_deselect = []
68         self.localpkgs = {}
69         self.repo_manager = None
70         self.repo_manager_options = None
71         self.Z = None
72         self.ts = None
73         self.probFilterFlags = []
74         self.incpkgs = {}
75         self.excpkgs = {}
76
77         self.has_prov_query = True
78
79     def doFileLogSetup(self, uid, logfile):
80         # don't do the file log for the livecd as it can lead to open fds
81         # being left and an inability to clean up after ourself
82         pass
83
84     def closeRpmDB(self):
85         pass
86
87     def close(self):
88         self.closeRpmDB()
89         if not os.path.exists("/etc/fedora-release") and not os.path.exists("/etc/meego-release"):
90             for i in range(3, os.sysconf("SC_OPEN_MAX")):
91                 try:
92                     os.close(i)
93                 except:
94                     pass
95         if self.ts:
96             self.ts.closeDB()
97             self.ts = None
98
99     def __del__(self):
100         self.close()
101
102     def _cleanupRpmdbLocks(self, installroot):
103         # cleans up temporary files left by bdb so that differing
104         # versions of rpm don't cause problems
105         import glob
106         for f in glob.glob(installroot + "/var/lib/rpm/__db*"):
107             os.unlink(f)
108
109     def setup(self, confpath, installroot):
110         self._cleanupRpmdbLocks(installroot)
111         self.installroot = installroot
112
113     def whatObsolete(self, pkg):
114         query = zypp.PoolQuery()
115         query.addKind(zypp.ResKind.package)
116         query.addAttribute(zypp.SolvAttr.obsoletes, pkg)
117         query.setMatchExact()
118         for pi in query.queryResults(self.Z.pool()):
119             return pi
120         return None
121
122     def _splitPkgString(self, pkg):
123         sp = pkg.rsplit(".",1)
124         name = sp[0]
125         arch = None
126         if len(sp) == 2:
127             arch = sp[1]
128             if self.creator.target_arch == None:
129                 # TODO, get the default_arch from conf or detected from global settings
130                 sysarch = zypp.Arch('i686')
131             else:
132                 sysarch = zypp.Arch(self.creator.target_arch)
133             if not zypp.Arch(arch).compatible_with (sysarch):
134                 arch = None
135                 name = ".".join(sp)
136         return name, arch
137
138     def selectPackage(self, pkg):
139         """ Select a given package or package pattern, can be specified with name.arch or name* or *name """
140         if not self.Z:
141             self.__initialize_zypp()
142
143         def markPoolItem(obs, pi):
144             if obs == None:
145                 pi.status().setToBeInstalled (zypp.ResStatus.USER)
146             else:
147                 obs.status().setToBeInstalled (zypp.ResStatus.USER)
148                 
149         found = False
150         startx = pkg.startswith("*")
151         endx = pkg.endswith("*")
152         ispattern = startx or endx
153         name, arch = self._splitPkgString(pkg)
154
155         q = zypp.PoolQuery()
156         q.addKind(zypp.ResKind.package)
157         if ispattern:
158             if startx and not endx:
159                 pattern = '%s$' % (pkg[1:])
160             if endx and not startx:
161                 pattern = '^%s' % (pkg[0:-1])
162             if endx and startx:
163                 pattern = '%s' % (pkg[1:-1])
164             q.setMatchRegex()
165             q.addAttribute(zypp.SolvAttr.name,pattern)
166         elif arch:
167             q.setMatchExact()
168             q.addAttribute(zypp.SolvAttr.name,name)
169         else:
170             q.setMatchExact()
171             q.addAttribute(zypp.SolvAttr.name,pkg)
172
173         for item in sorted(q.queryResults(self.Z.pool()), key=lambda item: str(item.edition()), reverse=True):
174             if item.name() in self.excpkgs.keys() and self.excpkgs[item.name()] == item.repoInfo().name():
175                 continue
176             if item.name() in self.incpkgs.keys() and self.incpkgs[item.name()] != item.repoInfo().name():
177                 continue
178             found = True
179             obspkg = self.whatObsolete(item.name())
180             if arch:
181                 if arch == str(item.arch()):
182                     item.status().setToBeInstalled (zypp.ResStatus.USER)
183             else:
184                 markPoolItem(obspkg, item)
185             if not ispattern:
186                 break
187         # Can't match using package name, then search from packge provides infomation
188         if found == False and not ispattern:
189             q.addAttribute(zypp.SolvAttr.provides, pkg)
190             q.addAttribute(zypp.SolvAttr.name,'')
191             for item in sorted(q.queryResults(self.Z.pool()), key=lambda item: str(item.edition()), reverse=True):
192                 if item.name() in self.excpkgs.keys() and self.excpkgs[item.name()] == item.repoInfo().name():
193                     continue
194                 if item.name() in self.incpkgs.keys() and self.incpkgs[item.name()] != item.repoInfo().name():
195                     continue
196                 found = True
197                 obspkg = self.whatObsolete(item.name())
198                 markPoolItem(obspkg, item)
199                 break
200         if found:
201             return None
202         else:
203             raise CreatorError("Unable to find package: %s" % (pkg,))
204
205     def inDeselectPackages(self, item):
206         """check if specified pacakges are in the list of inDeselectPackages"""
207         name = item.name()
208         for pkg in self.to_deselect:
209             startx = pkg.startswith("*")
210             endx = pkg.endswith("*")
211             ispattern = startx or endx
212             pkgname, pkgarch = self._splitPkgString(pkg)
213             if not ispattern:
214                 if pkgarch:
215                     if name == pkgname and str(item.arch()) == pkgarch:
216                         return True;
217                 else:
218                     if name == pkgname:
219                         return True;
220             else:
221                 if startx and name.endswith(pkg[1:]):
222                         return True;
223                 if endx and name.startswith(pkg[:-1]):
224                         return True;
225         return False;
226
227     def deselectPackage(self, pkg):
228         """collect packages should not be installed"""
229         self.to_deselect.append(pkg)
230
231     def selectGroup(self, grp, include = ksparser.GROUP_DEFAULT):
232         if not self.Z:
233             self.__initialize_zypp()
234         found = False
235         q=zypp.PoolQuery()
236         q.addKind(zypp.ResKind.pattern)
237         for item in q.queryResults(self.Z.pool()):
238             summary = "%s" % item.summary()
239             name = "%s" % item.name()
240             if name == grp or summary == grp:
241                 found = True
242                 item.status().setToBeInstalled (zypp.ResStatus.USER)
243                 break
244
245         if found:
246             if include == ksparser.GROUP_REQUIRED:
247                 map(lambda p: self.deselectPackage(p), grp.default_packages.keys())
248             return None
249         else:
250             raise CreatorError("Unable to find pattern: %s" % (grp,))
251
252     def addRepository(self, name, url = None, mirrorlist = None, proxy = None,
253                       proxy_username = None, proxy_password = None,
254                       inc = None, exc = None, ssl_verify = True):
255         if not self.repo_manager:
256             self.__initialize_repo_manager()
257
258         repo = RepositoryStub()
259         repo.name = name
260         repo.id = name
261         repo.proxy = proxy
262         repo.proxy_username = proxy_username
263         repo.proxy_password = proxy_password
264         repo.ssl_verify = ssl_verify
265         repo.baseurl.append(url)
266         if inc:
267             for pkg in inc:
268                 self.incpkgs[pkg] = name
269         if exc:
270             for pkg in exc:
271                 self.excpkgs[pkg] = name
272
273         # check LICENSE files
274         if not rpmmisc.checkRepositoryEULA(name, repo):
275             msger.warning('skip repo:%s for failed EULA confirmation' % name)
276             return None
277
278         if mirrorlist:
279             repo.mirrorlist = mirrorlist
280
281         # Enable gpg check for verifying corrupt packages
282         repo.gpgcheck = 1
283         self.repos.append(repo)
284
285         try:
286             repo_info = zypp.RepoInfo()
287             repo_info.setAlias(repo.name)
288             repo_info.setName(repo.name)
289             repo_info.setEnabled(repo.enabled)
290             repo_info.setAutorefresh(repo.autorefresh)
291             repo_info.setKeepPackages(repo.keeppackages)
292             baseurl = zypp.Url(repo.baseurl[0])
293             if not ssl_verify:
294                 baseurl.setQueryParam("ssl_verify", "no")
295             if proxy:
296                 (scheme, host, path, parm, query, frag) = urlparse.urlparse(proxy)
297                 proxyinfo = host.split(":")
298                 baseurl.setQueryParam ("proxy", proxyinfo[0])
299                 port = "80"
300                 if len(proxyinfo) > 1:
301                     port = proxyinfo[1]
302                 baseurl.setQueryParam ("proxyport", port)
303             repo_info.addBaseUrl(baseurl)
304             self.repo_manager.addRepository(repo_info)
305             self.__build_repo_cache(name)
306         except RuntimeError, e:
307             raise CreatorError(str(e))
308
309         msger.verbose('repo: %s was added' % name)
310         return repo
311
312     def installHasFile(self, file):
313         return False
314
315     def runInstall(self, checksize = 0):
316         os.environ["HOME"] = "/"
317         self.buildTransaction()
318
319         todo = zypp.GetResolvablesToInsDel(self.Z.pool())
320         installed_pkgs = todo._toInstall
321         dlpkgs = []
322         for item in installed_pkgs:
323             if not zypp.isKindPattern(item) and not self.inDeselectPackages(item):
324                 dlpkgs.append(item)
325
326         # record the total size of installed pkgs
327         pkgs_total_size = sum(map(lambda x: int(x.installSize()), dlpkgs))
328
329         # check needed size before actually download and install
330         if checksize and pkgs_total_size > checksize:
331             raise CreatorError("Size of specified root partition in kickstart file is too small to install all selected packages.")
332
333         # record all pkg and the content
334         localpkgs = self.localpkgs.keys()
335         for pkg in dlpkgs:
336             license = ''
337             if pkg.name() in localpkgs:
338                 hdr = rpmmisc.readRpmHeader(self.ts, self.localpkgs[pkg.name()])
339                 pkg_long_name = "%s-%s-%s.%s.rpm" % (hdr['name'], hdr['version'], hdr['release'], hdr['arch'])
340                 license = hdr['license']
341             else:
342                 pkg_long_name = "%s-%s.%s.rpm" % (pkg.name(), pkg.edition(), pkg.arch())
343                 package = zypp.asKindPackage(pkg)
344                 license = package.license()
345             self.__pkgs_content[pkg_long_name] = {} #TBD: to get file list
346             if license in self.__pkgs_license.keys():
347                 self.__pkgs_license[license].append(pkg_long_name)
348             else:
349                 self.__pkgs_license[license] = [pkg_long_name]
350
351         total_count = len(dlpkgs)
352         cached_count = 0
353         localpkgs = self.localpkgs.keys()
354         msger.info("Checking packages cache and packages integrity ...")
355         for po in dlpkgs:
356             """ Check if it is cached locally """
357             if po.name() in localpkgs:
358                 cached_count += 1
359             else:
360                 local = self.getLocalPkgPath(po)
361                 if os.path.exists(local):
362                     if self.checkPkg(local) != 0:
363                         os.unlink(local)
364                     else:
365                         cached_count += 1
366         download_count =  total_count - cached_count
367         msger.info("%d packages to be installed, %d packages gotten from cache, %d packages to be downloaded" % (total_count, cached_count, download_count))
368         try:
369             if download_count > 0:
370                 msger.info("Downloading packages ...")
371             self.downloadPkgs(dlpkgs, download_count)
372             self.installPkgs(dlpkgs)
373
374         except RepoError, e:
375             raise CreatorError("Unable to download from repo : %s" % (e,))
376         except RpmError, e:
377             raise CreatorError("Unable to install: %s" % (e,))
378
379     def getAllContent(self):
380         return self.__pkgs_content
381
382     def getPkgsLicense(self):
383         return self.__pkgs_license
384
385     def __initialize_repo_manager(self):
386         if self.repo_manager:
387             return
388
389         """ Clean up repo metadata """
390         shutil.rmtree(self.creator.cachedir + "/etc", ignore_errors = True)
391         shutil.rmtree(self.creator.cachedir + "/solv", ignore_errors = True)
392         shutil.rmtree(self.creator.cachedir + "/raw", ignore_errors = True)
393
394         zypp.KeyRing.setDefaultAccept( zypp.KeyRing.ACCEPT_UNSIGNED_FILE
395                                      | zypp.KeyRing.ACCEPT_VERIFICATION_FAILED
396                                      | zypp.KeyRing.ACCEPT_UNKNOWNKEY
397                                      | zypp.KeyRing.TRUST_KEY_TEMPORARILY
398                                      )
399         self.repo_manager_options = zypp.RepoManagerOptions(zypp.Pathname(self.creator._instroot))
400         self.repo_manager_options.knownReposPath = zypp.Pathname(self.creator.cachedir + "/etc/zypp/repos.d")
401         self.repo_manager_options.repoCachePath = zypp.Pathname(self.creator.cachedir)
402         self.repo_manager_options.repoRawCachePath = zypp.Pathname(self.creator.cachedir + "/raw")
403         self.repo_manager_options.repoSolvCachePath = zypp.Pathname(self.creator.cachedir + "/solv")
404         self.repo_manager_options.repoPackagesCachePath = zypp.Pathname(self.creator.cachedir + "/packages")
405
406         self.repo_manager = zypp.RepoManager(self.repo_manager_options)
407
408     def __build_repo_cache(self, name):
409         repo = self.repo_manager.getRepositoryInfo(name)
410         if self.repo_manager.isCached(repo) or not repo.enabled():
411             return
412         self.repo_manager.buildCache(repo, zypp.RepoManager.BuildIfNeeded)
413
414     def __initialize_zypp(self):
415         if self.Z:
416             return
417
418         zconfig = zypp.ZConfig_instance()
419
420         """ Set system architecture """
421         if self.creator.target_arch:
422             zconfig.setSystemArchitecture(zypp.Arch(self.creator.target_arch))
423
424         msger.info("zypp architecture is <%s>" % zconfig.systemArchitecture())
425
426         """ repoPackagesCachePath is corrected by this """
427         self.repo_manager = zypp.RepoManager(self.repo_manager_options)
428         repos = self.repo_manager.knownRepositories()
429         for repo in repos:
430             if not repo.enabled():
431                 continue
432             self.repo_manager.loadFromCache(repo)
433
434         self.Z = zypp.ZYppFactory_instance().getZYpp()
435         self.Z.initializeTarget(zypp.Pathname(self.creator._instroot))
436         self.Z.target().load()
437
438
439     def buildTransaction(self):
440         if not self.Z.resolver().resolvePool():
441             msger.warning("Problem count: %d" % len(self.Z.resolver().problems()))
442             for problem in self.Z.resolver().problems():
443                 msger.warning("Problem: %s, %s" % (problem.description().decode("utf-8"), problem.details().decode("utf-8")))
444
445     def getLocalPkgPath(self, po):
446         repoinfo = po.repoInfo()
447         cacheroot = repoinfo.packagesPath()
448         location= zypp.asKindPackage(po).location()
449         rpmpath = str(location.filename())
450         pkgpath = "%s/%s" % (cacheroot, rpmpath)
451         return pkgpath
452
453     def installLocal(self, pkg, po=None, updateonly=False):
454         if not self.ts:
455             self.__initialize_transaction()
456         solvfile = "%s/.solv" % (self.creator.cachedir)
457         rc, out = runner.runtool([fs_related.find_binary_path("rpms2solv"), pkg])
458         if rc == 0:
459             f = open(solvfile, "w+")
460             f.write(out)
461             f.close()
462             warnmsg = self.repo_manager.loadSolvFile(solvfile , os.path.basename(pkg))
463             if warnmsg:
464                 msger.warning(warnmsg)
465             os.unlink(solvfile)
466         else:
467             msger.warning('Can not get %s solv data.' % pkg)
468         hdr = rpmmisc.readRpmHeader(self.ts, pkg)
469         arch = zypp.Arch(hdr['arch'])
470         if self.creator.target_arch == None:
471             # TODO, get the default_arch from conf or detected from global settings
472             sysarch = zypp.Arch('i686')
473         else:
474             sysarch = zypp.Arch(self.creator.target_arch)
475         if arch.compatible_with (sysarch):
476             pkgname = hdr['name']
477             self.localpkgs[pkgname] = pkg
478             self.selectPackage(pkgname)
479             msger.info("Marking %s to be installed" % (pkg))
480         else:
481             msger.warning ("Cannot add package %s to transaction. Not a compatible architecture: %s" % (pkg, hdr['arch']))
482
483     def downloadPkgs(self, package_objects, count):
484         localpkgs = self.localpkgs.keys()
485         progress_obj = rpmmisc.TextProgress(count)
486         for po in package_objects:
487             if po.name() in localpkgs:
488                 continue
489             filename = self.getLocalPkgPath(po)
490             if os.path.exists(filename):
491                 if self.checkPkg(filename) == 0:
492                     continue
493             dirn = os.path.dirname(filename)
494             if not os.path.exists(dirn):
495                 os.makedirs(dirn)
496             baseurl = str(po.repoInfo().baseUrls()[0])
497             index = baseurl.find("?")
498             if index > -1:
499                 baseurl = baseurl[:index]
500             proxy = self.get_proxy(po.repoInfo())
501             proxies = {}
502             if proxy:
503                 proxies = {str(proxy.split(":")[0]):str(proxy)}
504
505             location = zypp.asKindPackage(po).location()
506             location = str(location.filename())
507             if location.startswith("./"):
508                 location = location[2:]
509             url = baseurl + "/%s" % location
510             try:
511                 filename = rpmmisc.myurlgrab(url, filename, proxies, progress_obj)
512             except CreatorError:
513                 self.close()
514                 raise
515
516     def installPkgs(self, package_objects):
517         if not self.ts:
518             self.__initialize_transaction()
519
520         """ Set filters """
521         probfilter = 0
522         for flag in self.probFilterFlags:
523             probfilter |= flag
524         self.ts.setProbFilter(probfilter)
525
526         localpkgs = self.localpkgs.keys()
527         for po in package_objects:
528             pkgname = po.name()
529             if pkgname in localpkgs:
530                 rpmpath = self.localpkgs[pkgname]
531             else:
532                 rpmpath = self.getLocalPkgPath(po)
533             if not os.path.exists(rpmpath):
534                 """ Maybe it is a local repo """
535                 baseurl = str(po.repoInfo().baseUrls()[0])
536                 baseurl = baseurl.strip()
537                 location = zypp.asKindPackage(po).location()
538                 location = str(location.filename())
539                 if baseurl.startswith("file:/"):
540                     rpmpath = baseurl[5:] + "/%s" % (location)
541             if not os.path.exists(rpmpath):
542                 raise RpmError("Error: %s doesn't exist" % rpmpath)
543             h = rpmmisc.readRpmHeader(self.ts, rpmpath)
544             self.ts.addInstall(h, rpmpath, 'u')
545
546         unresolved_dependencies = self.ts.check()
547         if not unresolved_dependencies:
548             self.ts.order()
549             cb = rpmmisc.RPMInstallCallback(self.ts)
550             installlogfile = "%s/__catched_stderr.buf" % (self.creator._instroot)
551             msger.enable_logstderr(installlogfile)
552             errors = self.ts.run(cb.callback, '')
553             if errors is None:
554                 pass
555             elif len(errors) == 0:
556                 msger.warning('scriptlet or other non-fatal errors occurred during transaction.')
557             else:
558                 for e in errors:
559                     msger.warning(e[0])
560                 msger.error('Could not run transaction.')
561             msger.disable_logstderr()
562
563             self.ts.closeDB()
564             self.ts = None
565         else:
566             for pkg, need, needflags, sense, key in unresolved_dependencies:
567                 package = '-'.join(pkg)
568                 if needflags == rpm.RPMSENSE_LESS:
569                     deppkg = ' < '.join(need)
570                 elif needflags == rpm.RPMSENSE_EQUAL:
571                     deppkg = ' = '.join(need)
572                 elif needflags == rpm.RPMSENSE_GREATER:
573                     deppkg = ' > '.join(need)
574                 else:
575                     deppkg = '-'.join(need)
576
577                 if sense == rpm.RPMDEP_SENSE_REQUIRES:
578                     msger.warning ("[%s] Requires [%s], which is not provided" % (package, deppkg))
579                 elif sense == rpm.RPMDEP_SENSE_CONFLICTS:
580                     msger.warning ("[%s] Conflicts with [%s]" % (package, deppkg))
581
582             raise RepoError("Unresolved dependencies, transaction failed.")
583
584     def __initialize_transaction(self):
585         if not self.ts:
586             self.ts = rpm.TransactionSet(self.creator._instroot)
587             # Set to not verify DSA signatures.
588             self.ts.setVSFlags(rpm._RPMVSF_NOSIGNATURES|rpm._RPMVSF_NODIGESTS)
589
590     def checkPkg(self, pkg):
591         ret = 1
592         if not os.path.exists(pkg):
593             return ret
594         ret = rpmmisc.checkRpmIntegrity('rpm', pkg)
595         if ret != 0:
596             msger.warning("package %s is damaged: %s" % (os.path.basename(pkg), pkg))
597
598         return ret
599
600     def _add_prob_flags(self, *flags):
601         for flag in flags:
602            if flag not in self.probFilterFlags:
603                self.probFilterFlags.append(flag)
604
605     def get_proxy(self, repoinfo):
606         proxy = None
607         reponame = "%s" % repoinfo.name()
608         for repo in self.repos:
609             if repo.name == reponame:
610                 proxy = repo.proxy
611                 break
612
613         if proxy:
614             return proxy
615         else:
616             repourl = str(repoinfo.baseUrls()[0])
617             return get_proxy_for(repourl)