check free space of cache filesystem for downloading.
[tools/mic.git] / plugins / backend / zypppkgmgr.py
1 #!/usr/bin/python -tt
2 #
3 # Copyright (c) 2010, 2011 Intel, Inc.
4 #
5 # This program is free software; you can redistribute it and/or modify it
6 # under the terms of the GNU General Public License as published by the Free
7 # Software Foundation; version 2 of the License
8 #
9 # This program is distributed in the hope that it will be useful, but
10 # WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
11 # or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
12 # for more details.
13 #
14 # You should have received a copy of the GNU General Public License along
15 # with this program; if not, write to the Free Software Foundation, Inc., 59
16 # Temple Place - Suite 330, Boston, MA 02111-1307, USA.
17
18 import os
19 import shutil
20 import urlparse
21 import rpm
22 from mic.utils import runner, fs_related
23
24 import zypp
25 if not hasattr(zypp, 'PoolQuery') or not hasattr(zypp.RepoManager, 'loadSolvFile'):
26     raise ImportError("python-zypp in host system cannot support PoolQuery or loadSolvFile interface, "
27                       "please update it to enhanced version which can be found in repo.meego.com/tools")
28
29 from mic import msger
30 from mic.kickstart import ksparser
31 from mic.utils import rpmmisc
32 from mic.utils import misc
33 from mic.utils.proxy import get_proxy_for
34 from mic.utils.errors import CreatorError
35 from mic.imager.baseimager import BaseImageCreator
36
37 class RepositoryStub:
38     def __init__(self):
39         self.name = None
40         self.baseurl = []
41         self.mirrorlist = None
42         self.proxy = None
43         self.proxy_username = None
44         self.proxy_password = None
45
46         self.enabled = True
47         self.autorefresh = True
48         self.keeppackages = True
49         self.priority = None
50
51 class RepoError(CreatorError):
52     pass
53
54 class RpmError(CreatorError):
55     pass
56
57 from mic.pluginbase import BackendPlugin
58 class Zypp(BackendPlugin):
59     name = 'zypp'
60
61     def __init__(self, target_arch, instroot, cachedir):
62         self.cachedir = cachedir
63         self.instroot  = instroot
64         self.target_arch = target_arch
65
66         self.__pkgs_license = {}
67         self.__pkgs_content = {}
68         self.repos = []
69         self.to_deselect = []
70         self.localpkgs = {}
71         self.repo_manager = None
72         self.repo_manager_options = None
73         self.Z = None
74         self.ts = None
75         self.probFilterFlags = []
76         self.incpkgs = {}
77         self.excpkgs = {}
78
79         self.has_prov_query = True
80
81     def doFileLogSetup(self, uid, logfile):
82         # don't do the file log for the livecd as it can lead to open fds
83         # being left and an inability to clean up after ourself
84         pass
85
86     def closeRpmDB(self):
87         pass
88
89     def close(self):
90         if self.ts:
91             self.ts.closeDB()
92             self.ts = None
93         self.closeRpmDB()
94         if not os.path.exists("/etc/fedora-release") and not os.path.exists("/etc/meego-release"):
95             for i in range(3, os.sysconf("SC_OPEN_MAX")):
96                 try:
97                     os.close(i)
98                 except:
99                     pass
100
101     def __del__(self):
102         self.close()
103
104     def _cleanupRpmdbLocks(self, installroot):
105         # cleans up temporary files left by bdb so that differing
106         # versions of rpm don't cause problems
107         import glob
108         for f in glob.glob(installroot + "/var/lib/rpm/__db*"):
109             os.unlink(f)
110
111     def setup(self, confpath=None, instroot=None):
112         if not self.instroot:
113             self.instroot = instroot
114         self._cleanupRpmdbLocks(self.instroot)
115
116     def whatObsolete(self, pkg):
117         query = zypp.PoolQuery()
118         query.addKind(zypp.ResKind.package)
119         query.addAttribute(zypp.SolvAttr.obsoletes, pkg)
120         query.setMatchExact()
121         for pi in query.queryResults(self.Z.pool()):
122             return pi
123         return None
124
125     def _splitPkgString(self, pkg):
126         sp = pkg.rsplit(".",1)
127         name = sp[0]
128         arch = None
129         if len(sp) == 2:
130             arch = sp[1]
131             sysarch = zypp.Arch(self.target_arch)
132             if not zypp.Arch(arch).compatible_with (sysarch):
133                 arch = None
134                 name = ".".join(sp)
135         return name, arch
136
137     def selectPackage(self, pkg):
138         """ Select a given package or package pattern, can be specified with name.arch or name* or *name """
139         if not self.Z:
140             self.__initialize_zypp()
141
142         def markPoolItem(obs, pi):
143             if obs == None:
144                 pi.status().setToBeInstalled (zypp.ResStatus.USER)
145             else:
146                 obs.status().setToBeInstalled (zypp.ResStatus.USER)
147
148         found = False
149         startx = pkg.startswith("*")
150         endx = pkg.endswith("*")
151         ispattern = startx or endx
152         name, arch = self._splitPkgString(pkg)
153
154         q = zypp.PoolQuery()
155         q.addKind(zypp.ResKind.package)
156         if ispattern:
157             if startx and not endx:
158                 pattern = '%s$' % (pkg[1:])
159             if endx and not startx:
160                 pattern = '^%s' % (pkg[0:-1])
161             if endx and startx:
162                 pattern = '%s' % (pkg[1:-1])
163             q.setMatchRegex()
164             q.addAttribute(zypp.SolvAttr.name,pattern)
165         elif arch:
166             q.setMatchExact()
167             q.addAttribute(zypp.SolvAttr.name,name)
168         else:
169             q.setMatchExact()
170             q.addAttribute(zypp.SolvAttr.name,pkg)
171
172         for item in sorted(q.queryResults(self.Z.pool()), key=lambda item: str(item.edition()), reverse=True):
173             if item.name() in self.excpkgs.keys() and self.excpkgs[item.name()] == item.repoInfo().name():
174                 continue
175             if item.name() in self.incpkgs.keys() and self.incpkgs[item.name()] != item.repoInfo().name():
176                 continue
177             found = True
178             obspkg = self.whatObsolete(item.name())
179             if arch:
180                 if arch == str(item.arch()):
181                     item.status().setToBeInstalled (zypp.ResStatus.USER)
182             else:
183                 markPoolItem(obspkg, item)
184             if not ispattern:
185                 break
186         # Can't match using package name, then search from packge provides infomation
187         if found == False and not ispattern:
188             q.addAttribute(zypp.SolvAttr.provides, pkg)
189             q.addAttribute(zypp.SolvAttr.name,'')
190             for item in sorted(q.queryResults(self.Z.pool()), key=lambda item: str(item.edition()), reverse=True):
191                 if item.name() in self.excpkgs.keys() and self.excpkgs[item.name()] == item.repoInfo().name():
192                     continue
193                 if item.name() in self.incpkgs.keys() and self.incpkgs[item.name()] != item.repoInfo().name():
194                     continue
195                 found = True
196                 obspkg = self.whatObsolete(item.name())
197                 markPoolItem(obspkg, item)
198                 break
199         if found:
200             return None
201         else:
202             raise CreatorError("Unable to find package: %s" % (pkg,))
203
204     def inDeselectPackages(self, item):
205         """check if specified pacakges are in the list of inDeselectPackages"""
206         name = item.name()
207         for pkg in self.to_deselect:
208             startx = pkg.startswith("*")
209             endx = pkg.endswith("*")
210             ispattern = startx or endx
211             pkgname, pkgarch = self._splitPkgString(pkg)
212             if not ispattern:
213                 if pkgarch:
214                     if name == pkgname and str(item.arch()) == pkgarch:
215                         return True;
216                 else:
217                     if name == pkgname:
218                         return True;
219             else:
220                 if startx and name.endswith(pkg[1:]):
221                         return True;
222                 if endx and name.startswith(pkg[:-1]):
223                         return True;
224         return False;
225
226     def deselectPackage(self, pkg):
227         """collect packages should not be installed"""
228         self.to_deselect.append(pkg)
229
230     def selectGroup(self, grp, include = ksparser.GROUP_DEFAULT):
231         if not self.Z:
232             self.__initialize_zypp()
233         found = False
234         q=zypp.PoolQuery()
235         q.addKind(zypp.ResKind.pattern)
236         for item in q.queryResults(self.Z.pool()):
237             summary = "%s" % item.summary()
238             name = "%s" % item.name()
239             if name == grp or summary == grp:
240                 found = True
241                 item.status().setToBeInstalled (zypp.ResStatus.USER)
242                 break
243
244         if found:
245             if include == ksparser.GROUP_REQUIRED:
246                 map(lambda p: self.deselectPackage(p), grp.default_packages.keys())
247             return None
248         else:
249             raise CreatorError("Unable to find pattern: %s" % (grp,))
250
251     def addRepository(self, name, url = None, mirrorlist = None, proxy = None,
252                       proxy_username = None, proxy_password = None,
253                       inc = None, exc = None, ssl_verify = True, cost=None,
254                       priority=None):
255         # TODO: Handle cost attribute for repos
256         if not self.repo_manager:
257             self.__initialize_repo_manager()
258
259         repo = RepositoryStub()
260         repo.name = name
261         repo.id = name
262         repo.proxy = proxy
263         repo.proxy_username = proxy_username
264         repo.proxy_password = proxy_password
265         repo.ssl_verify = ssl_verify
266         repo.baseurl.append(url)
267         if inc:
268             for pkg in inc:
269                 self.incpkgs[pkg] = name
270         if exc:
271             for pkg in exc:
272                 self.excpkgs[pkg] = name
273
274         # check LICENSE files
275         if not rpmmisc.checkRepositoryEULA(name, repo):
276             msger.warning('skip repo:%s for failed EULA confirmation' % name)
277             return None
278
279         if mirrorlist:
280             repo.mirrorlist = mirrorlist
281
282         # Enable gpg check for verifying corrupt packages
283         repo.gpgcheck = 1
284         if priority:
285             repo.priority = priority
286         self.repos.append(repo)
287
288         try:
289             repo_info = zypp.RepoInfo()
290             repo_info.setAlias(repo.name)
291             repo_info.setName(repo.name)
292             repo_info.setEnabled(repo.enabled)
293             repo_info.setAutorefresh(repo.autorefresh)
294             repo_info.setKeepPackages(repo.keeppackages)
295             baseurl = zypp.Url(repo.baseurl[0])
296             if not ssl_verify:
297                 baseurl.setQueryParam("ssl_verify", "no")
298             if proxy:
299                 (scheme, host, path, parm, query, frag) = urlparse.urlparse(proxy)
300                 proxyinfo = host.split(":")
301                 host = proxyinfo[0]
302                 port = "80"
303                 if len(proxyinfo) > 1:
304                     port = proxyinfo[1]
305                 if proxy.startswith("socks") and len(proxy.rsplit(':', 1)) == 2:
306                     host = proxy.rsplit(':', 1)[0]
307                     port = proxy.rsplit(':', 1)[1]
308                 baseurl.setQueryParam ("proxy", host)
309                 baseurl.setQueryParam ("proxyport", port)
310             repo_info.addBaseUrl(baseurl)
311             if repo.priority:
312                 repo_info.setPriority(repo.priority)
313             self.repo_manager.addRepository(repo_info)
314             self.__build_repo_cache(name)
315         except RuntimeError, e:
316             raise CreatorError(str(e))
317
318         msger.verbose('repo: %s was added' % name)
319         return repo
320
321     def installHasFile(self, file):
322         return False
323
324     def runInstall(self, checksize = 0):
325         os.environ["HOME"] = "/"
326         self.buildTransaction()
327
328         todo = zypp.GetResolvablesToInsDel(self.Z.pool())
329         installed_pkgs = todo._toInstall
330         dlpkgs = []
331         for item in installed_pkgs:
332             if not zypp.isKindPattern(item) and not self.inDeselectPackages(item):
333                 dlpkgs.append(item)
334
335         # record all pkg and the content
336         localpkgs = self.localpkgs.keys()
337         for pkg in dlpkgs:
338             license = ''
339             if pkg.name() in localpkgs:
340                 hdr = rpmmisc.readRpmHeader(self.ts, self.localpkgs[pkg.name()])
341                 pkg_long_name = "%s.%s %s-%s" % (hdr['name'], hdr['arch'], hdr['version'], hdr['release'])
342                 license = hdr['license']
343             else:
344                 pkg_long_name = "%s.%s %s" % (pkg.name(), pkg.arch(), pkg.edition())
345                 package = zypp.asKindPackage(pkg)
346                 license = package.license()
347             self.__pkgs_content[pkg_long_name] = {} #TBD: to get file list
348             if license in self.__pkgs_license.keys():
349                 self.__pkgs_license[license].append(pkg_long_name)
350             else:
351                 self.__pkgs_license[license] = [pkg_long_name]
352
353         total_count = len(dlpkgs)
354         cached_count = 0
355         download_total_size = sum(map(lambda x: int(x.downloadSize()), dlpkgs))
356         localpkgs = self.localpkgs.keys()
357         msger.info("Checking packages cache and packages integrity ...")
358         for po in dlpkgs:
359             """ Check if it is cached locally """
360             if po.name() in localpkgs:
361                 cached_count += 1
362             else:
363                 local = self.getLocalPkgPath(po)
364                 if os.path.exists(local):
365                     if self.checkPkg(local) != 0:
366                         os.unlink(local)
367                     else:
368                         download_total_size -= int(po.downloadSize())
369                         cached_count += 1
370         cache_avail_size = misc.get_filesystem_avail(self.cachedir)
371         if cache_avail_size < download_total_size:
372             raise CreatorError("No enough space used for downloading.")
373
374         # record the total size of installed pkgs
375         install_total_size = sum(map(lambda x: int(x.installSize()), dlpkgs))
376         # check needed size before actually download and install
377         if checksize and install_total_size > checksize:
378             raise CreatorError("No enough space used for installing, please resize partition size in ks file")
379
380         download_count =  total_count - cached_count
381         msger.info("%d packages to be installed, %d packages gotten from cache, %d packages to be downloaded" % (total_count, cached_count, download_count))
382         try:
383             if download_count > 0:
384                 msger.info("Downloading packages ...")
385             self.downloadPkgs(dlpkgs, download_count)
386             self.installPkgs(dlpkgs)
387
388         except RepoError, e:
389             raise CreatorError("Unable to download from repo : %s" % (e,))
390         except RpmError, e:
391             raise CreatorError("Unable to install: %s" % (e,))
392
393     def getAllContent(self):
394         return self.__pkgs_content
395
396     def getPkgsLicense(self):
397         return self.__pkgs_license
398
399     def __initialize_repo_manager(self):
400         if self.repo_manager:
401             return
402
403         """ Clean up repo metadata """
404         shutil.rmtree(self.cachedir + "/etc", ignore_errors = True)
405         shutil.rmtree(self.cachedir + "/solv", ignore_errors = True)
406         shutil.rmtree(self.cachedir + "/raw", ignore_errors = True)
407
408         zypp.KeyRing.setDefaultAccept( zypp.KeyRing.ACCEPT_UNSIGNED_FILE
409                                      | zypp.KeyRing.ACCEPT_VERIFICATION_FAILED
410                                      | zypp.KeyRing.ACCEPT_UNKNOWNKEY
411                                      | zypp.KeyRing.TRUST_KEY_TEMPORARILY
412                                      )
413         self.repo_manager_options = zypp.RepoManagerOptions(zypp.Pathname(self.instroot))
414         self.repo_manager_options.knownReposPath = zypp.Pathname(self.cachedir + "/etc/zypp/repos.d")
415         self.repo_manager_options.repoCachePath = zypp.Pathname(self.cachedir)
416         self.repo_manager_options.repoRawCachePath = zypp.Pathname(self.cachedir + "/raw")
417         self.repo_manager_options.repoSolvCachePath = zypp.Pathname(self.cachedir + "/solv")
418         self.repo_manager_options.repoPackagesCachePath = zypp.Pathname(self.cachedir + "/packages")
419
420         self.repo_manager = zypp.RepoManager(self.repo_manager_options)
421
422     def __build_repo_cache(self, name):
423         repo = self.repo_manager.getRepositoryInfo(name)
424         if self.repo_manager.isCached(repo) or not repo.enabled():
425             return
426         self.repo_manager.buildCache(repo, zypp.RepoManager.BuildIfNeeded)
427
428     def __initialize_zypp(self):
429         if self.Z:
430             return
431
432         zconfig = zypp.ZConfig_instance()
433
434         """ Set system architecture """
435         if self.target_arch:
436             zconfig.setSystemArchitecture(zypp.Arch(self.target_arch))
437
438         msger.info("zypp architecture is <%s>" % zconfig.systemArchitecture())
439
440         """ repoPackagesCachePath is corrected by this """
441         self.repo_manager = zypp.RepoManager(self.repo_manager_options)
442         repos = self.repo_manager.knownRepositories()
443         for repo in repos:
444             if not repo.enabled():
445                 continue
446             self.repo_manager.loadFromCache(repo)
447
448         self.Z = zypp.ZYppFactory_instance().getZYpp()
449         self.Z.initializeTarget(zypp.Pathname(self.instroot))
450         self.Z.target().load()
451
452
453     def buildTransaction(self):
454         if not self.Z.resolver().resolvePool():
455             msger.warning("Problem count: %d" % len(self.Z.resolver().problems()))
456             for problem in self.Z.resolver().problems():
457                 msger.warning("Problem: %s, %s" % (problem.description().decode("utf-8"), problem.details().decode("utf-8")))
458
459     def getLocalPkgPath(self, po):
460         repoinfo = po.repoInfo()
461         cacheroot = repoinfo.packagesPath()
462         location= zypp.asKindPackage(po).location()
463         rpmpath = str(location.filename())
464         pkgpath = "%s/%s" % (cacheroot, rpmpath)
465         return pkgpath
466
467     def installLocal(self, pkg, po=None, updateonly=False):
468         if not self.ts:
469             self.__initialize_transaction()
470         solvfile = "%s/.solv" % (self.cachedir)
471         rc, out = runner.runtool([fs_related.find_binary_path("rpms2solv"), pkg])
472         if rc == 0:
473             f = open(solvfile, "w+")
474             f.write(out)
475             f.close()
476             warnmsg = self.repo_manager.loadSolvFile(solvfile , os.path.basename(pkg))
477             if warnmsg:
478                 msger.warning(warnmsg)
479             os.unlink(solvfile)
480         else:
481             msger.warning('Can not get %s solv data.' % pkg)
482         hdr = rpmmisc.readRpmHeader(self.ts, pkg)
483         arch = zypp.Arch(hdr['arch'])
484         sysarch = zypp.Arch(self.target_arch)
485         if arch.compatible_with (sysarch):
486             pkgname = hdr['name']
487             self.localpkgs[pkgname] = pkg
488             self.selectPackage(pkgname)
489             msger.info("Marking %s to be installed" % (pkg))
490         else:
491             msger.warning ("Cannot add package %s to transaction. Not a compatible architecture: %s" % (pkg, hdr['arch']))
492
493     def downloadPkgs(self, package_objects, count):
494         localpkgs = self.localpkgs.keys()
495         progress_obj = rpmmisc.TextProgress(count)
496         for po in package_objects:
497             if po.name() in localpkgs:
498                 continue
499             filename = self.getLocalPkgPath(po)
500             if os.path.exists(filename):
501                 if self.checkPkg(filename) == 0:
502                     continue
503             dirn = os.path.dirname(filename)
504             if not os.path.exists(dirn):
505                 os.makedirs(dirn)
506             baseurl = str(po.repoInfo().baseUrls()[0])
507             index = baseurl.find("?")
508             if index > -1:
509                 baseurl = baseurl[:index]
510             proxy = self.get_proxy(po.repoInfo())
511             proxies = {}
512             if proxy:
513                 proxies = {str(proxy.split(":")[0]):str(proxy)}
514
515             location = zypp.asKindPackage(po).location()
516             location = str(location.filename())
517             if location.startswith("./"):
518                 location = location[2:]
519             url = baseurl + "/%s" % location
520             try:
521                 filename = rpmmisc.myurlgrab(url, filename, proxies, progress_obj)
522             except CreatorError:
523                 self.close()
524                 raise
525
526     def installPkgs(self, package_objects):
527         if not self.ts:
528             self.__initialize_transaction()
529
530         """ Set filters """
531         probfilter = 0
532         for flag in self.probFilterFlags:
533             probfilter |= flag
534         self.ts.setProbFilter(probfilter)
535
536         localpkgs = self.localpkgs.keys()
537         for po in package_objects:
538             pkgname = po.name()
539             if pkgname in localpkgs:
540                 rpmpath = self.localpkgs[pkgname]
541             else:
542                 rpmpath = self.getLocalPkgPath(po)
543             if not os.path.exists(rpmpath):
544                 """ Maybe it is a local repo """
545                 baseurl = str(po.repoInfo().baseUrls()[0])
546                 baseurl = baseurl.strip()
547                 location = zypp.asKindPackage(po).location()
548                 location = str(location.filename())
549                 if baseurl.startswith("file:/"):
550                     rpmpath = baseurl[5:] + "/%s" % (location)
551             if not os.path.exists(rpmpath):
552                 raise RpmError("Error: %s doesn't exist" % rpmpath)
553             h = rpmmisc.readRpmHeader(self.ts, rpmpath)
554             self.ts.addInstall(h, rpmpath, 'u')
555
556         unresolved_dependencies = self.ts.check()
557         if not unresolved_dependencies:
558             self.ts.order()
559             cb = rpmmisc.RPMInstallCallback(self.ts)
560             installlogfile = "%s/__catched_stderr.buf" % (self.instroot)
561             msger.enable_logstderr(installlogfile)
562             errors = self.ts.run(cb.callback, '')
563             if errors is None:
564                 pass
565             elif len(errors) == 0:
566                 msger.warning('scriptlet or other non-fatal errors occurred during transaction.')
567             else:
568                 for e in errors:
569                     msger.warning(e[0])
570                 msger.error('Could not run transaction.')
571             msger.disable_logstderr()
572
573             self.ts.closeDB()
574             self.ts = None
575         else:
576             for pkg, need, needflags, sense, key in unresolved_dependencies:
577                 package = '-'.join(pkg)
578                 if needflags == rpm.RPMSENSE_LESS:
579                     deppkg = ' < '.join(need)
580                 elif needflags == rpm.RPMSENSE_EQUAL:
581                     deppkg = ' = '.join(need)
582                 elif needflags == rpm.RPMSENSE_GREATER:
583                     deppkg = ' > '.join(need)
584                 else:
585                     deppkg = '-'.join(need)
586
587                 if sense == rpm.RPMDEP_SENSE_REQUIRES:
588                     msger.warning ("[%s] Requires [%s], which is not provided" % (package, deppkg))
589                 elif sense == rpm.RPMDEP_SENSE_CONFLICTS:
590                     msger.warning ("[%s] Conflicts with [%s]" % (package, deppkg))
591
592             raise RepoError("Unresolved dependencies, transaction failed.")
593
594     def __initialize_transaction(self):
595         if not self.ts:
596             self.ts = rpm.TransactionSet(self.instroot)
597             # Set to not verify DSA signatures.
598             self.ts.setVSFlags(rpm._RPMVSF_NOSIGNATURES|rpm._RPMVSF_NODIGESTS)
599
600     def checkPkg(self, pkg):
601         ret = 1
602         if not os.path.exists(pkg):
603             return ret
604         ret = rpmmisc.checkRpmIntegrity('rpm', pkg)
605         if ret != 0:
606             msger.warning("package %s is damaged: %s" % (os.path.basename(pkg), pkg))
607
608         return ret
609
610     def _add_prob_flags(self, *flags):
611         for flag in flags:
612            if flag not in self.probFilterFlags:
613                self.probFilterFlags.append(flag)
614
615     def get_proxy(self, repoinfo):
616         proxy = None
617         reponame = "%s" % repoinfo.name()
618         for repo in self.repos:
619             if repo.name == reponame:
620                 proxy = repo.proxy
621                 break
622
623         if proxy:
624             return proxy
625         else:
626             repourl = str(repoinfo.baseUrls()[0])
627             return get_proxy_for(repourl)