mic/utils/misc.py

   1 #!/usr/bin/python -tt
   2 #
   3 # Copyright (c) 2010, 2011 Intel Inc.
   4 #
   5 # This program is free software; you can redistribute it and/or modify it
   6 # under the terms of the GNU General Public License as published by the Free
   7 # Software Foundation; version 2 of the License
   8 #
   9 # This program is distributed in the hope that it will be useful, but
  10 # WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
  11 # or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12 # for more details.
  13 #
  14 # You should have received a copy of the GNU General Public License along
  15 # with this program; if not, write to the Free Software Foundation, Inc., 59
  16 # Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  17
  18 import os
  19 import sys
  20 import time
  21 import tempfile
  22 import re
  23 import shutil
  24 import glob
  25 import hashlib
  26 import subprocess
  27 import platform
  28 import rpmmisc
  29 import hashlib
  30 import sqlite3 as sqlite
  31
  32 try:
  33     import sqlite3 as sqlite
  34 except ImportError:
  35     import sqlite
  36
  37 try:
  38     from xml.etree import cElementTree
  39 except ImportError:
  40     import cElementTree
  41 xmlparse = cElementTree.parse
  42
  43 from errors import *
  44 from fs_related import *
  45 from grabber import myurlgrab
  46 from proxy import get_proxy_for
  47 import runner
  48
  49 from mic import msger
  50
  51 RPM_RE  = re.compile("(.*)\.(.*) (.*)-(.*)")
  52 RPM_FMT = "%(name)s.%(arch)s %(ver_rel)s"
  53 SRPM_RE = re.compile("(.*)-(\d+.*)-(\d+\.\d+).src.rpm")
  54
  55 def build_name(kscfg, release=None, prefix = None, suffix = None):
  56     """Construct and return an image name string.
  57
  58     This is a utility function to help create sensible name and fslabel
  59     strings. The name is constructed using the sans-prefix-and-extension
  60     kickstart filename and the supplied prefix and suffix.
  61
  62     kscfg -- a path to a kickstart file
  63     release --  a replacement to suffix for image release
  64     prefix -- a prefix to prepend to the name; defaults to None, which causes
  65               no prefix to be used
  66     suffix -- a suffix to append to the name; defaults to None, which causes
  67               a YYYYMMDDHHMM suffix to be used
  68
  69     Note, if maxlen is less then the len(suffix), you get to keep both pieces.
  70
  71     """
  72     name = os.path.basename(kscfg)
  73     idx = name.rfind('.')
  74     if idx >= 0:
  75         name = name[:idx]
  76
  77     if prefix is None:
  78         prefix = ""
  79     if suffix is None:
  80         suffix = time.strftime("%Y%m%d%H%M")
  81     if release is not None:
  82         suffix = release
  83
  84     if name.startswith(prefix):
  85         name = name[len(prefix):]
  86
  87     ret = prefix + name + "-" + suffix
  88     return ret
  89
  90 def get_distro():
  91     """Detect linux distribution, support "meego"
  92     """
  93
  94     support_dists = ('SuSE',
  95                      'debian',
  96                      'fedora',
  97                      'redhat',
  98                      'centos',
  99                      'meego',
 100                      'moblin',
 101                      'tizen')
 102     try:
 103         (dist, ver, id) = platform.linux_distribution( \
 104                               supported_dists = support_dists)
 105     except:
 106         (dist, ver, id) = platform.dist( \
 107                               supported_dists = support_dists)
 108
 109     return (dist, ver, id)
 110
 111 def get_distro_str():
 112     """Get composited string for current linux distribution
 113     """
 114     (dist, ver, id) = get_distro()
 115
 116     if not dist:
 117         return 'Unknown Linux Distro'
 118     else:
 119         distro_str = ' '.join(map(str.strip, (dist, ver, id)))
 120         return distro_str.strip()
 121
 122 _LOOP_RULE_PTH = None
 123 def hide_loopdev_presentation():
 124     udev_rules = "80-prevent-loop-present.rules"
 125     udev_rules_dir = [
 126                        '/usr/lib/udev/rules.d/',
 127                        '/lib/udev/rules.d/',
 128                        '/etc/udev/rules.d/'
 129                      ]
 130
 131     for rdir in udev_rules_dir:
 132         if os.path.exists(rdir):
 133             _LOOP_RULE_PTH = os.path.join(rdir, udev_rules)
 134
 135     if not _LOOP_RULE_PTH:
 136         return
 137
 138     try:
 139         with open(_LOOP_RULE_PTH, 'w') as wf:
 140             wf.write('KERNEL=="loop*", ENV{UDISKS_PRESENTATION_HIDE}="1"')
 141
 142         runner.quiet('udevadm trigger')
 143     except:
 144         pass
 145
 146 def unhide_loopdev_presentation():
 147     if not _LOOP_RULE_PTH:
 148         return
 149
 150     try:
 151         os.unlink(_LOOP_RULE_PTH)
 152         runner.quiet('udevadm trigger')
 153     except:
 154         pass
 155
 156 def extract_rpm(rpmfile, targetdir):
 157     rpm2cpio = find_binary_path("rpm2cpio")
 158     cpio = find_binary_path("cpio")
 159
 160     olddir = os.getcwd()
 161     os.chdir(targetdir)
 162
 163     msger.verbose("Extract rpm file with cpio: %s" % rpmfile)
 164     p1 = subprocess.Popen([rpm2cpio, rpmfile], stdout=subprocess.PIPE)
 165     p2 = subprocess.Popen([cpio, "-idv"], stdin=p1.stdout,
 166                           stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 167     (sout, serr) = p2.communicate()
 168     msger.verbose(sout or serr)
 169
 170     os.chdir(olddir)
 171
 172 def compressing(fpath, method):
 173     comp_map = {
 174         "gz": "gzip",
 175         "bz2": "bzip2"
 176     }
 177     if method not in comp_map:
 178         raise CreatorError("Unsupport compress format: %s, valid values: %s"
 179                            % (method, ','.join(comp_map.keys())))
 180     cmd = find_binary_path(comp_map[method])
 181     rc = runner.show([cmd, "-f", fpath])
 182     if rc:
 183         raise CreatorError("Failed to %s file: %s" % (comp_map[method], fpath))
 184
 185 def taring(dstfile, target):
 186     import tarfile
 187     basen, ext = os.path.splitext(dstfile)
 188     comp = {".tar": None,
 189             ".gz": "gz", # for .tar.gz
 190             ".bz2": "bz2", # for .tar.bz2
 191             ".tgz": "gz",
 192             ".tbz": "bz2"}[ext]
 193
 194     # specify tarball file path
 195     if not comp:
 196         tarpath = dstfile
 197     elif basen.endswith(".tar"):
 198         tarpath = basen
 199     else:
 200         tarpath = basen + ".tar"
 201     wf = tarfile.open(tarpath, 'w')
 202
 203     if os.path.isdir(target):
 204         for item in os.listdir(target):
 205             wf.add(os.path.join(target, item), item)
 206     else:
 207         wf.add(target, os.path.basename(target))
 208     wf.close()
 209
 210     if comp:
 211         compressing(tarpath, comp)
 212         # when dstfile ext is ".tgz" and ".tbz", should rename
 213         if not basen.endswith(".tar"):
 214             shutil.move("%s.%s" % (tarpath, comp), dstfile)
 215
 216 def ziping(dstfile, target):
 217     import zipfile
 218     wf = zipfile.ZipFile(dstfile, 'w', compression=zipfile.ZIP_DEFLATED)
 219     if os.path.isdir(target):
 220         for item in os.listdir(target):
 221             fpath = os.path.join(target, item)
 222             if not os.path.isfile(fpath):
 223                 continue
 224             wf.write(fpath, item, zipfile.ZIP_DEFLATED)
 225     else:
 226         wf.write(target, os.path.basename(target), zipfile.ZIP_DEFLATED)
 227     wf.close()
 228
 229 pack_formats = {
 230     ".tar": taring,
 231     ".tar.gz": taring,
 232     ".tar.bz2": taring,
 233     ".tgz": taring,
 234     ".tbz": taring,
 235     ".zip": ziping,
 236 }
 237
 238 def packing(dstfile, target):
 239     (base, ext) = os.path.splitext(dstfile)
 240     if ext in (".gz", ".bz2") and base.endswith(".tar"):
 241         ext = ".tar" + ext
 242     if ext not in pack_formats:
 243         raise CreatorError("Unsupport pack format: %s, valid values: %s"
 244                            % (ext, ','.join(pack_formats.keys())))
 245     func = pack_formats[ext]
 246     # func should be callable
 247     func(dstfile, target)
 248
 249 def human_size(size):
 250     """Return human readable string for Bytes size
 251     """
 252
 253     if size <= 0:
 254         return "0M"
 255     import math
 256     measure = ['B', 'K', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y']
 257     expo = int(math.log(size, 1024))
 258     mant = float(size/math.pow(1024, expo))
 259     return "{0:.1f}{1:s}".format(mant, measure[expo])
 260
 261 def check_space_pre_cp(src, dst):
 262     """Check whether disk space is enough before 'cp' like
 263     operations, else exception will be raised.
 264     """
 265
 266     srcsize  = get_file_size(src) * 1024 * 1024
 267     freesize = get_filesystem_avail(dst)
 268     if srcsize > freesize:
 269         raise CreatorError("space on %s(%s) is not enough for about %s files"
 270                            % (dst, human_size(freesize), human_size(srcsize)))
 271
 272 def calc_hashes(file_path, hash_names, start = 0, end = None):
 273     """ Calculate hashes for a file. The 'file_path' argument is the file
 274     to calculate hash functions for, 'start' and 'end' are the starting and
 275     ending file offset to calculate the has functions for. The 'hash_names'
 276     argument is a list of hash names to calculate. Returns the the list
 277     of calculated hash values in the hexadecimal form in the same order
 278     as 'hash_names'.
 279     """
 280     if end == None:
 281         end = os.path.getsize(file_path)
 282
 283     chunk_size = 65536
 284     to_read = end - start;
 285     read = 0
 286
 287     hashes = []
 288     for hash_name in hash_names:
 289         hashes.append(hashlib.new(hash_name))
 290
 291     with open(file_path, "rb") as f:
 292         f.seek(start)
 293
 294         while read < to_read:
 295             if read + chunk_size > to_read:
 296                 chunk_size = to_read - read
 297             chunk = f.read(chunk_size)
 298             for hash_obj in hashes:
 299                 hash_obj.update(chunk)
 300             read += chunk_size
 301
 302     result = []
 303     for hash_obj in hashes:
 304         result.append(hash_obj.hexdigest())
 305
 306     return result
 307
 308 def get_md5sum(fpath):
 309     return calc_hashes(fpath, ('md5', ))[0]
 310
 311 def normalize_ksfile(ksconf, release, arch):
 312     def _clrtempks():
 313         try:
 314             os.unlink(ksconf)
 315         except:
 316             pass
 317
 318     if not os.path.exists(ksconf):
 319         return
 320
 321     if not release:
 322         release = "latest"
 323     if not arch or re.match(r'i.86', arch):
 324         arch = "ia32"
 325
 326     with open(ksconf) as f:
 327         ksc = f.read()
 328
 329     if "@ARCH@" in ksc or "@BUILD_ID@" in ksc:
 330         msger.info("Substitute macro variable @BUILD_ID@/@ARCH@ in ks: %s" % ksconf)
 331         ksc = ksc.replace("@ARCH@", arch)
 332         ksc = ksc.replace("@BUILD_ID@", release)
 333         fd, ksconf = tempfile.mkstemp(prefix=os.path.basename(ksconf), dir="/tmp/")
 334         os.write(fd, ksc)
 335         os.close(fd)
 336
 337         msger.debug('new ks path %s' % ksconf)
 338
 339         import atexit
 340         atexit.register(_clrtempks)
 341
 342     return ksconf
 343
 344 def _check_mic_chroot(rootdir):
 345     def _path(path):
 346         return rootdir.rstrip('/') + path
 347
 348     release_files = map(_path, [ "/etc/moblin-release",
 349                                  "/etc/meego-release",
 350                                  "/etc/tizen-release"])
 351
 352     if not any(map(os.path.exists, release_files)):
 353         msger.warning("Dir %s is not a MeeGo/Tizen chroot env" % rootdir)
 354
 355     if not glob.glob(rootdir + "/boot/vmlinuz-*"):
 356         msger.warning("Failed to find kernel module under %s" % rootdir)
 357
 358     return
 359
 360 def selinux_check(arch, fstypes):
 361     try:
 362         getenforce = find_binary_path('getenforce')
 363     except CreatorError:
 364         return
 365
 366     selinux_status = runner.outs([getenforce])
 367     if arch and arch.startswith("arm") and selinux_status == "Enforcing":
 368         raise CreatorError("Can't create arm image if selinux is enabled, "
 369                            "please run 'setenforce 0' to disable selinux")
 370
 371     use_btrfs = filter(lambda typ: typ == 'btrfs', fstypes)
 372     if use_btrfs and selinux_status == "Enforcing":
 373         raise CreatorError("Can't create btrfs image if selinux is enabled,"
 374                            " please run 'setenforce 0' to disable selinux")
 375
 376 def get_image_type(path):
 377     def _get_extension_name(path):
 378         match = re.search("(?<=\.)\w+$", path)
 379         if match:
 380             return match.group(0)
 381         else:
 382             return None
 383
 384     if os.path.isdir(path):
 385         _check_mic_chroot(path)
 386         return "fs"
 387
 388     maptab = {
 389               "tar": "loop",
 390               "raw":"raw",
 391               "vmdk":"vmdk",
 392               "vdi":"vdi",
 393               "iso":"livecd",
 394               "usbimg":"liveusb",
 395              }
 396
 397     extension = _get_extension_name(path)
 398     if extension in maptab:
 399         return maptab[extension]
 400
 401     fd = open(path, "rb")
 402     file_header = fd.read(1024)
 403     fd.close()
 404     vdi_flag = "<<< Sun VirtualBox Disk Image >>>"
 405     if file_header[0:len(vdi_flag)] == vdi_flag:
 406         return maptab["vdi"]
 407
 408     output = runner.outs(['file', path])
 409     isoptn = re.compile(r".*ISO 9660 CD-ROM filesystem.*(bootable).*")
 410     usbimgptn = re.compile(r".*x86 boot sector.*active.*")
 411     rawptn = re.compile(r".*x86 boot sector.*")
 412     vmdkptn = re.compile(r".*VMware. disk image.*")
 413     ext3fsimgptn = re.compile(r".*Linux.*ext3 filesystem data.*")
 414     ext4fsimgptn = re.compile(r".*Linux.*ext4 filesystem data.*")
 415     btrfsimgptn = re.compile(r".*BTRFS.*")
 416     if isoptn.match(output):
 417         return maptab["iso"]
 418     elif usbimgptn.match(output):
 419         return maptab["usbimg"]
 420     elif rawptn.match(output):
 421         return maptab["raw"]
 422     elif vmdkptn.match(output):
 423         return maptab["vmdk"]
 424     elif ext3fsimgptn.match(output):
 425         return "ext3fsimg"
 426     elif ext4fsimgptn.match(output):
 427         return "ext4fsimg"
 428     elif btrfsimgptn.match(output):
 429         return "btrfsimg"
 430     else:
 431         raise CreatorError("Cannot detect the type of image: %s" % path)
 432
 433 def get_file_size(file):
 434     """ Return size in MB unit """
 435     rc, duOutput  = runner.runtool(['du', "-s", "-b", "-B", "1M", file])
 436     if rc != 0:
 437         raise CreatorError("Failed to run %s" % du)
 438
 439     size1 = int(duOutput.split()[0])
 440     rc, duOutput = runner.runtool(['du', "-s", "-B", "1M", file])
 441     if rc != 0:
 442         raise CreatorError("Failed to run %s" % du)
 443
 444     size2 = int(duOutput.split()[0])
 445     if size1 > size2:
 446         return size1
 447     else:
 448         return size2
 449
 450 def get_filesystem_avail(fs):
 451     vfstat = os.statvfs(fs)
 452     return vfstat.f_bavail * vfstat.f_bsize
 453
 454 def convert_image(srcimg, srcfmt, dstimg, dstfmt):
 455     #convert disk format
 456     if dstfmt != "raw":
 457         raise CreatorError("Invalid destination image format: %s" % dstfmt)
 458     msger.debug("converting %s image to %s" % (srcimg, dstimg))
 459     if srcfmt == "vmdk":
 460         path = find_binary_path("qemu-img")
 461         argv = [path, "convert", "-f", "vmdk", srcimg, "-O", dstfmt,  dstimg]
 462     elif srcfmt == "vdi":
 463         path = find_binary_path("VBoxManage")
 464         argv = [path, "internalcommands", "converttoraw", srcimg, dstimg]
 465     else:
 466         raise CreatorError("Invalid soure image format: %s" % srcfmt)
 467
 468     rc = runner.show(argv)
 469     if rc == 0:
 470         msger.debug("convert successful")
 471     if rc != 0:
 472         raise CreatorError("Unable to convert disk to %s" % dstfmt)
 473
 474 def uncompress_squashfs(squashfsimg, outdir):
 475     """Uncompress file system from squshfs image"""
 476     unsquashfs = find_binary_path("unsquashfs")
 477     args = [ unsquashfs, "-d", outdir, squashfsimg ]
 478     rc = runner.show(args)
 479     if (rc != 0):
 480         raise SquashfsError("Failed to uncompress %s." % squashfsimg)
 481
 482 def mkdtemp(dir = "/var/tmp", prefix = "mic-tmp-"):
 483     """ FIXME: use the dir in mic.conf instead """
 484
 485     makedirs(dir)
 486     return tempfile.mkdtemp(dir = dir, prefix = prefix)
 487
 488 def get_repostrs_from_ks(ks):
 489     def _get_temp_reponame(baseurl):
 490         md5obj = hashlib.md5(baseurl)
 491         tmpreponame = "%s" % md5obj.hexdigest()
 492         return tmpreponame
 493
 494     kickstart_repos = []
 495
 496     for repodata in ks.handler.repo.repoList:
 497         repo = {}
 498         for attr in ('name',
 499                      'baseurl',
 500                      'mirrorlist',
 501                      'includepkgs', # val is list
 502                      'excludepkgs', # val is list
 503                      'cost',    # int
 504                      'priority',# int
 505                      'save',
 506                      'proxy',
 507                      'proxyuser',
 508                      'proxypasswd',
 509                      'proxypasswd',
 510                      'debuginfo',
 511                      'source',
 512                      'gpgkey',
 513                      'ssl_verify'):
 514             if hasattr(repodata, attr) and getattr(repodata, attr):
 515                 repo[attr] = getattr(repodata, attr)
 516
 517         if 'name' not in repo:
 518             repo['name'] = _get_temp_reponame(repodata.baseurl)
 519
 520         kickstart_repos.append(repo)
 521
 522     return kickstart_repos
 523
 524 def _get_uncompressed_data_from_url(url, filename, proxies):
 525     filename = myurlgrab(url, filename, proxies)
 526     suffix = None
 527     if filename.endswith(".gz"):
 528         suffix = ".gz"
 529         runner.quiet(['gunzip', "-f", filename])
 530     elif filename.endswith(".bz2"):
 531         suffix = ".bz2"
 532         runner.quiet(['bunzip2', "-f", filename])
 533     if suffix:
 534         filename = filename.replace(suffix, "")
 535     return filename
 536
 537 def _get_metadata_from_repo(baseurl, proxies, cachedir, reponame, filename,
 538                             sumtype=None, checksum=None):
 539     url = os.path.join(baseurl, filename)
 540     filename_tmp = str("%s/%s/%s" % (cachedir, reponame, os.path.basename(filename)))
 541     if os.path.splitext(filename_tmp)[1] in (".gz", ".bz2"):
 542         filename = os.path.splitext(filename_tmp)[0]
 543     else:
 544         filename = filename_tmp
 545     if sumtype and checksum and os.path.exists(filename):
 546         try:
 547             sumcmd = find_binary_path("%ssum" % sumtype)
 548         except:
 549             file_checksum = None
 550         else:
 551             file_checksum = runner.outs([sumcmd, filename]).split()[0]
 552
 553         if file_checksum and file_checksum == checksum:
 554             return filename
 555
 556     return _get_uncompressed_data_from_url(url,filename_tmp,proxies)
 557
 558 def get_metadata_from_repos(repos, cachedir):
 559     my_repo_metadata = []
 560     for repo in repos:
 561         reponame = repo['name']
 562         baseurl  = repo['baseurl']
 563
 564
 565         if 'proxy' in repo:
 566             proxy = repo['proxy']
 567         else:
 568             proxy = get_proxy_for(baseurl)
 569
 570         proxies = None
 571         if proxy:
 572            proxies = {str(baseurl.split(":")[0]):str(proxy)}
 573
 574         makedirs(os.path.join(cachedir, reponame))
 575         url = os.path.join(baseurl, "repodata/repomd.xml")
 576         filename = os.path.join(cachedir, reponame, 'repomd.xml')
 577         repomd = myurlgrab(url, filename, proxies)
 578         try:
 579             root = xmlparse(repomd)
 580         except SyntaxError:
 581             raise CreatorError("repomd.xml syntax error.")
 582
 583         ns = root.getroot().tag
 584         ns = ns[0:ns.rindex("}")+1]
 585
 586         filepaths = {}
 587         checksums = {}
 588         sumtypes = {}
 589
 590         for elm in root.getiterator("%sdata" % ns):
 591             if elm.attrib["type"] == "patterns":
 592                 filepaths['patterns'] = elm.find("%slocation" % ns).attrib['href']
 593                 checksums['patterns'] = elm.find("%sopen-checksum" % ns).text
 594                 sumtypes['patterns'] = elm.find("%sopen-checksum" % ns).attrib['type']
 595                 break
 596
 597         for elm in root.getiterator("%sdata" % ns):
 598             if elm.attrib["type"] in ("group_gz", "group"):
 599                 filepaths['comps'] = elm.find("%slocation" % ns).attrib['href']
 600                 checksums['comps'] = elm.find("%sopen-checksum" % ns).text
 601                 sumtypes['comps'] = elm.find("%sopen-checksum" % ns).attrib['type']
 602                 break
 603
 604         primary_type = None
 605         for elm in root.getiterator("%sdata" % ns):
 606             if elm.attrib["type"] in ("primary_db", "primary"):
 607                 primary_type = elm.attrib["type"]
 608                 filepaths['primary'] = elm.find("%slocation" % ns).attrib['href']
 609                 checksums['primary'] = elm.find("%sopen-checksum" % ns).text
 610                 sumtypes['primary'] = elm.find("%sopen-checksum" % ns).attrib['type']
 611                 break
 612
 613         if not primary_type:
 614             continue
 615
 616         for item in ("primary", "patterns", "comps"):
 617             if item not in filepaths:
 618                 filepaths[item] = None
 619                 continue
 620             if not filepaths[item]:
 621                 continue
 622             filepaths[item] = _get_metadata_from_repo(baseurl,
 623                                                       proxies,
 624                                                       cachedir,
 625                                                       reponame,
 626                                                       filepaths[item],
 627                                                       sumtypes[item],
 628                                                       checksums[item])
 629
 630         """ Get repo key """
 631         try:
 632             repokey = _get_metadata_from_repo(baseurl,
 633                                               proxies,
 634                                               cachedir,
 635                                               reponame,
 636                                               "repodata/repomd.xml.key")
 637         except CreatorError:
 638             repokey = None
 639             msger.debug("\ncan't get %s/%s" % (baseurl, "repodata/repomd.xml.key"))
 640
 641         my_repo_metadata.append({"name":reponame,
 642                                  "baseurl":baseurl,
 643                                  "repomd":repomd,
 644                                  "primary":filepaths['primary'],
 645                                  "cachedir":cachedir,
 646                                  "proxies":proxies,
 647                                  "patterns":filepaths['patterns'],
 648                                  "comps":filepaths['comps'],
 649                                  "repokey":repokey})
 650
 651     return my_repo_metadata
 652
 653 def get_rpmver_in_repo(repometadata):
 654     for repo in repometadata:
 655         if repo["primary"].endswith(".xml"):
 656             root = xmlparse(repo["primary"])
 657             ns = root.getroot().tag
 658             ns = ns[0:ns.rindex("}")+1]
 659
 660             versionlist = []
 661             for elm in root.getiterator("%spackage" % ns):
 662                 if elm.find("%sname" % ns).text == 'rpm':
 663                     for node in elm.getchildren():
 664                         if node.tag == "%sversion" % ns:
 665                             versionlist.append(node.attrib['ver'])
 666
 667             if versionlist:
 668                 return reversed(
 669                          sorted(
 670                            versionlist,
 671                            key = lambda ver: map(int, ver.split('.')))).next()
 672
 673         elif repo["primary"].endswith(".sqlite"):
 674             con = sqlite.connect(repo["primary"])
 675             for row in con.execute("select version from packages where "
 676                                    "name=\"rpm\" ORDER by version DESC"):
 677                 con.close()
 678                 return row[0]
 679
 680     return None
 681
 682 def get_arch(repometadata):
 683     def uniqarch(archlist=[]):
 684         uniq_arch = []
 685         for i in range(len(archlist)):
 686             if archlist[i] not in rpmmisc.archPolicies.keys():
 687                 continue
 688             need_append = True
 689             j = 0
 690             while j < len(uniq_arch):
 691                 if archlist[i] in rpmmisc.archPolicies[uniq_arch[j]].split(':'):
 692                     need_append = False
 693                     break
 694                 if uniq_arch[j] in rpmmisc.archPolicies[archlist[i]].split(':'):
 695                     if need_append:
 696                         uniq_arch[j] = archlist[i]
 697                         need_append = False
 698                     else:
 699                         uniq_arch.remove(uniq_arch[j])
 700                         continue
 701                 j += 1
 702             if need_append:
 703                  uniq_arch.append(archlist[i])
 704
 705         return uniq_arch
 706
 707
 708     ret_uniq_arch = []
 709     ret_arch_list = []
 710     for repo in repometadata:
 711         archlist = []
 712         if repo["primary"].endswith(".xml"):
 713             root = xmlparse(repo["primary"])
 714             ns = root.getroot().tag
 715             ns = ns[0:ns.rindex("}")+1]
 716             for elm in root.getiterator("%spackage" % ns):
 717                 if elm.find("%sarch" % ns).text not in ("noarch", "src"):
 718                     arch = elm.find("%sarch" % ns).text
 719                     if arch not in archlist:
 720                         archlist.append(arch)
 721         elif repo["primary"].endswith(".sqlite"):
 722             con = sqlite.connect(repo["primary"])
 723             for row in con.execute("select arch from packages where arch not in (\"src\", \"noarch\")"):
 724                 if row[0] not in archlist:
 725                     archlist.append(row[0])
 726
 727             con.close()
 728
 729         uniq_arch = uniqarch(archlist)
 730         if not ret_uniq_arch and len(uniq_arch) == 1:
 731             ret_uniq_arch = uniq_arch
 732         ret_arch_list += uniq_arch
 733
 734     ret_arch_list = uniqarch(ret_arch_list)
 735     return ret_uniq_arch, ret_arch_list
 736
 737 def get_package(pkg, repometadata, arch = None):
 738     ver = ""
 739     target_repo = None
 740     if not arch:
 741         arches = []
 742     elif arch not in rpmmisc.archPolicies:
 743         arches = [arch]
 744     else:
 745         arches = rpmmisc.archPolicies[arch].split(':')
 746         arches.append('noarch')
 747
 748     for repo in repometadata:
 749         if repo["primary"].endswith(".xml"):
 750             root = xmlparse(repo["primary"])
 751             ns = root.getroot().tag
 752             ns = ns[0:ns.rindex("}")+1]
 753             for elm in root.getiterator("%spackage" % ns):
 754                 if elm.find("%sname" % ns).text == pkg:
 755                     if elm.find("%sarch" % ns).text in arches:
 756                         version = elm.find("%sversion" % ns)
 757                         tmpver = "%s-%s" % (version.attrib['ver'], version.attrib['rel'])
 758                         if tmpver > ver:
 759                             ver = tmpver
 760                             location = elm.find("%slocation" % ns)
 761                             pkgpath = "%s" % location.attrib['href']
 762                             target_repo = repo
 763                         break
 764         if repo["primary"].endswith(".sqlite"):
 765             con = sqlite.connect(repo["primary"])
 766             if arch:
 767                 sql = 'select version, release, location_href from packages ' \
 768                       'where name = "%s" and arch IN ("%s")' % \
 769                       (pkg, '","'.join(arches))
 770                 for row in con.execute(sql):
 771                     tmpver = "%s-%s" % (row[0], row[1])
 772                     if tmpver > ver:
 773                         ver = tmpver
 774                         pkgpath = "%s" % row[2]
 775                         target_repo = repo
 776                     break
 777             else:
 778                 sql = 'select version, release, location_href from packages ' \
 779                       'where name = "%s"' % pkg
 780                 for row in con.execute(sql):
 781                     tmpver = "%s-%s" % (row[0], row[1])
 782                     if tmpver > ver:
 783                         ver = tmpver
 784                         pkgpath = "%s" % row[2]
 785                         target_repo = repo
 786                     break
 787             con.close()
 788     if target_repo:
 789         makedirs("%s/packages/%s" % (target_repo["cachedir"], target_repo["name"]))
 790         url = os.path.join(target_repo["baseurl"], pkgpath)
 791         filename = str("%s/packages/%s/%s" % (target_repo["cachedir"], target_repo["name"], os.path.basename(pkgpath)))
 792         if os.path.exists(filename):
 793             ret = rpmmisc.checkRpmIntegrity('rpm', filename)
 794             if ret == 0:
 795                 return filename
 796
 797             msger.warning("package %s is damaged: %s" %
 798                           (os.path.basename(filename), filename))
 799             os.unlink(filename)
 800
 801         pkg = myurlgrab(str(url), filename, target_repo["proxies"])
 802         return pkg
 803     else:
 804         return None
 805
 806 def get_source_name(pkg, repometadata):
 807
 808     def get_bin_name(pkg):
 809         m = RPM_RE.match(pkg)
 810         if m:
 811             return m.group(1)
 812         return None
 813
 814     def get_src_name(srpm):
 815         m = SRPM_RE.match(srpm)
 816         if m:
 817             return m.group(1)
 818         return None
 819
 820     ver = ""
 821     target_repo = None
 822
 823     pkg_name = get_bin_name(pkg)
 824     if not pkg_name:
 825         return None
 826
 827     for repo in repometadata:
 828         if repo["primary"].endswith(".xml"):
 829             root = xmlparse(repo["primary"])
 830             ns = root.getroot().tag
 831             ns = ns[0:ns.rindex("}")+1]
 832             for elm in root.getiterator("%spackage" % ns):
 833                 if elm.find("%sname" % ns).text == pkg_name:
 834                     if elm.find("%sarch" % ns).text != "src":
 835                         version = elm.find("%sversion" % ns)
 836                         tmpver = "%s-%s" % (version.attrib['ver'], version.attrib['rel'])
 837                         if tmpver > ver:
 838                             ver = tmpver
 839                             fmt = elm.find("%sformat" % ns)
 840                             if fmt:
 841                                 fns = fmt.getchildren()[0].tag
 842                                 fns = fns[0:fns.rindex("}")+1]
 843                                 pkgpath = fmt.find("%ssourcerpm" % fns).text
 844                                 target_repo = repo
 845                         break
 846
 847         if repo["primary"].endswith(".sqlite"):
 848             con = sqlite.connect(repo["primary"])
 849             for row in con.execute("select version, release, rpm_sourcerpm from packages where name = \"%s\" and arch != \"src\"" % pkg_name):
 850                 tmpver = "%s-%s" % (row[0], row[1])
 851                 if tmpver > ver:
 852                     pkgpath = "%s" % row[2]
 853                     target_repo = repo
 854                 break
 855             con.close()
 856     if target_repo:
 857         return get_src_name(pkgpath)
 858     else:
 859         return None
 860
 861 def get_pkglist_in_patterns(group, patterns):
 862     found = False
 863     pkglist = []
 864     try:
 865         root = xmlparse(patterns)
 866     except SyntaxError:
 867         raise SyntaxError("%s syntax error." % patterns)
 868
 869     for elm in list(root.getroot()):
 870         ns = elm.tag
 871         ns = ns[0:ns.rindex("}")+1]
 872         name = elm.find("%sname" % ns)
 873         summary = elm.find("%ssummary" % ns)
 874         if name.text == group or summary.text == group:
 875             found = True
 876             break
 877
 878     if not found:
 879         return pkglist
 880
 881     found = False
 882     for requires in list(elm):
 883         if requires.tag.endswith("requires"):
 884             found = True
 885             break
 886
 887     if not found:
 888         return pkglist
 889
 890     for pkg in list(requires):
 891         pkgname = pkg.attrib["name"]
 892         if pkgname not in pkglist:
 893             pkglist.append(pkgname)
 894
 895     return pkglist
 896
 897 def get_pkglist_in_comps(group, comps):
 898     found = False
 899     pkglist = []
 900     try:
 901         root = xmlparse(comps)
 902     except SyntaxError:
 903         raise SyntaxError("%s syntax error." % comps)
 904
 905     for elm in root.getiterator("group"):
 906         id = elm.find("id")
 907         name = elm.find("name")
 908         if id.text == group or name.text == group:
 909             packagelist = elm.find("packagelist")
 910             found = True
 911             break
 912
 913     if not found:
 914         return pkglist
 915
 916     for require in elm.getiterator("packagereq"):
 917         if require.tag.endswith("packagereq"):
 918             pkgname = require.text
 919         if pkgname not in pkglist:
 920             pkglist.append(pkgname)
 921
 922     return pkglist
 923
 924 def is_statically_linked(binary):
 925     return ", statically linked, " in runner.outs(['file', binary])
 926
 927 def setup_qemu_emulator(rootdir, arch):
 928     # mount binfmt_misc if it doesn't exist
 929     if not os.path.exists("/proc/sys/fs/binfmt_misc"):
 930         modprobecmd = find_binary_path("modprobe")
 931         runner.show([modprobecmd, "binfmt_misc"])
 932     if not os.path.exists("/proc/sys/fs/binfmt_misc/register"):
 933         mountcmd = find_binary_path("mount")
 934         runner.show([mountcmd, "-t", "binfmt_misc", "none", "/proc/sys/fs/binfmt_misc"])
 935
 936     # qemu_emulator is a special case, we can't use find_binary_path
 937     # qemu emulator should be a statically-linked executable file
 938     qemu_emulator = "/usr/bin/qemu-arm"
 939     if not os.path.exists(qemu_emulator) or not is_statically_linked(qemu_emulator):
 940         qemu_emulator = "/usr/bin/qemu-arm-static"
 941     if not os.path.exists(qemu_emulator):
 942         raise CreatorError("Please install a statically-linked qemu-arm")
 943
 944     # qemu emulator version check
 945     armv7_list = [arch for arch in rpmmisc.archPolicies.keys() if arch.startswith('armv7')]
 946     if arch in armv7_list:  # need qemu (>=0.13.0)
 947         qemuout = runner.outs([qemu_emulator, "-h"])
 948         m = re.search("version\s*([.\d]+)", qemuout)
 949         if m:
 950             qemu_version = m.group(1)
 951             if qemu_version < "0.13":
 952                 raise CreatorError("Requires %s version >=0.13 for %s" % (qemu_emulator, arch))
 953         else:
 954             msger.warning("Can't get version info of %s, please make sure it's higher than 0.13.0" % qemu_emulator)
 955
 956     if not os.path.exists(rootdir + "/usr/bin"):
 957         makedirs(rootdir + "/usr/bin")
 958     shutil.copy(qemu_emulator, rootdir + qemu_emulator)
 959
 960     # disable selinux, selinux will block qemu emulator to run
 961     if os.path.exists("/usr/sbin/setenforce"):
 962         msger.info('Try to disable selinux')
 963         runner.show(["/usr/sbin/setenforce", "0"])
 964
 965     node = "/proc/sys/fs/binfmt_misc/arm"
 966     if is_statically_linked(qemu_emulator) and os.path.exists(node):
 967         return qemu_emulator
 968
 969     # unregister it if it has been registered and is a dynamically-linked executable
 970     if not is_statically_linked(qemu_emulator) and os.path.exists(node):
 971         qemu_unregister_string = "-1\n"
 972         fd = open("/proc/sys/fs/binfmt_misc/arm", "w")
 973         fd.write(qemu_unregister_string)
 974         fd.close()
 975
 976     # register qemu emulator for interpreting other arch executable file
 977     if not os.path.exists(node):
 978         qemu_arm_string = ":arm:M::\\x7fELF\\x01\\x01\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x02\\x00\\x28\\x00:\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\x00\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xfa\\xff\\xff\\xff:%s:\n" % qemu_emulator
 979         fd = open("/proc/sys/fs/binfmt_misc/register", "w")
 980         fd.write(qemu_arm_string)
 981         fd.close()
 982
 983     return qemu_emulator
 984
 985 def SrcpkgsDownload(pkgs, repometadata, instroot, cachedir):
 986     def get_source_repometadata(repometadata):
 987         src_repometadata=[]
 988         for repo in repometadata:
 989             if repo["name"].endswith("-source"):
 990                 src_repometadata.append(repo)
 991         if src_repometadata:
 992             return src_repometadata
 993         return None
 994
 995     def get_src_name(srpm):
 996         m = SRPM_RE.match(srpm)
 997         if m:
 998             return m.group(1)
 999         return None
1000
1001     src_repometadata = get_source_repometadata(repometadata)
1002
1003     if not src_repometadata:
1004         msger.warning("No source repo found")
1005         return None
1006
1007     src_pkgs = []
1008     lpkgs_dict = {}
1009     lpkgs_path = []
1010     for repo in src_repometadata:
1011         cachepath = "%s/%s/packages/*.src.rpm" %(cachedir, repo["name"])
1012         lpkgs_path += glob.glob(cachepath)
1013
1014     for lpkg in lpkgs_path:
1015         lpkg_name = get_src_name(os.path.basename(lpkg))
1016         lpkgs_dict[lpkg_name] = lpkg
1017     localpkgs = lpkgs_dict.keys()
1018
1019     cached_count = 0
1020     destdir = instroot+'/usr/src/SRPMS'
1021     if not os.path.exists(destdir):
1022         os.makedirs(destdir)
1023
1024     srcpkgset = set()
1025     for _pkg in pkgs:
1026         srcpkg_name = get_source_name(_pkg, repometadata)
1027         if not srcpkg_name:
1028             continue
1029         srcpkgset.add(srcpkg_name)
1030
1031     for pkg in list(srcpkgset):
1032         if pkg in localpkgs:
1033             cached_count += 1
1034             shutil.copy(lpkgs_dict[pkg], destdir)
1035             src_pkgs.append(os.path.basename(lpkgs_dict[pkg]))
1036         else:
1037             src_pkg = get_package(pkg, src_repometadata, 'src')
1038             if src_pkg:
1039                 shutil.copy(src_pkg, destdir)
1040                 src_pkgs.append(src_pkg)
1041     msger.info("%d source packages gotten from cache" % cached_count)
1042
1043     return src_pkgs
1044
1045 def strip_end(text, suffix):
1046     if not text.endswith(suffix):
1047         return text
1048     return text[:-len(suffix)]