tic/repo.py

   1 #!/usr/bin/python
   2 # Copyright (c) 2000 - 2016 Samsung Electronics Co., Ltd. All rights reserved.
   3 #
   4 # Contact:
   5 # @author Chulwoo Shin <cw1.shin@samsung.com>
   6 #
   7 # Licensed under the Apache License, Version 2.0 (the "License");
   8 # you may not use this file except in compliance with the License.
   9 # You may obtain a copy of the License at
  10 #
  11 # http://www.apache.org/licenses/LICENSE-2.0
  12 #
  13 # Unless required by applicable law or agreed to in writing, software
  14 # distributed under the License is distributed on an "AS IS" BASIS,
  15 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16 # See the License for the specific language governing permissions and
  17 # limitations under the License.
  18 #
  19 # Contributors:
  20 # - S-Core Co., Ltd
  21
  22 import os
  23 import base64
  24 import collections
  25 import hashlib
  26 from lxml import etree
  27 from tic.utils import file
  28 from tic.utils import process
  29 from tic.utils.error import TICError
  30 from tic.utils.grabber import myurlgrab
  31
  32
  33 def _get_uncompressed_data_from_url(url, filename, proxies):
  34     # download file
  35     filename = myurlgrab(url, filename, proxies)
  36     # Check if file compressed or not
  37     if filename.endswith(".gz"):
  38         decompress_filename = os.path.splitext(filename)[0]
  39         filename = file.decompress_gzip(filename, decompress_filename)
  40     elif filename.endswith(".bz2"):
  41         process.run(['bunzip2', "-f", filename])
  42         filename = os.path.splitext(filename)[0]
  43
  44     return filename
  45
  46 def _get_metadata_from_repo(baseurl, proxies, cachedir, reponame, filename,
  47                             sumtype=None, checksum=None):
  48     url = os.path.join(baseurl, filename)
  49     filename_tmp = str("%s/%s/%s" % (cachedir, reponame, os.path.basename(filename)))
  50     if os.path.splitext(filename_tmp)[1] in (".gz", ".bz2"):
  51         filename = os.path.splitext(filename_tmp)[0]
  52     else:
  53         filename = filename_tmp
  54
  55     if sumtype and checksum and os.path.exists(filename):
  56         if sumtype == 'sha256':
  57             file_checksum = hashlib.sha256(open(filename, 'rb').read()).hexdigest()
  58         elif sumtype == 'md5':
  59             file_checksum = hashlib.md5(open(filename, 'rb').read()).hexdigest()
  60         else:
  61             sumcmd = "%ssum" % sumtype
  62             result = process.run([sumcmd, filename])[1].strip()
  63             file_checksum = result.split()[0]
  64
  65         if file_checksum and file_checksum == checksum:
  66             return filename
  67
  68     return _get_uncompressed_data_from_url(url, filename_tmp, proxies)
  69
  70
  71 def get_repodata_from_repos(repos, cachedir):
  72     my_repodata = []
  73     for repo in repos:
  74         reponame = repo.name
  75         baseurl = repo.baseurl
  76         cache_dir = os.path.join(cachedir, reponame)
  77         cache_file = os.path.join(cache_dir, 'repomd.xml')
  78
  79         # make directory for caching
  80         file.make_dirs(cache_dir)
  81         #TODO: support proxy
  82
  83         url = os.path.join(baseurl, 'repodata/repomd.xml')
  84         repomd = myurlgrab(url, cache_file, None)
  85
  86         try:
  87             tree = etree.parse(repomd)
  88             root = tree.getroot()
  89         except etree.XMLSyntaxError:
  90             raise TICError("repomd.xml syntax error.")
  91
  92         ns = root.tag
  93         ns = ns[0:ns.rindex("}")+1]
  94
  95         filepaths = {}
  96         checksums = {}
  97         sumtypes = {}
  98
  99         for elm in root.findall("%sdata" % ns):
 100             if elm.attrib['type'] == 'patterns':
 101                 filepaths['patterns'] = elm.find("%slocation" % ns).attrib['href']
 102                 checksums['patterns'] = elm.find("%sopen-checksum" % ns).text
 103                 sumtypes['patterns'] = elm.find("%sopen-checksum" % ns).attrib['type']
 104             elif elm.attrib['type'] == 'group':
 105                 filepaths['comps'] = elm.find("%slocation" % ns).attrib['href']
 106                 checksums['comps'] = elm.find("%sopen-checksum" % ns).text
 107                 sumtypes['comps'] = elm.find("%sopen-checksum" % ns).attrib['type']
 108             elif elm.attrib["type"] == 'primary':
 109                 filepaths['primary'] = elm.find("%slocation" % ns).attrib['href']
 110                 checksums['primary'] = elm.find("%sopen-checksum" % ns).text
 111                 sumtypes['primary'] = elm.find("%sopen-checksum" % ns).attrib['type']
 112
 113         for item in ("primary", "patterns", "comps"):
 114             if item not in filepaths:
 115                 filepaths[item] = None
 116                 continue
 117
 118             filepaths[item] = _get_metadata_from_repo(baseurl,
 119                                                       None,
 120                                                       cachedir,
 121                                                       reponame,
 122                                                       filepaths[item],
 123                                                       sumtypes[item],
 124                                                       checksums[item])
 125
 126         my_repodata.append({"name":reponame,
 127                             "baseurl":baseurl,
 128                             "repomd":repomd,
 129                             "primary":filepaths['primary'],
 130                             "cachedir":cachedir,
 131                             "proxies":None,
 132                             "patterns":filepaths['patterns'],
 133                             "comps":filepaths['comps']})
 134
 135     return my_repodata
 136
 137
 138 RepoType = collections.namedtuple('Repo', 'name, baseurl')
 139 def Repo(name, baseurl):
 140     return RepoType(name, baseurl)
 141
 142 if __name__ == '__main__':
 143     repo_url_1 = 'https://download.tizen.org/snapshots/tizen/base/latest/repos/arm64/packagesaaa'
 144     repo_url_2 = 'https://download.tizen.org/snapshots/tizen/mobile/latest/repos/arm64-wayland/packages'
 145     repos = []
 146     repos.append(Repo(base64.urlsafe_b64encode(repo_url_1), repo_url_1))
 147     repos.append(Repo(base64.urlsafe_b64encode(repo_url_2), repo_url_2))
 148     cachedir = '/var/tmp/tic-core/cached'
 149     repodata = get_repodata_from_repos(repos, cachedir)
 150     print(repodata)
 151
 152
 153