tic/repo.py

   1 #!/usr/bin/python
   2 # Copyright (c) 2000 - 2016 Samsung Electronics Co., Ltd. All rights reserved.
   3 #
   4 # Contact:
   5 # @author Chulwoo Shin <cw1.shin@samsung.com>
   6 #
   7 # Licensed under the Apache License, Version 2.0 (the "License");
   8 # you may not use this file except in compliance with the License.
   9 # You may obtain a copy of the License at
  10 #
  11 # http://www.apache.org/licenses/LICENSE-2.0
  12 #
  13 # Unless required by applicable law or agreed to in writing, software
  14 # distributed under the License is distributed on an "AS IS" BASIS,
  15 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16 # See the License for the specific language governing permissions and
  17 # limitations under the License.
  18 #
  19 # Contributors:
  20 # - S-Core Co., Ltd
  21
  22 import logging
  23 import os
  24 import base64
  25 import hashlib
  26 import collections
  27 from lxml import etree
  28 from tic.utils import file
  29 from tic.utils import process
  30 from tic.utils.error import TICError
  31 from tic.utils.grabber import myurlgrab2
  32 from tic.utils import misc
  33
  34 def _get_uncompressed_data_from_url(url, filename, proxies=None):
  35     # download file
  36     filename = myurlgrab2(url, filename)
  37     # Check if file compressed or not
  38     if filename.endswith(".gz"):
  39         decompress_filename = os.path.splitext(filename)[0]
  40         filename = file.decompress_gzip(filename, decompress_filename)
  41     elif filename.endswith(".bz2"):
  42         process.run(['bunzip2', "-f", filename])
  43         filename = os.path.splitext(filename)[0]
  44     return filename
  45
  46 def _get_metadata_from_repo(baseurl, proxies, tempdir, cachedir, reponame, filehref,
  47                             sumtype=None, checksum=None):
  48     logger = logging.getLogger(__name__)
  49     url = os.path.join(baseurl, filehref)
  50     filename_tmp = str("%s/%s" % (cachedir, os.path.basename(filehref)))
  51     if os.path.splitext(filename_tmp)[1] in (".gz", ".bz2"):
  52         filename = os.path.splitext(filename_tmp)[0]
  53     else:
  54         filename = filename_tmp
  55     if sumtype and checksum and os.path.exists(filename):
  56         if sumtype == 'sha256':
  57             file_checksum = hashlib.sha256(open(filename, 'rb').read()).hexdigest()
  58         elif sumtype == 'md5':
  59             file_checksum = hashlib.md5(open(filename, 'rb').read()).hexdigest()
  60         else:
  61             sumcmd = "%ssum" % sumtype
  62             result = process.run([sumcmd, filename])[1].strip()
  63             file_checksum = result.split()[0]
  64         # use cached file
  65         if file_checksum and file_checksum == checksum:
  66             logger.info('use a cache file - ' + str(filename))
  67             return filename
  68
  69     temp_file = os.path.join(tempdir, os.path.basename(filehref))
  70     file_path =_get_uncompressed_data_from_url(url, temp_file, proxies)
  71     return file.copyfile_flock(file_path, filename)
  72
  73 def get_repodata_from_repos(repos, cachedir):
  74     my_repodata = []
  75     temp_path = os.path.join(cachedir, 'temp', str(misc.get_timestamp()))
  76     for repo in repos:
  77         reponame = repo.get('name')
  78         baseurl = repo.get('url')
  79
  80         # make temp_dir
  81         base64url = base64.urlsafe_b64encode(baseurl)
  82         temp_dir = os.path.join(temp_path, base64url);
  83         repomd_file = os.path.join(temp_dir, 'repomd.xml')
  84         file.make_dirs(temp_dir);
  85
  86         #TODO: support proxy
  87         url = os.path.join(baseurl, 'repodata/repomd.xml')
  88         repomd = myurlgrab2(url, repomd_file)
  89
  90         try:
  91             tree = etree.parse(repomd)
  92             root = tree.getroot()
  93         except etree.XMLSyntaxError:
  94             raise TICError("repomd.xml syntax error.")
  95
  96         # make cache_dir
  97         repo_checksum = hashlib.sha256(open(repomd_file, 'rb').read()).hexdigest();
  98         cache_dir = os.path.join(cachedir, 'cached', base64url, repo_checksum)
  99         file.make_dirs(cache_dir)
 100
 101         ns = root.tag
 102         ns = ns[0:ns.rindex("}")+1]
 103
 104         filepaths = {}
 105         checksums = {}
 106         sumtypes = {}
 107
 108         for elm in root.findall("%sdata" % ns):
 109             if elm.attrib['type'] == 'patterns':
 110                 filepaths['patterns'] = elm.find("%slocation" % ns).attrib['href']
 111                 checksums['patterns'] = elm.find("%sopen-checksum" % ns).text
 112                 sumtypes['patterns'] = elm.find("%sopen-checksum" % ns).attrib['type']
 113             elif elm.attrib['type'] == 'group':
 114                 filepaths['comps'] = elm.find("%slocation" % ns).attrib['href']
 115                 checksums['comps'] = elm.find("%sopen-checksum" % ns).text
 116                 sumtypes['comps'] = elm.find("%sopen-checksum" % ns).attrib['type']
 117             elif elm.attrib["type"] == 'primary':
 118                 filepaths['primary'] = elm.find("%slocation" % ns).attrib['href']
 119                 checksums['primary'] = elm.find("%sopen-checksum" % ns).text
 120                 sumtypes['primary'] = elm.find("%sopen-checksum" % ns).attrib['type']
 121
 122         for item in ("primary", "patterns", "comps"):
 123             if item not in filepaths:
 124                 filepaths[item] = None
 125                 continue
 126             filepaths[item] = _get_metadata_from_repo(baseurl,
 127                                                       None,
 128                                                       temp_dir,
 129                                                       cache_dir,
 130                                                       reponame,
 131                                                       filepaths[item],
 132                                                       sumtypes[item],
 133                                                       checksums[item])
 134         my_repodata.append({"name":reponame,
 135                             "baseurl":baseurl,
 136                             "checksum":repo_checksum,
 137                             "repomd":repomd,
 138                             "primary":filepaths['primary'],
 139                             "cachedir":cache_dir,
 140                             "proxies":None,
 141                             "patterns":filepaths['patterns'],
 142                             "comps":filepaths['comps']})
 143     return my_repodata
 144
 145
 146 RepoType = collections.namedtuple('Repo', 'name, url')
 147 def Repo(name, baseurl):
 148     return RepoType(name, baseurl)