2 # Copyright (c) 2000 - 2016 Samsung Electronics Co., Ltd. All rights reserved.
5 # @author Chulwoo Shin <cw1.shin@samsung.com>
7 # Licensed under the Apache License, Version 2.0 (the "License");
8 # you may not use this file except in compliance with the License.
9 # You may obtain a copy of the License at
11 # http://www.apache.org/licenses/LICENSE-2.0
13 # Unless required by applicable law or agreed to in writing, software
14 # distributed under the License is distributed on an "AS IS" BASIS,
15 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 # See the License for the specific language governing permissions and
17 # limitations under the License.
27 from lxml import etree
28 from tic.utils import file
29 from tic.utils import process
30 from tic.utils.error import TICError
31 from tic.utils.grabber import myurlgrab2
32 from tic.utils import misc
33 from tic.config import configmgr
35 def _get_uncompressed_data_from_url(url, filename, proxies=None):
37 filename = myurlgrab2(url, filename)
38 # Check if file compressed or not
39 if filename.endswith(".gz"):
40 decompress_filename = os.path.splitext(filename)[0]
41 filename = file.decompress_gzip(filename, decompress_filename)
42 elif filename.endswith(".bz2"):
43 process.run(['bunzip2', "-f", filename])
44 filename = os.path.splitext(filename)[0]
47 def _get_metadata_from_repo(baseurl, proxies, tempdir, cachedir, reponame, filehref,
48 sumtype=None, checksum=None):
49 logger = logging.getLogger(__name__)
50 url = os.path.join(baseurl, filehref)
51 filename_tmp = str("%s/%s" % (cachedir, os.path.basename(filehref)))
52 if os.path.splitext(filename_tmp)[1] in (".gz", ".bz2"):
53 filename = os.path.splitext(filename_tmp)[0]
55 filename = filename_tmp
56 if sumtype and checksum and os.path.exists(filename):
57 if sumtype == 'sha256':
58 file_checksum = hashlib.sha256(open(filename, 'rb').read()).hexdigest()
59 elif sumtype == 'md5':
60 file_checksum = hashlib.md5(open(filename, 'rb').read()).hexdigest()
62 sumcmd = "%ssum" % sumtype
63 result = process.run([sumcmd, filename])[1].strip()
64 file_checksum = result.split()[0]
66 if file_checksum and file_checksum == checksum:
67 logger.info('use a cache file - ' + str(filename))
70 temp_file = os.path.join(tempdir, os.path.basename(filehref))
71 file_path =_get_uncompressed_data_from_url(url, temp_file, proxies)
72 return file.copyfile_flock(file_path, filename)
74 def get_repodata_from_repos(repos, cachedir):
75 logger = logging.getLogger(__name__)
77 temp_path = os.path.join(cachedir, 'temp', str(misc.get_timestamp()))
79 reponame = repo.get('name')
80 baseurl = repo.get('url')
83 base64url = base64.urlsafe_b64encode(baseurl)
84 temp_dir = os.path.join(temp_path, base64url);
85 repomd_file = os.path.join(temp_dir, 'repomd.xml')
86 file.make_dirs(temp_dir);
88 #TODO: support local files(local directory)
89 # local/remote repository
90 url = os.path.join(baseurl, 'repodata/repomd.xml')
91 repomd = myurlgrab2(url, repomd_file)
94 tree = etree.parse(repomd)
96 except etree.XMLSyntaxError as e:
98 raise TICError(configmgr.message['xml_parse_error'] % ('repomd.xml', url))
101 repo_checksum = hashlib.sha256(open(repomd_file, 'rb').read()).hexdigest()
102 cache_dir = os.path.join(cachedir, 'cached', base64url, repo_checksum)
103 file.make_dirs(cache_dir)
106 ns = ns[0:ns.rindex("}")+1]
112 for elm in root.findall("%sdata" % ns):
113 if elm.attrib['type'] == 'patterns':
114 filepaths['patterns'] = elm.find("%slocation" % ns).attrib['href']
115 checksums['patterns'] = elm.find("%sopen-checksum" % ns).text
116 sumtypes['patterns'] = elm.find("%sopen-checksum" % ns).attrib['type']
117 elif elm.attrib['type'] == 'group':
118 filepaths['comps'] = elm.find("%slocation" % ns).attrib['href']
119 checksums['comps'] = elm.find("%sopen-checksum" % ns).text
120 sumtypes['comps'] = elm.find("%sopen-checksum" % ns).attrib['type']
121 elif elm.attrib["type"] == 'primary':
122 filepaths['primary'] = elm.find("%slocation" % ns).attrib['href']
123 checksums['primary'] = elm.find("%sopen-checksum" % ns).text
124 sumtypes['primary'] = elm.find("%sopen-checksum" % ns).attrib['type']
126 for item in ("primary", "patterns", "comps"):
127 if item not in filepaths:
128 filepaths[item] = None
130 filepaths[item] = _get_metadata_from_repo(baseurl,
138 my_repodata.append({"name":reponame,
140 "checksum":repo_checksum,
142 "primary":filepaths['primary'],
143 "cachedir":cache_dir,
145 "patterns":filepaths['patterns'],
146 "comps":filepaths['comps']})
150 RepoType = collections.namedtuple('Repo', 'name, url')
151 def Repo(name, baseurl):
152 return RepoType(name, baseurl)