tic/parser/repo_parser.py

   1 #!/usr/bin/python
   2 # Copyright (c) 2000 - 2016 Samsung Electronics Co., Ltd. All rights reserved.
   3 #
   4 # Contact:
   5 # @author Chulwoo Shin <cw1.shin@samsung.com>
   6 #
   7 # Licensed under the Apache License, Version 2.0 (the "License");
   8 # you may not use this file except in compliance with the License.
   9 # You may obtain a copy of the License at
  10 #
  11 # http://www.apache.org/licenses/LICENSE-2.0
  12 #
  13 # Unless required by applicable law or agreed to in writing, software
  14 # distributed under the License is distributed on an "AS IS" BASIS,
  15 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16 # See the License for the specific language governing permissions and
  17 # limitations under the License.
  18 #
  19 # Contributors:
  20 # - S-Core Co., Ltd
  21
  22 import re
  23 import logging
  24 from lxml import etree
  25 from tic.utils.error import TICError
  26 from tic.utils.rpmmisc import archPolicies, default_arch
  27 from tic.config import configmgr
  28
  29 # meta pkg
  30 META_PREFIX = configmgr.regularexp['meta_prefix']
  31 META_PREFIX_ROOT = configmgr.regularexp['meta_prefix_root']
  32 META_PREFIX_SUB1 = configmgr.regularexp['meta_prefix_sub1']
  33 META_PATTERN = re.compile(''.join(['^', META_PREFIX, configmgr.regularexp['meta_pattern']]), re.I)
  34 META_SUB1_PATTERN = re.compile(configmgr.regularexp['meta_sub1_pattern'])
  35 META_SUB2_PATTERN = re.compile(configmgr.regularexp['meta_sub2_pattern'])
  36 # profile pkg
  37 PROFILE_PATTERN = re.compile(configmgr.regularexp['profile_pattern'])
  38
  39 class RepodataParser(object):
  40
  41     def __init__(self, arch, repodata_list):
  42         self.arch = arch
  43         self.repodata_list = repodata_list
  44
  45     def _xml_parse(self, pkg_group, pkg_list, tag_dic):
  46
  47         def _set_version(element, tag):
  48             if 'ver' in tag.attrib:
  49                 element['ver'] = tag.attrib['ver']
  50             if 'rel' in tag.attrib:
  51                 element['rel'] = tag.attrib['rel']
  52             if 'epoch' in tag.attrib:
  53                 element['epoch'] = tag.attrib['epoch']
  54             if 'flags' in tag.attrib:
  55                 element['flags'] = tag.attrib['flags']
  56
  57         pkg_dict = pkg_group.get('pkg_dict')
  58         provides_dict = pkg_group.get('provides')
  59         files_dict = pkg_group.get('files')
  60         meta_info = pkg_group.get('meta_info')
  61         pkg_id = len(pkg_dict)
  62
  63         for pkg in pkg_list:
  64             pkg_name = pkg.findtext(tag_dic['name'])
  65
  66             # check whether a package is duplicated.
  67             if pkg_name in pkg_dict:
  68                 # TODO: Apply to policy of duplication
  69                 # logger.warning('package(%s) is duplicated. exclude this package', pkg_name)
  70                 continue
  71
  72             pkg_info = {}
  73             pkg_info['id'] = pkg_id
  74             pkg_info['name'] = pkg_name
  75             pkg_info['arch'] = pkg.findtext(tag_dic['arch'])
  76             pkg_info['selfChecked'] = False # for web-ui tree
  77
  78             # Parsing meta-pkg using meta naming rule
  79             meta_match = META_PATTERN.search(pkg_info['name'])
  80             if meta_match is not None:
  81                 #print(meta_match.group(0), ', ', meta_match.group('meta'), ', ', meta_match.group('pkgname'))
  82                 if meta_match.group('meta') == 'root':
  83                     meta_info['root'].append([pkg_info['name']])
  84                     pkg_info['meta'] = 'root'
  85                 elif meta_match.group('meta') == 'sub1':
  86                     sub1_match = META_SUB1_PATTERN.search(meta_match.group('pkgname'))
  87                     meta_info['sub1'].append([pkg_info['name'],
  88                                               ''.join([META_PREFIX_ROOT, sub1_match.group('root')])])
  89                     pkg_info['meta'] = 'sub1'
  90                 elif meta_match.group('meta') == 'sub2':
  91                     sub2_match = META_SUB2_PATTERN.search(meta_match.group('pkgname'))
  92                     meta_info['sub2'].append([pkg_info['name'],
  93                                               ''.join([META_PREFIX_ROOT, sub2_match.group('root')]),
  94                                               ''.join([META_PREFIX_SUB1, sub2_match.group('root'),'-', sub2_match.group('sub1')])])
  95                     pkg_info['meta'] = 'sub2'
  96
  97             # check profile pkg
  98             profile_match = PROFILE_PATTERN.search(pkg_info['name']);
  99             if profile_match and profile_match.group('profile'):
 100                 pkg_info['profile'] = profile_match.group('profile')
 101             else:
 102                 pkg_info['profile'] = None
 103
 104             ver_tag = pkg.find(tag_dic['version'])
 105             pkg_info['version'] = {'epoch':ver_tag.attrib['epoch'],
 106                                    'ver':ver_tag.attrib['ver'],
 107                                    'rel':ver_tag.attrib['rel']}
 108             pkg_info['checksum'] = pkg.findtext(tag_dic['checksum'])
 109             pkg_info['summary'] = pkg.findtext(tag_dic['summary'])
 110             pkg_info['description'] = pkg.findtext(tag_dic['description'])
 111             pkg_info['location'] = pkg.find(tag_dic['location']).attrib['href']
 112             size_tag = pkg.find(tag_dic['size'])
 113             pkg_info['size'] = size_tag.attrib['package']
 114             pkg_info['installed'] = size_tag.attrib['installed']
 115
 116             format_tag = pkg.find(tag_dic['format'])
 117             if(format_tag is not None):
 118                 requires_tag = format_tag.find(tag_dic['requires'])
 119                 if requires_tag is not None:
 120                     dep_list = []
 121                     for rpm in requires_tag:
 122                         require = {}
 123                         require['name'] = rpm.attrib['name']
 124                         _set_version(require, rpm)
 125                         dep_list.append(require)
 126                     pkg_info['requires'] = dep_list;
 127                 provides_tag = format_tag.find(tag_dic['provides'])
 128                 if provides_tag is not None:
 129                     dep_list = []
 130                     for rpm in provides_tag:
 131                         provide = {}
 132                         provide['name'] = rpm.attrib['name']
 133                         _set_version(provide, rpm)
 134                         if provide.get('ver') and not provide.get('rel') and pkg_info['version']['rel']:
 135                             provide['rel'] = pkg_info['version']['rel'];
 136
 137                         if rpm.attrib['name'] in provides_dict:
 138                             provides_dict[rpm.attrib['name']].append({'name': pkg_name, 'data': provide})
 139                         else:
 140                             provides_dict[rpm.attrib['name']] = [{'name': pkg_name, 'data': provide}]
 141                         dep_list.append(provide)
 142                     pkg_info['provides'] = dep_list;
 143                 conflicts_tag = format_tag.find(tag_dic['conflicts'])
 144                 if conflicts_tag is not None:
 145                     dep_list = []
 146                     for rpm in conflicts_tag:
 147                         conflict = {}
 148                         conflict['name'] = rpm.attrib['name']
 149                         _set_version(conflict, rpm)
 150                         dep_list.append(conflict)
 151                     pkg_info['conflicts'] = dep_list;
 152                 recommends_tag = format_tag.find(tag_dic['recommends'])
 153                 if recommends_tag is not None:
 154                     dep_list = []
 155                     for rpm in recommends_tag:
 156                         recommend = {}
 157                         recommend['name'] = rpm.attrib['name']
 158                         _set_version(recommend, rpm)
 159                         dep_list.append(recommend)
 160                     pkg_info['recommends'] = dep_list;
 161                 suggests_tag = format_tag.find(tag_dic['suggests'])
 162                 if suggests_tag is not None:
 163                     dep_list = []
 164                     for rpm in suggests_tag:
 165                         dep_list.append(rpm.attrib['name'])
 166                     pkg_info['suggests'] = dep_list;
 167                 file_tag = format_tag.find(tag_dic['file'])
 168                 if file_tag is not None:
 169                     dep_list = []
 170                     for file_t in format_tag.findall(tag_dic['file']):
 171                         if file_t.text in files_dict:
 172                             files_dict[file_t.text].append(pkg_name)
 173                         else:
 174                             files_dict[file_t.text] = [pkg_name]
 175                         dep_list.append(file_t.text)
 176                     pkg_info['file'] = dep_list;
 177             pkg_dict[pkg_name] = pkg_info
 178             pkg_id += 1
 179
 180     def _prepare_requires_id(self, pkg_group):
 181         logger = logging.getLogger(__name__)
 182         pkg_list = pkg_group.get('pkg_list')
 183         pkg2id = pkg_group.get('pkg2id')
 184         provides2id = pkg_group.get('provides2id')
 185         file2id = pkg_group.get('file2id')
 186
 187         for pkg_id in range(len(pkg_list)):
 188             requires = pkg_list[pkg_id].get('requires')
 189             if requires is not None:
 190                 for req_info in requires:
 191                     req_name = req_info['name']
 192                     #TODO: Determine dependency search order
 193                     if req_name in provides2id:
 194                         #TODO: Select a package in provides (version?)
 195                         req_info['id'] = provides2id[req_name][0]['id']
 196                     elif req_name in pkg2id:
 197                         req_info['id'] = pkg2id[req_name]
 198                     elif req_name in file2id:
 199                         req_info['id'] = file2id[req_name][0]['id']
 200                     else:
 201                         #TODO: Exception Check
 202                         # the dependent package does not exist.
 203                         logger.warning('"%s" required by "%s" does not exist.', req_name, pkg_list[pkg_id].get('name'))
 204
 205     def _get_tagname(self, root):
 206         tags = {}
 207         # xmlns = re.sub('metadata$', '', root.tag)
 208         tags['metadata'] = '{%s}metadata' % root.nsmap[None]
 209         tags['package'] = '{%s}package' % root.nsmap[None]
 210         tags['name'] = '{%s}name' % root.nsmap[None]
 211         tags['arch'] = '{%s}arch' % root.nsmap[None]
 212         tags['version'] = '{%s}version' % root.nsmap[None]
 213         tags['checksum'] = '{%s}checksum' % root.nsmap[None]
 214         tags['summary'] = '{%s}summary' % root.nsmap[None]
 215         tags['description'] = '{%s}description' % root.nsmap[None]
 216         tags['location'] = '{%s}location' % root.nsmap[None]
 217         tags['size'] = '{%s}size' % root.nsmap[None]
 218         tags['format'] = '{%s}format' % root.nsmap[None]
 219         tags['requires'] = '{%s}requires' % root.nsmap['rpm']
 220         tags['provides'] = '{%s}provides' % root.nsmap['rpm']
 221         tags['conflicts'] = '{%s}conflicts' % root.nsmap['rpm']
 222         tags['suggests'] = '{%s}suggests' % root.nsmap['rpm']
 223         tags['recommends'] = '{%s}recommends' % root.nsmap['rpm']
 224         tags['file'] = '{%s}file' % root.nsmap[None]
 225         return tags
 226
 227     def _filtering_data_based_arch(self, xml_list, tag_dic):
 228         ret_list = []
 229         for xml_root in xml_list:
 230             pkg_data = {}
 231             for pkg_elm in xml_root.findall(tag_dic['package']):
 232                 pkg_arch = pkg_elm.findtext(tag_dic['arch'])
 233                 if pkg_arch not in archPolicies[self.arch] and pkg_arch not in default_arch:
 234                     continue;
 235                 if not pkg_data.get(pkg_arch):
 236                     pkg_data[pkg_arch] = []
 237                 pkg_data[pkg_arch].append(pkg_elm)
 238             ret_list.append(pkg_data)
 239         return ret_list
 240
 241     def parse(self):
 242         logger = logging.getLogger(__name__)
 243         if not self.repodata_list:
 244             return None
 245
 246         xml_list = []
 247         try:
 248             for repodata in self.repodata_list:
 249                 tree = etree.parse(repodata['primary'])
 250                 xml_list.append(tree.getroot())
 251         except etree.XMLSyntaxError as e:
 252             logger.info(e)
 253             raise TICError(configmgr.message['xml_parse_error'] % ('primary', repodata['baseurl']))
 254
 255         tag_dic = self._get_tagname(xml_list[0])
 256
 257         # TODO: temporary code (should be deleted)
 258         # meta_string='<?xml version="1.0" encoding="UTF-8"?><metadata xmlns="http://linux.duke.edu/metadata/common" xmlns:rpm="http://linux.duke.edu/metadata/rpm" packages="1467"><package type="rpm"><name>building-blocks-root-chooseonlyone_Kernel</name><arch>armv7l</arch><version epoch="0" ver="2.0.4" rel="4.2" vcs="platform/upstream/SDL#5f9405ba696ad79a0e05150430fe69e874f8280d" /><checksum type="sha256" pkgid="YES">6f74666eb89e1addc4ce75e25f3e639bbfdd8798fc848a6b7027501070567dcf</checksum><summary>Linux Kernel</summary><description>Include Linux Kernel in the Platform Image</description><packager /><url>http://tizen.org</url><time file="1487645040" build="1487645034" /><size package="2088" installed="0" archive="124" /><location href="armv7l/building-blocks-root-Kernel-0.0.1-1.1.armv7l.rpm" /><format><rpm:license>Apache-2</rpm:license><rpm:vendor>tizen</rpm:vendor><rpm:group>Meta</rpm:group><rpm:buildhost>w08</rpm:buildhost><rpm:sourcerpm>building-blocks-0.0.1-1.1.src.rpm</rpm:sourcerpm><rpm:header-range start="280" end="1964" /><rpm:requires><rpm:entry name="linux-kernel" flags="GE" epoch="0" ver="3.10" /></rpm:requires></format></package></metadata>'
 259         # xml_list.append(etree.fromstring(meta_string))
 260
 261         filter_data = self._filtering_data_based_arch(xml_list, tag_dic)
 262
 263         pkg_group = dict(pkg_dict={},
 264                          provides={},
 265                          files={},
 266                          groups={},
 267                          conflicts={},
 268                          meta_info=dict(root=[], sub1=[], sub2=[]))
 269
 270         # parses the repodata (primary.xml)
 271         # for xml_root in xml_list:
 272         #    self._xml_parse(pkg_group, xml_root, tag_dic)
 273         for xml_data in filter_data:
 274             for arch in archPolicies[self.arch]:
 275                 if arch in xml_data:
 276                     self._xml_parse(pkg_group, xml_data[arch], tag_dic)
 277             # noarch, src
 278             for arch in default_arch:
 279                 if arch in xml_data:
 280                     self._xml_parse(pkg_group, xml_data[arch], tag_dic)
 281
 282         # set the requires id
 283         #self._prepare_requires_id(pkg_group)
 284
 285         return pkg_group