tic/parser/recipe_parser.py

   1 #!/usr/bin/python
   2 # Copyright (c) 2000 - 2016 Samsung Electronics Co., Ltd. All rights reserved.
   3 #
   4 # Contact:
   5 # @author Chulwoo Shin <cw1.shin@samsung.com>
   6 #
   7 # Licensed under the Apache License, Version 2.0 (the "License");
   8 # you may not use this file except in compliance with the License.
   9 # You may obtain a copy of the License at
  10 #
  11 # http://www.apache.org/licenses/LICENSE-2.0
  12 #
  13 # Unless required by applicable law or agreed to in writing, software
  14 # distributed under the License is distributed on an "AS IS" BASIS,
  15 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16 # See the License for the specific language governing permissions and
  17 # limitations under the License.
  18 #
  19 # Contributors:
  20 # - S-Core Co., Ltd
  21
  22 import copy
  23 import os
  24 import yaml
  25 import urllib2
  26 import contextlib
  27 import collections
  28 import logging
  29 from datetime import datetime
  30 from tic.utils.error import TICError
  31 from tic.utils.file import write, make_dirs
  32 from tic.config import configmgr
  33
  34 DUMMY_PLATFORM = 'DummyPlatform'
  35 DEFAULT_RECIPE_NAME = 'default_recipe'
  36 DEFAULT_RECIPE_PATH = configmgr.setting['default_recipe']
  37 RECIPE_EXTEND_FIELD = {'Repos', 'Groups', 'Repositories', 'Partitions', 'ExtraPackages', 'RemovePackages', 'PostScripts', 'NoChrootScripts'}
  38
  39 class DefaultRecipe(object):
  40     DEFAULT_RECIPE = {'NoChrootScripts': [{'Contents': 'if [ -n "$IMG_NAME" ]; then\n    echo "BUILD_ID=$IMG_NAME" >> $INSTALL_ROOT/etc/tizen-release\n    echo "BUILD_ID=$IMG_NAME" >> $INSTALL_ROOT/etc/os-release\nfi\n',
  41                                            'Name': 'buildname'}],
  42           'Partitions': [{'Contents': 'part / --size=2000 --ondisk mmcblk0p --fstype=ext4 --label=rootfs --extoptions="-J size=16"\npart /opt/ --size=1000 --ondisk mmcblk0p --fstype=ext4 --label=system-data --extoptions="-m 0"\npart /boot/kernel/mod_tizen_tm1/lib/modules --size=12 --ondisk mmcblk0p --fstype=ext4 --label=modules\n',
  43                           'Name': 'default-part'}],
  44           'PostScripts': [{'Contents': '#!/bin/sh\necho "#################### generic-base.post ####################"\n\ntest ! -e /opt/var && mkdir -p /opt/var\ntest -d /var && cp -arf /var/* /opt/var/\nrm -rf /var\nln -snf opt/var /var\n\ntest ! -e /opt/usr/home && mkdir -p /opt/usr/home\ntest -d /home && cp -arf /home/* /opt/usr/home/\nrm -rf /home\nln -snf opt/usr/home /home\n\nbuild_ts=$(date -u +%s)\nbuild_date=$(date -u --date @$build_ts +%Y%m%d_%H%M%S)\nbuild_time=$(date -u --date @$build_ts +%H:%M:%S)\n\nsed -ri \\\n\t-e \'s|@BUILD_ID[@]|@BUILD_ID@|g\' \\\n\t-e "s|@BUILD_DATE[@]|$build_date|g" \\\n\t-e "s|@BUILD_TIME[@]|$build_time|g" \\\n\t-e "s|@BUILD_TS[@]|$build_ts|g" \\\n\t/etc/tizen-build.conf\n\n# setup systemd default target for user session\ncat <<\'EOF\' >>/usr/lib/systemd/user/default.target\n[Unit]\nDescription=User session default target\nEOF\nmkdir -p /usr/lib/systemd/user/default.target.wants\n\n# sdx: fix smack labels on /var/log\nchsmack -a \'*\' /var/log\n\n# create appfw dirs inside homes\nfunction generic_base_user_exists() {\n        user=$1\n        getent passwd | grep -q ^${user}:\n}\n\nfunction generic_base_user_home() {\n        user=$1\n        getent passwd | grep ^${user}: | cut -f6 -d\':\'\n}\n\nfunction generic_base_fix_user_homedir() {\n        user=$1\n        generic_base_user_exists $user || return 1\n\nhomedir=$(generic_base_user_home $user)\n        mkdir -p $homedir/apps_rw\n        for appdir in desktop manifest dbspace; do\n                mkdir -p $homedir/.applications/$appdir\n        done\n        find $homedir -type d -exec chsmack -a User {} \\;\n        chown -R $user:users $homedir\n        return 0\n}\n\n# fix TC-320 for SDK\n. /etc/tizen-build.conf\n[ "${TZ_BUILD_WITH_EMULATOR}" == "1" ] && generic_base_fix_user_homedir developer\n\n# Add info.ini for system-info CAPI (TC-2047)\n/etc/make_info_file.sh',
  45                            'Name': 'generic-base'}],
  46           'Recipe': {'Active': True,
  47                      'Architecture': 'armv7l',
  48                      'Baseline': 'tizen',
  49                      'BootLoader': True,
  50                      'BootloaderAppend': 'rw vga=current splash rootwait rootfstype=ext4 plymouth.enable=0',
  51                      'BootloaderOptions': '--ptable=gpt --menus="install:Wipe and Install:systemd.unit=system-installer.service:test"',
  52                      'BootloaderTimeout': 3,
  53                      'DefaultUser': 'guest',
  54                      'DefaultUserPass': 'tizen',
  55                      'Desktop': 'None',
  56                      'ExtraPackages': [],
  57                      'FileName': 'default-armv7l',
  58                      'Groups': [],
  59                      'Keyboard': 'us',
  60                      'Language': 'en_US.UTF-8',
  61                      'Mic2Options': '-f raw --fstab=uuid --copy-kernel --compress-disk-image=bz2 --generate-bmap',
  62                      'Name': 'default-recipe',
  63                      'NoChrootScripts': ['buildname'],
  64                      'Part': 'default-part',
  65                      'PostScripts': ['generic-base'],
  66                      'RemovePackages': [],
  67                      'Repos': ['tizen_unified', 'tizen_base_armv7l'],
  68                      'RootPass': 'tizen',
  69                      'SaveRepos': False,
  70                      'Schedule': '*',
  71                      'StartX': False,
  72                      'Timezone': 'Asia/Seoul',
  73                      'UserGroups': 'audio,video'},
  74           'Repositories': [{'Name': 'tizen_unified',
  75                             'Options': '--ssl_verify=no',
  76                             'Url': 'http://download.tizen.org/snapshots/tizen/unified/latest/repos/standard/packages/'},
  77                            {'Name': 'tizen_base_armv7l',
  78                             'Options': '--ssl_verify=no',
  79                             'Url': 'http://download.tizen.org/snapshots/tizen/base/latest/repos/arm/packages/'}]}
  80     _instance = None
  81     def __new__(cls, *args, **kwargs):
  82         if not cls._instance:
  83             cls._instance = super(DefaultRecipe, cls).__new__(cls, *args, **kwargs)
  84         return cls._instance
  85     def __init__(self):
  86         logger = logging.getLogger(__name__)
  87         if os.path.exists(DEFAULT_RECIPE_PATH):
  88             try:
  89                 with file(DEFAULT_RECIPE_PATH) as f:
  90                     self.DEFAULT_RECIPE = yaml.load(f)
  91                     logger.info('Read default recipe from %s' % DEFAULT_RECIPE_PATH)
  92             except IOError as err:
  93                 logger.info(err)
  94             except yaml.YAMLError as err:
  95                 logger.info(err)
  96     def getDefaultRecipe(self):
  97         return copy.deepcopy(self.DEFAULT_RECIPE)
  98     def getSystemConfig(self):
  99         data = copy.deepcopy(self.DEFAULT_RECIPE)
 100         for field in RECIPE_EXTEND_FIELD:
 101             if field == 'Partitions':
 102                 continue
 103             if data['Recipe'].get(field):
 104                 data['Recipe'][field] = []
 105             if data.get(field):
 106                 data[field] = []
 107         return data
 108     def getDefaultParameter(self):
 109         return [dict(url=DEFAULT_RECIPE_NAME, type='recipe')]
 110
 111 default_recipe = DefaultRecipe()
 112
 113 class RecipeParser(object):
 114     def __init__(self, inputs):
 115         # in order to priority definition
 116         self.inputs = []
 117         self.recipes = {}
 118         self._repositories = None
 119         self._recipe = None
 120         # add recipe input
 121         self.addRecipes(inputs)
 122
 123     def parse(self):
 124         logger = logging.getLogger(__name__)
 125         if not self.inputs:
 126             return
 127         self._repositories = None
 128         self._recipe = None
 129         repo_count = 1
 130         try:
 131             for data in self.inputs:
 132                 data_type = data.get('type')
 133                 # type: recipe or repository
 134                 if data_type == 'recipe':
 135                     # default recipe
 136                     if data.get('url') == DEFAULT_RECIPE_NAME:
 137                         self.recipes[data.get('url')] = default_recipe.getDefaultRecipe()
 138                     else:
 139                         with contextlib.closing(urllib2.urlopen(data.get('url'))) as op:
 140                             self.recipes[data.get('url')] = yaml.load(op.read())
 141                 elif data_type == 'repository':
 142                     data['name'] = 'repository_%s' % repo_count
 143                     repo_count += 1
 144         except urllib2.HTTPError as err:
 145             if err.code == 404:
 146                 msg = configmgr.message['recipe_not_found'] % data.get('url')
 147             else:
 148                 msg = str(err)
 149             logger.error(err)
 150             raise TICError(msg)
 151         except urllib2.URLError as err:
 152             logger.error(err)
 153             raise TICError(configmgr.message['server_error'])
 154         except yaml.YAMLError as err:
 155             logger.error(err)
 156             raise TICError(configmgr.message['recipe_parse_error'] % data.get('url'))
 157
 158     def addRecipes(self, inputs):
 159         if inputs:
 160             if isinstance(inputs, list):
 161                 for data in inputs:
 162                     self.inputs.append(data)
 163             else:
 164                 self.inputs.append(inputs)
 165
 166     def getRepositories(self):
 167         if not self._repositories:
 168             self._repositories = self._getAllRepositories()
 169         return self._repositories
 170
 171     def _getAllRepositories(self):
 172         repos = []
 173         name_count = 1
 174         for data in self.inputs:
 175             if data.get('type') == 'recipe':
 176                 recipe_repos = []
 177                 recipe_info = self.recipes[data['url']]
 178                 recipe_name = None
 179                 if recipe_info.get('Recipe'):
 180                     if recipe_info['Recipe'].get('Name'):
 181                         recipe_name = recipe_info['Recipe'].get('Name')
 182                     if recipe_info['Recipe'].get('Repos'):
 183                         for repo_name in recipe_info['Recipe'].get('Repos'):
 184                             isExist = False
 185                             if recipe_info.get('Repositories'):
 186                                 for repo_info in recipe_info.get('Repositories'):
 187                                     if repo_info.get('Name') == repo_name:
 188                                         recipe_repos.append(dict(name=repo_name,
 189                                                                  url=repo_info.get('Url'),
 190                                                                  options=repo_info.get('Options')))
 191                                         isExist = True
 192                                         break
 193                             # repository does not exist
 194                             if not isExist:
 195                                 raise TICError(configmgr.message['recipe_repo_not_exist'] % repo_name)
 196                 if not recipe_name:
 197                     recipe_name = 'recipe_%s' % name_count
 198                     name_count += 1
 199                 repos.append(dict(name=recipe_name,
 200                                   url=data['url'],
 201                                   repos=recipe_repos,
 202                                   type='recipe'))
 203             else:
 204                 repos.append(data)
 205         return repos
 206
 207     def _renameRepository(self, repo_dict, repo_name):
 208         number = repo_dict.get(repo_name)
 209         new_name = ''.join([repo_name, '_', str(number)])
 210         while(new_name in repo_dict):
 211             number += 1
 212             new_name = ''.join([repo_name, '_', str(number)])
 213         repo_dict[repo_name] = number + 1
 214         return new_name
 215
 216     def getMergedRepositories(self):
 217         result = []
 218         repositories = self.getRepositories()
 219         repo_name = {} # 'name': count
 220         repo_url = {} # 'url': exist
 221         for target in repositories:
 222             if target.get('type') == 'recipe':
 223                 if target.get('repos'):
 224                     for repo in target.get('repos'):
 225                         # if repo's url is duplicated, remove it.
 226                         if repo.get('url') in repo_url:
 227                             continue
 228                         # if repo's name is duplicated, rename it (postfix '_count')
 229                         if repo.get('name') in repo_name:
 230                             repo['name'] = self._renameRepository(repo_name, repo['name'])
 231                         else:
 232                             repo_name[repo['name']] = 1
 233                         repo_url[repo['url']] = 1
 234                         result.append(repo)
 235                 else:
 236                     # recipe does not have repository information
 237                     pass
 238             elif(target.get('type') == 'repository'):
 239                 # if repo's url is duplicated, remove it.
 240                 if target.get('url') in repo_url:
 241                     continue
 242                 if target['name'] in repo_name:
 243                     target['name'] = self._renameRepository(repo_name, target['name'])
 244                 else:
 245                     repo_name[target['name']] = 1
 246                 repo_url[target['url']] = 1
 247                 result.append(target)
 248         return result
 249
 250     def getMergedRecipe(self):
 251         if self._recipe:
 252             return self._recipe
 253
 254         mergedInfo = default_recipe.getSystemConfig()
 255         # merge recipe info
 256         for i in xrange(len(self.inputs), 0, -1):
 257             if self.inputs[i-1].get('type') == 'recipe':
 258                 recipe = self.recipes[self.inputs[i-1].get('url')]
 259                 if recipe.get('Recipe'):
 260                     for k, v in recipe.get('Recipe').iteritems():
 261                         if not v:
 262                             continue
 263                         if k in RECIPE_EXTEND_FIELD:
 264                             if k == 'Repos':
 265                                 continue
 266                             for j in xrange(len(v), 0, -1):
 267                                 mergedInfo['Recipe'][k].append(v[j-1])
 268                         else:
 269                             mergedInfo['Recipe'][k] = v
 270                 for fieldName in RECIPE_EXTEND_FIELD:
 271                     if recipe.get(fieldName):
 272                         if fieldName == 'Repositories':
 273                             continue
 274                         for data in recipe.get(fieldName):
 275                             mergedInfo[fieldName].append(data)
 276         # reverse order
 277         for extName in RECIPE_EXTEND_FIELD:
 278             if mergedInfo['Recipe'].get(extName):
 279                 mergedInfo['Recipe'][extName].reverse()
 280             if mergedInfo.get(extName):
 281                 mergedInfo[extName].reverse()
 282
 283         # set repositories
 284         mergedInfo['Repositories'] = self.getMergedRepositories()
 285         if mergedInfo.get('Repositories'):
 286             for repo in mergedInfo['Repositories']:
 287                 mergedInfo['Recipe']['Repos'].append(repo['name'])
 288         return mergedInfo
 289
 290     def export2Recipe(self, packages, outdir, filename='recipe.yaml'):
 291         logger = logging.getLogger(__name__)
 292         recipe = self.getMergedRecipe()
 293         make_dirs(outdir)
 294         reciep_path = os.path.join(outdir, filename)
 295         # set packages
 296         if packages:
 297             recipe['Recipe']['ExtraPackages'] = packages
 298         # set repositories
 299         if 'Repositories' in recipe:
 300             repos = []
 301             for repo in recipe.get('Repositories'):
 302                 repos.append(dict(Name= repo.get('name'),
 303                                   Url= repo.get('url'),
 304                                   Options = repo.get('options')))
 305             recipe['Repositories'] = repos
 306
 307         try:
 308             with open(reciep_path, 'w') as outfile:
 309                 yaml.safe_dump(recipe, outfile, line_break="\n", width=1000, default_flow_style=False)
 310                 #outfile.write(stream.replace('\n', '\n\n'))
 311                 if not os.path.exists(reciep_path):
 312                     raise TICError('No recipe file was created')
 313         except IOError as err:
 314             logger.info(err)
 315             raise TICError('Could not read the recipe files')
 316         except yaml.YAMLError as err:
 317             logger.info(err)
 318             raise TICError(configmgr.message['recipe_convert_error'])
 319         return reciep_path
 320
 321     def export2Yaml(self, packages, filepath):
 322         logger = logging.getLogger(__name__)
 323         recipe = self.getMergedRecipe()
 324         # config.yaml
 325         config = dict(Default=None, Configurations=[])
 326         config['Default'] = recipe.get('Recipe')
 327         if packages:
 328             config['Default']['ExtraPackages'] = packages
 329         # targets (only one target)
 330         extraconfs = dict(Platform=DUMMY_PLATFORM,
 331                           ExtraPackages=[],
 332                           Name= recipe['Recipe'].get('Name'),
 333                           FileName= recipe['Recipe'].get('FileName'),
 334                           Part= recipe['Recipe'].get('Part'))
 335         config['Configurations'].append(extraconfs)
 336         config[DUMMY_PLATFORM] = dict(ExtraPackages=[])
 337
 338         dir_path = os.path.join(filepath, datetime.now().strftime('%Y%m%d%H%M%S%f'))
 339         make_dirs(dir_path)
 340         logger.info('kickstart cache dir=%s' % dir_path)
 341
 342         yamlinfo = YamlInfo(dir_path,
 343                             os.path.join(dir_path, 'configs.yaml'),
 344                             os.path.join(dir_path, 'repos.yaml'))
 345
 346         # configs.yaml
 347         with open(yamlinfo.configs, 'w') as outfile:
 348             yaml.safe_dump(config, outfile, default_flow_style=False)
 349
 350         # repo.yaml
 351         if 'Repositories' in recipe:
 352             repos = dict(Repositories= [])
 353             for repo in recipe.get('Repositories'):
 354                 repos['Repositories'].append(dict(Name= repo.get('name'),
 355                                                   Url= repo.get('url'),
 356                                                   Options = repo.get('options')))
 357             with open(yamlinfo.repos, 'w') as outfile:
 358                 yaml.safe_dump(repos, outfile, default_flow_style=False)
 359
 360         # partition info
 361         if 'Partitions' in recipe:
 362             for partition in recipe.get('Partitions'):
 363                 partition_path = os.path.join(dir_path, 'partitions')
 364                 file_name = partition.get('Name')
 365                 temp = os.path.join(partition_path, file_name)
 366                 write(temp, partition['Contents'])
 367
 368         # script.post
 369         if 'PostScripts' in recipe:
 370             for script in recipe.get('PostScripts'):
 371                 script_path = os.path.join(dir_path, 'scripts')
 372                 file_name = '%s.post' % script.get('Name')
 373                 write(os.path.join(script_path, file_name), script['Contents'])
 374         if 'NoChrootScripts' in recipe:
 375             for script in recipe.get('NoChrootScripts'):
 376                 script_path = os.path.join(dir_path, 'scripts')
 377                 file_name = '%s.nochroot' % script.get('Name')
 378                 write(os.path.join(script_path, file_name), script['Contents'])
 379         return yamlinfo
 380
 381 def load_yaml(path):
 382     logger = logging.getLogger(__name__)
 383     try:
 384         with file(path) as f:
 385             return yaml.load(f)
 386     except IOError as err:
 387         logger.info(err)
 388         raise TICError(configmgr.message['server_error'])
 389     except yaml.YAMLError as err:
 390         logger.info(err)
 391         raise TICError(configmgr.message['recipe_parse_error'] % os.path.basename(path))
 392
 393 YamlType = collections.namedtuple('YamlInfo', 'cachedir, configs, repos')
 394 def YamlInfo(cachedir, configs, repos):
 395     return YamlType(cachedir, configs, repos)
 396
 397 if __name__ == '__main__':
 398     inputs = [{'url': DEFAULT_RECIPE_NAME, 'type': 'recipe'}, {'url': 'http://localhost/repo/recipe/recipe1.yaml', 'type': 'recipe'}]
 399     parser = RecipeParser()
 400     parser.addRecipes(inputs)
 401     parser.parse()
 402     print(parser.repositories)