tic/parser/recipe_parser.py

   1 #!/usr/bin/python
   2 # Copyright (c) 2016 Samsung Electronics Co., Ltd
   3 #
   4 # Licensed under the Flora License, Version 1.1 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at
   7 #
   8 #     http://floralicense.org/license/
   9 #
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15 #
  16 # Contributors:
  17 # - S-Core Co., Ltd
  18
  19 import copy
  20 import os
  21 import yaml
  22 import urllib2
  23 import contextlib
  24 import collections
  25 import logging
  26 from datetime import datetime
  27 from tic.utils.error import TICError
  28 from tic.utils.file import write, make_dirs
  29 from tic.config import configmgr
  30
  31 DUMMY_PLATFORM = 'DummyPlatform'
  32 DEFAULT_RECIPE_NAME = 'default_recipe'
  33 DEFAULT_RECIPE_PATH = configmgr.setting['default_recipe']
  34 RECIPE_EXTEND_FIELD = {'Repos', 'Groups', 'Repositories', 'Partitions', 'ExtraPackages', 'RemovePackages', 'PostScripts', 'NoChrootScripts'}
  35
  36 class DefaultRecipe(object):
  37     DEFAULT_RECIPE = {'NoChrootScripts': [{'Contents': 'if [ -n "$IMG_NAME" ]; then\n    echo "BUILD_ID=$IMG_NAME" >> $INSTALL_ROOT/etc/tizen-release\n    echo "BUILD_ID=$IMG_NAME" >> $INSTALL_ROOT/etc/os-release\nfi\n',
  38                                            'Name': 'buildname'}],
  39           'Partitions': [{'Contents': 'part / --size=2000 --ondisk mmcblk0p --fstype=ext4 --label=rootfs --extoptions="-J size=16"\npart /opt/ --size=1000 --ondisk mmcblk0p --fstype=ext4 --label=system-data --extoptions="-m 0"\npart /boot/kernel/mod_tizen_tm1/lib/modules --size=12 --ondisk mmcblk0p --fstype=ext4 --label=modules\n',
  40                           'Name': 'default-part'}],
  41           'PostScripts': [{'Contents': '#!/bin/sh\necho "#################### generic-base.post ####################"\n\ntest ! -e /opt/var && mkdir -p /opt/var\ntest -d /var && cp -arf /var/* /opt/var/\nrm -rf /var\nln -snf opt/var /var\n\ntest ! -e /opt/usr/home && mkdir -p /opt/usr/home\ntest -d /home && cp -arf /home/* /opt/usr/home/\nrm -rf /home\nln -snf opt/usr/home /home\n\nbuild_ts=$(date -u +%s)\nbuild_date=$(date -u --date @$build_ts +%Y%m%d_%H%M%S)\nbuild_time=$(date -u --date @$build_ts +%H:%M:%S)\n\nsed -ri \\\n\t-e \'s|@BUILD_ID[@]|@BUILD_ID@|g\' \\\n\t-e "s|@BUILD_DATE[@]|$build_date|g" \\\n\t-e "s|@BUILD_TIME[@]|$build_time|g" \\\n\t-e "s|@BUILD_TS[@]|$build_ts|g" \\\n\t/etc/tizen-build.conf\n\n# setup systemd default target for user session\ncat <<\'EOF\' >>/usr/lib/systemd/user/default.target\n[Unit]\nDescription=User session default target\nEOF\nmkdir -p /usr/lib/systemd/user/default.target.wants\n\n# sdx: fix smack labels on /var/log\nchsmack -a \'*\' /var/log\n\n# create appfw dirs inside homes\nfunction generic_base_user_exists() {\n        user=$1\n        getent passwd | grep -q ^${user}:\n}\n\nfunction generic_base_user_home() {\n        user=$1\n        getent passwd | grep ^${user}: | cut -f6 -d\':\'\n}\n\nfunction generic_base_fix_user_homedir() {\n        user=$1\n        generic_base_user_exists $user || return 1\n\nhomedir=$(generic_base_user_home $user)\n        mkdir -p $homedir/apps_rw\n        for appdir in desktop manifest dbspace; do\n                mkdir -p $homedir/.applications/$appdir\n        done\n        find $homedir -type d -exec chsmack -a User {} \\;\n        chown -R $user:users $homedir\n        return 0\n}\n\n# fix TC-320 for SDK\n. /etc/tizen-build.conf\n[ "${TZ_BUILD_WITH_EMULATOR}" == "1" ] && generic_base_fix_user_homedir developer\n\n# Add info.ini for system-info CAPI (TC-2047)\n/etc/make_info_file.sh',
  42                            'Name': 'generic-base'}],
  43           'Recipe': {'Active': True,
  44                      'Architecture': 'armv7l',
  45                      'Baseline': 'tizen',
  46                      'BootLoader': True,
  47                      'BootloaderAppend': 'rw vga=current splash rootwait rootfstype=ext4 plymouth.enable=0',
  48                      'BootloaderOptions': '--ptable=gpt --menus="install:Wipe and Install:systemd.unit=system-installer.service:test"',
  49                      'BootloaderTimeout': 3,
  50                      'DefaultUser': 'guest',
  51                      'DefaultUserPass': 'tizen',
  52                      'Desktop': 'None',
  53                      'ExtraPackages': [],
  54                      'FileName': 'default-armv7l',
  55                      'Groups': [],
  56                      'Keyboard': 'us',
  57                      'Language': 'en_US.UTF-8',
  58                      'Mic2Options': '-f raw --fstab=uuid --copy-kernel --compress-disk-image=bz2 --generate-bmap',
  59                      'Name': 'default-recipe',
  60                      'NoChrootScripts': ['buildname'],
  61                      'Part': 'default-part',
  62                      'PostScripts': ['generic-base'],
  63                      'RemovePackages': [],
  64                      'Repos': ['tizen_unified', 'tizen_base_armv7l'],
  65                      'RootPass': 'tizen',
  66                      'SaveRepos': False,
  67                      'Schedule': '*',
  68                      'StartX': False,
  69                      'Timezone': 'Asia/Seoul',
  70                      'UserGroups': 'audio,video'},
  71           'Repositories': [{'Name': 'tizen_unified',
  72                             'Options': '--ssl_verify=no',
  73                             'Url': 'http://download.tizen.org/snapshots/tizen/unified/latest/repos/standard/packages/'},
  74                            {'Name': 'tizen_base_armv7l',
  75                             'Options': '--ssl_verify=no',
  76                             'Url': 'http://download.tizen.org/snapshots/tizen/base/latest/repos/arm/packages/'}]}
  77     _instance = None
  78     def __new__(cls, *args, **kwargs):
  79         if not cls._instance:
  80             cls._instance = super(DefaultRecipe, cls).__new__(cls, *args, **kwargs)
  81         return cls._instance
  82     def __init__(self):
  83         logger = logging.getLogger(__name__)
  84         if os.path.exists(DEFAULT_RECIPE_PATH):
  85             try:
  86                 with file(DEFAULT_RECIPE_PATH) as f:
  87                     self.DEFAULT_RECIPE = yaml.load(f)
  88                     logger.info('Read default recipe from %s' % DEFAULT_RECIPE_PATH)
  89             except IOError as err:
  90                 logger.info(err)
  91             except yaml.YAMLError as err:
  92                 logger.info(err)
  93     def getDefaultRecipe(self):
  94         return copy.deepcopy(self.DEFAULT_RECIPE)
  95     def getSystemConfig(self):
  96         data = copy.deepcopy(self.DEFAULT_RECIPE)
  97         for field in RECIPE_EXTEND_FIELD:
  98             if field == 'Partitions':
  99                 continue
 100             if data['Recipe'].get(field):
 101                 data['Recipe'][field] = []
 102             if data.get(field):
 103                 data[field] = []
 104         return data
 105     def getDefaultParameter(self):
 106         return [dict(url=DEFAULT_RECIPE_NAME, type='recipe')]
 107
 108 default_recipe = DefaultRecipe()
 109
 110 class RecipeParser(object):
 111     def __init__(self, inputs):
 112         # in order to priority definition
 113         self.inputs = []
 114         self.recipes = {}
 115         self._repositories = None
 116         self._recipe = None
 117         # add recipe input
 118         self.addRecipes(inputs)
 119
 120     def parse(self):
 121         logger = logging.getLogger(__name__)
 122         if not self.inputs:
 123             return
 124         self._repositories = None
 125         self._recipe = None
 126         repo_count = 1
 127         try:
 128             for data in self.inputs:
 129                 data_type = data.get('type')
 130                 # type: recipe or repository
 131                 if data_type == 'recipe':
 132                     # default recipe
 133                     if data.get('url') == DEFAULT_RECIPE_NAME:
 134                         self.recipes[data.get('url')] = default_recipe.getDefaultRecipe()
 135                     else:
 136                         with contextlib.closing(urllib2.urlopen(data.get('url'))) as op:
 137                             self.recipes[data.get('url')] = yaml.load(op.read())
 138                 elif data_type == 'repository':
 139                     data['name'] = 'repository_%s' % repo_count
 140                     repo_count += 1
 141         except urllib2.HTTPError as err:
 142             if err.code == 404:
 143                 msg = configmgr.message['recipe_not_found'] % data.get('url')
 144             else:
 145                 msg = str(err)
 146             logger.error(err)
 147             raise TICError(msg)
 148         except urllib2.URLError as err:
 149             logger.error(err)
 150             raise TICError(configmgr.message['server_error'])
 151         except yaml.YAMLError as err:
 152             logger.error(err)
 153             raise TICError(configmgr.message['recipe_parse_error'] % data.get('url'))
 154
 155     def addRecipes(self, inputs):
 156         if inputs:
 157             if isinstance(inputs, list):
 158                 for data in inputs:
 159                     self.inputs.append(data)
 160             else:
 161                 self.inputs.append(inputs)
 162
 163     def getRepositories(self):
 164         if not self._repositories:
 165             self._repositories = self._getAllRepositories()
 166         return self._repositories
 167
 168     def _getAllRepositories(self):
 169         repos = []
 170         name_count = 1
 171         for data in self.inputs:
 172             if data.get('type') == 'recipe':
 173                 recipe_repos = []
 174                 recipe_info = self.recipes[data['url']]
 175                 recipe_name = None
 176                 if recipe_info.get('Recipe'):
 177                     if recipe_info['Recipe'].get('Name'):
 178                         recipe_name = recipe_info['Recipe'].get('Name')
 179                     if recipe_info['Recipe'].get('Repos'):
 180                         for repo_name in recipe_info['Recipe'].get('Repos'):
 181                             isExist = False
 182                             if recipe_info.get('Repositories'):
 183                                 for repo_info in recipe_info.get('Repositories'):
 184                                     if repo_info.get('Name') == repo_name:
 185                                         recipe_repos.append(dict(name=repo_name,
 186                                                                  url=repo_info.get('Url'),
 187                                                                  options=repo_info.get('Options')))
 188                                         isExist = True
 189                                         break
 190                             # repository does not exist
 191                             if not isExist:
 192                                 raise TICError(configmgr.message['recipe_repo_not_exist'] % repo_name)
 193                 if not recipe_name:
 194                     recipe_name = 'recipe_%s' % name_count
 195                     name_count += 1
 196                 repos.append(dict(name=recipe_name,
 197                                   url=data['url'],
 198                                   repos=recipe_repos,
 199                                   type='recipe'))
 200             else:
 201                 repos.append(data)
 202         return repos
 203
 204     def _renameRepository(self, repo_dict, repo_name):
 205         number = repo_dict.get(repo_name)
 206         new_name = ''.join([repo_name, '_', str(number)])
 207         while(new_name in repo_dict):
 208             number += 1
 209             new_name = ''.join([repo_name, '_', str(number)])
 210         repo_dict[repo_name] = number + 1
 211         return new_name
 212
 213     def getMergedRepositories(self):
 214         result = []
 215         repositories = self.getRepositories()
 216         repo_name = {} # 'name': count
 217         repo_url = {} # 'url': exist
 218         for target in repositories:
 219             if target.get('type') == 'recipe':
 220                 if target.get('repos'):
 221                     for repo in target.get('repos'):
 222                         # if repo's url is duplicated, remove it.
 223                         if repo.get('url') in repo_url:
 224                             continue
 225                         # if repo's name is duplicated, rename it (postfix '_count')
 226                         if repo.get('name') in repo_name:
 227                             repo['name'] = self._renameRepository(repo_name, repo['name'])
 228                         else:
 229                             repo_name[repo['name']] = 1
 230                         repo_url[repo['url']] = 1
 231                         result.append(repo)
 232                 else:
 233                     # recipe does not have repository information
 234                     pass
 235             elif(target.get('type') == 'repository'):
 236                 # if repo's url is duplicated, remove it.
 237                 if target.get('url') in repo_url:
 238                     continue
 239                 if target['name'] in repo_name:
 240                     target['name'] = self._renameRepository(repo_name, target['name'])
 241                 else:
 242                     repo_name[target['name']] = 1
 243                 repo_url[target['url']] = 1
 244                 result.append(target)
 245         return result
 246
 247     def getMergedRecipe(self):
 248         if self._recipe:
 249             return self._recipe
 250
 251         mergedInfo = default_recipe.getSystemConfig()
 252         # merge recipe info
 253         for i in xrange(len(self.inputs), 0, -1):
 254             if self.inputs[i-1].get('type') == 'recipe':
 255                 recipe = self.recipes[self.inputs[i-1].get('url')]
 256                 if recipe.get('Recipe'):
 257                     for k, v in recipe.get('Recipe').iteritems():
 258                         if not v:
 259                             continue
 260                         if k in RECIPE_EXTEND_FIELD:
 261                             if k == 'Repos':
 262                                 continue
 263                             for j in xrange(len(v), 0, -1):
 264                                 mergedInfo['Recipe'][k].append(v[j-1])
 265                         else:
 266                             mergedInfo['Recipe'][k] = v
 267                 for fieldName in RECIPE_EXTEND_FIELD:
 268                     if recipe.get(fieldName):
 269                         if fieldName == 'Repositories':
 270                             continue
 271                         for data in recipe.get(fieldName):
 272                             mergedInfo[fieldName].append(data)
 273         # reverse order
 274         for extName in RECIPE_EXTEND_FIELD:
 275             if mergedInfo['Recipe'].get(extName):
 276                 mergedInfo['Recipe'][extName].reverse()
 277             if mergedInfo.get(extName):
 278                 mergedInfo[extName].reverse()
 279
 280         # set repositories
 281         mergedInfo['Repositories'] = self.getMergedRepositories()
 282         if mergedInfo.get('Repositories'):
 283             for repo in mergedInfo['Repositories']:
 284                 mergedInfo['Recipe']['Repos'].append(repo['name'])
 285         return mergedInfo
 286
 287     def export2Recipe(self, packages, outdir, filename='recipe.yaml'):
 288         logger = logging.getLogger(__name__)
 289         recipe = self.getMergedRecipe()
 290         make_dirs(outdir)
 291         reciep_path = os.path.join(outdir, filename)
 292         # set packages
 293         if packages:
 294             recipe['Recipe']['ExtraPackages'] = packages
 295         # set repositories
 296         if 'Repositories' in recipe:
 297             repos = []
 298             for repo in recipe.get('Repositories'):
 299                 repos.append(dict(Name= repo.get('name'),
 300                                   Url= repo.get('url'),
 301                                   Options = repo.get('options')))
 302             recipe['Repositories'] = repos
 303
 304         try:
 305             with open(reciep_path, 'w') as outfile:
 306                 yaml.safe_dump(recipe, outfile, line_break="\n", width=1000, default_flow_style=False)
 307                 #outfile.write(stream.replace('\n', '\n\n'))
 308                 if not os.path.exists(reciep_path):
 309                     raise TICError('No recipe file was created')
 310         except IOError as err:
 311             logger.info(err)
 312             raise TICError('Could not read the recipe files')
 313         except yaml.YAMLError as err:
 314             logger.info(err)
 315             raise TICError(configmgr.message['recipe_convert_error'])
 316         return reciep_path
 317
 318     def export2Yaml(self, packages, filepath):
 319         logger = logging.getLogger(__name__)
 320         recipe = self.getMergedRecipe()
 321         # config.yaml
 322         config = dict(Default=None, Configurations=[])
 323         config['Default'] = recipe.get('Recipe')
 324         if packages:
 325             config['Default']['ExtraPackages'] = packages
 326         # targets (only one target)
 327         extraconfs = dict(Platform=DUMMY_PLATFORM,
 328                           ExtraPackages=[],
 329                           Name= recipe['Recipe'].get('Name'),
 330                           FileName= recipe['Recipe'].get('FileName'),
 331                           Part= recipe['Recipe'].get('Part'))
 332         config['Configurations'].append(extraconfs)
 333         config[DUMMY_PLATFORM] = dict(ExtraPackages=[])
 334
 335         dir_path = os.path.join(filepath, datetime.now().strftime('%Y%m%d%H%M%S%f'))
 336         make_dirs(dir_path)
 337         logger.info('kickstart cache dir=%s' % dir_path)
 338
 339         yamlinfo = YamlInfo(dir_path,
 340                             os.path.join(dir_path, 'configs.yaml'),
 341                             os.path.join(dir_path, 'repos.yaml'))
 342
 343         # configs.yaml
 344         with open(yamlinfo.configs, 'w') as outfile:
 345             yaml.safe_dump(config, outfile, default_flow_style=False)
 346
 347         # repo.yaml
 348         if 'Repositories' in recipe:
 349             repos = dict(Repositories= [])
 350             for repo in recipe.get('Repositories'):
 351                 repos['Repositories'].append(dict(Name= repo.get('name'),
 352                                                   Url= repo.get('url'),
 353                                                   Options = repo.get('options')))
 354             with open(yamlinfo.repos, 'w') as outfile:
 355                 yaml.safe_dump(repos, outfile, default_flow_style=False)
 356
 357         # partition info
 358         if 'Partitions' in recipe:
 359             for partition in recipe.get('Partitions'):
 360                 partition_path = os.path.join(dir_path, 'partitions')
 361                 file_name = partition.get('Name')
 362                 temp = os.path.join(partition_path, file_name)
 363                 write(temp, partition['Contents'])
 364
 365         # script.post
 366         if 'PostScripts' in recipe:
 367             for script in recipe.get('PostScripts'):
 368                 script_path = os.path.join(dir_path, 'scripts')
 369                 file_name = '%s.post' % script.get('Name')
 370                 write(os.path.join(script_path, file_name), script['Contents'])
 371         if 'NoChrootScripts' in recipe:
 372             for script in recipe.get('NoChrootScripts'):
 373                 script_path = os.path.join(dir_path, 'scripts')
 374                 file_name = '%s.nochroot' % script.get('Name')
 375                 write(os.path.join(script_path, file_name), script['Contents'])
 376         return yamlinfo
 377
 378 def load_yaml(path):
 379     logger = logging.getLogger(__name__)
 380     try:
 381         with file(path) as f:
 382             return yaml.load(f)
 383     except IOError as err:
 384         logger.info(err)
 385         raise TICError(configmgr.message['server_error'])
 386     except yaml.YAMLError as err:
 387         logger.info(err)
 388         raise TICError(configmgr.message['recipe_parse_error'] % os.path.basename(path))
 389
 390 YamlType = collections.namedtuple('YamlInfo', 'cachedir, configs, repos')
 391 def YamlInfo(cachedir, configs, repos):
 392     return YamlType(cachedir, configs, repos)
 393
 394 if __name__ == '__main__':
 395     inputs = [{'url': DEFAULT_RECIPE_NAME, 'type': 'recipe'}, {'url': 'http://localhost/repo/recipe/recipe1.yaml', 'type': 'recipe'}]
 396     parser = RecipeParser()
 397     parser.addRecipes(inputs)
 398     parser.parse()
 399     print(parser.repositories)