From 95e35418419d443b87102155fa2822e6c63220a5 Mon Sep 17 00:00:00 2001 From: Chulwoo Shin Date: Tue, 13 Dec 2016 16:13:09 +0900 Subject: [PATCH] [TIC-CORE] provides default-recipe for 1st-release - provides default-recipe for creating image - provides the 1st draft recipe parser - modify response data of analysis Change-Id: I30af364856a732af2f19076e6b1072287a4498f2 Signed-off-by: Chulwoo Shin --- test/test_dependency.py | 14 ++-- test/test_repodata.py | 25 +++---- tic/command.py | 36 +++++++--- tic/dependency.py | 50 +++++++++++++- tic/parser/recipe_parser.py | 159 ++++++++++++++++++++++++++++++++++++++++++++ tic/repo.py | 20 +++--- tic/server/tic_server.py | 48 +++++++------ tic/utils/file.py | 5 +- tools/tic-core | 4 +- 9 files changed, 293 insertions(+), 68 deletions(-) create mode 100644 tic/parser/recipe_parser.py diff --git a/test/test_dependency.py b/test/test_dependency.py index bfcae8d..cf9fa0f 100644 --- a/test/test_dependency.py +++ b/test/test_dependency.py @@ -20,16 +20,13 @@ # - S-Core Co., Ltd import os -import base64 import time import unittest from tic.dependency import analyze_dependency from tic.parser.repo_parser import RepodataParser -from tic.repo import Repo from tic.repo import get_repodata_from_repos - current_milli_time = lambda: int(round(time.time() * 1000)) CWD = os.path.dirname(__file__) or '.' TEST_REPODATA_LOC=os.path.join(CWD, 'dependency_fixtures') @@ -42,13 +39,12 @@ def suite(): class DependencyTest(unittest.TestCase): def setUp(self): # test environment setup - self.repo_list = ['file:/' + TEST_REPODATA_LOC + '/base', - 'file:/' + TEST_REPODATA_LOC + '/mobile'] + self.repo_list = [{'name': 'local_base', + 'url': 'file:/' + TEST_REPODATA_LOC + '/base'}, + {'name': 'local_mobile', + 'url': 'file:/' + TEST_REPODATA_LOC + '/mobile'}] - repos = [] - for repo_url in self.repo_list: - repos.append(Repo(base64.urlsafe_b64encode(repo_url), repo_url)) - self.repodata_list = get_repodata_from_repos(repos, DEFAULT_CACHEDIR) + self.repodata_list = get_repodata_from_repos(self.repo_list, DEFAULT_CACHEDIR) self.pkg_group = None try: diff --git a/test/test_repodata.py b/test/test_repodata.py index 16ccbf8..fc36f8b 100644 --- a/test/test_repodata.py +++ b/test/test_repodata.py @@ -20,10 +20,7 @@ # - S-Core Co., Ltd import os -import base64 import unittest - -from tic.repo import Repo from tic.repo import get_repodata_from_repos CWD = os.path.dirname(__file__) or '.' @@ -36,11 +33,15 @@ def suite(): class RepodataTest(unittest.TestCase): def setUp(self): # test environment setup - self.local_repo = ['file:/' + TEST_REPODATA_LOC + '/base', - 'file:/' + TEST_REPODATA_LOC + '/mobile'] + self.local_repo = [{'name': 'local_base', + 'url': 'file:/' + TEST_REPODATA_LOC + '/base'}, + {'name': 'local_mobile', + 'url': 'file:/' + TEST_REPODATA_LOC + '/mobile'}] - self.remote_repo = ['http://download.tizen.org/snapshots/tizen/base/latest/repos/arm64/packages', - 'http://download.tizen.org/snapshots/tizen/mobile/latest/repos/arm64-wayland/packages'] + self.remote_repo = [{'name': 'local_base', + 'url': 'http://download.tizen.org/snapshots/tizen/base/latest/repos/arm64/packages'}, + {'name': 'local_mobile', + 'url': 'http://download.tizen.org/snapshots/tizen/mobile/latest/repos/arm64-wayland/packages'}] def tearDown(self): # clear environment after test @@ -48,10 +49,7 @@ class RepodataTest(unittest.TestCase): del self.remote_repo def test_local_repodata(self): - repos = [] - for repo_url in self.local_repo: - repos.append(Repo(base64.urlsafe_b64encode(repo_url), repo_url)) - repodata_list = get_repodata_from_repos(repos, DEFAULT_CACHEDIR) + repodata_list = get_repodata_from_repos(self.local_repo, DEFAULT_CACHEDIR) for repo_info in repodata_list: self.assertNotEqual(repo_info, None) @@ -63,10 +61,7 @@ class RepodataTest(unittest.TestCase): raise self.failureException def test_remote_repodata(self): - repos = [] - for repo_url in self.remote_repo: - repos.append(Repo(base64.urlsafe_b64encode(repo_url), repo_url)) - repodata_list = get_repodata_from_repos(repos, DEFAULT_CACHEDIR) + repodata_list = get_repodata_from_repos(self.remote_repo, DEFAULT_CACHEDIR) for repo_info in repodata_list: self.assertNotEqual(repo_info, None) diff --git a/tic/command.py b/tic/command.py index ea70c6e..65570d1 100644 --- a/tic/command.py +++ b/tic/command.py @@ -1,11 +1,11 @@ -import base64 import logging -from tic.dependency import analyze_dependency +from tic.dependency import analyze_dependency, get_installed_packages +from tic.parser.recipe_parser import get_default_recipe from tic.parser.repo_parser import RepodataParser from tic.parser.view_parser import make_view_data -from tic.repo import Repo +from tic.utils.error import TICError from tic.repo import get_repodata_from_repos DEFAULT_CACHEDIR='/var/tmp/tic-core/cached' @@ -13,15 +13,30 @@ DEFAULT_CACHEDIR='/var/tmp/tic-core/cached' def analyze(repo_list, recipe_list=None): logger = logging.getLogger(__name__) + if not repo_list and not recipe_list: + raise TICError('No repositories defined') + repos = [] - for repo_url in repo_list: - repos.append(Repo(base64.urlsafe_b64encode(repo_url), repo_url)) + recipe = None + #TODO Repository check + # using default recipe (Temporary Code) + if recipe_list and recipe_list[0] == 'default': + recipe = get_default_recipe() + for repo_url in recipe.get('Repositories'): + repos.append({'name': repo_url.get('Name'), + 'url': repo_url.get('Url')}) + else: + number=1 + for repo_url in repo_list: + repos.append({'name': 'repository_%d' % number, + 'url': repo_url}) + number = number + 1 #Download repodata from repositories (Remote/Local) - repodata_list = get_repodata_from_repos(repos, DEFAULT_CACHEDIR) + repoinfo = get_repodata_from_repos(repos, DEFAULT_CACHEDIR) # Parse the xml files for the analysis of package (.rpm) - repo_parser = RepodataParser(repodata_list) + repo_parser = RepodataParser(repoinfo) pkg_group = repo_parser.parse() logger.info('pkg_list: %d, pkg2id: %d', len(pkg_group['pkg_list']), len(pkg_group['pkg2id'])) @@ -29,5 +44,10 @@ def analyze(repo_list, recipe_list=None): analyze_dependency(pkg_group) # Make a data for TIC (Tizen image creation) view_data = make_view_data(pkg_group) + inst_packages = get_installed_packages(recipe, repoinfo, pkg_group) + + result = {'packages': view_data, + 'repos': repos, + 'defaultpackages': inst_packages} - return view_data \ No newline at end of file + return result \ No newline at end of file diff --git a/tic/dependency.py b/tic/dependency.py index 28a4f0e..ce3c3c9 100644 --- a/tic/dependency.py +++ b/tic/dependency.py @@ -18,7 +18,8 @@ # # Contributors: # - S-Core Co., Ltd - +from lxml import etree +from tic.utils.error import TICError import logging def analyze_dependency(pkg_group): @@ -89,4 +90,51 @@ def analyze_dependency(pkg_group): return analyze() +def get_installed_packages(recipe, repoinfo, pkg_group): + logger = logging.getLogger(__name__) + + if not recipe or not repoinfo: + return [] + + default = recipe.get('Default') + config = recipe.get('Configurations')[0] + platform_name = config.get('Platform') + platform = recipe.get(platform_name) + + # check groups/extraPackages + group_set = set([]) + extrapkg_set = set([]) + for g in [default, platform, config]: + if g.has_key('Groups'): + group_set.update(g.get('Groups')) + if g.has_key('ExtraPackages'): + extrapkg_set.update(g.get('ExtraPackages')) + group_dict = dict.fromkeys(group_set) + + # parsing group.xml + try: + tree = etree.parse(repoinfo[0].get('comps')) + root = tree.getroot() + except etree.XMLSyntaxError as e: + raise TICError('primary.xml syntax error. %s', e) + + # Convert groups to packages + pkg_set = set([]) + for elm in root.findall('group'): + group_name = elm.find('name').text + if group_dict.has_key(group_name): + pkglist = elm.find('packagelist') + plist = [] + for pkgreq in pkglist.findall('packagereq'): + plist.append(pkgreq.text) + pkg_set.update(set(plist)) + + # set up required package from group packages + pkg2id = pkg_group.get('pkg2id') + pkg_list = pkg_group.get('pkg_list') + for pkg in pkg_set: + pkg_id = pkg2id.get(pkg) + if pkg_id and pkg_list[pkg_id].get('dependency'): + extrapkg_set.update(set(pkg_list[pkg_id].get('dependency'))) + return list(extrapkg_set) \ No newline at end of file diff --git a/tic/parser/recipe_parser.py b/tic/parser/recipe_parser.py new file mode 100644 index 0000000..7de426a --- /dev/null +++ b/tic/parser/recipe_parser.py @@ -0,0 +1,159 @@ +#!/usr/bin/python +# Copyright (c) 2000 - 2016 Samsung Electronics Co., Ltd. All rights reserved. +# +# Contact: +# @author Chulwoo Shin +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Contributors: +# - S-Core Co., Ltd + +import os +import yaml +from tic.utils import error +from tic.utils.file import write + +def get_default_recipe(): + recipe = dict( + Default=dict( + Baseline= 'tizen-3.0', + Active= True, + Mic2Options= '-f raw --fstab=uuid --copy-kernel --compress-disk-image=bz2 --generate-bmap', + Part='mobile-mbr', + Language= 'en_US.UTF-8', + Keyboard= 'us', + Timezone= 'Asia/Seoul', + RootPass= 'tizen', + DefaultUser= 'guest', + DefaultUserPass= 'tizen', + BootLoader= True, + BootloaderAppend= "rw vga=current splash rootwait rootfstype=ext4 plymouth.enable=0", + BootloaderTimeout= 3, + BootloaderOptions= '--ptable=gpt --menus="install:Wipe and Install:systemd.unit=system-installer.service:test"', + StartX= False, + Desktop= 'None', + SaveRepos= False, + UserGroups= "audio,video" + ), + Emulator64wayland=dict( + Part='mobile-mbr', + UserGroups='audio,video', + Groups=[ + 'Generic Base', + 'Mobile Base', + 'Mobile Console Tools', + 'Mobile Adaptation', + 'Mobile Wayland', + 'Mobile Middleware', + 'Mobile Applications', + 'Generic Multimedia', + 'Mobile Multimedia', + 'Generic Desktop Applications', + 'Mobile Dali', + 'Mobile EFL', + 'Mobile Enlightenment', + 'Mobile Input Framework', + 'Mobile Connectivity Framework', + 'Mobile Bluetooth', + 'Mobile Web Framework', + 'Mobile Telephony'], + PostScripts=[], + Repos= [], + NoChrootScripts=[] + ), + Configurations=[ + dict( + Name='mobile-emulator64-wayland', + Architecture='x86_64', + Schedule= "*", + Active= True, + Platform= 'Emulator64wayland', + Mic2Options= '-f loop --pack-to=@NAME@.tar.gz,', + FileName= 'mobile-emulator64-wayland', + Repos=['mobile-emulator64-wayland', 'base_emulator64'], + Groups=['Mobile Adaptation Emulator'], + ExtraPackages= [], + RemovePackages=[] + ) + ], + Repositories=[ + dict(Name='mobile-emulator64-wayland', + Url='http://download.tizen.org/snapshots/tizen/mobile/latest/repos/emulator64-wayland/packages/', + Options='--ssl_verify=no'), + dict(Name='base_emulator64', + Url='http://download.tizen.org/snapshots/tizen/base/latest/repos/emulator64/packages/', + Options='--ssl_verify=no') + ], + Partitions=[ + dict(Name='mobile-mbr', + Contents='part / --fstype="ext4" --size=3584 --ondisk=sda --active --label platform --fsoptions=defaults,noatime') + ] + ) + return recipe + +def load_yaml(path): + try: + with file(path) as f: + return yaml.load(f) + except IOError: + raise error.TICError('cannot read meta file: %s' % path) + except: + raise error.TICError('yaml format error of meta file: %s' % path) + + +def separate_recipe_file_for_ks(recipe): + path = '/home/shinchulwoo/recipe/test' + + # config.yaml + config = dict(Default=None, Configurations=[]) + config['Default'] = recipe.get('Default') + # targets (only one target) + config['Configurations'].append(recipe.get('Configurations')[0]) + platform_name = config['Configurations'][0].get('Platform') + config[platform_name] = recipe.get(platform_name) + with open(os.path.join(path, 'config.yaml'), 'w') as outfile: + yaml.dump(config, outfile, default_flow_style=False) + + # repo.yaml + if 'Repositories' in recipe: + repos = {} + repos['Repositories'] = recipe['Repositories'] + with open(os.path.join(path, 'repos.yaml'), 'w') as outfile: + yaml.dump(repos, outfile, default_flow_style=False) + + # partition info + if 'Partitions' in recipe: + for partition in recipe.get('Partitions'): + partition_path = os.path.join(path, 'partitions') + file_name = partition.get('Name') + temp = os.path.join(partition_path, file_name) + write(temp, partition['Contents']) + + # script.post + if 'PostScripts' in recipe: + for script in recipe.get('PostScripts'): + script_path = os.path.join(path, 'scripts') + script_type = script.get('Type') + if script_type and script_type == 'nochroot': + file_name = '%s.nochroot' % script.get('Name') + else: + file_name = '%s.post' % script.get('Name') + write(os.path.join(script_path, file_name), script['Contents']) + +if __name__ == "__main__": + get_default_recipe() + recipe = load_yaml('/home/shinchulwoo/recipe/test/test.yaml') + separate_recipe_file_for_ks(recipe) + + print('test') diff --git a/tic/repo.py b/tic/repo.py index 6a28659..faeb817 100644 --- a/tic/repo.py +++ b/tic/repo.py @@ -22,8 +22,8 @@ import logging import os import base64 -import collections import hashlib +import collections from lxml import etree from tic.utils import file from tic.utils import process @@ -48,7 +48,7 @@ def _get_metadata_from_repo(baseurl, proxies, cachedir, reponame, filename, sumtype=None, checksum=None): logger = logging.getLogger(__name__) url = os.path.join(baseurl, filename) - filename_tmp = str("%s/%s/%s" % (cachedir, reponame, os.path.basename(filename))) + filename_tmp = str("%s/%s" % (cachedir, os.path.basename(filename))) if os.path.splitext(filename_tmp)[1] in (".gz", ".bz2"): filename = os.path.splitext(filename_tmp)[0] else: @@ -74,9 +74,9 @@ def _get_metadata_from_repo(baseurl, proxies, cachedir, reponame, filename, def get_repodata_from_repos(repos, cachedir): my_repodata = [] for repo in repos: - reponame = repo.name - baseurl = repo.baseurl - cache_dir = os.path.join(cachedir, reponame) + reponame = repo.get('name') + baseurl = repo.get('url') + cache_dir = os.path.join(cachedir, base64.urlsafe_b64encode(baseurl)) cache_file = os.path.join(cache_dir, 'repomd.xml') # make directory for caching @@ -119,7 +119,7 @@ def get_repodata_from_repos(repos, cachedir): continue filepaths[item] = _get_metadata_from_repo(baseurl, None, - cachedir, + cache_dir, reponame, filepaths[item], sumtypes[item], @@ -129,7 +129,7 @@ def get_repodata_from_repos(repos, cachedir): "baseurl":baseurl, "repomd":repomd, "primary":filepaths['primary'], - "cachedir":cachedir, + "cachedir":cache_dir, "proxies":None, "patterns":filepaths['patterns'], "comps":filepaths['comps']}) @@ -137,7 +137,7 @@ def get_repodata_from_repos(repos, cachedir): return my_repodata -RepoType = collections.namedtuple('Repo', 'name, baseurl') +RepoType = collections.namedtuple('Repo', 'name, url') def Repo(name, baseurl): return RepoType(name, baseurl) @@ -145,8 +145,8 @@ if __name__ == '__main__': repo_url_1 = 'https://download.tizen.org/snapshots/tizen/base/latest/repos/arm64/packagesaaa' repo_url_2 = 'https://download.tizen.org/snapshots/tizen/mobile/latest/repos/arm64-wayland/packages' repos = [] - repos.append(Repo(base64.urlsafe_b64encode(repo_url_1), repo_url_1)) - repos.append(Repo(base64.urlsafe_b64encode(repo_url_2), repo_url_2)) + repos.append(Repo('repo_1', repo_url_1)) + repos.append(Repo('repo_2', repo_url_2)) cachedir = '/var/tmp/tic-core/cached' repodata = get_repodata_from_repos(repos, cachedir) print(repodata) diff --git a/tic/server/tic_server.py b/tic/server/tic_server.py index ed6f301..a1a6b4c 100644 --- a/tic/server/tic_server.py +++ b/tic/server/tic_server.py @@ -8,12 +8,10 @@ import os import logging from tic import command from tic.utils import error - - -#from flask_cors import CORS +from flask_cors import CORS app = Flask(__name__) -#CORS(app) +CORS(app) @app.route('/') def index(): @@ -22,23 +20,13 @@ def index():

TIC-Core web server is working

''' -@app.route('/analysis', methods=['GET', 'POST']) -def anlayze(): +@app.route('/analysis', methods=['POST']) +def analysis(): try: logger = logging.getLogger(__name__) - repo_list = [] - recipe_list = [] - if request.method == 'GET': - repo_list = request.args.getlist('repos') - recipe_list = request.args.getlist('recipes') - else: - logger.info('%s - %s %s : data=%s' % (request.remote_addr, request.method, request.path, request.data)) - repo_info = json.loads(request.data) - if 'repos' in repo_info: - repo_list = repo_info['repos'] - if 'recipes' in repo_info: - recipe_list = repo_info['recipes'] - view_data = command.analyze(repo_list, recipe_list) + logger.info('%s - %s %s : data=%s' % (request.remote_addr, request.method, request.path, request.data)) + repo_info = json.loads(request.data) + view_data = command.analyze(repo_info.get('repos'), repo_info.get('recipes')) resp = makeresponse(view_data, None) except error.TICError as err: logger.error(err) @@ -52,6 +40,24 @@ def anlayze(): return resp +@app.route('/exports', methods=['POST']) +def exports(): + try: + logger = logging.getLogger(__name__) + logger.info('%s - %s %s : data=%s' % (request.remote_addr, request.method, request.path, request.data)) + + except error.TICError as err: + logger.error(err) + resp = makeresponse(str(err), err) + except ValueError as ve: + logger.error(ve) + resp = makeresponse(str(ve), ve) + except Exception as ex: + logger.error(ex) + resp = makeresponse(str(ex), ex) + + return resp + def start(port_num=59001): # cryptographic random generator @@ -59,7 +65,7 @@ def start(port_num=59001): with app.test_request_context(): print(url_for('index')) - print(url_for('anlayze')) + print(url_for('analysis')) app.run(host='0.0.0.0', port=port_num) @@ -85,4 +91,4 @@ def ResultInfo(result='false', data=None, message=None): return ResultInfoType(result, data, message) if __name__ == '__main__': - start() + start(59003) diff --git a/tic/utils/file.py b/tic/utils/file.py index daaee91..9e51ffb 100644 --- a/tic/utils/file.py +++ b/tic/utils/file.py @@ -32,8 +32,9 @@ def make_dirs(dirname): raise def write(path, data): - file_path = os.path.join(path, 'tic_view.json') - with(open(file_path, 'w')) as f: + # make directory + make_dirs(os.path.dirname(path)) + with(open(path, 'w')) as f: f.write(data) def decompress_gzip(intput_path, output_path): diff --git a/tools/tic-core b/tools/tic-core index 0bdf1bd..8400c77 100644 --- a/tools/tic-core +++ b/tools/tic-core @@ -82,8 +82,8 @@ def main(argv): return 0 except error.TICError as err: logger.error(err) - except Exception as e: - logger.error(err) + except Exception as ex: + logger.error(ex) return 2 if __name__ == "__main__": -- 2.7.4