#!/usr/bin/python import argparse import bz2 import ftplib import getpass import os import os.path import string import sqlite3 import subprocess import sys import urllib2 from xml.dom.minidom import parseString from types import * this='tizen-dev-tool' tool_base_dir='/var/tmp/tizen-dev-tool' def updateOsceBuildRoot(arch): os.environ['OSCE_BUILD_ROOT'] = tool_base_dir + '/build_root_%s' % arch def getOsceBuildRoot(): return os.environ['OSCE_BUILD_ROOT'] def getOsceArchType(arch): if arch == 'ia32': return 'i586' else: return 'armv7el' def getRepoInformation(repo_type, snapshot_base_url, arch): information = {} information['repo_url'] = snapshot_base_url + '/repos/' + repo_type information['bin'] = {} information['bin']['db_file_name'] = repo_type + '.sqlite' information['bin']['base_url'] = information['repo_url'] + '/' + arch + '/packages' information['dbg'] = {} information['dbg']['db_file_name'] = repo_type + '-dbg.sqlite' information['dbg']['base_url'] = information['repo_url'] + '/' + arch + '/debug' return information def guessReleaseTypeWithSnapshotID(snapshot_id): key = snapshot_id.split('_')[0] if key == 'SLPRelease': return 'release' elif key == 'SLP': return 'standard' elif key == 'tizen-SDK': return 'SDK' else: print 'Cannot guess release type with ' + snapshot_id print 'Please update guessReleaseTypeWithSnapshotID() function in ' + this + ' script' sys.exit() def getInformation(snapshot_id, release_type, arch): information = {} information['snapshot_id'] = snapshot_id if not release_type: release_type = guessReleaseTypeWithSnapshotID(snapshot_id) information['release_type'] = release_type information['base_dir'] = '%s/%s' % (snapshot_id, arch) information['base_dir_abs'] = tool_base_dir + '/' + information['base_dir'] if not os.path.exists(information['base_dir_abs']): os.makedirs(information['base_dir_abs']) if release_type == 'SDK': information['server'] = 'download.tizendev.org' information['snapshot_base_url'] = 'https://%s/snapshots/SDK/common/%s' % (information['server'], information['snapshot_id']) information['repo_type_list'] = ['tizen-base', 'tizen-main'] information['auth'] = { 'user': None, 'pass': None } else: information['server'] = 'slp-build.sec.samsung.net' information['snapshot_base_url'] = 'http://%s:8008/snapshots/%s/%s' % (information['server'], information['release_type'], information['snapshot_id']) information['repo_type_list'] = ['slp-base', 'slp-system', 'slp-%s' % information['release_type']] information['auth'] = None for repo_type in information['repo_type_list']: information[repo_type] = getRepoInformation(repo_type, information['snapshot_base_url'], arch) return information def needAuth(information): return information['auth'] def getUser(information): if not information['auth']: return None if not information['auth']['user']: print information['server'] information['auth']['user'] = raw_input('user : ') information['auth']['pass'] = getpass.getpass('password: ') return information['auth']['user'] def getPass(information): if not information['auth']: return None if not information['auth']['user']: print information['server'] information['auth']['user'] = raw_input('user : ') information['auth']['pass'] = getpass.getpass('password: ') return information['auth']['pass'] def getDBFilePath(information, repo_type, pkg_type): return information['base_dir_abs'] + '/' + information[repo_type][pkg_type]['db_file_name'] def getDB(information, repo_type, pkg_type): if needAuth(information): authinfo = urllib2.HTTPPasswordMgrWithDefaultRealm() authinfo.add_password(None, information['server'], getUser(information), getPass(information)) handler = urllib2.HTTPBasicAuthHandler(authinfo) myopener = urllib2.build_opener(handler) opened = urllib2.install_opener(myopener) file = urllib2.urlopen(information[repo_type][pkg_type]['base_url'] + '/repodata/repomd.xml') data = file.read() file.close() dom = parseString(data) for element in dom.getElementsByTagName('data'): if element.hasAttribute('type') and element.getAttribute('type') == 'primary_db': db_bz2_url = information[repo_type][pkg_type]['base_url'] + '/' + element.getElementsByTagName('location')[0].getAttribute('href') file = urllib2.urlopen(db_bz2_url) data = file.read() file.close() file = open(getDBFilePath(information, repo_type, pkg_type), 'wb') file.write(bz2.decompress(data)) file.close() return True return False def downloadPackages(information, repo_type, pkg_type, package_list): conn = sqlite3.connect(getDBFilePath(information, repo_type, pkg_type)) c = conn.cursor() downloaded = set([]) for package in package_list: c.execute('SELECT location_href, version, release, arch FROM packages WHERE name = "%s"' % package) result = c.fetchone() if result: package_file_name = package + '-' + result[1] + '-' + result[2] + '.' + result[3] + '.rpm' package_file_path = information['base_dir_abs'] + '/' + package_file_name package_url = information[repo_type][pkg_type]['base_url'] + '/' + result[0] if not os.path.isfile(package_file_path): print 'download ' + package_file_name if needAuth(information): command = 'curl -o%s %s -u%s:%s -k' % (package_file_path, package_url, getUser(information), getPass(information)) else: command = 'curl -o%s %s -k' % (package_file_path, package_url) subprocess.call(command.split()) if os.path.isfile(package_file_path): current_dir = os.getcwd() os.chdir(information['base_dir_abs']) print 'extracting ' + package_file_name pipe = subprocess.Popen(['rpm2cpio', package_file_name], stdout=subprocess.PIPE) subprocess.Popen(['cpio', '-mid'], stdin=subprocess.PIPE).communicate(input=pipe.stdout.read()) os.chdir(current_dir) downloaded |= set([package]) conn.close() return downloaded def getPackages(snapshot_id, release_type, arch, package_list): information = getInformation(snapshot_id, release_type, arch) for repo_type in information['repo_type_list']: for pkg_type in ['bin', 'dbg']: if not os.path.isfile(getDBFilePath(information, repo_type, pkg_type)): if not getDB(information, repo_type, pkg_type): print 'Failed to get DB' return package_list -= downloadPackages(information, repo_type, pkg_type, package_list) if len(package_list) == 0: return print 'Cannot find packages : %r' % package_list def commandGetPkgs(program, args): parser = argparse.ArgumentParser(prog=program, description='Download packages') parser.add_argument('snapshot_id', action='store') parser.add_argument('package_name', nargs='+') parser.add_argument('-a', '--arch', choices=['armv7l', 'ia32'], default='armv7l') parser.add_argument('-t', '--type', choices=['standard', 'release', 'SDK'], dest='release_type', help='release type. If this is not specified, ' + this + 'tries to guess it with snapshot_id') parsedArgs = parser.parse_args(args); getPackages(parsedArgs.snapshot_id, parsedArgs.release_type, parsedArgs.arch, set(parsedArgs.package_name)) def getAllPackages(dbfile): conn = sqlite3.connect(dbfile) c = conn.cursor() pkg_list = set() for row in c.execute('SELECT name, version, release, arch FROM packages'): pkg_list |= set([row[0] + '-' + row[1] + '-' + row[2] + '.' + row[3] + '.rpm']) return pkg_list def listPackages(snapshot_id, release_type, arch): information = getInformation(snapshot_id, release_type, arch) pkg_list = set() for repo_type in information['repo_type_list']: for pkg_type in ['bin', 'dbg']: dbfile = getDBFilePath(information, repo_type, pkg_type) if not os.path.isfile(dbfile): if not getDB(information, repo_type, pkg_type): print 'Failed to get DB' return pkg_list |= getAllPackages(dbfile) for pkg_name in sorted(pkg_list): print pkg_name def commandPkgList(program, args): parser = argparse.ArgumentParser(prog=program, description='Show package list') parser.add_argument('snapshot_id', action='store') parser.add_argument('-a', '--arch', choices=['armv7l', 'ia32'], default='armv7l') parser.add_argument('-t', '--type', choices=['standard', 'release', 'SDK'], dest='release_type', help='release type. If this is not specified, ' + this + 'tries to guess it with snapshot_id') parsedArgs = parser.parse_args(args); listPackages(parsedArgs.snapshot_id, parsedArgs.release_type, parsedArgs.arch) def GT_I8800_e4412(key_list): if key_list[2] >= '20121017' and key_list[2] < '20121024': return None else: return 'slp-release.sec.samsung.net', '/pub/GT-I8800_E4412_Release/release/GT-I8800_e4412_%s_1/information' % key_list[2], 'GT-I8800_e4412_%s_1.ks' % key_list[2] def GT_I8800_e4412_cluster(key_list): return 'slp-release.sec.samsung.net', '/pub/GT-I8800_E4412_Release/release/GT-I8800_e4412_%s_1/Redwood_Cluster' % key_list[3], 'GT-I8800_e4412_cluster_%s_1.ks' % key_list[3] def GT_I8800_e4412v34_cluster(key_list): return 'slp-release.sec.samsung.net', '/pub/GT-I8800_E4412_Release/release/GT-I8800_e4412_%s_1/Redwoodv34_Cluster' % key_list[3], 'GT-I8800_e4412v34_cluster_%s_1.ks' % key_list[3] def getKsPathInfo(key_list): search_dict = {} search_dict['GT-I8800'] = {} search_dict['GT-I8800']['e4412'] = {'default': GT_I8800_e4412} search_dict['GT-I8800']['e4412']['cluster'] = GT_I8800_e4412_cluster search_dict['GT-I8800']['e4412v34'] = {} search_dict['GT-I8800']['e4412v34']['cluster'] = GT_I8800_e4412v34_cluster search = search_dict for key in key_list: if key not in search.keys(): if type(search['default']) is FunctionType: return search['default'](key_list) return None search = search[key] if not search: return None if type(search) is FunctionType: return search(key_list) return None def findSnapshotID(release_version): ksPathInfo = getKsPathInfo(release_version.split('_')) if not ksPathInfo: print 'Have no information of ' + release_version print 'Please update search_dict in the ' + this + ' script' return None ksdir = tool_base_dir + '/ks' if not os.path.exists(ksdir): os.makedirs(ksdir) ks_server = ksPathInfo[0] ks_server_dir = ksPathInfo[1] ks_file_name = ksPathInfo[2] ks_path = ksdir + '/' + ks_file_name if not os.path.isfile(ks_path): ftp = ftplib.FTP(ks_server) print ks_server ks_user = raw_input('user : ') ks_pass = getpass.getpass('password: ') ftp.login(ks_user, ks_pass) try: ftp.retrbinary('RETR %s/%s' % (ks_server_dir, ks_file_name), open(ks_path, 'wb').write) except: os.remove(ks_path) print 'Failed to get %s:%s/%s' % (ks_server, ks_server_dir, ks_file_name) print 'Maybe there is not such path in the server.' print 'Please update search_dict in the ' + this + ' script' return None ftp.quit() for line in open(ks_path).readlines(): if line.startswith("repo "): snapshot_id = line.split('/repos/')[0].split('/')[-1] print 'SnapshotID: ' + snapshot_id return snapshot_id def commandFindSnapshotID(program, args): parser = argparse.ArgumentParser(prog=program, description='Find repo id from release version') parser.add_argument('release_version', action='store', help='release version of target binary or SDK') parsedArgs = parser.parse_args(args); return findSnapshotID(parsedArgs.release_version) def __findProvides(information, repo_type, pkg_type, found_pkgs, file_path_list): conn = sqlite3.connect(getDBFilePath(information, repo_type, pkg_type)) c = conn.cursor() found_files = set() for file_path in file_path_list: file_name = file_path.split('/')[-1] isLibrary = False if file_name.endswith('.so'): isLibrary = True else: file_token = file_name.split('.so.') if len(file_token) is 2: isLibrary = True file_name = file_token[0] + '.so.' + file_token[1].split('.')[0] if isLibrary: for row in c.execute('select provides.name, packages.name from provides, packages where provides.name = "' + file_name + '" and provides.pkgKey == packages.pkgKey;'): if row[0] not in found_pkgs.keys(): found_pkgs[row[0]] = set() found_pkgs[row[0]] |= set([row[1]]) found_files |= set([file_path]) else: for row in c.execute('select files.name, packages.name from files, packages where files.name = "' + file_path + '" and files.pkgKey == packages.pkgKey;'): if row[0] not in found_pkgs.keys(): found_pkgs[row[0]] = set() found_pkgs[row[0]] |= set([row[1]]) found_files |= set([file_path]) conn.close() return found_files def _findProvides(information, file_path_list): found_pkgs = {} found_files = set() for repo_type in information['repo_type_list']: for pkg_type in ['bin', 'dbg']: dbfile = getDBFilePath(information, repo_type, pkg_type) if not os.path.isfile(dbfile): if not getDB(information, repo_type, pkg_type): print 'Failed to get DB' return found_files |= __findProvides(information, repo_type, pkg_type, found_pkgs, file_path_list) return found_pkgs, found_files def findProvides(snapshot_id, release_type, arch, file_path_list): information = getInformation(snapshot_id, release_type, arch) ret = _findProvides(information, file_path_list) for key in sorted(ret[0].keys()): for package_name in sorted(ret[0][key]): print key + ' is provided by [' + package_name + ']' file_path_list -= ret[1] if len(file_path_list) > 0: print 'Cannot find provider for %r' % sorted(file_path_list) def commandProvides(program, args): parser = argparse.ArgumentParser(prog=program, description='Find package that provides certain library') parser.add_argument('snapshot_id', action='store') parser.add_argument('file_path', nargs='+') parser.add_argument('-a', '--arch', choices=['armv7l', 'ia32'], default='armv7l') parser.add_argument('-t', '--type', choices=['standard', 'release', 'SDK'], dest='release_type', help='release type. If this is not specified, ' + this + 'tries to guess it with snapshot_id') parsedArgs = parser.parse_args(args); findProvides(parsedArgs.snapshot_id, parsedArgs.release_type, parsedArgs.arch, set(parsedArgs.file_path)) def findPID(parse_dict, line): if parse_dict['FindPID']['finished']: return if line.startswith('Pid :') or line.startswith('pid :'): parse_dict['FindPID']['finished'] = True parse_dict['FindPID']['result'] = line.split()[-1] def findExecPath(parse_dict, line): if parse_dict['FindExecPath']['finished']: return if line.startswith('ExePath:'): parse_dict['FindExecPath']['finished'] = True parse_dict['FindExecPath']['result'] = line.split()[-1] elif parse_dict['FindMapsStart']['finished'] and len(line.split()) is 4: parse_dict['FindExecPath']['finished'] = True parse_dict['FindExecPath']['result'] = line.split()[3] def findMapsStart(parse_dict, line): if parse_dict['FindMapsStart']['finished']: return if line.startswith('maps information'): parse_dict['FindMapsStart']['finished'] = True def findReleaseVersion(parse_dict, line): if parse_dict['FindReleaseVersion']['finished']: return if line.startswith('Build='): parse_dict['FindReleaseVersion']['finished'] = True parse_dict['FindReleaseVersion']['result'] = line.split('=')[1].split(';')[0] def findRelatedFiles(parse_dict, line): if parse_dict['FindRelatedFiles']['finished']: return if line.startswith('end of maps information'): parse_dict['FindRelatedFiles']['finished'] = True else: line_tokens = line.split() if parse_dict['FindMapsStart']['finished'] and len(line_tokens) is 4: if line_tokens[3].startswith('['): return if parse_dict['FindRelatedFiles']['result'] is None: parse_dict['FindRelatedFiles']['result'] = [] else: parse_dict['FindRelatedFiles']['result'].append(line_tokens[3]) def parseCSFile(cs_file): parse_dict = {} parse_dict['FindPID'] = {'finished': False, 'function': findPID, 'result': None} parse_dict['FindExecPath'] = {'finished': False, 'function': findExecPath, 'result': None} parse_dict['FindMapsStart'] = {'finished': False, 'function': findMapsStart} parse_dict['FindReleaseVersion'] = {'finished': False, 'function': findReleaseVersion, 'result': None} parse_dict['FindRelatedFiles'] = {'finished': False, 'function': findRelatedFiles, 'result': None} if not os.path.isfile(cs_file): print cs_file + ' is not valid file' return None for line in open(cs_file).readlines(): for key in parse_dict.keys(): parse_dict[key]['function'](parse_dict, line) if not parse_dict['FindPID']['result']: print 'Failed to get PID' print 'Please update parse_dict in ' + this + ' script' return None if not parse_dict['FindExecPath']['result']: print 'Failed to get exec path' print 'Please update parse_dict in ' + this + ' script' return None if not parse_dict['FindReleaseVersion']['result']: print 'Failed to get release version' print 'Please update parse_dict in ' + this + ' script' return None print 'Pid: ' + parse_dict['FindPID']['result'] print 'ExecPath: ' + parse_dict['FindExecPath']['result'] print 'ReleaseVersion: ' + parse_dict['FindReleaseVersion']['result'] #print 'RelatedFiles: %r' % parse_dict['FindRelatedFiles']['result'] return parse_dict['FindPID']['result'], parse_dict['FindExecPath']['result'], parse_dict['FindReleaseVersion']['result'], parse_dict['FindRelatedFiles']['result'] def commandCsInfo(program, args): parser = argparse.ArgumentParser(prog=program) parser.add_argument('cs_file', action='store') parsedArgs = parser.parse_args(args); return parseCSFile(parsedArgs.cs_file) def doChroot(arch): updateOsceBuildRoot(arch) getbt_dir = tool_base_dir + '/getbt' current_dir = os.getcwd() os.chdir(getbt_dir) subprocess.call(['osce', 'chroot', getOsceArchType(arch)]) os.chdir(current_dir) def commandChroot(program, args): parser = argparse.ArgumentParser(prog=program) parser.add_argument('-a', '--arch', choices=['armv7l', 'ia32'], default='armv7l') parsedArgs = parser.parse_args(args); doChroot(parsedArgs.arch) def getBackTrace(snapshot_id, release_type, arch, core_file, exec_path, force_copy, related_files): if not os.path.isfile(core_file): print core_file + ' is not valid file' core_file_name = core_file.split('/')[-1] information = getInformation(snapshot_id, release_type, arch) if not os.path.isfile(information['base_dir_abs'] + exec_path): print 'executive file is not exists : ' + information['base_dir_abs'] + exec_path ret = _findProvides(information, [exec_path]) if len(ret[0]) > 0: print 'please install ' + string.join(ret[0][exec_path]) + ' with below command' print ' ' + this + ' getpkgs ' + snapshot_id + ' -a' + arch + ' ' + string.join(ret[0][exec_path]) else: print 'cannot find package that provides ' + exec_path if len(related_files) > 0: print '\nrelated packages from memory map' ret = _findProvides(information, related_files) pkg_name_list = [] for pkg_name_set in ret[0].values(): for pkg_name in pkg_name_set: pkg_name_list.append(pkg_name) print string.join(sorted(pkg_name_list)) return getbt_dir = tool_base_dir + '/getbt' if not os.path.exists(getbt_dir): os.makedirs(getbt_dir) current_dir = os.getcwd() os.chdir(getbt_dir) if not os.path.exists('.git'): os.makedirs('.git') if not os.path.exists('packaging'): os.makedirs('packaging') updateOsceBuildRoot(arch) package_file = 'packaging/webkit2-efl.spec' if not os.path.exists(package_file): spec_data = [ 'Name: webkit2-efl', 'Summary: get bt', 'Version: 1', 'Release: 1', 'Group: System/Libraries', 'License: BSD', 'Source0: %{name}-%{version}.tar.gz', '', 'BuildRequires: gdb cmake python bison flex gperf perl gettext-tools vi', '', '%description', 'Browser Engine based on Webkit2 EFL (Shared Library)', '', '%prep', '%%setup -q', '', '%build', 'echo Finished to create getbt environment', ] file = open(package_file, 'w') for line in spec_data: file.write(line) file.write('\n') file.close() if not os.path.exists(getOsceBuildRoot()): subprocess.call(['osce', 'build', getOsceArchType(arch)]) os.chdir(current_dir) osce_home_abuild = getOsceBuildRoot() + '/home/abuild' subprocess.call(['sudo', 'mkdir', '-p', osce_home_abuild + '/' + snapshot_id]) if force_copy or not os.path.exists(osce_home_abuild + '/' + information['base_dir']): print 'Coping snapshot data to build root' subprocess.call(['sudo', 'cp', '-rfa', information['base_dir_abs'], osce_home_abuild + '/' + snapshot_id + '/']) subprocess.call(['sudo', 'cp', core_file, osce_home_abuild]) gdb_sysroot = '/home/abuild/' + information['base_dir'] file = open('/tmp/cmd', 'w') cmd_data = [ 'exec-file ' + gdb_sysroot + exec_path, 'set sysroot ' + gdb_sysroot, 'set debug-file-directory ' + gdb_sysroot + '/usr/lib/debug', 'set substitute-path /usr/src ' + gdb_sysroot + '/usr/src', 'core-file ' + core_file_name, 'bt', ] for line in cmd_data: file.write(line) file.write('\n') file.close() subprocess.call(['sudo', 'cp', '/tmp/cmd', osce_home_abuild]) print '\n\n=====================================================================' print '- type \'gdb -x cmd\' to open ' + core_file_name + ' with gdb' print '=====================================================================\n\n' doChroot(arch) def commandGetBt(program, args): parser = argparse.ArgumentParser(prog=program) parser.add_argument('snapshot_id', action='store') parser.add_argument('core_file', action='store') parser.add_argument('exec_path', action='store') parser.add_argument('-a', '--arch', choices=['armv7l', 'ia32'], default='armv7l') parser.add_argument('-t', '--type', choices=['standard', 'release', 'SDK'], dest='release_type', help='release type. If this is not specified, ' + this + 'tries to guess it with snapshot_id') parser.add_argument('-f', '--force', dest='force_copy', default=False, action='store_true', help='Copy snapshot data to the qemu forcely. This will take some time, but this need to be set when new packages are downloaded.') parsedArgs = parser.parse_args(args); getBackTrace(parsedArgs.snapshot_id, parsedArgs.release_type, parsedArgs.arch, parsedArgs.core_file, parsedArgs.exec_path, parsedArgs.force_copy, None) def checkCrash(arch, core_file, cs_file, force_copy): if not os.path.isfile(core_file): print core_file + ' is not valid file' result = parseCSFile(cs_file) if not result: print 'Failed to parse cs file' return crash_pid = result[0] exec_path = result[1] release_version = result[2] related_files = result[3] core_file_name = 'core.' + crash_pid if core_file.split('/')[-1] != core_file_name: print 'core file is not matched with cs file' return snapshot_id = findSnapshotID(release_version) if not snapshot_id: print 'Failed to find snapshot ID with ' + release_version return getBackTrace(snapshot_id, None, arch, core_file, exec_path, force_copy, set(related_files)) def commandCrash(program, args): parser = argparse.ArgumentParser(prog=program) parser.add_argument('core_file', action='store') parser.add_argument('cs_file', action='store') parser.add_argument('-a', '--arch', choices=['armv7l', 'ia32'], default='armv7l') parser.add_argument('-f', '--force', dest='force_copy', default=False, action='store_true', help='Copy snapshot data to the qemu forcely. This will take some time, but this need to be set when new packages are downloaded.') parsedArgs = parser.parse_args(args); checkCrash(parsedArgs.arch, parsedArgs.core_file, parsedArgs.cs_file, parsedArgs.force_copy) def main(): commandMap = {} commandMap['getpkgs'] = commandGetPkgs commandMap['pkglist'] = commandPkgList commandMap['findsnapshot'] = commandFindSnapshotID commandMap['provides'] = commandProvides commandMap['csinfo'] = commandCsInfo commandMap['getbt'] = commandGetBt commandMap['crash'] = commandCrash commandMap['chroot'] = commandChroot parser = argparse.ArgumentParser() parser.add_argument('command', choices=commandMap.keys(), help='sub command. Type ' + this + ' -h for help') parser.add_argument('args', nargs=argparse.REMAINDER, help='arguments of command') parsedArgs = parser.parse_args() commandMap[parsedArgs.command](this + ' ' + parsedArgs.command, parsedArgs.args) if __name__ == '__main__': main()