From: biao716.wang Date: Thu, 2 Jul 2020 06:14:10 +0000 (+0900) Subject: refine code with SAM tool check X-Git-Tag: submit/trunk/20200723.141412~2 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=954b397e3bfa4fa108329e4f79f03a28d5777d3d;p=tools%2Fgbs.git refine code with SAM tool check Change-Id: I4f5bab1e6d9303b9a08423c200036cc1aaa7807d Signed-off-by: biao716.wang --- diff --git a/gitbuildsys/cmd_build.py b/gitbuildsys/cmd_build.py index bf17172..52a25f8 100644 --- a/gitbuildsys/cmd_build.py +++ b/gitbuildsys/cmd_build.py @@ -205,9 +205,10 @@ def prepare_repos_and_build_conf(args, arch, profile): return cmd_opts -def prepare_depanneur_opts(args): - '''generate extra options for depanneur''' +#refine code with SAM check. +def prepare_depanneur_simple_opts(args): + '''generate extra simple options for depanneur''' cmd_opts = [] if args.exclude: cmd_opts += ['--exclude=%s' % i for i in args.exclude.split(',')] @@ -245,6 +246,13 @@ def prepare_depanneur_opts(args): cmd_opts += ['--style=%s' % args.style] if args.export_only: cmd_opts += ['--export-only'] + + return cmd_opts + +def prepare_depanneur_opts(args): + '''generate extra options for depanneur''' + + cmd_opts = prepare_depanneur_simple_opts(args) # if args.package_list: package_list = args.package_list.split(',') @@ -350,11 +358,11 @@ def get_local_archs(repos): return archs -def create_autoconf(arch, snapshot, full_build): +def get_profile_url(snapshot): """ - Create ~/.gbs.conf.auto for user + get profile url according to snapshot. """ - url = '' + profile_url = '' if snapshot: if snapshot.startswith('tizen-3.0-mobile'): profile_url = 'http://download.tizen.org/snapshots/tizen/3.0-mobile/' + snapshot @@ -373,6 +381,17 @@ def create_autoconf(arch, snapshot, full_build): if res.status_code == 404: raise GbsError('specified snapshot: %s does not exist, please check' %profile_url) + + return profile_url + +def create_autoconf(arch, snapshot, full_build): + """ + Create ~/.gbs.conf.auto for user + """ + url = '' + + profile_url = get_profile_url(snapshot) + log.info("sync git-ref-mapping from review.tizen.org to get reference binary id") refparser = GitRefMappingParser() ref_meta = refparser.parse() @@ -566,9 +585,120 @@ def prepare_depsbuild_source(gnmapper, profile, arch, pkgs, url, download_path): sync_source(None, deps_path, url, download_path) + +def get_local_pkgs(args): + '''get local pkgs''' + exclude_pkgs = [] + if args.exclude: + exclude_pkgs = args.exclude.split(',') + gnmapper = GerritNameMapper(r.content, repoparser.primaryxml) + for spec_file in gitf.specs: + try: + spec = SpecFile(spec_file) + if spec.name in exclude_pkgs: + continue + + if args.full_build: + pkg = gnmapper.get_gerritname_by_srcname(spec.name) + else: + pkg = gnmapper.get_pkgname_by_srcname(spec.name) + if pkg != None: + local_pkgs.append(pkg) + else: + log.error('package %s parse failed' %spec.name) + except GbpError as err: + log.warning('gbp parse spec failed. %s' % err) + + return local_pkgs + +def prepare_depanneur_cmd(args, buildarch, profile, workdir): + '''Prepare depanneur commond''' + # get virtual env from system env first + if 'VIRTUAL_ENV' in os.environ: + cmd = ['%s/usr/bin/depanneur' % os.environ['VIRTUAL_ENV']] + else: + cmd = ['depanneur'] + + cmd += ['--arch=%s' % buildarch] + + if args.clean: + cmd += ['--clean'] + + # check & prepare repos and build conf + if not args.noinit: + cmd += prepare_repos_and_build_conf(args, buildarch, profile) + else: + cmd += ['--noinit'] + + cmd += ['--path=%s' % "'"+str(workdir)+"'"] + + if args.ccache: + cmd += ['--ccache'] + + if args.extra_packs: + cmd += ['--extra-packs=%s' % args.extra_packs] + + hostarch = os.uname()[4] + if hostarch != buildarch and buildarch in CHANGE_PERSONALITY: + cmd = [CHANGE_PERSONALITY[buildarch]] + cmd + + # Extra depanneur special command options + cmd += prepare_depanneur_opts(args) + + # Extra options for gbs export + if args.include_all: + cmd += ['--include-all'] + if args.commit: + cmd += ['--commit=%s' % args.commit] + + if args.upstream_branch: + cmd += ['--upstream-branch=%s' % args.upstream_branch] + if args.upstream_tag: + cmd += ['--upstream-tag=%s' % args.upstream_tag] + + if args.conf and args.conf != '.gbs.conf': + fallback = configmgr.get('fallback_to_native') + elif args.full_build or args.deps_build: + fallback = configmgr.get('fallback_to_native') + else: + fallback = '' + if args.fallback_to_native or config_is_true(fallback): + cmd += ['--fallback-to-native'] + + if args.squash_patches_until: + cmd += ['--squash-patches-until=%s' % args.squash_patches_until] + if args.no_patch_export: + cmd += ['--no-patch-export'] + + if args.define: + cmd += [('--define="%s"' % i) for i in args.define] + if args.spec: + cmd += ['--spec=%s' % args.spec] + + # Determine if we're on devel branch + orphan_packaging = configmgr.get('packaging_branch', 'orphan-devel') + if orphan_packaging: + cmd += ['--spec-commit=%s' % orphan_packaging] + + return cmd + +def init_buildroot(args, profile): + '''init build root''' + if args.buildroot: + build_root = args.buildroot + elif 'TIZEN_BUILD_ROOT' in os.environ: + build_root = os.environ['TIZEN_BUILD_ROOT'] + elif profile.buildroot: + build_root = profile.buildroot + else: + build_root = configmgr.get('buildroot', 'general') + build_root = os.path.expanduser(build_root) + + return build_root + def main(args): """gbs build entry point.""" - + print (args) global TMPDIR TMPDIR = os.path.join(configmgr.get('tmpdir', 'general'), '%s-gbs' % USERID) @@ -645,27 +775,8 @@ def main(args): if r.status_code == 404: raise GbsError('get pkg xml from %s failed' %profile.pkgs.url) - exclude_pkgs = [] - if args.exclude: - exclude_pkgs = args.exclude.split(',') + local_pkgs = get_local_pkgs(args) gnmapper = GerritNameMapper(r.content, repoparser.primaryxml) - for spec_file in gitf.specs: - try: - spec = SpecFile(spec_file) - if spec.name in exclude_pkgs: - continue - - if args.full_build: - pkg = gnmapper.get_gerritname_by_srcname(spec.name) - else: - pkg = gnmapper.get_pkgname_by_srcname(spec.name) - if pkg != None: - local_pkgs.append(pkg) - else: - log.error('package %s parse failed' %spec.name) - except GbpError as err: - log.warning('gbp parse spec failed. %s' % err) - if args.full_build: prepare_fullbuild_source(profile, local_pkgs, profile.source.url, download_path.path) else: @@ -681,15 +792,7 @@ def main(args): curdir = os.getcwd() os.chdir(workdir) - if args.buildroot: - build_root = args.buildroot - elif 'TIZEN_BUILD_ROOT' in os.environ: - build_root = os.environ['TIZEN_BUILD_ROOT'] - elif profile.buildroot: - build_root = profile.buildroot - else: - build_root = configmgr.get('buildroot', 'general') - build_root = os.path.expanduser(build_root) + build_root = init_buildroot(args, profile) # transform variables from shell to python convention ${xxx} -> %(xxx)s build_root = re.sub(r'\$\{([^}]+)\}', r'%(\1)s', build_root) sanitized_profile_name = re.sub("[^a-zA-Z0-9:._-]", "_", profile.name) @@ -704,71 +807,8 @@ def main(args): args.exclude = ','.join(profile.exclude_packages) os.environ['TIZEN_BUILD_ROOT'] = os.path.abspath(build_root) - # get virtual env from system env first - if 'VIRTUAL_ENV' in os.environ: - cmd = ['%s/usr/bin/depanneur' % os.environ['VIRTUAL_ENV']] - else: - cmd = ['depanneur'] - - cmd += ['--arch=%s' % buildarch] - - if args.clean: - cmd += ['--clean'] - - # check & prepare repos and build conf - if not args.noinit: - cmd += prepare_repos_and_build_conf(args, buildarch, profile) - else: - cmd += ['--noinit'] - - cmd += ['--path=%s' % "'"+str(workdir)+"'"] - - if args.ccache: - cmd += ['--ccache'] - - if args.extra_packs: - cmd += ['--extra-packs=%s' % args.extra_packs] - - if hostarch != buildarch and buildarch in CHANGE_PERSONALITY: - cmd = [CHANGE_PERSONALITY[buildarch]] + cmd - - # Extra depanneur special command options - cmd += prepare_depanneur_opts(args) - - # Extra options for gbs export - if args.include_all: - cmd += ['--include-all'] - if args.commit: - cmd += ['--commit=%s' % args.commit] - - if args.upstream_branch: - cmd += ['--upstream-branch=%s' % args.upstream_branch] - if args.upstream_tag: - cmd += ['--upstream-tag=%s' % args.upstream_tag] - - if args.conf and args.conf != '.gbs.conf': - fallback = configmgr.get('fallback_to_native') - elif args.full_build or args.deps_build: - fallback = configmgr.get('fallback_to_native') - else: - fallback = '' - if args.fallback_to_native or config_is_true(fallback): - cmd += ['--fallback-to-native'] - - if args.squash_patches_until: - cmd += ['--squash-patches-until=%s' % args.squash_patches_until] - if args.no_patch_export: - cmd += ['--no-patch-export'] - - if args.define: - cmd += [('--define="%s"' % i) for i in args.define] - if args.spec: - cmd += ['--spec=%s' % args.spec] - - # Determine if we're on devel branch - orphan_packaging = configmgr.get('packaging_branch', 'orphan-devel') - if orphan_packaging: - cmd += ['--spec-commit=%s' % orphan_packaging] + #prepare depanneur commond + cmd = prepare_depanneur_cmd(args, buildarch, profile, workdir) log.debug("running command: %s" % ' '.join(cmd)) retcode = os.system(' '.join(cmd)) diff --git a/gitbuildsys/cmd_depends.py b/gitbuildsys/cmd_depends.py index f67c6db..fe4a849 100644 --- a/gitbuildsys/cmd_depends.py +++ b/gitbuildsys/cmd_depends.py @@ -19,28 +19,21 @@ """Implementation of subcmd: depends """ - import os import shutil import pwd import re import urlparse import glob -import gzip import requests -from lxml import etree -import xml.etree.cElementTree as ET -import xml.etree.ElementTree as ETP import subprocess import re -from gitbuildsys.utils import Temp, Workdir, RepoParser, read_localconf, \ - guess_spec, show_file_from_rev, \ - GitRefMappingParser, GitDirFinder, GerritNameMapper +from gitbuildsys.utils import Temp, RepoParser, read_localconf from gitbuildsys.errors import GbsError, Usage -from gitbuildsys.conf import configmgr, MappingConfigParser, encode_passwd +from gitbuildsys.conf import configmgr from gitbuildsys.safe_url import SafeURL -from gitbuildsys.cmd_export import get_packaging_dir, config_is_true +from gitbuildsys.cmd_export import get_packaging_dir from gitbuildsys.log import LOGGER as log from gitbuildsys.oscapi import OSC, OSCError from gitbuildsys.log import DEBUG @@ -50,43 +43,23 @@ from gbp import rpm from gbp.rpm import SpecFile from gbp.errors import GbpError +from gitbuildsys.cmd_build import CHANGE_PERSONALITY, SUPPORTEDARCHS, formalize_build_conf, get_profile, get_local_archs -CHANGE_PERSONALITY = { - 'ia32': 'linux32', - 'i686': 'linux32', - 'i586': 'linux32', - 'i386': 'linux32', - 'ppc': 'powerpc32', - 's390': 's390', - 'sparc': 'linux32', - 'sparcv8': 'linux32', - } - -SUPPORTEDARCHS = [ - 'x86_64', - 'i586', - 'armv6l', - 'armv7hl', - 'armv7l', - 'aarch64', - 'mips', - 'mipsel', - ] USERID = pwd.getpwuid(os.getuid())[0] TMPDIR = None -def formalize_build_conf(profile): - ''' formalize build conf file name from profile''' +def prepare_depanneur_opts(args): + '''generate extra options for depanneur''' + + cmd_opts = [] + if args.debug: + cmd_opts += ['--debug'] - # build conf file name should not start with digital, see: - # obs-build/Build.pm:read_config_dist() - start_digital_re = re.compile(r'^[0-9]') - if start_digital_re.match(profile): - profile = 'tizen%s' % profile + cmd_opts += ['--packaging-dir=%s' % get_packaging_dir(args)] + cmd_opts += ['--depends'] - # '-' is not allowed, so replace with '_' - return profile.replace('-', '_') + return cmd_opts def prepare_repos_and_build_conf(args, arch, profile): '''generate repos and build conf options for depanneur''' @@ -161,77 +134,13 @@ def prepare_repos_and_build_conf(args, arch, profile): if not distconf.endswith('.conf') or '-' in os.path.basename(distconf): raise GbsError("build config file must end with .conf, and can't " "contain '-'") - dist = os.path.basename(distconf)[:-len('.conf')] + dist = "'"+os.path.basename(distconf)[:-len('.conf')]+"'" cmd_opts += ['--dist=%s' % dist] - cmd_opts += ['--configdir=%s' % os.path.dirname(distconf)] - - return cmd_opts - -def prepare_depanneur_opts(args): - '''generate extra options for depanneur''' - - cmd_opts = [] - if args.debug: - cmd_opts += ['--debug'] - - cmd_opts += ['--packaging-dir=%s' % get_packaging_dir(args)] - cmd_opts += ['--depends'] + path = "'"+os.path.dirname(distconf)+"'" + cmd_opts += ['--configdir=%s' % path] return cmd_opts -def get_profile(args): - """ - Get the build profile to be used - """ - if args.profile: - profile_name = args.profile if args.profile.startswith("profile.") \ - else "profile." + args.profile - profile = configmgr.build_profile_by_name(profile_name) - else: - profile = configmgr.get_current_profile() - return profile - -def get_local_archs(repos): - """ - Get the supported arch from prebuilt toolchains - > get primary file - > get archs - - Each toolchain should contain about 128 packages, - it is insufficient if less than that. - """ - def get_primary_file_from_local(repos): - def find_primary(repo): - pattern = os.path.join(repo, 'repodata', '*primary.*.gz') - files = glob.glob(pattern) - if files: - return files[0] - - for repo in repos: - if not repo.startswith('http'): - pri = find_primary(repo) - if pri: - yield pri - - def extract_arch(primary): - with gzip.open(primary) as fobj: - root = ET.fromstring(fobj.read()) - - xmlns = re.sub(r'metadata$', '', root.tag) - for elm in root.getiterator('%spackage' % xmlns): - arch = elm.find('%sarch' % xmlns).text - if re.match(r'i[3-6]86', arch): - yield 'i586' - elif arch not in ('noarch', 'src'): - yield arch - - archs = set() - for pri in get_primary_file_from_local(repos): - for arch in extract_arch(pri): - archs.add(arch) - - return archs - def main(args): """gbs depends entry point."""