From ddfcc4b1e6e92e8c069397ab7b546adc76b64877 Mon Sep 17 00:00:00 2001 From: "y0169.zhang" Date: Tue, 9 Jan 2018 13:39:47 +0800 Subject: [PATCH] Add --full-build and --deps-build features With gbs build --full-build option, gbs will download the whole profile's source codes except the local package(s) from remote snapshot and perform building. With gbs build --deps-build option, gbs will analyse the reverse dependency of local package(s) and download them from remote snapshot and perform building. Change-Id: I1f62fa5e2f4741f706d4f7fab5c378a9b5859edf --- data/gbs.sh | 3 +- data/mapping.conf | 71 +++++++++++ debian/control | 2 + debian/rules | 1 + gitbuildsys/cmd_build.py | 315 ++++++++++++++++++++++++++++++++++++++++++++++- gitbuildsys/conf.py | 96 +++++++++++++++ gitbuildsys/utils.py | 150 ++++++++++++++++++++++ packaging/gbs.spec | 3 + tools/gbs | 5 + 9 files changed, 641 insertions(+), 5 deletions(-) create mode 100644 data/mapping.conf diff --git a/data/gbs.sh b/data/gbs.sh index 3fe75c5..a14c0dd 100644 --- a/data/gbs.sh +++ b/data/gbs.sh @@ -84,7 +84,8 @@ __gbs () --arch= --repository= --dist= --buildroot= --clean --include-all --extra-packs= --spec= --commit= --cache --skip-conf-repos --profile= --noinit --keep-packs --use-higher-deps - --not-export-source --clean-repos --define --baselib --disable-debuginfo + --not-export-source --clean-repos --define --baselibs --disable-debuginfo + --full-build --deps-build --snapshot " cr_opts=" --profile= --tmpfs --ks-file diff --git a/data/mapping.conf b/data/mapping.conf new file mode 100644 index 0000000..c8676ce --- /dev/null +++ b/data/mapping.conf @@ -0,0 +1,71 @@ +[obs_mapping] +mobile3.0 = Tizen:3.0:Mobile +wearable3.0 = Tizen:3.0:Wearable +tv3.0 = Tizen:3.0:TV +ivi3.0 = Tizen:3.0:IVI +unified = Tizen:Unified +base3.0 = Tizen:3.0:Base +base = Tizen:Base + +[prefix_mapping] +mobile3.0 = http://download.tizen.org/snapshots/tizen/3.0-mobile/tizen-3.0-mobile_ +wearable3.0 = http://download.tizen.org/snapshots/tizen/3.0-wearable/tizen-3.0-wearable_ +tv3.0 = http://download.tizen.org/snapshots/tizen/3.0-tv/tizen-3.0-tv_ +ivi3.0 = http://download.tizen.org/snapshots/tizen/3.0-ivi/tizen-3.0-ivi_ +unified = http://download.tizen.org/snapshots/tizen/unified/tizen-unified_ +base3.0 = http://download.tizen.org/snapshots/tizen/3.0-base/tizen-3.0-base_ +base = http://download.tizen.org/snapshots/tizen/base/tizen-base_ + +[repo_mapping] +mobile3.0 = arm-wayland,emulator32-wayland,emulator64-wayland,target-TM1 +wearable3.0 = emulator32-wayland,emulator-circle,target-circle +tv3.0 = arm-wayland,emulator32-wayland,emulator64-wayland +ivi3.0 = emulator,arm +unified = standard,emulator +base3.0 = arm,arm64,emulator32,emulator64,ia32,x86_64 +base = standard + +[profile_mapping] +unified_standard = repo.base_standard,repo.base_standard_debug,repo.unified_standard,repo.unified_standard_debug +unified_emulator = repo.base_standard,repo.base_standard_debug,repo.unified_emulator,repo.unified_emulator_debug +mobile3.0_arm-wayland = repo.base3.0_arm,repo.base3.0_arm_debug,repo.mobile3.0_arm-wayland,repo.mobile3.0_arm-wayland_debug +mobile3.0_emulator32-wayland = repo.base3.0_emulator32,repo.base3.0_emulator32_debug,repo.mobile3.0_emulator32-wayland,repo.mobile3.0_emulator32-wayland_debug +mobile3.0_target-TM1 = repo.base3.0_arm,repo.base3.0_arm_debug,repo.mobile3.0_target-TM1,repo.mobile3.0_target-TM1_debug +wearable3.0_emulator32-wayland = repo.base3.0_emulator32,repo.base3.0_emulator32_debug,repo.wearable3.0_emulator32-wayland,repo.wearable3.0_emulator32-wayland_debug +wearable3.0_emulator-circle = repo.base3.0_emulator32,repo.base3.0_emulator32_debug,repo.wearable3.0_emulator-circle,repo.wearable3.0_emulator-circle_debug +wearable3.0_target-circle = repo.base3.0_arm,repo.base3.0_arm_debug,repo.wearable3.0_target-circle,repo.wearable3.0_target-circle_debug +tv3.0_arm-wayland = repo.base3.0_arm,repo.base3.0_arm_debug,repo.tv3.0_arm-wayland,repo.tv3.0_arm-wayland_debug +tv3.0_emulator32-wayland = repo.base3.0_emulator32,repo.base3.0_emulator32_debug,repo.tv3.0_emulator32-wayland,repo.tv3.0_emulator32-wayland_debug +tv3.0_emulator64-wayland = repo.base3.0_emulator64,repo.base3.0_emulator64_debug,repo.tv3.0_emulator64-wayland,repo.tv3.0_emulator64-wayland_debug +ivi3.0_emulator = repo.base3.0_emulator32,repo.base3.0_emulator32_debug,repo.ivi3.0_emulator,repo.ivi3.0_emulator_debug +ivi3.0_arm = repo.base3.0_arm,repo.base3.0_arm_debug,repo.ivi3.0_arm,repo.ivi3.0_arm_debug + +[source_mapping] +unified_standard = repo.unified_standard_source, repo.unified_standard_depends, repo.unified_standard_pkgs +unified_emulator = repo.unified_emulator_source, repo.unified_emulator_depends, repo.unified_emulator_pkgs +mobile3.0_arm-wayland = repo.mobile3.0_arm-wayland_source, repo.mobile3.0_arm-wayland_depends, repo.mobile3.0_arm-wayland_pkgs +mobile3.0_emulator32-wayland = repo.mobile3.0_emulator32-wayland_source, repo.mobile3.0_emulator32-wayland_depends, repo.mobile3.0_emulator32-wayland_pkgs +mobile3.0_target-TM1 = repo.mobile3.0_target-TM1_source, repo.mobile3.0_target-TM1_depends, repo.mobile3.0_target-TM1_pkgs +wearable3.0_emulator32-wayland = repo.wearable3.0_emulator32-wayland_source, repo.wearable3.0_emulator32-wayland_depends, repo.wearable3.0_emulator32-wayland_pkgs +wearable3.0_emulator-circle = repo.wearable3.0_emulator-circle_source, repo.wearable3.0_emulator-circle_depends, repo.wearable3.0_emulator-circle_pkgs +wearable3.0_target-circle = repo.wearable3.0_target-circle_source, repo.wearable3.0_target-circle_depends, repo.wearable3.0_target-circle_pkgs +tv3.0_arm-wayland = repo.tv3.0_arm-wayland_source, repo.tv3.0_arm-wayland_depends, repo.tv3.0_arm-wayland_pkgs +tv3.0_emulator32-wayland = repo.tv3.0_emulator32-wayland_source, repo.tv3.0_emulator32-wayland_depends, repo.tv3.0_emulator32-wayland_pkgs +tv3.0_emulator64-wayland = repo.tv3.0_emulator64-wayland_source, repo.tv3.0_emulator64-wayland_depends, repo.tv3.0_emulator64-wayland_pkgs +ivi3.0_emulator = repo.ivi3.0_emulator_source, repo.ivi3.0_emulator_depends, repo.ivi3.0_emulator_pkgs +ivi3.0_arm = repo.ivi3.0_arm_source, repo.ivi3.0_arm_depends, repo.ivi3.0_arm_pkgs + +[osc_mapping] +unified_standard = Tizen:Unified, standard +unified_emulator = Tizen:Unified, emulator +mobile3.0_arm-wayland = Tizen:3.0:Mobile, arm-wayland +mobile3.0_emulator32-wayland = Tizen:3.0:Mobile, emulator32-wayland +mobile3.0_target-TM1 = Tizen:3.0:Mobile, target-TM1 +wearable3.0_emulator32-wayland = Tizen:3.0:Wearable, emulator32-wayland +wearable3.0_emulator-circle = Tizen:3.0:Wearable, emulator-circle +wearable3.0_target-circle = Tizen:3.0:Wearable, target-circle +tv3.0_arm-wayland = Tizen:3.0:TV, arm-wayland +tv3.0_emulator32-wayland = Tizen:3.0:TV, emulator32-wayland +tv3.0_emulator64-wayland = Tizen:3.0:TV, emulator64-wayland +ivi3.0_emulator = Tizen:3.0:IVI, emulator +ivi3.0_arm = Tizen:3.0:IVI, arm diff --git a/debian/control b/debian/control index a7b0543..f849c5b 100755 --- a/debian/control +++ b/debian/control @@ -11,6 +11,8 @@ Package: gbs Architecture: all Depends: ${misc:Depends}, ${python:Depends}, python-pycurl, + python-requests, + python-lxml, sudo, osc (>= 0.132.6), git-buildpackage-rpm (>= 0.7.5-tizen20161231), diff --git a/debian/rules b/debian/rules index 0e50bdd..42c25b3 100644 --- a/debian/rules +++ b/debian/rules @@ -11,6 +11,7 @@ override_dh_auto_install: install -m644 docs/gbs.1 debian/tmp/usr/share/man/man1 install -m644 data/initrd debian/tmp/usr/share/gbs install -m644 data/vmlinuz debian/tmp/usr/share/gbs + install -m644 data/mapping.conf debian/tmp/usr/share/gbs for job_name in $(shell ls jenkins-jobs/configs); do \ mkdir -p debian/tmp/var/lib/jenkins/jobs/$$job_name; \ install -m644 jenkins-jobs/configs/$$job_name/config.xml debian/tmp/var/lib/jenkins/jobs/$$job_name; \ diff --git a/gitbuildsys/cmd_build.py b/gitbuildsys/cmd_build.py index 3120e8b..fcd0ad8 100644 --- a/gitbuildsys/cmd_build.py +++ b/gitbuildsys/cmd_build.py @@ -26,18 +26,27 @@ import re import urlparse import glob import gzip +import requests +from lxml import etree import xml.etree.cElementTree as ET +import xml.etree.ElementTree as ETP +import subprocess +import re -from gitbuildsys.utils import Temp, RepoParser, read_localconf, \ - guess_spec, show_file_from_rev +from gitbuildsys.utils import Temp, Workdir, RepoParser, read_localconf, \ + guess_spec, show_file_from_rev, \ + GitRefMappingParser, GitDirFinder, GerritNameMapper from gitbuildsys.errors import GbsError, Usage -from gitbuildsys.conf import configmgr +from gitbuildsys.conf import configmgr, MappingConfigParser, encode_passwd from gitbuildsys.safe_url import SafeURL from gitbuildsys.cmd_export import get_packaging_dir, config_is_true from gitbuildsys.log import LOGGER as log +from gitbuildsys.oscapi import OSC, OSCError +from gitbuildsys.log import DEBUG from gbp.rpm.git import GitRepositoryError, RpmGitRepository from gbp import rpm +from gbp.rpm import SpecFile from gbp.errors import GbpError @@ -158,7 +167,6 @@ def prepare_repos_and_build_conf(args, arch, profile): 'following repos:\n%s' % (arch, '\n'.join(repos))) cmd_opts += [('--repository=%s' % url.full) for url in repourls] - profile = get_profile(args) profile_name = formalize_build_conf(profile.name.replace('profile.', '', 1)) distconf = os.path.join(TMPDIR, '%s.conf' % profile_name) @@ -337,6 +345,222 @@ def get_local_archs(repos): return archs +def create_autoconf(arch, snapshot, full_build): + """ + Create ~/.gbs.conf.auto for user + """ + url = '' + if snapshot: + if snapshot.startswith('tizen-3.0-mobile'): + profile_url = 'http://download.tizen.org/snapshots/tizen/3.0-mobile/' + snapshot + elif snapshot.startswith('tizen-3.0-tv'): + profile_url = 'http://download.tizen.org/snapshots/tizen/3.0-tv/' + snapshot + elif snapshot.startswith('tizen-3.0-wearable'): + profile_url = 'http://download.tizen.org/snapshots/tizen/3.0-wearable/' + snapshot + elif snapshot.startswith('tizen-3.0-ivi'): + profile_url = 'http://download.tizen.org/snapshots/tizen/3.0-ivi/' + snapshot + elif snapshot.startswith('tizen-unified'): + profile_url = 'http://download.tizen.org/snapshots/tizen/unified/' + snapshot + else: + raise GbsError('unkown snapshot: %s, please check' %snapshot) + + res = requests.get(profile_url) + if res.status_code == 404: + raise GbsError('specified snapshot: %s does not exist, please check' %profile_url) + + log.info("sync git-ref-mapping from review.tizen.org to get reference binary id") + refparser = GitRefMappingParser() + ref_meta = refparser.parse() + + mapparser = MappingConfigParser('/usr/share/gbs/mapping.conf') + obs_meta = mapparser.GetObsMapping() + prefix_meta = mapparser.GetPrefixMapping() + repo_meta = mapparser.GetRepoMapping() + profile_meta = mapparser.GetProfileMapping() + source_meta = mapparser.GetSourceMapping() + osc_meta = mapparser.GetOscMapping() + + content = '' + default = '' + if arch in ['armv7l', 'aarch64']: + default = 'profile.unified_standard' + elif arch in ['i586', 'x86_64']: + default = 'profile.unified_emulator' + else: + default = 'profile.error' + + content += '[general]\nfallback_to_native = true\nprofile = ' + default + '\n' + repos_map = {} + for k, v in obs_meta.iteritems(): + ref_id = ref_meta.get(v) + if ref_id == None: + ref_id = 'latest' + + prefix = prefix_meta.get(k) + if prefix == None: + continue + + if ref_id == 'latest': + prefix = os.path.dirname(prefix) + '/' + + url = prefix + ref_id + if snapshot: + if os.path.dirname(url) == os.path.dirname(profile_url): + url = profile_url + + repos = repo_meta.get(k) + if repos == None: + continue + + + repotypes = repos.split(',') + bid = os.path.basename(url) + for repo in repotypes: + repos_map[k + '_' + repo.replace('-', '_')] = url + '/repos/' + repo + '/packages/' + content += '[repo.' + k + '_' + repo + ']\n' + content += 'url = ' + url + '/repos/' + repo + '/packages/\n' + content += '[repo.' + k + '_' + repo + '_source]\n' + content += 'url = ' + url + '/builddata/manifest/' + bid + '_' + repo + '.xml\n' + content += '[repo.' + k + '_' + repo + '_debug]\n' + content += 'url = ' + url + '/repos/' + repo + '/debug/\n' + content += '[repo.' + k + '_' + repo + '_depends]\n' + content += 'url = ' + url + '/builddata/depends/dep_graph/' + repo + '/' + arch + '/\n' + content += '[repo.' + k + '_' + repo + '_pkgs]\n' + content += 'url = ' + url + '/builddata/depends/' + v + '_' + repo + '_' + arch + '_revpkgdepends.xml\n' + + for k, v in profile_meta.iteritems(): + content += '[profile.' + k + ']\n' + if full_build: + v = v[:v.index('repo.' + k) - 1] + + content += 'repos = ' + v + '\n' + source = source_meta.get(k) + if source != None: + attrs = source.split(',') + content += 'source = ' + attrs[0] + '\n' + content += 'depends = ' + attrs[1] + '\n' + content += 'pkgs = ' + attrs[2] + '\n' + + content += 'obs = obs.tizen\n' + v = osc_meta.get(k) + if v != None: + attrs = v.split(',') + content += 'obs_prj = ' + attrs[0] + '\n' + content += 'obs_repo = ' + attrs[1] + '\n' + + content += '[obs.tizen]\nurl = https://build.tizen.org\nuser = obs_viewer\npasswd = obs_viewer\n' + + fpath = os.path.expanduser('~/.gbs.conf.auto') + with open(fpath, 'w') as wfile: + wfile.write(content) + + configmgr.add_conf(fpath) + + return repos_map + +def sync_source(exclude_pkgs, pkgs, manifest_url, path): + if pkgs != None: + if len(pkgs) == 0: + return + + cmd = 'repo init -u https://git.tizen.org/cgit/scm/manifest -b tizen -m unified_standard.xml' + with Workdir(path): + ret = os.system(cmd) + if ret != 0: + raise GbsError("failed to run %s in %s" % (' '.join(cmd), path)) + + out_file = path + '/.repo/manifests/unified_standard.xml.new' + in_file = path + '/.repo/manifests/unified_standard.xml' + with open(out_file, 'w') as f: + with open(in_file, 'r') as i: + for line in i.readlines(): + if 'metadata.xml' not in line and 'prebuilt.xml' not in line: + f.write(line) + + shutil.move(out_file, in_file) + r = requests.get(manifest_url) + if r.status_code == 404: + log.error("manifest %s not found" %manifest_url) + return + + old = path + '/.repo/manifests/unified/standard/projects.xml' + new = path + '/.repo/manifests/unified/standard/projects.xml.new' + with open(new, "w") as f: + f.write(r.content) + + log.info('use %s as projects.xml' %manifest_url) + with open(new, 'r') as f: + with open(old, 'w') as i: + for line in f.readlines(): + if exclude_pkgs != None: + if 'revision' in line: + k = line.index('"', line.index('path')) + abs_path = line[k + 1:line.index('"', k + 1)] + if abs_path in exclude_pkgs: + continue + + i.write(line) + + if pkgs != None: + if 'revision' in line: + k = line.index('"', line.index('path')) + abs_path = line[k + 1:line.index('"', k + 1)] + if abs_path not in pkgs: + continue + + i.write(line) + + cmd = 'repo sync' + with Workdir(path): + ret = os.system(cmd) + if ret != 0: + raise GbsError("failed to run %s in %s" % (' '.join(cmd), path)) + +def prepare_fullbuild_source(profile, pkgs, url, download_path): + """ + prepare full build source + """ + sync_source(pkgs, None, url, download_path) + +def prepare_depsbuild_source(gnmapper, profile, arch, pkgs, url, download_path): + """ + prepare deps build source + """ + deps = set([]) + deps_path = [] + try: + for pkg in pkgs: + depurl = profile.depends.url + pkg + '.full_edges.vis_input.js' + r = requests.get(depurl) + if r.status_code == 404: + log.error('get depends from %s failed' %depurl) + continue + + match = re.findall("label: '.*'", r.content) + if not match: + continue + + for m in match: + dep = m[m.index("'") + 1:m.rindex("'")] + if dep == pkg: + continue + + if dep not in deps: + deps.add(dep) + + log.info("what depends on number:%d --> %s" %(len(deps), deps)) + for pkg in deps: + gerrit_name = gnmapper.get_gerritname_by_obsname(pkg) + if gerrit_name == None: + log.warning('can not get gerrit name for pkg:%s' %pkg) + continue + + deps_path.append(gnmapper.get_gerritname_by_obsname(pkg)) + except OSCError, err: + raise GbsError(str(err)) + + sync_source(None, deps_path, url, download_path) + def main(args): """gbs build entry point.""" @@ -359,6 +583,12 @@ def main(args): raise GbsError("git project can't be found for --spec, " "give it in argument or cd into it") + repos_map = {} + if not args.conf: + if args.full_build or args.deps_build: + repos_map = create_autoconf(args.arch, args.snapshot, args.full_build) + log.info("Create ~/.gbs.conf.auto using reference binary id") + read_localconf(workdir) hostarch = os.uname()[4] @@ -373,6 +603,79 @@ def main(args): (buildarch, ','.join(SUPPORTEDARCHS))) profile = get_profile(args) + if args.full_build or args.deps_build: + if profile.source == None: + raise GbsError('full build/deps build option must specify source repo in gbs.conf') + + download_path = Temp(prefix=os.path.expanduser('~/gbs-build'), + directory=True) + local_pkgs = [] + gitf = GitDirFinder(workdir) + + profile_name = formalize_build_conf(profile.name.replace('profile.', '', 1)) + profile_repo = repos_map.get(profile_name) + + cache = Temp(prefix=os.path.join(TMPDIR, 'gbscache'), + directory=True) + cachedir = cache.path + repoparser = RepoParser([SafeURL(profile_repo)], cachedir) + distconf = os.path.join(download_path.path, '%s.conf' % profile_name) + + if repoparser.buildconf is None: + raise GbsError('failed to get build conf from repos, please ' + 'use snapshot repo or specify build config using ' + '-D option') + else: + buildconf = repoparser.buildconf + try: + shutil.copy(buildconf, distconf) + log.info('build conf has been downloaded at:\n %s' \ + % distconf) + except IOError, err: + raise GbsError("Failed to copy build conf: %s" % (str(err))) + + profile.buildconf = distconf + + r = requests.get(profile.pkgs.url) + if r.status_code == 404: + raise GbsError('get pkg xml from %s failed' %profile.pkgs.url) + + exclude_pkgs = [] + if args.exclude: + exclude_pkgs = args.exclude.split(',') + gnmapper = GerritNameMapper(r.content, repoparser.primaryxml) + for spec_file in gitf.specs: + try: + spec = SpecFile(spec_file) + if spec.name in exclude_pkgs: + continue + + if args.full_build: + pkg = gnmapper.get_gerritname_by_srcname(spec.name) + else: + pkg = gnmapper.get_pkgname_by_srcname(spec.name) + if pkg != None: + local_pkgs.append(pkg) + else: + log.error('package %s parse failed' %spec.name) + except GbpError, err: + log.warning('gbp parse spec failed. %s' % err) + + if args.full_build: + prepare_fullbuild_source(profile, local_pkgs, profile.source.url, download_path.path) + else: + if len(local_pkgs) == 0: + raise GbsError('deps build option must has local packages') + + prepare_depsbuild_source(gnmapper, profile, args.arch, local_pkgs, profile.source.url, download_path.path) + + for path in gitf.paths: + shutil.copytree(path, os.path.join(download_path.path, os.path.basename(path))) + + workdir = download_path.path + curdir = os.getcwd() + os.chdir(workdir) + if args.buildroot: build_root = args.buildroot elif 'TIZEN_BUILD_ROOT' in os.environ: @@ -440,6 +743,8 @@ def main(args): if args.conf and args.conf != '.gbs.conf': fallback = configmgr.get('fallback_to_native') + elif args.full_build or args.deps_build: + fallback = configmgr.get('fallback_to_native') else: fallback = '' if args.fallback_to_native or config_is_true(fallback): @@ -462,6 +767,8 @@ def main(args): log.debug("running command: %s" % ' '.join(cmd)) retcode = os.system(' '.join(cmd)) + if args.full_build or args.deps_build: + os.chdir(curdir) if retcode != 0: raise GbsError('some packages failed to be built') else: diff --git a/gitbuildsys/conf.py b/gitbuildsys/conf.py index e405bb3..5c01f28 100644 --- a/gitbuildsys/conf.py +++ b/gitbuildsys/conf.py @@ -506,6 +506,11 @@ class Profile(object): self.common_user = user self.common_password = password self.repos = [] + self.source = None + self.obs_prj = None + self.obs_repo = None + self.depends = None + self.pkgs = None self.obs = None self.buildroot = None self.buildconf = None @@ -515,6 +520,26 @@ class Profile(object): '''add a repo to repo list of the profile''' self.repos.append(repoconf) + def set_source(self, repoconf): + '''set a repo to the profile''' + self.source = repoconf + + def set_obs_prj(self, prj): + '''set a obs project to the profile''' + self.obs_prj = prj + + def set_obs_repo(self, repo): + '''set a obs repo to the profile''' + self.obs_repo = repo + + def set_depends(self, deps): + '''set depends to the profile''' + self.depends = deps + + def set_pkgs(self, pkgs): + '''set pkgs to the profile''' + self.pkgs = pkgs + def set_obs(self, obsconf): '''set OBS api of the profile''' self.obs = obsconf @@ -646,7 +671,11 @@ class BizConfigManager(ConfigMgr): if g_passwd: password = g_passwd + obs_prj = self.get_optional_item(name, 'obs_prj') + obs_repo = self.get_optional_item(name, 'obs_repo') profile = Profile(name, user, password) + profile.set_obs_prj(obs_prj) + profile.set_obs_repo(obs_repo) obs = self.get_optional_item(name, 'obs') if obs: @@ -673,6 +702,24 @@ class BizConfigManager(ConfigMgr): self._get_url_options(repo)) profile.add_repo(repoconf) + source = self.get_optional_item(name, 'source') + if source: + repoconf = SectionConf(profile, source, + self._get_url_options(source)) + profile.set_source(repoconf) + + depends = self.get_optional_item(name, 'depends') + if depends: + repoconf = SectionConf(profile, depends, + self._get_url_options(depends)) + profile.set_depends(repoconf) + + pkgs = self.get_optional_item(name, 'pkgs') + if pkgs: + repoconf = SectionConf(profile, pkgs, + self._get_url_options(pkgs)) + profile.set_pkgs(repoconf) + profile.buildroot = self.get_optional_item(name, 'buildroot') if self.get_optional_item(name, 'buildconf'): profile.buildconf = os.path.expanduser(self._interpolate( @@ -751,5 +798,54 @@ class BizConfigManager(ConfigMgr): return profile +class MappingConfigParser(SafeConfigParser): + def __init__(self, file, defaults=None): + SafeConfigParser.__init__(self, defaults=None) + self._obs_meta = {} + self._prefix_meta = {} + self._repo_meta = {} + self._profile_meta = {} + self._source_meta = {} + self._osc_meta = {} + self.read(file) + kvs = self.items('obs_mapping') + for kv in kvs: + self._obs_meta[kv[0]] = kv[1] + kvs = self.items('prefix_mapping') + for kv in kvs: + self._prefix_meta[kv[0]] = kv[1] + kvs = self.items('repo_mapping') + for kv in kvs: + self._repo_meta[kv[0]] = kv[1] + kvs = self.items('profile_mapping') + for kv in kvs: + self._profile_meta[kv[0]] = kv[1] + kvs = self.items('source_mapping') + for kv in kvs: + self._source_meta[kv[0]] = kv[1] + kvs = self.items('osc_mapping') + for kv in kvs: + self._osc_meta[kv[0]] = kv[1] + + def optionxform(self, optionstr): + return optionstr + + def GetObsMapping(self): + return self._obs_meta + + def GetPrefixMapping(self): + return self._prefix_meta + + def GetRepoMapping(self): + return self._repo_meta + + def GetProfileMapping(self): + return self._profile_meta + + def GetSourceMapping(self): + return self._source_meta + + def GetOscMapping(self): + return self._osc_meta configmgr = BizConfigManager() diff --git a/gitbuildsys/utils.py b/gitbuildsys/utils.py index 262293a..63a9165 100644 --- a/gitbuildsys/utils.py +++ b/gitbuildsys/utils.py @@ -289,6 +289,7 @@ class RepoParser(object): self.cachedir = cachedir self.repourls = defaultdict(list) self.buildconf = None + self.primaryxml = None self.standardrepos = [] self.urlgrabber = URLGrabber() @@ -441,6 +442,20 @@ class RepoParser(object): fh_gz.close() buildconf_fh.close() self.buildconf = buildconf_file + for elem in root.findall('%sdata' % xmlns): + if elem.attrib['type'] == 'primary': + location_elem = elem.find('%slocation' % xmlns) + break + if location_elem is not None and 'href' in location_elem.attrib: + primary_url = baseurl.pathjoin(location_elem.attrib['href']) + fname = self.fetch(primary_url) + if fname: + if fname[-3:] == '.gz': + fh_gz = gzip.open(fname, 'r') + else: + fh_gz = open(fname, 'r') + self.primaryxml = fh_gz.read() + fh_gz.close() def parse(self, remotes): """Parse each remote repo, try to fetch build.xml and build.conf""" @@ -517,6 +532,141 @@ class RepoParser(object): return filter_valid_repo(repos) +class GitRefMappingParser(object): + """git-ref-mapping parser for get reference binary id.""" + + def __init__(self, giturl='https://git.tizen.org/cgit/scm/git-ref-mapping'): + self._giturl = giturl + + def parse(self): + cmd = [] + workdir = os.path.expanduser('~/.ref-gbs/git-ref-mapping/') + refxml = os.path.expanduser('~/.ref-gbs/git-ref-mapping/git-ref-mapping.xml') + if os.path.exists(refxml): + cmd = ['git', 'pull'] + else: + if not os.path.exists(workdir): + os.makedirs(workdir) + + cmd = ['git', 'clone', self._giturl, workdir] + try: + with Workdir(workdir): + output = subprocess.Popen(cmd, stdout=subprocess.PIPE) + output.wait() + except (subprocess.CalledProcessError, OSError): + raise GbsError("failed to run %s in %s" % (' '.join(cmd), workdir)) + + try: + etree = ET.parse(refxml) + except ET.ParseError: + log.warning('Not well formed xml: %s' % refxml) + return + + root = etree.getroot() + meta = {} + for elem in root.iter(tag='branch'): + attr = elem.attrib + if attr['name'] in ['tizen_unified', 'tizen_3.0']: + obs_prj = attr['OBS_project'] + #obs_prj = obs_prj.replace(':', '') + #obs_prj = obs_prj.lower() + ref_prj = attr['OBS_staging_project'] + arr = ref_prj.split(':') + meta[obs_prj] = arr[len(arr) - 1] + + return meta + +class GitDirFinder(object): + def __init__(self, dir=None): + self.paths = [] + self.specs = [] + if dir: + self.find(dir) + + def find(self, dir): + if os.path.isdir(os.path.join(dir, '.git')): + self.paths.append(dir) + f = glob.glob('%s/packaging/*.spec' % dir) + self.specs.extend(f) + return + + files = os.listdir(dir) + for file in files: + d = os.path.join(dir, file) + git_d = os.path.join(d, '.git') + if os.path.isdir(d): + if os.path.isdir(git_d): + self.paths.append(d) + f = glob.glob('%s/packaging/*.spec' % d) + self.specs.extend(f) + else: + self.find(d) + +class GerritNameMapper(object): + def __init__(self, pkgxml, primaryxml): + self._pkg2src = {} + self._pkg2subpkgs = {} + self._src2gerrit = {} + self._src2pkg = {} + self.parse_pkgxml(pkgxml) + self.parse_primaryxml(primaryxml) + + def parse_pkgxml(self, pkgxml): + try: + root = ET.fromstring(pkgxml) + except ET.ParseError: + log.warning('Not well formed xml pkgxml') + return + + lst_node = root.getiterator("package") + for node in lst_node: + if node.attrib.has_key("name"): + for child in node.getchildren(): + if child.tag == 'source': + self._pkg2src[node.attrib['name']] = child.text + self._src2pkg[child.text] = node.attrib['name'] + if child.tag == 'subpkg': + if child.text.endswith('debugsource') or child.text.endswith('debuginfo'): + continue + + self._pkg2subpkgs[node.attrib['name']] = child.text + break + + def parse_primaryxml(self, primaryxml): + try: + root = ET.fromstring(primaryxml) + except ET.ParseError: + log.warning('Not well formed xml primaryxml') + return + + xmlns = re.sub('metadata$', '', root.tag) + for elem in root.findall('%spackage' % xmlns): + name = elem.find('%sname' % xmlns).text + vcs = elem.find('%sversion' % xmlns).attrib['vcs'] + self._src2gerrit[name] = vcs[:vcs.index('#')] + + def get_gerritname_by_obsname(self, obsname): + src = self._pkg2src.get(obsname) + if src == None: + return None + + name = self._src2gerrit.get(src) + if name == None: + src = self._pkg2subpkgs.get(obsname) + name = self._src2gerrit.get(src) + + return name + + def get_gerritname_by_srcname(self, srcname): + name = self._src2gerrit.get(srcname) + if name == None: + pkg = self._src2pkg.get(srcname) + name = self.get_gerritname_by_obsname(pkg) + + return name + + def get_pkgname_by_srcname(self, srcname): + return self._src2pkg.get(srcname) def read_localconf(workdir): """Read local configuration file from project directory.""" diff --git a/packaging/gbs.spec b/packaging/gbs.spec index e1dbe83..4e86c3f 100755 --- a/packaging/gbs.spec +++ b/packaging/gbs.spec @@ -19,6 +19,8 @@ URL: http://www.tizen.org Source0: %{name}_%{version}.tar.gz Requires: python >= 2.6 Requires: python-pycurl +Requires: python-requests +Requires: python-lxml Requires: sudo Requires: osc >= 0.132.6 Requires: tizen-gbp-rpm >= 20161231 @@ -116,6 +118,7 @@ mkdir -p %{buildroot}/%{_prefix}/share/gbs install -m644 docs/gbs.1 %{buildroot}/%{_prefix}/share/man/man1 install -m644 data/initrd %{buildroot}/%{_prefix}/share/gbs install -m644 data/vmlinuz %{buildroot}/%{_prefix}/share/gbs +install -m644 data/mapping.conf %{buildroot}/%{_prefix}/share/gbs # Install Jenkins Jobs for job_name in $(ls jenkins-jobs/configs) diff --git a/tools/gbs b/tools/gbs index 06d3867..f580b52 100755 --- a/tools/gbs +++ b/tools/gbs @@ -234,6 +234,11 @@ def build_parser(parser): help='The kernel of kvm machine') group.add_argument('--not-export-source', action='store_true', help='Do not export source, use git source to build directly') + group.add_argument('--full-build', action='store_true', + help='Download all the package sources except local package in gbs.conf, and do build') + group.add_argument('--deps-build', action='store_true', + help='Download packages depends on local package from gbs.conf, and do build') + group.add_argument('--snapshot', type=str, help='Specify snapshot id to use') group = parser.add_argument_group('speed up building options') group.add_argument('--incremental', action='store_true', -- 2.7.4