From: SoonKyu Park Date: Tue, 12 Sep 2017 05:52:24 +0000 (+0900) Subject: Update gbs reference fullbuild jenkins job which is verified by staging aws system X-Git-Tag: submit/trunk/20190927.012743~294 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=ac8a1996b8b66cdbdc10a45bf43815c5c81dd489;p=services%2Fjenkins-scripts.git Update gbs reference fullbuild jenkins job which is verified by staging aws system Change-Id: I63075cdaf6c6f7eb23a3f3c9b885913e7e24d261 --- diff --git a/job_gbs_ref_fullbuild.py b/job_gbs_ref_fullbuild.py index bff03a5..ef0c526 100755 --- a/job_gbs_ref_fullbuild.py +++ b/job_gbs_ref_fullbuild.py @@ -15,17 +15,18 @@ from common.buildservice import BuildService # prepare related global variables workspace = os.getenv('WORKSPACE') -basic_url= os.getenv('URL_PUBLIC_REPO_BASE') + '/snapshots/tizen/' +#basic_url= os.getenv('URL_PUBLIC_REPO_BASE') + '/snapshots/tizen/' +basic_url = 'http://download.tizen.org/snapshots/tizen/' gbs_meta_default_profile = os.getenv('GBS_META_DEFAULT_PROFILE') TIZEN_PROFILE = os.getenv('TIZEN_PROFILE') REPOSITORY = os.getenv('REPOSITORY') ARCHITECTURE = os.getenv('ARCHITECTURE') #gbs_ref_fullbuild_root = workspace -gbs_ref_fullbuild_root = '/data/jenkins_fullbuild/' +gbs_ref_fullbuild_root = os.getenv('HOME') + '/gbsfullbuild-ROOT/' BUILD_ROOT = gbs_ref_fullbuild_root + '/GBS-ROOT/' SRC_ROOT = gbs_ref_fullbuild_root + '/SRC-ROOT/' -gbs_default_build_arg='timeout 6h gbs build --threads=28 --define "jobs 8" --define "_smp_mflags -j8" --baselibs --clean-once' -mic_default_build_arg='sudo mic --non-interactive cr auto ' +gbs_default_build_arg='timeout 6h gbs build --threads=16 --define "jobs 8" --define "_smp_mflags -j8" --baselibs --clean-once' +mic_default_build_arg='sudo mic --non-interactive cr auto -o '+workspace+'/mic/out ' ref_profile={} all_repo_arch_build=0 @@ -77,6 +78,11 @@ def _update_ref_bin_index(ref_binary): else: ref_snapshot_url = basic_snapshot_url + "tizen-" + t_ver + profile + "_" + ref_snapshot_number + "/" + if ref_snapshot_url.split('/')[-1]: + ref_snapshot_build_id = ref_snapshot_url.split('/')[-1] + else: + ref_snapshot_build_id = ref_snapshot_url.split('/')[-2] + repository = [] arch_list= {} obs_meta_file = prjdir + '/' + obs_prj + '/_meta' @@ -90,6 +96,7 @@ def _update_ref_bin_index(ref_binary): arch_tmp = line.split("")[1].split("")[0] arch_list[repo_tmp].append(arch_tmp) + ref_profile['profile']=profile ref_profile['obs_prj']=obs_prj ref_profile['t_ver']=t_ver @@ -98,104 +105,13 @@ def _update_ref_bin_index(ref_binary): ref_profile['basic_snapshot_url']=basic_snapshot_url ref_profile['ref_snapshot_number'] = ref_snapshot_number ref_profile['ref_snapshot_url'] = ref_snapshot_url + ref_profile['ref_snapshot_build_id'] = ref_snapshot_build_id ref_profile['repository'] = repository ref_profile['arch_list'] = arch_list print 'reference profile %s' %ref_profile -def replace_depanneur_file(): - tmp_python_file=workspace+'/tmp_python' - pytext=""" -#!/usr/bin/env python - -import sys -import os -import subprocess - -def replace_depanneur(): - print 'start' - patch1='$packs_queue->enqueue({\\n\\ - filename => "$pkg_path/$cache_key/$spec_file",\\n\\ - project_base_path => $base,\\n\\ - });' - origin1='push(@packs, {\\n\\ - filename => "$pkg_path/$cache_key/$spec_file",\\n\\ - project_base_path => $base,\\n\\ - });' - patch2='my $packs_queue = Thread::Queue->new();\\n\\ -\\tmy $data_queue = Thread::Queue->new();\\n\\ -\\tforeach my $pack (@pre_packs) {\\n\\ - if ($not_export_source == 1) {\\n\\ - my $name = basename($pack->{"project_base_path"});\\n\\ - my $r = grep /^$name$/, @not_export;\\n\\ - if ($vmtype eq "kvm") {\\n\\ - $r = 0;\\n\\ - }\\n\\ - if ($r) {\\n\\ - info("skip export $name for accel...");\\n\\ - push @packs, $pack;\\n\\ - } else {\\n\\ - info("package $name not support skip export source");\\n\\ - $data_queue->enqueue($pack);\\n\\ - }\\n\\ - } else {\\n\\ - $data_queue->enqueue($pack);\\n\\ - }\\n\\ -\\t}\\n\\ -\\n\\ -\\tmy $thread_num = int(sysconf(SC_NPROCESSORS_ONLN));\\n\\ -\\tfor (0..$thread_num) {\\n\\ -\\t\\t$data_queue->enqueue(undef);\\n\\ -\\t\\tthreads->create(sub {\\n\\ -\\t\\t\\twhile (my $pack = $data_queue->dequeue()) {\\n\\ -\\t\\t\\t\\tprepare_git($config, $pack->{"project_base_path"}, $pack->{"filename"}, $packs_queue);\\n\\ -\\t\\t\\t}\\n\\ -\\t\\t});\\n\\ -\\t}\\n\\ -\\tforeach (threads->list()) { $_->join(); }\\n\\ -\\t# Check error\\n\\ -\\tforeach (threads->list()) {\\n\\ -\\t\\tif (my $chk_err = $_->error()){\\n\\ -\\t\\t\\twarning("export thread error: $chk_err\\\\n");\\n\\ -\\t\\t}\\n\\ -\\t}\\n\\ -\\t$packs_queue->enqueue(undef);\\n\\ -\\twhile (my $pack = $packs_queue->dequeue()) {\\n\\ -\\t\\tpush @packs, $pack;\\n\\ -\\t}' - - origin2='foreach my $pack (@pre_packs) {\\n\\ - prepare_git($config, $pack->{"project_base_path"}, $pack->{"filename"});\\n\\ - }' - - depanneur_file='/usr/bin/depanneur' - - f = open(depanneur_file,'rb') - text = f.read() - if text.find(patch1) != -1 and text.find(patch2) != -1: - print depanneur_file+' will be replaced' - f.close() - new_text=text.replace(patch1,origin1).replace(patch2,origin2) - f = open(depanneur_file,'wb') - f.write(new_text) - f.close() - -def main(): - replace_depanneur() - -if __name__ == "__main__": - sys.exit(main()) -""" - - f = open(tmp_python_file,'wb') - f.write(pytext) - f.close() - - cmd='sudo python '+tmp_python_file - subprocess.call(cmd, stdout=sys.stdout,stderr=sys.stderr, shell=True) - - def _do_repo_init_sync(repo): print '----start _do_repo_init_sync-----------------------------------' @@ -286,12 +202,23 @@ def _do_repo_arch_gbs_fullbuild(repo,arch): retry_cnt -= 1 if __is_gbs_fullbuild_result_fail(repo,arch): + # TRIGGER NEXT BUILD-MONITOR + if False: + update_message="Successed:B(X)I(-)" + if len(update_message) < 119: + trigger_next("BUILD-MONITOR", \ + {'bm_stage':'GBSFULLBUILD_SNAPSHOT', + 'snapshot_name':ref_profile['ref_snapshot_build_id'], + 'gbsfullbuild_string': update_message}) raise LocalError('There are errors on GBS fullbuild for repo:%s, arch:%s' %(repo,arch)) def do_gbs_build(): print '----start do_gbs_build-----------------------------------' + if not os.path.exists(gbs_ref_fullbuild_root): + os.mkdir(gbs_ref_fullbuild_root) + if os.path.exists(BUILD_ROOT): cmd = 'sudo rm -rf '+BUILD_ROOT subprocess.call(cmd, stdout=sys.stdout,stderr=sys.stderr, shell=True) @@ -314,12 +241,7 @@ def do_gbs_build(): def _ks_add_local_repo(remote_ks_content,repo): print '----start _ks_add_local_repo-----------------------------------' - if ref_profile['ref_snapshot_url'].split('/')[-1]: - ref_snapshot_build_id = ref_profile['ref_snapshot_url'].split('/')[-1] - else: - ref_snapshot_build_id = ref_profile['ref_snapshot_url'].split('/')[-2] - - remote_ks_content=remote_ks_content.replace('@BUILD_ID@',ref_snapshot_build_id) + remote_ks_content=remote_ks_content.replace('@BUILD_ID@',ref_profile['ref_snapshot_build_id']) profile_list=os.listdir(BUILD_ROOT+'/local/repos/') for profile in profile_list: if profile.find(repo) != -1: @@ -360,6 +282,14 @@ def _do_mic_build(ks_path): print 'mic build command is %s' %cmd ret = subprocess.call(cmd, stdout=sys.stdout,stderr=sys.stderr, shell=True) if ret: + # TRIGGER NEXT BUILD-MONITOR + if False: + update_message="Successed:B(O)I(X)" + if len(update_message) < 119: + trigger_next("BUILD-MONITOR", \ + {'bm_stage':'GBSFULLBUILD_SNAPSHOT', + 'snapshot_name':ref_profile['ref_snapshot_build_id'], + 'gbsfullbuild_string': update_message}) raise LocalError('Image creation error for ks:%s' %ks_path) @@ -383,6 +313,7 @@ def do_image_creation(): f = open(ks_dir+ks_file,'wb') remote_ks_content=urllib2.urlopen(ks_url).read() local_ks_content=_ks_add_local_repo(remote_ks_content,repo) + local_ks_content=local_ks_content.replace('%runscript\n\n%end','') print 'file content:=============================================================' print local_ks_content print '==========================================================================' @@ -444,7 +375,6 @@ def main(): print "skip fullbuild...\nGBS reference fullbuild is triggered only for %s" %TIZEN_PROFILE return 0 - replace_depanneur_file() do_gbs_build() do_image_creation() @@ -454,7 +384,7 @@ def main(): if len(update_message) < 119: trigger_next("BUILD-MONITOR", \ {'bm_stage':'GBSFULLBUILD_SNAPSHOT', - 'snapshot_name':release_id, + 'snapshot_name':ref_profile['ref_snapshot_build_id'], 'gbsfullbuild_string': update_message}) if __name__ == '__main__':