From a1453500d709a783c7beadc56c1d10f903458df3 Mon Sep 17 00:00:00 2001 From: SoonKyu Park Date: Thu, 6 Jul 2017 16:47:01 +0900 Subject: [PATCH] Add new jenkins job : 'update-gbs-meta' when master branch of scm/git-ref-mapping is updated Update following gbs fullbuild related meta file scm/manifest scm/meta/gbs-config scm/meta/build-config Change-Id: Id148aa998c11be10ca769f83e53bc3778c5bde5d --- debian/jenkins-scripts.install | 1 + job_update_gbs_meta.py | 461 +++++++++++++++++++++++++++++++++++++++++ packaging/jenkins-scripts.spec | 1 + 3 files changed, 463 insertions(+) create mode 100755 job_update_gbs_meta.py diff --git a/debian/jenkins-scripts.install b/debian/jenkins-scripts.install index c55bb52..ff19c0a 100644 --- a/debian/jenkins-scripts.install +++ b/debian/jenkins-scripts.install @@ -35,3 +35,4 @@ debian/tmp/job_trigger_obs_sync.py /var/lib/jenkins/jenkins-scripts/ debian/tmp/job_update_public_git.py /var/lib/jenkins/jenkins-scripts/ debian/tmp/job_control_ondemand_slaves.groovy /var/lib/jenkins/jenkins-scripts/ debian/tmp/job_test_trigger_automation.py /var/lib/jenkins/jenkins-scripts/ +debian/tmp/job_update_gbs_meta.py /var/lib/jenkins/jenkins-scripts/ diff --git a/job_update_gbs_meta.py b/job_update_gbs_meta.py new file mode 100755 index 0000000..452e0db --- /dev/null +++ b/job_update_gbs_meta.py @@ -0,0 +1,461 @@ +#!/usr/bin/env python + +import sys +import os +import tempfile +import atexit +import shutil +import urllib2 +import gzip +from common.git import Git, clone_gitproject +from common.gerrit import Gerrit, get_gerrit_event, GerritError, is_ref_deleted + +# prepare related global variables +workspace = os.getenv('WORKSPACE') +basic_url= os.getenv('BASIC_SNAPSHOT_URL') +gbs_meta_default_profile = os.getenv('GBS_META_DEFAULT_PROFILE') +gbs_meta_tizen_base_prj = os.getenv('GBS_META_TIZEN_BASE_PRJ') +gbs_meta_support_tizen_ver = os.getenv('GBS_META_SUPPORT_TIZEN_VER') + +def _update_ref_bin_index(ref_binary): + """ Update Reference Binary Index""" + print "-----------------------------------------------------" + print "Update Reference Binary Index" + print "-----------------------------------------------------" + + ref_list = [] + + # prepare separate temp directory for each build + git_prj = 'scm/meta/obs' + git_branch="master" + tmpdir = tempfile.mkdtemp(prefix=workspace+'/') + atexit.register(shutil.rmtree, tmpdir) + prjdir = os.path.join(tmpdir, git_prj) + + # clone gerrit project to local dir + if not clone_gitproject(git_prj, prjdir): + print >> sys.stderr, 'Error cloning %s' %git_prj + return 1 + mygit = Git(prjdir) + mygit.checkout(git_branch) + + for obs_prj in ref_binary.keys(): + print '\nobs_prj: %s' %obs_prj + profile = obs_prj.split(':')[-1].lower() + + if obs_prj.split(':')[-2] == 'Tizen': + t_ver = '' + t_ver_path = '/' + else: + t_ver = obs_prj.split(':')[-2] + '-' + t_ver_path = '/' + obs_prj.split(':')[-2] + '/' + + ref_snapshot_number = ref_binary[obs_prj].split('ref:')[1] + if ref_snapshot_number == 'latest': + ref_snapshot_url = basic_url + t_ver + profile + "/" + ref_snapshot_number + "/" + else: + ref_snapshot_url = basic_url + t_ver + profile + "/" + "tizen-" + t_ver + profile + "_" + ref_snapshot_number + "/" + + repository = [] + path_prj_list= {} + obs_meta_file = prjdir + '/' + obs_prj + '/_meta' + lines = open(obs_meta_file).readlines() + for line in lines: + if line.find('repository name=') != -1: + repo_tmp=line.split('"')[1] + repository.append(repo_tmp) + path_prj_list[repo_tmp] = [] + if line.find('path project=') != -1: + path_prj_arch = line.split('"')[1]+'##'+line.split('"')[3] + path_prj_list[repo_tmp].append(path_prj_arch) + + ref_list.append({'profile':profile}) + ref_list[-1]['t_ver']=t_ver + ref_list[-1]['t_ver_path']=t_ver_path + ref_list[-1]['ref_snapshot_number'] = ref_snapshot_number + ref_list[-1]['ref_snapshot_url'] = ref_snapshot_url + ref_list[-1]['repository'] = repository + ref_list[-1]['path_prj_list'] = path_prj_list + + print 'reference list %s' %ref_list + return ref_list + + +def _update_base_prj_index(): + """ Update Tizen Base Project Index""" + print "-----------------------------------------------------" + print "Update Tizen Base Project Index" + print "-----------------------------------------------------" + + base_prj_list = [] + + # prepare separate temp directory for each build + git_prj = 'scm/meta/obs' + git_branch="master" + tmpdir = tempfile.mkdtemp(prefix=workspace+'/') + atexit.register(shutil.rmtree, tmpdir) + prjdir = os.path.join(tmpdir, git_prj) + + # clone gerrit project to local dir + if not clone_gitproject(git_prj, prjdir): + print >> sys.stderr, 'Error cloning %s' %git_prj + return 1 + mygit = Git(prjdir) + mygit.checkout(git_branch) + + latest_t_ver = gbs_meta_support_tizen_ver.split(' ')[0] + for ver in gbs_meta_support_tizen_ver.split(' '): + if ver > latest_t_ver: + latest_t_ver = ver + + for base_ver in gbs_meta_support_tizen_ver.split(' '): + if base_ver == latest_t_ver: + obs_prj = 'Tizen:Base' + t_ver = '' + git_branch = 'tizen' + else: + obs_prj = 'Tizen:'+base_ver+':Base' + t_ver = base_ver+'-' + git_branch = 'tizen_'+base_ver + + + repository = [] + obs_meta_file = prjdir + '/' + obs_prj + '/_meta' + lines = open(obs_meta_file).readlines() + for line in lines: + if line.find('repository name=') != -1: + repo_tmp=line.split('"')[1] + repository.append(repo_tmp) + + base_prj_list.append({'base_ver':base_ver}) + base_prj_list[-1]['t_ver']=t_ver + base_prj_list[-1]['repository'] = repository + base_prj_list[-1]['git_branch'] = git_branch + + print 'base_prj_list = %s' %base_prj_list + return base_prj_list + + +def _update_build_conf(ref_list,commit_msg): + """ Update scm/meta/build-config after ref.binary update""" + print "-----------------------------------------------------" + print "Update scm/meta/build-config after ref.binary update" + print "-----------------------------------------------------" + + # prepare separate temp directory for each build + git_prj = 'scm/meta/build-config' + git_branch="tizen" + + tmpdir = tempfile.mkdtemp(prefix=workspace+'/') + atexit.register(shutil.rmtree, tmpdir) + prjdir = os.path.join(tmpdir, git_prj) + + # clone gerrit project to local dir + if not clone_gitproject(git_prj, prjdir): + print >> sys.stderr, 'Error cloning %s' %git_prj + return 1 + mygit = Git(prjdir) + mygit.checkout(git_branch) + + # Update build.conf file + for ref_profile in ref_list: + if not os.path.exists(prjdir + ref_profile['t_ver_path'] + ref_profile['profile']): + print 'No dir exists for %s. Create it' %(ref_profile['t_ver_path'] + ref_profile['profile']) + os.makedirs(prjdir + ref_profile['t_ver_path'] + ref_profile['profile']) + + for repository in ref_profile['repository']: + repo_build_conf_file = prjdir + ref_profile['t_ver_path'] + ref_profile['profile'] + '/' + repository + '_build.conf' + repodata_url = ref_profile['ref_snapshot_url'] + "repos/" + repository + "/packages/repodata/" + print 'profile: %s, repository:%s' %(ref_profile['profile'], repository) + print 'repodata_url : %s' %repodata_url + # fine url of build.conf.gz file + res = urllib2.urlopen(repodata_url) + for line in res.read().split('"'): + if line.find('build.conf.gz') != -1: + build_conf_file=line + build_conf_url = repodata_url + build_conf_file + print 'build_conf_url: %s' %build_conf_url + # download xxxx.build.conf.gz file -> decompress -> save it to repo_build.conf inside scm/meta/build-config git + tmp_build_conf_file = tmpdir + '/tmp_build.conf.gz' + f = open(tmp_build_conf_file,'wb') + res2 = urllib2.urlopen(build_conf_url) + f.write(res2.read()) + f.close() + with gzip.GzipFile(tmp_build_conf_file, 'rb') as inF: + with file(repo_build_conf_file, 'wb') as outF: + s = inF.read() + outF.write(s.encode('utf-8')) + + if mygit.is_clean()[0] == 0: + print '%s, branch %s has some changes' %(git_prj,git_branch) + mygit.add_files(prjdir) + mygit.commit_all(commit_msg) + mygit.push(repo = 'origin', src = git_branch) + else: + print '%s, branch %s has nothing to commit' %(git_prj,git_branch) + + +def _update_gbs_conf(ref_list,base_prj_list,commit_msg): + """ Update scm/meta/gbs-config after ref.binary update""" + print "-----------------------------------------------------" + print "Update scm/meta/gbs-config after ref.binary update" + print "-----------------------------------------------------" + + # prepare separate temp directory for each build + git_prj = 'scm/meta/gbs-config' + git_branch="tizen" + + tmpdir = tempfile.mkdtemp(prefix=workspace+'/') + atexit.register(shutil.rmtree, tmpdir) + prjdir = os.path.join(tmpdir, git_prj) + + # clone gerrit project to local dir + if not clone_gitproject(git_prj, prjdir): + print >> sys.stderr, 'Error cloning %s' %git_prj + return 1 + mygit = Git(prjdir) + mygit.checkout(git_branch) + + # clone gerrit project to local dir + if not clone_gitproject(git_prj, prjdir): + print >> sys.stderr, 'Error cloning %s' %git_prj + return 1 + mygit = Git(prjdir) + mygit.checkout(git_branch) + + gbs_conf_file = prjdir+'/gbs.conf' + default_profile = ref_list[0]['profile'] + default_repo = ref_list[0]['repository'][0] + default_t_ver = ref_list[0]['t_ver'] + + #[general] section + gbs_conf_text = '[general]\nfallback_to_native = true\nprofile = profile.'+default_t_ver+default_profile+'_'+default_repo+'\n\n\n' + + #Profile section + gbs_conf_text += '#########################################################\n################## Profile Section ##################\n#########################################################\n\n' + + for ref_profile in ref_list: + profile = ref_profile['profile'] + t_ver = ref_profile['t_ver'] + t_ver_path = ref_profile['t_ver_path'] + gbs_conf_text = gbs_conf_text + '############# '+ t_ver+profile+' #############\n' + for repository in ref_profile['repository']: + gbs_conf_text += '[profile.'+t_ver+profile+'_'+repository+']\n' + gbs_conf_text += 'buildconf=./scm/meta/build-config'+t_ver_path+profile+'/'+repository+'_build.conf\n' + gbs_conf_text += 'repos = ' + for path_prj_arch in ref_profile['path_prj_list'][repository]: + path_prj=path_prj_arch.split('##')[0].replace('Tizen:','').replace(':','-').lower() + path_arch=path_prj_arch.split('##')[1] + gbs_conf_text += 'repo.'+path_prj+'_'+path_arch+',repo.'+path_prj+'_'+path_arch+'_debug,' + gbs_conf_text = gbs_conf_text[:-1] + + gbs_conf_text += '\n\n' + + #repo section + gbs_conf_text += '\n\n\n#########################################################\n################## Repo Section##################\n#########################################################\n' + + #base repo + for base_prj in base_prj_list: + profile = 'base' + t_ver = base_prj['t_ver'] + repository = base_prj['repository'] + gbs_conf_text += '\n############# '+ t_ver + profile + ' #############\n' + for repository in base_prj['repository']: + gbs_conf_text += '[repo.'+t_ver+profile+'_'+repository+']\n' + gbs_conf_text += 'url = '+basic_url+t_ver+profile+'/latest/repos/'+repository+'/packages/\n' + gbs_conf_text += '[repo.'+t_ver+profile+'_'+repository+'_debug]\n' + gbs_conf_text += 'url = '+basic_url+t_ver+profile+'/latest/repos/'+repository+'/debug/\n\n' + + #profile repo + for ref_profile in ref_list: + profile = ref_profile['profile'] + t_ver = ref_profile['t_ver'] + + gbs_conf_text += '\n############# '+ t_ver+profile+' #############\n' + for repository in ref_profile['repository']: + gbs_conf_text += '[repo.'+t_ver+profile+'_'+repository+']\n' + gbs_conf_text += 'url = '+basic_url+t_ver+profile+'/latest/repos/'+repository+'/packages/\n' + gbs_conf_text += '[repo.'+t_ver+profile+'_'+repository+'_debug]\n' + gbs_conf_text += 'url = '+basic_url+t_ver+profile+'/latest/repos/'+repository+'/debug/\n\n' + + #write gbs_conf_text to default gbs_conf_file, which is scm/meta/gbs-config/gbs.conf + print '==================================================\n=========== default gbs.conf file==========\n==================================================\n' + print gbs_conf_text + with open(gbs_conf_file,"wb") as f: + f.write(gbs_conf_text) + + #Copy default gbs.conf file to each version/profile/repository_gbs.conf file and replace default profile + for ref_profile in ref_list: + profile = ref_profile['profile'] + t_ver = ref_profile['t_ver'] + t_ver_path = ref_profile['t_ver_path'] + + if not os.path.exists(prjdir + t_ver_path + profile): + print 'No dir exists for %s. Create it' %(t_ver_path + profile) + os.makedirs(prjdir + t_ver_path + profile) + + for repository in ref_profile['repository']: + file_path_name = prjdir + t_ver_path + profile + '/' + repository + '_gbs.conf' + print 'filename and path: %s' %file_path_name + shutil.copy(gbs_conf_file,file_path_name) + old_profile='profile = profile.'+default_t_ver+default_profile+'_'+default_repo + new_profile='profile = profile.'+t_ver+profile+'_'+repository + print 'new_profile: %s' %new_profile + with open(file_path_name,"rb") as f: + newText = f.read().replace(old_profile,new_profile) + with open(file_path_name,"wb") as f: + f.write(newText) + + #git push if there are any changes + if mygit.is_clean()[0] == 0: + print '%s, branch %s has some changes' %(git_prj,git_branch) + mygit.add_files(prjdir) + mygit.commit_all(commit_msg) + mygit.push(repo = 'origin', src = git_branch) + else: + print '%s, branch %s has nothing to commit' %(git_prj,git_branch) + + +def _update_scm_manifest(ref_list,base_prj_list,commit_msg): + """ Update scm/manifest after ref.binary update""" + print "-----------------------------------------------------" + print "/n/n/nUpdate scm/manifest after ref.binary update" + print "-----------------------------------------------------" + + # prepare separate temp directory for each build + git_prj = 'scm/manifest' + + for base_prj in base_prj_list: + tmpdir = tempfile.mkdtemp(prefix=workspace+'/') + atexit.register(shutil.rmtree, tmpdir) + prjdir = os.path.join(tmpdir, git_prj) + + # clone gerrit project to local dir + if not clone_gitproject(git_prj, prjdir): + print >> sys.stderr, 'Error cloning %s' %git_prj + return 1 + mygit = Git(prjdir) + + git_branch = base_prj['git_branch'] + + print 'git_prj: %s git_branch: %s' %(git_prj, git_branch) + mygit.checkout(git_branch) + + for ref_profile in ref_list: + if ref_profile['t_ver'] == base_prj['t_ver']: + profile = ref_profile['profile'] + t_ver = ref_profile['t_ver'] + t_ver_path = ref_profile['t_ver_path'] + ref_snapshot_number = ref_profile['ref_snapshot_number'] + ref_snapshot_url = ref_profile['ref_snapshot_url'] + + profile_path = os.path.join(prjdir,profile) + for repository in ref_profile['repository']: + if not os.path.exists(prjdir+'/'+profile+'/'+repository): + print 'No dir exists for %s. Create it' %(profile+'/'+repository) + os.makedirs(prjdir+'/'+profile+'/'+repository) + + #projects.xml + projects_file = profile_path + '/' + repository + '/projects.xml' + + manifest_url = ref_snapshot_url + '/builddata/manifest/' + res = urllib2.urlopen(manifest_url) + for index in res.read().split('a href="'): + if index.find(repository+'.xml') != -1: + manifest_file=index.split('">')[0] + manifest_file_url = manifest_url + manifest_file + print 'projects_file: %s' %projects_file + print 'manifest_file_url: %s' %manifest_file_url + + with open(projects_file,'wb') as f: + res2 = urllib2.urlopen(manifest_file_url) + f.write(res2.read()) + + #metadata.xml + metadata_file = profile_path + '/' + repository + '/metadata.xml' + metadata_text = '\n\n \n \n \n \n' + print 'repository: %s' %repository + print 'metadata_text: \n%s' %metadata_text + with open(metadata_file,'wb') as f: + f.write(metadata_text) + + #profile_repository.xml + manifest_xml_file = prjdir+'/'+profile+'_'+repository+'.xml' + manifest_text = '\n\n \n \n \n \n\n' + print 'repository: %s' %repository + print 'manifest_text: \n%s' %manifest_text + + with open(manifest_xml_file,'wb') as f: + f.write(manifest_text) + + if mygit.is_clean()[0] == 0: + print '%s, branch %s has some changes' %(git_prj,git_branch) + mygit.add_files(prjdir) + mygit.commit_all(commit_msg) + mygit.push(repo = 'origin', src = git_branch) + else: + print '%s, branch %s has nothing to commit' %(git_prj,git_branch) + + +def main(): + """script entry point""" + + print '---[JOB STARTED]----------------------------------------' + + event = get_gerrit_event() + + # prepare separate temp directory for each build + + if event['event_type'] != "ref-updated" or event['project'] != "scm/git-ref-mapping": + # This is just a sanity check as ref-updated is the only event we + # react on and it's configured in the job configuraion + print >> sys.stderr, "Configuration error: This job can't process"\ + "project %s! Only scm/git-ref-mapping is allowed and " \ + "event %s! Only ref-updated events are allowed" \ + %(event['event_type'], event['project']) + return 1 + + + git_prj = event['project'] + git_branch = event['refname'] + filename = 'git-ref-mapping.xml' + tmpdir = tempfile.mkdtemp(prefix=workspace+'/') + atexit.register(shutil.rmtree, tmpdir) + prjdir = os.path.join(tmpdir, git_prj) + + # clone gerrit project to local dir + if not clone_gitproject(git_prj, prjdir): + print >> sys.stderr, 'Error cloning %s' %git_prj + return 1 + mygit = Git(prjdir) + mygit.checkout(git_branch) + commit_msg=mygit.show(mygit.get_commits()[0]).split('\n')[4].replace(" ","") + + ref_binary = {} + base_prj_list = {} + + lines = open(prjdir+'/'+filename).readlines() + for line in lines: + if line.find('branch OBS_project') != -1: + ref_binary[line.split('"')[1]] = line.split('"')[3] + + #Add default profile supported by gbs + for profile in gbs_meta_default_profile.split(' '): + ref_binary[profile]=profile+':ref:latest' + + print 'Each reference snapshot numbers are like below' + print ref_binary + + ref_list = _update_ref_bin_index(ref_binary) + base_prj_list = _update_base_prj_index() + _update_build_conf(ref_list,commit_msg) + _update_gbs_conf(ref_list,base_prj_list,commit_msg) + _update_scm_manifest(ref_list,base_prj_list,commit_msg) + +if __name__ == '__main__': + sys.exit(main()) diff --git a/packaging/jenkins-scripts.spec b/packaging/jenkins-scripts.spec index 0a8e5fb..cf091da 100644 --- a/packaging/jenkins-scripts.spec +++ b/packaging/jenkins-scripts.spec @@ -163,6 +163,7 @@ fi %{destdir}/job_update_public_git.py %{destdir}/job_control_ondemand_slaves.groovy %{destdir}/job_test_trigger_automation.py +%{destdir}/job_update_gbs_meta.py %files common %defattr(-,jenkins,jenkins) -- 2.7.4