import shutil
import urllib2
import gzip
+import re
from common.git import Git, clone_gitproject
from common.gerrit import Gerrit, get_gerrit_event, GerritError, is_ref_deleted
from common.buildtrigger import trigger_info, trigger_next
# prepare related global variables
workspace = os.getenv('WORKSPACE')
basic_url= os.getenv('URL_PUBLIC_REPO_BASE') + '/snapshots/tizen/'
+public_basic_url = 'http://download.tizen.org/snapshots/tizen/'
daily_release_url= os.getenv('URL_PUBLIC_REPO_BASE') + '/releases/daily/tizen/'
+public_daily_release_url = 'http://download.tizen.org/releases/daily/tizen/'
+public_daily_release_url = 'http://download.tizen.org/releases/daily/tizen/'
gbs_meta_default_profile = os.getenv('GBS_META_DEFAULT_PROFILE')
gbs_meta_support_tizen_ver = os.getenv('GBS_META_SUPPORT_TIZEN_VER')
+
+def __get_index_from_path_prj_arch(path_prj_arch):
+ """ Get base repo url from obs project & repository"""
+ print "-------__get_base_repo_url start-----------------"
+
+ base_repo = {}
+ obs_prj=path_prj_arch.split('##')[0]
+ path_arch=path_prj_arch.split('##')[1]
+ path_prj=obs_prj.replace('Tizen:','').replace(':','-').lower()
+
+ if obs_prj.find(':ref:') != -1:
+ ref_build_id=obs_prj.split(':ref:')[1]
+ obs_prj=obs_prj.split(':ref:')[0]
+ snapshot_id=obs_prj.replace('Tizen:','').replace(':','-').lower()
+ temp_id = '/'+snapshot_id+'/tizen-'+snapshot_id+'_'+ref_build_id+'/repos/'+path_arch
+ else:
+ snapshot_id=path_prj
+ temp_id = '/'+snapshot_id+'/latest/repos/'+path_arch
+
+ url_candidates = []
+ url_candidates.append(basic_url+temp_id)
+ url_candidates.append(daily_release_url+temp_id)
+ url_candidates.append(public_basic_url+temp_id)
+ url_candidates.append(public_daily_release_url+temp_id)
+ for url in url_candidates:
+ try:
+ urllib2.urlopen(url)
+ except:
+ continue
+ else:
+ base_url = url
+ break
+
+ print 'obs_prj: %s, snapshot_id: %s' %(obs_prj,snapshot_id)
+ print 'path_prj: %s, path_arch: %s, base_url: %s' %(path_prj,path_arch,base_url)
+
+ base_repo['repo_name']='repo.'+path_prj+'_'+path_arch
+ base_repo['debug_repo_name']='repo.'+path_prj+'_'+path_arch+'_debug'
+ base_repo['repo_url']=base_url+'/packages/'
+ base_repo['debug_repo_url']=base_url+'/debug/'
+
+ print 'base_repo: %s' %base_repo
+ return base_repo
+
+
def _update_ref_bin_index(ref_binary):
""" Update Reference Binary Index"""
print "-----------------------------------------------------"
for obs_prj in ref_binary.keys():
print '\nobs_prj: %s' %obs_prj
- profile = obs_prj.split(':')[-1].lower()
- if obs_prj.split(':')[-2] == 'Tizen':
+ if len(re.findall('\D', obs_prj.split(':')[1].replace('.',''))) != 0:
t_ver = ''
t_ver_path = '/'
+ profile_array = obs_prj.split(':')[1:]
else:
- t_ver = obs_prj.split(':')[-2] + '-'
- t_ver_path = '/' + obs_prj.split(':')[-2] + '/'
-
+ t_ver = obs_prj.split(':')[1] + '-'
+ t_ver_path = '/' + obs_prj.split(':')[1] + '/'
+ profile_array = obs_prj.split(':')[2:]
+ #find profile name
+ i=0
+ profile = ''
+ while i < len(profile_array):
+ profile += profile_array[i].lower() + '-'
+ i += 1
+ profile = profile.rstrip('-')
+
+ print 'psk-test'
+ print obs_prj
+ print ref_binary
ref_snapshot_number = ref_binary[obs_prj].split('ref:')[1]
if ref_snapshot_number == 'latest':
ref_snapshot_url = basic_url + t_ver + profile + "/" + ref_snapshot_number + "/"
print 'repodata_url : %s' %repodata_url
# fine url of build.conf.gz file
res = urllib2.urlopen(repodata_url)
- for line in res.read().split('"'):
+ for line in res.read().split('a href="'):
if line.find('build.conf.gz') != -1:
- build_conf_file=line
+ build_conf_file=line.split('"')[0]
build_conf_url = repodata_url + build_conf_file
print 'build_conf_url: %s' %build_conf_url
# download xxxx.build.conf.gz file -> decompress -> save it to repo_build.conf inside scm/meta/build-config git
print '%s, branch %s has nothing to commit' %(git_prj,git_branch)
-def _update_gbs_conf(ref_list,base_prj_list,commit_msg,is_staging):
+def _update_gbs_conf(ref_list,commit_msg,is_staging,is_fullbuild):
""" Update scm/meta/gbs-config after ref.binary update"""
print "-----------------------------------------------------"
print "Update scm/meta/gbs-config after ref.binary update"
# prepare separate temp directory for each build
git_prj = 'scm/meta/gbs-config'
- git_branch="tizen"+is_staging
+ if is_fullbuild == 'true':
+ git_branch="tizen"+is_staging
+ else:
+ git_branch="tizen_ref"
tmpdir = tempfile.mkdtemp(prefix=workspace+'/')
atexit.register(shutil.rmtree, tmpdir)
#Profile section
gbs_conf_text += '#########################################################\n################## Profile Section ##################\n#########################################################\n\n'
+ path_prj_arch_overall_list = []
for ref_profile in ref_list:
profile = ref_profile['profile']
t_ver = ref_profile['t_ver']
gbs_conf_text = gbs_conf_text + '############# '+ t_ver+profile+' #############\n'
for repository in ref_profile['repository']:
gbs_conf_text += '[profile.'+t_ver+profile+'_'+repository+']\n'
- gbs_conf_text += 'buildconf=./scm/meta/build-config'+t_ver_path+profile+'/'+repository+'_build.conf\n'
+ if is_fullbuild == 'true':
+ gbs_conf_text += 'buildconf=./scm/meta/build-config'\
+ +t_ver_path+profile+'/'+repository+'_build.conf\n'
gbs_conf_text += 'repos = '
for path_prj_arch in ref_profile['path_prj_list'][repository]:
- path_prj=path_prj_arch.split('##')[0].replace('Tizen:','').replace(':','-').lower()
- path_arch=path_prj_arch.split('##')[1]
- gbs_conf_text += 'repo.'+path_prj+'_'+path_arch+',repo.'+path_prj+'_'+path_arch+'_debug,'
+ path_prj_arch_overall_list.append(path_prj_arch)
+ base_repo = __get_index_from_path_prj_arch(path_prj_arch)
+ gbs_conf_text += base_repo['repo_name']+','+base_repo['debug_repo_name']+','
+ if is_fullbuild == 'false':
+ gbs_conf_text += 'repo.'+t_ver+profile+'_'+repository+','
+ gbs_conf_text += 'repo.'+t_ver+profile+'_'+repository+'_debug,'
gbs_conf_text = gbs_conf_text[:-1]
gbs_conf_text += '\n\n'
#repo section
gbs_conf_text += '\n\n\n#########################################################\n################## Repo Section##################\n#########################################################\n'
+ path_prj_arch_overall_list = list(set(path_prj_arch_overall_list))
#base repo
- for base_prj in base_prj_list:
- profile = 'base'
- t_ver = base_prj['t_ver']
- repository = base_prj['repository']
- gbs_conf_text += '\n############# '+ t_ver + profile + ' #############\n'
- for repository in base_prj['repository']:
- gbs_conf_text += '[repo.'+t_ver+profile+'_'+repository+']\n'
- gbs_conf_text += 'url = '+basic_url+t_ver+profile+'/latest/repos/'+repository+'/packages/\n'
- gbs_conf_text += '[repo.'+t_ver+profile+'_'+repository+'_debug]\n'
- gbs_conf_text += 'url = '+basic_url+t_ver+profile+'/latest/repos/'+repository+'/debug/\n\n'
+ gbs_conf_text += '\n############# base #############\n'
+
+ for path_prj_arch in path_prj_arch_overall_list:
+ base_repo = __get_index_from_path_prj_arch(path_prj_arch)
+ gbs_conf_text += '['+base_repo['repo_name']+']\n'
+ gbs_conf_text += 'url = '+base_repo['repo_url']+'\n'
+ gbs_conf_text += '['+base_repo['debug_repo_name']+']\n'
+ gbs_conf_text += 'url = '+base_repo['debug_repo_url']+'\n\n'
#profile repo
for ref_profile in ref_list:
def _update_scm_manifest(ref_list,base_prj_list,commit_msg,is_staging):
""" Update scm/manifest after ref.binary update"""
print "-----------------------------------------------------"
- print "/n/n/nUpdate scm/manifest after ref.binary update"
+ print "\n\n\nUpdate scm/manifest after ref.binary update"
print "-----------------------------------------------------"
# prepare separate temp directory for each build
print 'git_prj: %s git_branch: %s' %(git_prj, git_branch)
mygit.checkout(git_branch)
+ #_remote.xml
+ remote_file = prjdir + '/_remote.xml'
+ gerrit_hostname = os.getenv('GERRIT_HOSTNAME_EXTERNAL')
+ remote_text = '<?xml version="1.0" encoding="UTF-8"?>\n<manifest>\n <remote name="tizen-gerrit"\n fetch="ssh://'+gerrit_hostname+'/"\n review="https://'+gerrit_hostname+'/gerrit"/>\n</manifest>'
+ with open(remote_file,'wb') as f:
+ f.write(remote_text)
+
for ref_profile in ref_list:
if ref_profile['t_ver'] == base_prj['t_ver']:
profile = ref_profile['profile']
print '%s, branch %s has nothing to commit' %(git_prj,git_branch)
-def main():
- """script entry point"""
-
- print '---[JOB STARTED]----------------------------------------'
-
- #comment out because this job is triggered by gerrit event
- event = get_gerrit_event()
-# event = trigger_info(os.getenv("TRIGGER_INFO"))
-
- # prepare separate temp directory for each build
-
- # In case of this job is triggered by gerrit event, first submit meta files to staging branch
- # Otherwise, submit gbs meta files to original branch
- if 'project' in event and 'event_type' in event:
- print 'update-gbs-meta job is triggered by gerrit event'
- is_staging = '_staging'
- if event['event_type'] != "ref-updated" or event['project'] != "scm/git-ref-mapping":
- # This is just a sanity check as ref-updated is the only event we
- # react on and it's configured in the job configuraion
- print >> sys.stderr, "Configuration error: This job can't process"\
- "project %s! Only scm/git-ref-mapping is allowed and " \
- "event %s! Only ref-updated events are allowed" \
- %(event['event_type'], event['project'])
- return 1
- else:
- print 'update-gbs-meta job is triggered by gbs-ref-fullbuild'
- event = trigger_info(os.getenv("TRIGGER_INFO"))
- is_staging = ''
- print 'is_staging is %s' %is_staging
+def prepare_ref_binary_by_git(event):
+ """ prepare_ref_binary_by_git"""
+ print "----------prepare_ref_binary_by_git start--------"
git_prj = event['project']
git_branch = event['refname']
# clone gerrit project to local dir
if not clone_gitproject(git_prj, prjdir):
- print >> sys.stderr, 'Error cloning %s' %git_prj
+ print >> sys.stderr, 'Error cloning %s' %git_prj
return 1
mygit = Git(prjdir)
mygit.checkout(git_branch)
- commit_msg=mygit.show(mygit.get_commits()[0]).split('\n')[4].replace(" ","")
+ commit_msg=mygit.show(event['newrev']).split('\n')[4].replace(" ","")
ref_binary = {}
base_prj_list = {}
for profile in gbs_meta_default_profile.split(' '):
ref_binary[profile]=profile+':ref:latest'
+ return (ref_binary,commit_msg)
+
+
+def prepare_ref_binary_by_build_id(content):
+ """ prepare_ref_binary_by_build_id"""
+ print "----------prepare_ref_binary_by_build_id start--------"
+ ref_binary = {}
+
+ support_obs_prj = os.getenv('SUPPORT_OBS_PRJ')
+ for obs_prj in support_obs_prj.split(' '):
+ path_prj = obs_prj.replace(':','-').lower()
+ url = basic_url + '/' + path_prj.replace('tizen-','') + '/'
+ print url
+ url_content=urllib2.urlopen(url).read()
+ build_id=0
+ for snapshot_id in url_content.split(path_prj+'_')[1:]:
+ tmp_id=snapshot_id.split('/">')[0]
+ if tmp_id[-1] == '1' and tmp_id.split('.')[0] >= build_id:
+ build_id = tmp_id
+ ref_binary[obs_prj] = obs_prj+':ref:'+build_id
+
+ return (ref_binary,content['build_id'])
+
+
+def main():
+ """script entry point"""
+
+ print '---[JOB STARTED]----------------------------------------'
+
+ #comment out because this job is triggered by gerrit event
+ event = get_gerrit_event()
+# event = trigger_info(os.getenv("TRIGGER_INFO"))
+
+ # prepare separate temp directory for each build
+
+ # In case of this job is triggered by gerrit event, first submit meta files to staging branch
+ # Otherwise, submit gbs meta files to original branch
+ if 'project' in event and 'event_type' in event:
+ print 'update-gbs-meta job is triggered by gerrit event'
+ triggered_by_snapshot_creation = 'false'
+ is_staging = '_staging'
+ if event['event_type'] != "ref-updated" or event['project'] != "scm/git-ref-mapping":
+ # This is just a sanity check as ref-updated is the only event we
+ # react on and it's configured in the job configuraion
+ print >> sys.stderr, "Configuration error: This job can't process"\
+ "project %s! Only scm/git-ref-mapping is allowed and " \
+ "event %s! Only ref-updated events are allowed" \
+ %(event['event_type'], event['project'])
+ return 1
+ else:
+ print 'update-gbs-meta job is triggered by other jenkins job\n\
+ gbs-ref-fullbuild or create-snapshot'
+ content = trigger_info(os.getenv("TRIGGER_INFO"))
+ if content["triggered_from"] == 'gbs_ref_fullbuild':
+ print 'update-gbs-meta job is triggered by gbs-ref-fullbuild'
+ triggered_by_snapshot_creation = 'false'
+ event = content['event']
+ is_staging = ''
+ else:
+ print 'update-gbs-meta job is triggered by create-snapshot'
+ support_obs_prj = os.getenv('SUPPORT_OBS_PRJ')
+ if support_obs_prj.find(content['project']) == -1 or\
+ content['build_id'][-1] != '1':
+ print 'update-gbs-meta is triggered only for first snapshot of %s' %support_obs_prj
+ return 1
+ triggered_by_snapshot_creation = 'true'
+ is_staging = '_staging'
+
+ if triggered_by_snapshot_creation == 'false':
+ (ref_binary,commit_msg) = prepare_ref_binary_by_git(event)
+ else:
+ (ref_binary,commit_msg) = prepare_ref_binary_by_build_id(content)
+
print 'Each reference snapshot numbers are like below'
print ref_binary
ref_list = _update_ref_bin_index(ref_binary)
base_prj_list = _update_base_prj_index()
_update_build_conf(ref_list,commit_msg,is_staging)
- _update_gbs_conf(ref_list,base_prj_list,commit_msg,is_staging)
+ _update_gbs_conf(ref_list,commit_msg,is_staging,is_fullbuild='true')
+ _update_gbs_conf(ref_list,commit_msg,is_staging,is_fullbuild='false')
_update_scm_manifest(ref_list,base_prj_list,commit_msg,is_staging)
if is_staging == '_staging':
projects / services / jenkins-scripts.git / commitdiff