From c7240f6b813fb0cc6ce8f2b7466047918e434f39 Mon Sep 17 00:00:00 2001 From: Yonghee Han Date: Mon, 5 Jun 2017 14:16:17 +0900 Subject: [PATCH] 1) Add action for create,remote-create,copy in ref_create_prj_obs.py 2) change the trigger data for repo_sync 3) Fixed some error for autoremove jobs of trbs exclude a devel project in autoremove jobs of trbs We have to check the "Tizen:" project 4) Do not send the rsync for devel and test project. Change-Id: I4fc014f1253d9c55a274f1aecd72cb3df0ac8a31 --- job_ref_create_prj_obs.py | 313 ++++++++++++++++++++++++++++++++++------------ job_ref_purge_prj_obs.py | 3 +- job_rsync_download.py | 8 ++ job_sync_repo.py | 2 +- 4 files changed, 244 insertions(+), 82 deletions(-) diff --git a/job_ref_create_prj_obs.py b/job_ref_create_prj_obs.py index 4914b02..7a1e607 100644 --- a/job_ref_create_prj_obs.py +++ b/job_ref_create_prj_obs.py @@ -39,7 +39,10 @@ class LocalError(Exception): class ref_create_project_obs(object): """ create reference project to obs """ - profiles = ast.literal_eval(os.getenv("REF_ACTIVE_PRJS")) + if os.getenv("REF_ACTIVE_PRJS"): + profiles = ast.literal_eval(os.getenv("REF_ACTIVE_PRJS")) + else: + profiles = '' pattern = r'tizen[0-9a-zA-Z_\-\.]*[0-9]{8}.[0-9]{1,2}' def setup_profile(self, project, repo_path=None): @@ -156,6 +159,17 @@ class ref_create_project_obs(object): else: return [results[-1]] + def update_build_flag_project(self, build, target, flag=None): + """ + Update a build flag on project + """ + if flag is None: # None is disable + build.disable_build_flag(target, repo = None, flag="publish", status="disable") + build.disable_build_flag(target, repo = None, flag="build", status="disable") + else: # default + build.default_build_flag(target, repo = None, flag="build") + build.default_build_flag(target, repo = None, flag="publish") + def create_project(self, build, target, info, meta, config, baserefproject=None): """ create project @@ -187,7 +201,8 @@ class ref_create_project_obs(object): #print ElementTree.tostring(xml_meta) #add target person - element = ElementTree.Element('person', {"userid": "%s" % (os.getenv("GERRIT_USERNAME")),"role": "maintainer"}) + element = ElementTree.Element('person', {"userid": "%s" % (os.getenv("OBS_API_USERNAME")), + "role": "maintainer"}) xml_meta.append(element) print ElementTree.tostring(xml_meta) build.set_meta(ElementTree.tostring(xml_meta), target) @@ -199,10 +214,6 @@ class ref_create_project_obs(object): build.disable_build_flag(target, repo = None, flag="publish", status="disable") #disable build flag build.disable_build_flag(target, repo = None, flag="build", status="disable") - #disable debuginfo flag - build.disable_build_flag(target, repo = None, flag="debuginfo", status="disable") - #disable useforbuild flag - build.disable_build_flag(target, repo = None, flag="useforbuild", status="disable") print "\nTarget project %s created" %(target) return True else: @@ -221,7 +232,6 @@ class ref_create_project_obs(object): "" """ ret_dict = {} - sourceinfo = remote_build.get_sourceinfo_list(profile) upload_packages="" if packages: @@ -278,6 +288,8 @@ class ref_create_project_obs(object): _git = _url.split(os.getenv('GIT_CACHE_DIR')+'/')[1] elif len(_url.split(os.getenv('GERRIT_FETCH_URL')+'/')) == 2: _git = _url.split(os.getenv('GIT_CACHE_DIR')+'/')[1] + elif len(_url.split(os.getenv('GERRIT_SSHPORT')+'/')) == 2: + _git = _url.split(os.getenv('GERRIT_SSHPORT')+'/')[1] if param.get('name') == 'revision': _rev = param.text @@ -390,8 +402,6 @@ class ref_create_project_obs(object): build.disable_build_flag(this_project, repo = None, flag='build', status='disable') build.disable_build_flag(this_project, repo = None, flag='publish', status='disable') - build.disable_build_flag(this_project, repo = None, flag="debuginfo", status="disable") - build.disable_build_flag(this_project, repo = None, flag="useforbuild", status="disable") #build.link_project(this_project, src=target, linktype=None) @@ -414,25 +424,56 @@ class ref_create_project_obs(object): print 'Profile %s updated to %s' % (this_project, target) - def run_ref_create_project_obs(self): - - force_trigger = False + def precheck_all_error_package(self, build, target): + """ + Precheck all error package list from project + """ - print '---[JOB STARTED]-------------------------' + need_runservice = [] + for _wait in range(0,30): + sys.stdout.flush() + sleep(10) # Wait 10 seconds... + viewinfofile = build.get_source_viewinfo(target, nofilename=0) + root = ElementTree.parse(viewinfofile).getroot() + errpackages = '' + errpackages = [ s.get('package') for s in root.iter('sourceinfo') if s.findall('error') ] + for x in root.iter('sourceinfo'): + for y in x.findall('filename'): + if '_service:gbs:service-error.spec' in y.text: + errpackages.append(x.get('package')) + break + for y in x.findall('error'): + print x.get('package'),y.text + if 'bad build configuration, no build type' in y.text: + errpackages.remove(x.get('package')) + print errpackages + if errpackages: + print ' 5-1) Under packages are still... (%d)\n %s' % (len(errpackages), errpackages) + # Retrigger git sync + for item in errpackages: + pkgview = ElementTree.fromstring(build.get_source_info(target, item)) + for sv in pkgview.findall('serviceinfo'): + if sv.get('code') != 'failed': continue + for er in sv.findall('error'): + print ' %s %s with cause: (%s)' % (item, sv.get('code'), er.text) + need_runservice.append(item) + sys.stdout.flush() + sleep(30) # Wait 30 seconds... + for item in need_runservice: + print ' runservice for %s' % item + build.runservice(target, item) + need_runservice = [] + else: + print ' 5-2) All packages imported.' + break - obs_api = os.getenv("REF_TARGET_OBS_API_URL") - obs_user = os.getenv("REF_TARGET_OBS_API_USERNAME") - obs_passwd = os.getenv("REF_TARGET_OBS_API_PASSWD") + return errpackages - remote_obs_api = os.getenv("REF_REMOTE_OBS_API_URL") - remote_obs_user = os.getenv("REF_REMOTE_OBS_API_USERNAME") - remote_obs_passwd = os.getenv("REF_REMOTE_OBS_API_PASSWD") + def run_ref_create_project_obs(self, obs_api, obs_user, obs_passwd, + remote_obs_api, remote_obs_user, remote_obs_passwd, + gerrit_env, fields=None): - # default value is null - if os.getenv("REF_GERRIT_NAME"): - gerrit_env = GerritEnv(os.getenv("REF_GERRIT_NAME")) - else: - gerrit_env = GerritEnv("") + print '---[JOB STARTED]-------------------------' remote_meta = '' config = '' @@ -440,16 +481,6 @@ class ref_create_project_obs(object): todo_dict = {} - # precheck profile - fields = '' - fields = trigger_info(os.getenv('TRIGGER_INFO')) - - # Check if we've got required field in TRIGGER_INFO - for field in ('repo_path', 'build_id', 'project'): - if field not in fields: - print 'Error: TRIGGER_INFO doesn\'t contain %s' % field - return -1 - self.profile = self.setup_profile(fields['project'], fields['repo_path']) if not self.profile: print 'Skip Sync OBS project %s' % fields['project'] @@ -537,7 +568,7 @@ class ref_create_project_obs(object): todo_dict = self.list_packages_from_remote(build_src, build, src, target) else: print "\n 0-2) project already exists: %s" %(target) - todo_dict = self.list_packages_from_remote(build_src, build, target, target, existing=True) + todo_dict = self.list_packages_from_remote(build_src, build, src, target, existing=True) print '\nListing from Remote Done at %s' % (str(datetime.now())) print 'todo_dict(%d):' % (len(todo_dict)) @@ -563,7 +594,7 @@ class ref_create_project_obs(object): if ret_dict_add: todo_dict.update(ret_dict_add) print '\nAdd Remove Done at %s' % (str(datetime.now())) - + print "\n********" print " 4) compare package project " manifest_packages = self.get_manifest_filelists_snapshot(self.profile, \ @@ -580,44 +611,97 @@ class ref_create_project_obs(object): print "\n********" print " 5) Precheck all error package list from project" + errpackages = self.precheck_all_error_package(build, target) - need_runservice = [] - for _wait in range(0,30): + print "\n********" + print " 6) create related packages \n" + self.create_related_packages(remote_build, build, profile, target) + + print "\n********" + print " 7) Sync Done..." + + if errpackages: + return 1 + return 0 + + def run_copy_project_obs(self, obs_api, obs_user, obs_passwd, + remote_obs_api, remote_obs_user, remote_obs_passwd, + gerrit_env, fields=None): + """ copy project """ + + print '---[JOB STARTED]-------------------------' + + remote_meta = '' + config = '' + remote_package_list = '' + + todo_dict = {} + + print '\nJOB Started at %s' % (str(datetime.now())) + + #### remote buildservice #### + remote_build = BuildService(remote_obs_api, remote_obs_user, remote_obs_passwd) + #### target buildservice #### + build = BuildService(obs_api, obs_user, obs_passwd, \ + remote_obs_api, remote_obs_user, remote_obs_passwd) + + src = profile = fields['SRC_PROJECT'] + target = fields['DEST_PROJECT'] + + build_src = build + print "src = %s , target = %s" %(src, target) + + print " 0) Get meta, config, package list from remote [ %s ]" % (profile) + remote_meta = remote_build.get_meta(profile) + config = remote_build.get_project_config(profile) + remote_package_list = [ p for p in remote_build.get_sourceinfo_list(profile) ] + print '\nGet Package List from Remote Done at %s' % (str(datetime.now())) + if 'patchinfo' in remote_package_list: + remote_package_list.remove('patchinfo') + print 'Please check patchinfo' + + print "\n 0-1) copy package list (%d):\n %s" %(len(remote_package_list), remote_package_list) + if not build.exists(target): + print "\n 0-2) create project: %s" %(target) + self.create_project(build, target, None, remote_meta, config) + todo_dict = self.list_packages_from_remote(build_src, build, src, target) + else: + print "\n 0-2) project already exists: %s" %(target) + todo_dict = self.list_packages_from_remote(build_src, build, target, target, existing=True) + + print '\nListing from Remote Done at %s' % (str(datetime.now())) + print 'todo_dict(%d):' % (len(todo_dict)) + + if True: + package_list = [ x for x in todo_dict ] + print "\n********" + print " 1) package list of target project \n %s" %(package_list) + + packages = self.remove_duplicates(package_list, remote_package_list) + print "\n********" + print " 2) remove package %s" %(packages) + for pkgname in packages: + del todo_dict[pkgname] + #build.delete_package(target, pkgname) + + packages = self.remove_duplicates(remote_package_list, package_list) + print "\n********" + print " 3) add packages %s" %(packages) + if packages: + ret_dict_add = self.list_packages_from_remote(\ + remote_build, build, profile, target, packages=packages) + if ret_dict_add: + todo_dict.update(ret_dict_add) + print '\nAdd Remove Done at %s' % (str(datetime.now())) + + print '\n 4-1) Final packages to be updated %d' % len(todo_dict) sys.stdout.flush() - sleep(10) # Wait 10 seconds... - viewinfofile = build.get_source_viewinfo(target, nofilename=0) - root = ElementTree.parse(viewinfofile).getroot() - errpackages = '' - errpackages = [ s.get('package') for s in root.iter('sourceinfo') if s.findall('error') ] - for x in root.iter('sourceinfo'): - for y in x.findall('filename'): - if '_service:gbs:service-error.spec' in y.text: - errpackages.append(x.get('package')) - break - for y in x.findall('error'): - print x.get('package'),y.text - if 'bad build configuration, no build type' in y.text: - errpackages.remove(x.get('package')) - print errpackages - if errpackages: - print ' 5-1) Under packages are still... (%d)\n %s' % (len(errpackages), errpackages) - # Retrigger git sync - for item in errpackages: - pkgview = ElementTree.fromstring(build.get_source_info(target, item)) - for sv in pkgview.findall('serviceinfo'): - if sv.get('code') != 'failed': continue - for er in sv.findall('error'): - print ' %s %s with cause: (%s)' % (item, sv.get('code'), er.text) - need_runservice.append(item) - sys.stdout.flush() - sleep(30) # Wait 30 seconds... - for item in need_runservice: - print ' runservice for %s' % item - build.runservice(target, item) - need_runservice = [] - else: - print ' 5-2) All packages imported.' - break + self.update_packages(remote_build, build, profile, target, todo_dict) + print '\nUpdate Packages Done at %s' % (str(datetime.now())) + + print "\n********" + print " 5) Precheck all error package list from project" + errpackages = self.precheck_all_error_package(build, target) print "\n********" print " 6) create related packages \n" @@ -626,23 +710,92 @@ class ref_create_project_obs(object): print "\n********" print " 7) Sync Done..." - # importrpm trigger next - fields['profile'] = profile - fields['target'] = target - trigger_next("REF_IMPORT_RPM_OBS", fields) + self.update_build_flag_project(build, target, flag=True) + print "\n********" + print " 8) Update build flag=default..." if errpackages: return 1 return 0 - def main(self): + def main(self, action=None): + """ + main + """ print('-----[JOB STARTED: ref_create_project_obs ]-----') + for loop in range(1): try: - return self.run_ref_create_project_obs() + if action == 'create' or action is None: + + remote_obs_api = obs_api = os.getenv("REF_TARGET_OBS_API_URL") + remote_obs_user = obs_user = os.getenv("REF_TARGET_OBS_API_USERNAME") + remote_obs_passwd = obs_passwd = os.getenv("REF_TARGET_OBS_API_PASSWD") + gerrit_env = GerritEnv("") + + fields = trigger_info(os.getenv('TRIGGER_INFO')) + # Check if we've got required field in TRIGGER_INFO + for field in ('repo_path', 'build_id', 'project'): + if field not in fields: + print 'Error: TRIGGER_INFO doesn\'t contain %s' % field + return -1 + + return self.run_ref_create_project_obs(obs_api, obs_user, obs_passwd, + remote_obs_api, remote_obs_user, remote_obs_passwd, + gerrit_env, fields) + elif action == 'copy': + + remote_obs_api = obs_api = os.getenv("REF_TARGET_OBS_API_URL") + remote_obs_user = obs_user = os.getenv("REF_TARGET_OBS_API_USERNAME") + remote_obs_passwd = obs_passwd = os.getenv("REF_TARGET_OBS_API_PASSWD") + gerrit_env = GerritEnv("") + + if os.getenv('SRC_PROJECT') and os.getenv('DEST_PROJECT'): + fields = {'SRC_PROJECT': os.getenv('SRC_PROJECT'), + 'DEST_PROJECT': os.getenv('DEST_PROJECT')} + else: + fields = trigger_info(os.getenv('TRIGGER_INFO')) + # Check if we've got required field in TRIGGER_INFO + for field in ('SRC_PROJECT', 'DEST_PROJECT'): + if field not in fields: + print 'Error: TRIGGER_INFO doesn\'t contain %s' % field + return -1 + + return self.run_copy_project_obs(obs_api, obs_user, obs_passwd, + remote_obs_api, remote_obs_user, remote_obs_passwd, + gerrit_env, fields) + elif action == 'remote-create': + + obs_api = os.getenv("REF_TARGET_OBS_API_URL") + obs_user = os.getenv("REF_TARGET_OBS_API_USERNAME") + obs_passwd = os.getenv("REF_TARGET_OBS_API_PASSWD") + + remote_obs_api = os.getenv("REF_REMOTE_OBS_API_URL") + remote_obs_user = os.getenv("REF_REMOTE_OBS_API_USERNAME") + remote_obs_passwd = os.getenv("REF_REMOTE_OBS_API_PASSWD") + # default value is null + if os.getenv("REF_GERRIT_NAME"): + gerrit_env = GerritEnv(os.getenv("REF_GERRIT_NAME")) + else: + gerrit_env = GerritEnv("") + + fields = trigger_info(os.getenv('TRIGGER_INFO')) + # Check if we've got required field in TRIGGER_INFO + for field in ('repo_path', 'build_id', 'project'): + if field not in fields: + print 'Error: TRIGGER_INFO doesn\'t contain %s' % field + return -1 + + return self.run_ref_create_project_obs(obs_api, obs_user, obs_passwd, + remote_obs_api, remote_obs_user, remote_obs_passwd, + gerrit_env, fields) + + else: + print 'not enable action = %s' %(action) + return -1 except Exception, err: - print 'run_ref_create_project_obs operation failed, retrying...' + print 'run_ef_create_project_obs operation failed, retrying...' print err raise LocalError("FAIL %s" % (err)) sleep(5) @@ -653,7 +806,7 @@ if __name__ == '__main__': try: trigger = ref_create_project_obs() - sys.exit(trigger.main()) + sys.exit(trigger.main(sys.argv[1])) except Exception as e: print(e) sys.exit(1) diff --git a/job_ref_purge_prj_obs.py b/job_ref_purge_prj_obs.py index 4e88e9c..9cd9910 100644 --- a/job_ref_purge_prj_obs.py +++ b/job_ref_purge_prj_obs.py @@ -61,7 +61,8 @@ def main(): build = BuildService(obs_api, obs_user, obs_passwd) all_prj_list = [ obj for obj in build.get_package_list("") \ - if not obj.find(":ref:") == -1 \ + if not obj.find("Tizen:") == -1 \ + and not obj.find(":ref:") == -1 \ and obj.find(":Base:") == -1 ] # Limit purge project , remove the project list diff --git a/job_rsync_download.py b/job_rsync_download.py index 98b78ff..532a384 100755 --- a/job_rsync_download.py +++ b/job_rsync_download.py @@ -67,6 +67,14 @@ def main(): print '---[JOB STARTED]-------------------------' content = trigger_info(os.getenv("TRIGGER_INFO")) + try: + if content['project'].split(':')[0] != 'Tizen' and \ + content['project'].split(':')[0] != 'tizen': + print 'Skip Sync OBS project %s' % content['project'] + return + except Exception, err: + print 'Skip Sync OBS project %s' % content['project'] + return # for prerelease and prebuild if content['repo_path'].find(os.getenv("PATH_REPO_BASE")) == 0: diff --git a/job_sync_repo.py b/job_sync_repo.py index c897670..a479988 100755 --- a/job_sync_repo.py +++ b/job_sync_repo.py @@ -266,7 +266,7 @@ def main(): # trigger sync obs job syncobs_data = { "project": project, - "snapshotdir": os.path.join(dest_dir,snapshot_version), + "repo_path": os.path.join(dest_dir,snapshot_version), "build_id": snapshot_version } remote_jenkins_build_job(os.getenv('REMOTE_TRBS_JENKINS_URL'), \ -- 2.7.4