From: Yonghee Han Date: Wed, 3 Jan 2018 02:00:22 +0000 (+0900) Subject: Add feature : Sync for mapping table X-Git-Tag: submit/trunk/20190927.012743~180 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=refs%2Fchanges%2F25%2F165625%2F1;p=services%2Fjenkins-scripts.git Add feature : Sync for mapping table Process : remote-create-project -> copy-project-obs-sync-mapping Change-Id: Ie79c0b78359152f58b5a10b786ca36d5fe6d36fe --- diff --git a/common/mapping.py b/common/mapping.py index e2d6fe2..c9e6c20 100644 --- a/common/mapping.py +++ b/common/mapping.py @@ -221,6 +221,25 @@ class MappingV2(object): return mapping + def get_sync_mapping_list(self, project): + """Get Sync mapping list""" + mapping = [] + + if self.mapping_obj.configure: + # if configure.enable is false then return [] + if self.mapping_obj.configure.enable == 'false': + return mapping + + if self.mapping_obj.sync: + for prjs in self.mapping_obj.sync: + if (project and prjs.name != project): + continue + for prj in prjs.project: + mapping.append(prj.name) + return mapping + + return mapping + def remove_overlaps(orig_list): """docstring for make_unique""" result = [] @@ -453,3 +472,37 @@ def get_ref_map(gerrit_prj, gerrit_branch=None, gitcache=None, \ return obs_prjs +def get_sync_map_list(obs_project=None, gitcache=None, \ + gerrit_hostname=None, gerrit_username=None, gerrit_sshport=None): + """ + Find an sync project in git-obs-mapping + """ + + + if gitcache: + git_cache = gitcache + else: + git_cache = os.getenv("GIT_CACHE_DIR") + + mapping_prj = os.getenv("MAPPING_PRJ") + + git_obs_mapping_path = os.path.join(git_cache, mapping_prj) + + if not clone_gitproject(mapping_prj, \ + os.path.join(git_cache, mapping_prj), \ + gerrit_hostname=gerrit_hostname, gerrit_username=gerrit_username, gerrit_sshport=gerrit_sshport): + raise MappingError('Error cloning %s' % mapping_prj) + + sync_prjs = [] + + # get sync mapping files list + mapping_path = '{0}/sync/'.format(git_obs_mapping_path) + mapping_file_lists = get_xml_file_list(mapping_path) + + # get mappings + for file in mapping_file_lists: + mymapping = MappingV2(file) + sync_prjs.extend(mymapping.get_sync_mapping_list(obs_project)) + + return sync_prjs + diff --git a/job_ref_create_prj_obs.py b/job_ref_create_prj_obs.py index 2f06991..8334ba2 100644 --- a/job_ref_create_prj_obs.py +++ b/job_ref_create_prj_obs.py @@ -33,7 +33,7 @@ from common.buildtrigger import trigger_info, trigger_next, remote_jenkins_build from gitbuildsys.errors import ObsError import xml.etree.ElementTree as ET import xml.etree.cElementTree as ElementTree -from common.mapping import git_obs_map, get_ref_map, git_obs_map_full_list +from common.mapping import get_ref_map, get_sync_map_list from common.gerrit import GerritEnv class LocalError(Exception): @@ -522,6 +522,17 @@ class ref_create_project_obs(object): print 'Profile %s updated to %s' % (this_project, target) + def update_title(self, build, target, title): + """ + Update title + """ + target_meta = build.get_meta(target) + print 'OK we got target_meta...\n%s' % target_meta + target_xml_meta = ElementTree.fromstringlist(target_meta) + target_xml_meta.find('title').text = 'Reference from %s' % title + print 'set meta started...\n%s' % ElementTree.tostring(target_xml_meta) + build.set_meta(ElementTree.tostring(target_xml_meta), target) + def precheck_all_error_package(self, build, target): """ Precheck all error package list from project @@ -971,6 +982,18 @@ class ref_create_project_obs(object): fields['profile'] = profile fields['target'] = target trigger_next("REF_IMPORT_RPM_OBS", fields) + print "\n********" + print " 8) Sync Copy Project." + if os.getenv('SR_SYNC_PROJECTS'): + enabled_projects = ast.literal_eval(os.getenv('SR_SYNC_PROJECTS')) + count = 0 + for project in enabled_projects[profile]: + print project + if project.get('obs-sync') == 'Yes': + data={} + data['source'] = target + data['target'] = project.get('target') + trigger_next("REF_COPY_SYNC_%s" %(count), data) if errpackages: return 1 @@ -1112,6 +1135,152 @@ class ref_create_project_obs(object): return 1 return 0 + def run_obs_sync_mapping_project_obs(self, obs_api, obs_user, obs_passwd, + remote_obs_api, remote_obs_user, remote_obs_passwd, + gerrit_env, fields=None): + """ Sync Mapping project """ + + print '---[JOB STARTED]-------------------------' + + remote_meta = '' + config = '' + remote_package_list = '' + + todo_dict = {} + + print '\nJOB Started at %s' % (str(datetime.now())) + + #### remote buildservice #### + remote_build = BuildService(remote_obs_api, remote_obs_user, remote_obs_passwd) + #### target buildservice #### + build = BuildService(obs_api, obs_user, obs_passwd, \ + remote_obs_api, remote_obs_user, remote_obs_passwd) + + src = profile = fields.get('source') + target = fields.get('target') + replace_gerrit_url = None + addPrefix = 'Sync' + # get sync list + # ex) source : Tizen:4.0:Unified:ref:20170724.2 + # ex) target : Tizen:4.0:MCD:AI_SPeaker + # Add prefix : Sync + sync_prj_list = get_sync_map_list(target) + target = "%s:%s" %(target, addPrefix) + + if not sync_prj_list: + print "\nSKIP Sync Copy for the (%s) !!!\n" %(target) + return + + build_src = build + print "src = %s , target = %s" %(src, target) + + print " 0) Get meta, config, package list from remote [ %s ]" % (profile) + remote_meta = remote_build.get_meta(profile) + config = remote_build.get_project_config(profile) + remote_package_list = [ p for p in remote_build.get_sourceinfo_list(profile) ] + print '\nGet Package List from Remote Done at %s' % (str(datetime.now())) + if 'patchinfo' in remote_package_list: + remote_package_list.remove('patchinfo') + print 'Please check patchinfo' + + print "\n 0-1) copy package list (%d):\n %s" %(len(remote_package_list), remote_package_list) + if not build.exists(target): + print "\n 0-2) create project: %s" %(target) + self.create_project(build, target, None, remote_meta, config) + src_todo_dict = self.list_packages_from_remote(build_src, build, src, target) + todo_dict = self.list_packages_from_remote(build_src, build, target, target) + self.update_title(build_src, target, src) + else: + print "\n 0-2) project already exists: %s" %(target) + src_todo_dict = self.list_packages_from_remote(build_src, build, src, target) + todo_dict = self.list_packages_from_remote(build_src, build, target, target, existing=True) + self.update_title(build_src, target, src) + + print '\nListing from Remote Done at %s' % (str(datetime.now())) + print 'todo_dict(%d):' % (len(todo_dict)) + + if True: + package_list = [ x for x in todo_dict ] + print "\n********" + print " 1) package list of target project \n %s" %(package_list) + + packages = self.remove_duplicates(package_list, remote_package_list) + print "\n********" + print " 2) remove package %s" %(packages) + for pkgname in packages: + del todo_dict[pkgname] + if build.exists(target, pkgname): + print 'Deleting %s' % pkgname + build.delete_package(target, pkgname) + + packages = self.remove_duplicates(remote_package_list, package_list) + print "\n********" + print " 3) add packages %s" %(packages) + sys.stdout.flush() + if packages: + add_src_packages = {} + print 'Checking %s' % packages + for x in packages: + add_src_packages[x] = src_todo_dict[x] + todo_dict.update(add_src_packages) + + print '\nAdd Remove Done at %s' % (str(datetime.now())) + # Compare to Sync mapping + print " 4) compare package project in Sync mapping" + sync_remove_packages = [] + for x in todo_dict: + if todo_dict[x].get('git', None) not in sync_prj_list: + print 'Git repository is not in Sync mapping! %s' % todo_dict[x].get('git',None) + sync_remove_packages.append(x) + for x in sync_remove_packages: + del todo_dict[x] + + print '\nCompare With Sync mapping Done at %s' % (str(datetime.now())) + print " 4-1) compare package project " + todo_dict_merge = copy.deepcopy(todo_dict) + cnt = 0 + for item in todo_dict_merge: + if 'git' in todo_dict_merge[item] and todo_dict_merge[item]['git'] \ + and item in src_todo_dict: + rev_my = todo_dict_merge[item]['rev'] + rev_snapshot = src_todo_dict[item]['rev'] + if rev_my != rev_snapshot: + print ' >> DIFFER (%s) -> (%s) %s' % (rev_my, rev_snapshot, item) + todo_dict_merge[item]['rev'] = rev_snapshot + todo_dict_merge[item]['content'] = todo_dict_merge[item]['content'].replace(rev_my, rev_snapshot) + todo_dict_merge[item]['exist'] = False + cnt += 1 + print 'Differs %d' % cnt + + # Remove packages that are already exists which do not need to update + for k, v in todo_dict_merge.items(): + if 'exist' in v and v['exist'] == True: + del todo_dict_merge[k] + todo_dict = todo_dict_merge + + print '\nCompare With project Done at %s' % (str(datetime.now())) + + print '\n 4-2) Final packages to be updated %d' % len(todo_dict) + sys.stdout.flush() + self.update_packages(remote_build, build, profile, target, todo_dict) + print '\nUpdate Packages Done at %s' % (str(datetime.now())) + + print "\n********" + print " 5) Precheck all error package list from project" + errpackages = self.precheck_all_error_package(build, target) + + print "\n********" + print " 6) create related packages \n" + self.create_related_packages(remote_build, build, profile, target) + + print "\n********" + print " 7) Sync Done..." + + if errpackages: + return 1 + return 0 + + def main(self, action=None): """ main @@ -1213,6 +1382,28 @@ class ref_create_project_obs(object): current_ref, new_ref, snapshot_dir, is_the_same_snapshot ) + elif action == 'obs_sync_mapping': + + remote_obs_api = obs_api = os.getenv("REF_TARGET_OBS_API_URL") + remote_obs_user = obs_user = os.getenv("REF_TARGET_OBS_API_USERNAME") + remote_obs_passwd = obs_passwd = os.getenv("REF_TARGET_OBS_API_PASSWD") + gerrit_env = GerritEnv("") + + if os.getenv('source') and os.getenv('target'): + fields = {'source': os.getenv('source'), + 'target': os.getenv('target'), + } + else: + fields = trigger_info(os.getenv('TRIGGER_INFO')) + # Check if we've got required field in TRIGGER_INFO + for field in ('source', 'target'): + if field not in fields: + print 'Error: TRIGGER_INFO doesn\'t contain %s' % field + return -1 + + return self.run_obs_sync_mapping_project_obs(obs_api, obs_user, obs_passwd, + remote_obs_api, remote_obs_user, remote_obs_passwd, + gerrit_env, fields) else: print 'not enable action = %s' %(action)