Add feature : Sync for mapping table

Process : remote-create-project -> copy-project-obs-sync-mapping

Change-Id: Ie79c0b78359152f58b5a10b786ca36d5fe6d36fe

diff --git a/common/mapping.py b/common/mapping.py
index e2d6fe2..c9e6c20 100644 (file)
--- a/common/mapping.py
+++ b/common/mapping.py
@@ -221,6 +221,25 @@ class MappingV2(object):
 
         return mapping
 
+    def get_sync_mapping_list(self, project):
+        """Get Sync mapping list"""
+        mapping = []
+
+        if self.mapping_obj.configure:
+            # if configure.enable is false then return []
+            if self.mapping_obj.configure.enable == 'false':
+                return mapping
+
+        if self.mapping_obj.sync:
+            for prjs in self.mapping_obj.sync:
+                if (project and prjs.name != project):
+                    continue
+                for prj in prjs.project:
+                    mapping.append(prj.name)
+            return mapping
+
+        return mapping
+
 def remove_overlaps(orig_list):
     """docstring for make_unique"""
     result = []
@@ -453,3 +472,37 @@ def get_ref_map(gerrit_prj, gerrit_branch=None, gitcache=None, \
 
     return obs_prjs
 
+def get_sync_map_list(obs_project=None, gitcache=None, \
+                gerrit_hostname=None, gerrit_username=None, gerrit_sshport=None):
+    """
+    Find an sync project in git-obs-mapping
+    """
+    
+
+    if gitcache:
+        git_cache = gitcache
+    else:
+        git_cache = os.getenv("GIT_CACHE_DIR")
+
+    mapping_prj = os.getenv("MAPPING_PRJ")
+
+    git_obs_mapping_path = os.path.join(git_cache, mapping_prj)
+
+    if not clone_gitproject(mapping_prj, \
+            os.path.join(git_cache, mapping_prj), \
+            gerrit_hostname=gerrit_hostname, gerrit_username=gerrit_username, gerrit_sshport=gerrit_sshport):
+        raise MappingError('Error cloning %s' % mapping_prj)
+
+    sync_prjs = []
+
+    # get sync mapping files list
+    mapping_path = '{0}/sync/'.format(git_obs_mapping_path)
+    mapping_file_lists = get_xml_file_list(mapping_path)
+
+    # get mappings
+    for file in mapping_file_lists:
+        mymapping = MappingV2(file)
+        sync_prjs.extend(mymapping.get_sync_mapping_list(obs_project))
+
+    return sync_prjs
+

diff --git a/job_ref_create_prj_obs.py b/job_ref_create_prj_obs.py
index 2f06991..8334ba2 100644 (file)
--- a/job_ref_create_prj_obs.py
+++ b/job_ref_create_prj_obs.py
@@ -33,7 +33,7 @@ from common.buildtrigger import trigger_info, trigger_next, remote_jenkins_build
 from gitbuildsys.errors import ObsError
 import xml.etree.ElementTree as ET
 import xml.etree.cElementTree as ElementTree
-from common.mapping import git_obs_map, get_ref_map, git_obs_map_full_list
+from common.mapping import get_ref_map, get_sync_map_list
 from common.gerrit import GerritEnv
 
 class LocalError(Exception):
@@ -522,6 +522,17 @@ class ref_create_project_obs(object):
 
         print 'Profile %s updated to %s' % (this_project, target)
 
+    def update_title(self, build, target, title):
+        """
+        Update title 
+        """
+        target_meta = build.get_meta(target)
+        print 'OK we got target_meta...\n%s' % target_meta
+        target_xml_meta = ElementTree.fromstringlist(target_meta)
+        target_xml_meta.find('title').text = 'Reference from %s' % title
+        print 'set meta started...\n%s' % ElementTree.tostring(target_xml_meta)
+        build.set_meta(ElementTree.tostring(target_xml_meta), target)
+
     def precheck_all_error_package(self, build, target):
         """
         Precheck all error package list from project
@@ -971,6 +982,18 @@ class ref_create_project_obs(object):
         fields['profile'] = profile
         fields['target'] = target
         trigger_next("REF_IMPORT_RPM_OBS", fields)
+        print "\n********"
+        print "  8) Sync Copy Project."
+        if os.getenv('SR_SYNC_PROJECTS'):
+            enabled_projects = ast.literal_eval(os.getenv('SR_SYNC_PROJECTS'))
+            count = 0
+            for project in enabled_projects[profile]:
+                print project
+                if project.get('obs-sync') == 'Yes':
+                    data={}
+                    data['source'] = target
+                    data['target'] = project.get('target')
+                    trigger_next("REF_COPY_SYNC_%s" %(count), data)
 
         if errpackages:
             return 1
@@ -1112,6 +1135,152 @@ class ref_create_project_obs(object):
             return 1
         return 0
 
+    def run_obs_sync_mapping_project_obs(self, obs_api, obs_user, obs_passwd,
+                             remote_obs_api, remote_obs_user, remote_obs_passwd,
+                             gerrit_env, fields=None):
+        """ Sync Mapping project """
+
+        print '---[JOB STARTED]-------------------------'
+
+        remote_meta = ''
+        config = ''
+        remote_package_list = ''
+
+        todo_dict = {}
+
+        print '\nJOB Started at %s' % (str(datetime.now()))
+
+        #### remote buildservice ####
+        remote_build = BuildService(remote_obs_api, remote_obs_user, remote_obs_passwd)
+        #### target buildservice ####
+        build = BuildService(obs_api, obs_user, obs_passwd, \
+                             remote_obs_api, remote_obs_user, remote_obs_passwd)
+
+        src = profile = fields.get('source')
+        target = fields.get('target')
+        replace_gerrit_url = None
+        addPrefix = 'Sync'
+        # get sync list
+        # ex) source : Tizen:4.0:Unified:ref:20170724.2
+        # ex) target : Tizen:4.0:MCD:AI_SPeaker
+        #     Add prefix : Sync
+        sync_prj_list = get_sync_map_list(target)          
+        target = "%s:%s" %(target, addPrefix)
+        
+        if not sync_prj_list:
+            print "\nSKIP Sync Copy for the (%s) !!!\n" %(target)
+            return
+
+        build_src = build
+        print "src = %s , target = %s" %(src, target)
+
+        print "  0) Get meta, config, package list from remote [ %s ]" % (profile)
+        remote_meta = remote_build.get_meta(profile)
+        config = remote_build.get_project_config(profile)
+        remote_package_list = [ p for p in remote_build.get_sourceinfo_list(profile) ]
+        print '\nGet Package List from Remote Done at %s' % (str(datetime.now()))
+        if 'patchinfo' in remote_package_list:
+            remote_package_list.remove('patchinfo')
+            print 'Please check patchinfo'
+
+        print "\n    0-1) copy package list (%d):\n %s" %(len(remote_package_list), remote_package_list)
+        if not build.exists(target):
+            print "\n    0-2) create project: %s" %(target)
+            self.create_project(build, target, None, remote_meta, config)
+            src_todo_dict = self.list_packages_from_remote(build_src, build, src, target)
+            todo_dict = self.list_packages_from_remote(build_src, build, target, target)
+            self.update_title(build_src, target, src)
+        else:
+            print "\n    0-2) project already exists: %s" %(target)
+            src_todo_dict = self.list_packages_from_remote(build_src, build, src, target)
+            todo_dict = self.list_packages_from_remote(build_src, build, target, target, existing=True)
+            self.update_title(build_src, target, src)
+
+        print '\nListing from Remote Done at %s' % (str(datetime.now()))
+        print 'todo_dict(%d):' % (len(todo_dict))
+
+        if True:
+            package_list = [ x for x in todo_dict ]
+            print "\n********"
+            print "  1) package list of target project \n %s" %(package_list)
+
+            packages = self.remove_duplicates(package_list, remote_package_list)
+            print "\n********"
+            print "  2) remove package %s" %(packages)
+            for pkgname in packages:
+                del todo_dict[pkgname]
+                if build.exists(target, pkgname):
+                    print 'Deleting %s' % pkgname
+                    build.delete_package(target, pkgname)
+
+            packages = self.remove_duplicates(remote_package_list, package_list)
+            print "\n********"
+            print "  3) add packages %s" %(packages)
+            sys.stdout.flush()
+            if packages:
+                add_src_packages = {}
+                print 'Checking %s' % packages
+                for x in packages:
+                    add_src_packages[x] = src_todo_dict[x]
+                todo_dict.update(add_src_packages)
+
+            print '\nAdd Remove Done at %s' % (str(datetime.now()))
+            # Compare to Sync mapping
+            print "  4) compare package project in Sync mapping"
+            sync_remove_packages = []
+            for x in todo_dict:
+                if todo_dict[x].get('git', None) not in sync_prj_list:
+                    print 'Git repository is not in Sync mapping! %s' % todo_dict[x].get('git',None)
+                    sync_remove_packages.append(x)
+            for x in sync_remove_packages:
+                del todo_dict[x]
+
+            print '\nCompare With Sync mapping Done at %s' % (str(datetime.now()))
+            print "  4-1) compare package project "
+            todo_dict_merge = copy.deepcopy(todo_dict)
+            cnt = 0
+            for item in todo_dict_merge:
+                if 'git' in todo_dict_merge[item] and todo_dict_merge[item]['git'] \
+                    and item in src_todo_dict:
+                    rev_my = todo_dict_merge[item]['rev']
+                    rev_snapshot = src_todo_dict[item]['rev']
+                    if rev_my != rev_snapshot:
+                        print '  >> DIFFER (%s) -> (%s) %s' % (rev_my, rev_snapshot, item)
+                        todo_dict_merge[item]['rev'] = rev_snapshot
+                        todo_dict_merge[item]['content'] = todo_dict_merge[item]['content'].replace(rev_my, rev_snapshot)
+                        todo_dict_merge[item]['exist'] = False
+                        cnt += 1
+            print 'Differs %d' % cnt
+
+            # Remove packages that are already exists which do not need to update
+            for k, v in todo_dict_merge.items():
+                if 'exist' in v and v['exist'] == True:
+                    del todo_dict_merge[k]
+            todo_dict = todo_dict_merge
+
+            print '\nCompare With project Done at %s' % (str(datetime.now()))
+
+            print '\n  4-2) Final packages to be updated %d' % len(todo_dict)
+            sys.stdout.flush()
+            self.update_packages(remote_build, build, profile, target, todo_dict)
+            print '\nUpdate Packages Done at %s' % (str(datetime.now()))
+
+        print "\n********"
+        print "  5) Precheck all error package list from project"
+        errpackages = self.precheck_all_error_package(build, target)
+
+        print "\n********"
+        print "  6) create related packages \n"
+        self.create_related_packages(remote_build, build, profile, target)
+
+        print "\n********"
+        print "  7) Sync Done..."
+
+        if errpackages:
+            return 1
+        return 0
+
+
     def main(self, action=None):
         """
         main
@@ -1213,6 +1382,28 @@ class ref_create_project_obs(object):
                                                    current_ref, new_ref, snapshot_dir,
                                                    is_the_same_snapshot
                                                    )
+                elif action == 'obs_sync_mapping':
+
+                    remote_obs_api = obs_api = os.getenv("REF_TARGET_OBS_API_URL")
+                    remote_obs_user = obs_user = os.getenv("REF_TARGET_OBS_API_USERNAME")
+                    remote_obs_passwd = obs_passwd = os.getenv("REF_TARGET_OBS_API_PASSWD")
+                    gerrit_env = GerritEnv("")
+
+                    if os.getenv('source') and os.getenv('target'):
+                        fields = {'source': os.getenv('source'),
+                                  'target': os.getenv('target'),
+                                 }
+                    else:
+                        fields = trigger_info(os.getenv('TRIGGER_INFO'))
+                        # Check if we've got required field in TRIGGER_INFO
+                        for field in ('source', 'target'):
+                            if field not in fields:
+                                print 'Error: TRIGGER_INFO doesn\'t contain %s' % field
+                                return -1
+
+                    return self.run_obs_sync_mapping_project_obs(obs_api, obs_user, obs_passwd,
+                                                     remote_obs_api, remote_obs_user, remote_obs_passwd,
+                                                     gerrit_env, fields)
 
                 else:
                     print 'not enable action = %s' %(action)