NEW JOB: update_git_obs_mapping_for_dashboard

This job stores git-obs-mapping information by parsing
scm/git-obs-mapping for dashboard.

Change-Id: I5d1c63219d0b4cea59377b9bde27214f551f572a
Signed-off-by: Junghyun Kim <jh0822.kim@samsung.com>

diff --git a/common/git_obs_mapping.py b/common/git_obs_mapping.py
new file mode 100644 (file)
index 0000000..572e777
--- /dev/null
+++ b/common/git_obs_mapping.py
@@ -0,0 +1,219 @@
+import xml.etree.ElementTree as ET
+import sys
+import os
+import re
+import json
+
+#================================================================================
+# class GitObsMapping
+#================================================================================
+class GitObsMapping:
+
+    def __init__(self, prjdir):
+        self.OBS_project = ""
+        self.mapping = {"path":{}, "project":{}}
+        self.branches = set()
+        self.cur_path = ""
+        self.cur_branches = ""
+        self.cur_project = ""
+        self.cur_default = 0
+        self.cur_submission = "Y"
+
+        for root,dirs,files in os.walk(prjdir):
+            for f in files:
+                if f.endswith('.xml'):
+                    fpath = os.path.join(root,f)
+                    self.parse_tag(fpath)
+
+    def _add_branches(self, branches):
+        for b in branches.split(','):
+            self.branches.add(b)
+
+    def _add_mapping(self, proj_or_path, name, branches, attrib):
+        for b in branches.split(','):
+            #print "add mapping ", proj_or_path, name, b, attrib
+            if b not in self.mapping[proj_or_path][name]:
+                self.mapping[proj_or_path][name][b] = {}
+
+            self.mapping[proj_or_path][name][b][attrib["OBS_project"]] = attrib
+
+    def _parse_tag(self, xml):
+        #####################################
+        # <configure>
+        if xml.tag == "configure":
+            if not re.search("true", xml.attrib["enable"], re.IGNORECASE):
+                #print "configure false!"
+                return 0
+
+        #####################################
+        # <default>
+        elif xml.tag == "default":
+            self.cur_default = 1
+
+            for child in xml:
+                self._parse_tag(child)
+
+            self.cur_default = 0
+
+        #####################################
+        # <virtual>
+        elif xml.tag == "virtual":
+            pass
+
+        #####################################
+        # <path>
+        elif xml.tag == "path":
+            path_name = xml.attrib["name"]
+
+            old_submission = self.cur_submission
+            if "submission" in xml.attrib:
+                self.cur_submission = xml.attrib["submission"]
+
+            self.cur_path = path_name
+            if path_name not in self.mapping["path"]:
+                self.mapping["path"][path_name]={}
+                self.mapping["path"][path_name]["submission"] = self.cur_submission
+
+            for child in xml:
+                self._parse_tag(child)
+
+            self.cur_path = ""
+            self.cur_submission = old_submission
+
+        #####################################
+        # <branch>
+        elif xml.tag == "branch":
+            branches = xml.attrib["name"]
+
+            self.cur_branches = branches
+            self._add_branches(branches)
+
+            if self.cur_path == "":
+                if self.cur_project == "":
+                    self.cur_OBS_project = xml.attrib["OBS_project"]
+                    self.cur_OBS_staging_project = xml.attrib["OBS_staging_project"]
+
+                    for child in xml:
+                        self._parse_tag(child)
+
+                    self.cur_OBS_project = ""
+                    self.cur_OBS_staging_project = ""
+                else:
+                    attrib = {}
+                    attrib["submission"] = self.cur_submission
+                    attrib["OBS_project"] = xml.attrib["OBS_project"]
+                    attrib["OBS_staging_project"] = xml.attrib["OBS_staging_project"]
+                    if "OBS_package" in xml.attrib:
+                        attrib["OBS_package"] = xml.attrib["OBS_package"]
+                    self._add_mapping("project", self.cur_project, branches, attrib)
+
+            else:
+                # associated with path
+                attrib = {}
+                attrib["submission"] = self.cur_submission
+                attrib["OBS_project"] = xml.attrib["OBS_project"]
+                if "OBS_staging_project" in xml.attrib:
+                    attrib["OBS_staging_project"] = xml.attrib["OBS_staging_project"]
+                self._add_mapping("path", self.cur_path, branches, attrib)
+
+            self.cur_branches = ""
+
+        #####################################
+        # <project>
+        elif xml.tag == "project":
+            project_name = xml.attrib["name"]
+
+            self.cur_project = project_name
+
+            old_submission = self.cur_submission
+            if "submission" in xml.attrib:
+                self.cur_submission = xml.attrib["submission"]
+
+            if project_name not in self.mapping["project"]:
+                self.mapping["project"][project_name] = {}
+                self.mapping["project"][project_name]["submission"] = self.cur_submission
+
+            if self.cur_branches == "":
+                for child in xml:
+                    self._parse_tag(child)
+            else:
+                attrib = {}
+                attrib["submission"] = self.cur_submission
+                attrib["OBS_project"] = self.cur_OBS_project
+                attrib["OBS_staging_project"] = self.cur_OBS_staging_project
+                attrib["OBS_package"] = xml.attrib["OBS_package"]
+                self._add_mapping("project", project_name, self.cur_branches, attrib)
+
+            self.cur_project = ""
+            self.cur_submission = old_submission
+
+        #####################################
+        # others...
+        else:
+            for child in xml:
+                self._parse_tag(child)
+
+        return 1
+
+    def get_git_obs_map(self, proj, branch):
+        return_obj = {}
+        # first check if there is the same proj
+        if proj in self.mapping["project"]:
+            if self.mapping["project"][proj]["submission"] == "Y" and branch in self.mapping["project"][proj]:
+                return_obj.update(self.mapping["project"][proj][branch])
+
+        # going up pathes
+        path = "/"+proj
+        while path != "/":
+            path = path[0:path.rfind("/", 0, len(path)-1)+1]
+            if path in self.mapping["path"]:
+                if self.mapping["path"][path]["submission"] == "N":
+                    return None
+                if branch in self.mapping["path"][path]:
+                    return_obj.update(self.mapping["path"][path][branch])
+                    return return_obj
+                return None
+
+        return None
+
+    def parse_tag(self, filename):
+        if not os.path.isfile(filename):
+            print "No such file: %s\n" % (filename)
+            exit(0)
+        mapping = ET.parse(filename).getroot()
+
+        for child in mapping:
+            if not self._parse_tag(child):
+                break
+
+    def dump(self):
+        print self.mapping
+
+    def print_json(self):
+        print json.dumps(self.mapping)
+
+    def print_to_file(self, filename):
+        with open(filename, "w") as f:
+            f.write(json.dumps(self.mapping))
+
+    def print_pretty(self):
+        for proj_name in self.mapping:
+            print proj_name
+            for proj in self.mapping[proj_name]:
+                print proj
+                print self.mapping[proj_name][proj]["submission"]
+                for b in self.mapping[proj_name][proj]:
+                    print b
+                    print self.mapping[proj_name][proj][b]
+
+    def print_branches(self):
+        print json.dumps(list(self.branches))
+
+    def get_branches(self):
+        return self.branches
+
+    def print_branches_to_file(self, filename):
+        with open(filename, "w") as f:
+            f.write(json.dumps(list(self.branches)))
+
+

diff --git a/job_update_git_branch_project_mapping_for_dashboard.py b/job_update_git_branch_project_mapping_for_dashboard.py
new file mode 100644 (file)
index 0000000..94d0a68
--- /dev/null
+++ b/job_update_git_branch_project_mapping_for_dashboard.py
@@ -0,0 +1,78 @@
+#!/usr/bin/python
+import sys
+import os
+import re
+import json
+import subprocess
+
+from common.utils import sync
+from common.git_obs_mapping import GitObsMapping
+from common.git import clone_gitproject
+
+#================================================================================
+# generate_mapping()
+#================================================================================
+def generate_mapping(prjdir, target_dir):
+    mapping = GitObsMapping(prjdir)
+
+    if not os.path.exists(target_dir):
+        os.makedirs(target_dir)
+
+    branches = mapping.get_branches()
+    #mapping.print_pretty()
+
+    arg = ' -b '+' -b '.join(branches)
+
+    branch_project_map = {}
+
+    #proj_branch_info = gerrit.ls_projects()
+    cmd = "ssh -p %s %s gerrit ls-projects %s > proj_list" % (os.getenv('GERRIT_SSHPORT', 29418),
+                                                              os.getenv('GERRIT_HOSTNAME', "review.tizen.org"),
+                                                              arg)
+    subprocess.check_output(cmd, shell=True)
+    with open("proj_list", "r") as f:
+        for l in f:
+            branches_available = l.strip().split(' ')
+            gerrit_project = branches_available[-1]
+            i = -1
+            for b in branches:
+                i=i+1
+                # skip if no commits for that branch
+                if branches_available[i][0] == '-':
+                    continue
+                if b not in branch_project_map:
+                    branch_project_map[b] = []
+                branch_project_map[b].append(gerrit_project)
+
+
+    final_map = {"branches": list(branches), "branch_projects": branch_project_map}
+
+    target_filename = os.path.join(target_dir, "mapping.json")
+
+    with open(target_filename, "w") as f:
+        f.write(json.dumps(final_map))
+
+###########################################################
+# test
+#prjdir = os.path.join('.')
+#target_dir = ".dashboard/branch_project_mapping"
+#generate_mapping(prjdir, target_dir)
+#exit(0)
+
+###########################################################
+# main
+WORKSPACE = os.getenv('WORKSPACE', '.')
+MAPPING_PRJ = os.getenv("MAPPING_PRJ", "scm/git-obs-mapping")
+
+print "proj = ", MAPPING_PRJ
+
+prjdir = os.path.join(WORKSPACE, MAPPING_PRJ)
+
+clone_gitproject(MAPPING_PRJ, prjdir)
+target_dir = ".dashboard/branch_project_mapping"
+
+generate_mapping(prjdir, target_dir)
+
+sync_dest = os.path.join(os.getenv("IMG_SYNC_DEST_BASE"), "snapshots", target_dir)
+# sync to the download server.
+sync(target_dir, sync_dest)

diff --git a/job_update_git_obs_mapping_for_dashboard.py b/job_update_git_obs_mapping_for_dashboard.py
new file mode 100755 (executable)
index 0000000..4471b29
--- /dev/null
+++ b/job_update_git_obs_mapping_for_dashboard.py
@@ -0,0 +1,88 @@
+#!/usr/bin/python
+
+import xml.etree.ElementTree as ET
+import sys
+import os
+import re
+import json
+import subprocess
+
+from common.utils import sync
+from common.gerrit import Gerrit, get_gerrit_event
+from common.git import clone_gitproject
+from common.git_obs_mapping import GitObsMapping
+
+#================================================================================
+# generate_mapping()
+#================================================================================
+def generate_mapping(prjdir, target_dir):
+    mapping = GitObsMapping(prjdir)
+
+    if not os.path.exists(target_dir):
+        os.makedirs(target_dir)
+
+    #mapping.print_to_file(os.path.join(target_dir, "mapping.json"))
+
+    branches = mapping.get_branches()
+    #mapping.print_pretty()
+
+    arg = ' -b '+' -b '.join(branches)
+
+    git_obs_mapping_info = {}
+
+    #proj_branch_info = gerrit.ls_projects()
+    cmd = "ssh -p %s %s gerrit ls-projects %s > proj_list" % (os.getenv('GERRIT_SSHPORT', 29418),
+                                                              os.getenv('GERRIT_HOSTNAME', "review.tizen.org"),
+                                                              arg)
+    subprocess.check_output(cmd, shell=True)
+    with open("proj_list", "r") as f:
+        for l in f:
+            branches_available = l.strip().split(' ')
+            gerrit_project = branches_available[-1]
+            git_obs_mapping_info[gerrit_project] = {}
+            i = -1
+            for b in branches:
+                i=i+1
+                if branches_available[i][0] == '-':
+                    continue
+                m = mapping.get_git_obs_map(gerrit_project, b)
+                if m is not None:
+                    for p in m:
+                        package = os.path.basename(gerrit_project)
+                        if "OBS_package" in m[p]:
+                            package = m[p]["OBS_package"]
+                        git_obs_mapping_info[gerrit_project][p] = package
+
+    target_filename = os.path.join(target_dir, "mapping.json")
+
+    with open(target_filename, "w") as f:
+        f.write(json.dumps(git_obs_mapping_info))
+
+###########################################################
+# test code
+#target_dir = ".dashboard/git_obs_mapping"
+#generate_mapping('.', target_dir)
+#exit(0)
+
+###########################################################
+# main
+GERRIT_PROJECT = os.getenv('GERRIT_PROJECT')
+WORKSPACE = os.getenv('WORKSPACE')
+GERRIT_HOSTNAME = os.getenv('GERRIT_HOSTNAME')
+GERRIT_USERNAME = os.getenv('GERRIT_USERNAME')
+GERRIT_SSHPORT = os.getenv('GERRIT_SSHPORT')
+GERRIT_SILENT_MODE = int(os.getenv('GERRIT_SILENT_MODE'))
+
+events = get_gerrit_event()
+print(events)
+
+prjdir = os.path.join(WORKSPACE, "git-obs-mapping")
+
+clone_gitproject(GERRIT_PROJECT, prjdir)
+target_dir = ".dashboard/git_obs_mapping"
+
+generate_mapping(prjdir, target_dir)
+
+sync_dest = os.path.join(os.getenv("IMG_SYNC_DEST_BASE"), "snapshots", target_dir)
+# sync to the download server.
+sync(target_dir, sync_dest)

diff --git a/packaging/jenkins-scripts.spec b/packaging/jenkins-scripts.spec
index be0e60e..f4a9f4e 100644 (file)
--- a/packaging/jenkins-scripts.spec
+++ b/packaging/jenkins-scripts.spec
@@ -125,6 +125,7 @@ fi
 %{destdir}/common/check_scm_meta_git.py
 %{destdir}/common/git_diff_parse.py
 %{destdir}/common/buildmonitor_db.py
+%{destdir}/common/git_obs_mapping.py
 %{destdir}/job_re.py
 %{destdir}/job_create_snapshot.py
 %{destdir}/job_release_snapshot.py
@@ -141,6 +142,8 @@ fi
 %{destdir}/job_repa.py
 %{destdir}/job_trbs_test_result_receiver.py
 %{destdir}/job_update_scm_meta_git_for_dashboard.py
+%{destdir}/job_update_git_obs_mapping_for_dashboard.py
+%{destdir}/job_update_git_branch_project_mapping_for_dashboard.py
 %{destdir}/job_litmus_jira_issue_receiver.py
 %{destdir}/job_litmus_tct_file_receiver.py
 %dir %{destdir}/templates