From: Junghyun Kim Date: Thu, 9 Mar 2017 03:45:27 +0000 (+0900) Subject: NEW JOB: update_git_obs_mapping_for_dashboard X-Git-Tag: submit/trunk/20190927.012743~511^2 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=refs%2Fchanges%2F83%2F118283%2F5;p=services%2Fjenkins-scripts.git NEW JOB: update_git_obs_mapping_for_dashboard This job stores git-obs-mapping information by parsing scm/git-obs-mapping for dashboard. Change-Id: I5d1c63219d0b4cea59377b9bde27214f551f572a Signed-off-by: Junghyun Kim --- diff --git a/common/git_obs_mapping.py b/common/git_obs_mapping.py new file mode 100644 index 0000000..572e777 --- /dev/null +++ b/common/git_obs_mapping.py @@ -0,0 +1,219 @@ +import xml.etree.ElementTree as ET +import sys +import os +import re +import json + +#================================================================================ +# class GitObsMapping +#================================================================================ +class GitObsMapping: + + def __init__(self, prjdir): + self.OBS_project = "" + self.mapping = {"path":{}, "project":{}} + self.branches = set() + self.cur_path = "" + self.cur_branches = "" + self.cur_project = "" + self.cur_default = 0 + self.cur_submission = "Y" + + for root,dirs,files in os.walk(prjdir): + for f in files: + if f.endswith('.xml'): + fpath = os.path.join(root,f) + self.parse_tag(fpath) + + def _add_branches(self, branches): + for b in branches.split(','): + self.branches.add(b) + + def _add_mapping(self, proj_or_path, name, branches, attrib): + for b in branches.split(','): + #print "add mapping ", proj_or_path, name, b, attrib + if b not in self.mapping[proj_or_path][name]: + self.mapping[proj_or_path][name][b] = {} + + self.mapping[proj_or_path][name][b][attrib["OBS_project"]] = attrib + + def _parse_tag(self, xml): + ##################################### + # + if xml.tag == "configure": + if not re.search("true", xml.attrib["enable"], re.IGNORECASE): + #print "configure false!" + return 0 + + ##################################### + # + elif xml.tag == "default": + self.cur_default = 1 + + for child in xml: + self._parse_tag(child) + + self.cur_default = 0 + + ##################################### + # + elif xml.tag == "virtual": + pass + + ##################################### + # + elif xml.tag == "path": + path_name = xml.attrib["name"] + + old_submission = self.cur_submission + if "submission" in xml.attrib: + self.cur_submission = xml.attrib["submission"] + + self.cur_path = path_name + if path_name not in self.mapping["path"]: + self.mapping["path"][path_name]={} + self.mapping["path"][path_name]["submission"] = self.cur_submission + + for child in xml: + self._parse_tag(child) + + self.cur_path = "" + self.cur_submission = old_submission + + ##################################### + # + elif xml.tag == "branch": + branches = xml.attrib["name"] + + self.cur_branches = branches + self._add_branches(branches) + + if self.cur_path == "": + if self.cur_project == "": + self.cur_OBS_project = xml.attrib["OBS_project"] + self.cur_OBS_staging_project = xml.attrib["OBS_staging_project"] + + for child in xml: + self._parse_tag(child) + + self.cur_OBS_project = "" + self.cur_OBS_staging_project = "" + else: + attrib = {} + attrib["submission"] = self.cur_submission + attrib["OBS_project"] = xml.attrib["OBS_project"] + attrib["OBS_staging_project"] = xml.attrib["OBS_staging_project"] + if "OBS_package" in xml.attrib: + attrib["OBS_package"] = xml.attrib["OBS_package"] + self._add_mapping("project", self.cur_project, branches, attrib) + + else: + # associated with path + attrib = {} + attrib["submission"] = self.cur_submission + attrib["OBS_project"] = xml.attrib["OBS_project"] + if "OBS_staging_project" in xml.attrib: + attrib["OBS_staging_project"] = xml.attrib["OBS_staging_project"] + self._add_mapping("path", self.cur_path, branches, attrib) + + self.cur_branches = "" + + ##################################### + # + elif xml.tag == "project": + project_name = xml.attrib["name"] + + self.cur_project = project_name + + old_submission = self.cur_submission + if "submission" in xml.attrib: + self.cur_submission = xml.attrib["submission"] + + if project_name not in self.mapping["project"]: + self.mapping["project"][project_name] = {} + self.mapping["project"][project_name]["submission"] = self.cur_submission + + if self.cur_branches == "": + for child in xml: + self._parse_tag(child) + else: + attrib = {} + attrib["submission"] = self.cur_submission + attrib["OBS_project"] = self.cur_OBS_project + attrib["OBS_staging_project"] = self.cur_OBS_staging_project + attrib["OBS_package"] = xml.attrib["OBS_package"] + self._add_mapping("project", project_name, self.cur_branches, attrib) + + self.cur_project = "" + self.cur_submission = old_submission + + ##################################### + # others... + else: + for child in xml: + self._parse_tag(child) + + return 1 + + def get_git_obs_map(self, proj, branch): + return_obj = {} + # first check if there is the same proj + if proj in self.mapping["project"]: + if self.mapping["project"][proj]["submission"] == "Y" and branch in self.mapping["project"][proj]: + return_obj.update(self.mapping["project"][proj][branch]) + + # going up pathes + path = "/"+proj + while path != "/": + path = path[0:path.rfind("/", 0, len(path)-1)+1] + if path in self.mapping["path"]: + if self.mapping["path"][path]["submission"] == "N": + return None + if branch in self.mapping["path"][path]: + return_obj.update(self.mapping["path"][path][branch]) + return return_obj + return None + + return None + + def parse_tag(self, filename): + if not os.path.isfile(filename): + print "No such file: %s\n" % (filename) + exit(0) + mapping = ET.parse(filename).getroot() + + for child in mapping: + if not self._parse_tag(child): + break + + def dump(self): + print self.mapping + + def print_json(self): + print json.dumps(self.mapping) + + def print_to_file(self, filename): + with open(filename, "w") as f: + f.write(json.dumps(self.mapping)) + + def print_pretty(self): + for proj_name in self.mapping: + print proj_name + for proj in self.mapping[proj_name]: + print proj + print self.mapping[proj_name][proj]["submission"] + for b in self.mapping[proj_name][proj]: + print b + print self.mapping[proj_name][proj][b] + + def print_branches(self): + print json.dumps(list(self.branches)) + + def get_branches(self): + return self.branches + + def print_branches_to_file(self, filename): + with open(filename, "w") as f: + f.write(json.dumps(list(self.branches))) + + diff --git a/job_update_git_branch_project_mapping_for_dashboard.py b/job_update_git_branch_project_mapping_for_dashboard.py new file mode 100644 index 0000000..94d0a68 --- /dev/null +++ b/job_update_git_branch_project_mapping_for_dashboard.py @@ -0,0 +1,78 @@ +#!/usr/bin/python +import sys +import os +import re +import json +import subprocess + +from common.utils import sync +from common.git_obs_mapping import GitObsMapping +from common.git import clone_gitproject + +#================================================================================ +# generate_mapping() +#================================================================================ +def generate_mapping(prjdir, target_dir): + mapping = GitObsMapping(prjdir) + + if not os.path.exists(target_dir): + os.makedirs(target_dir) + + branches = mapping.get_branches() + #mapping.print_pretty() + + arg = ' -b '+' -b '.join(branches) + + branch_project_map = {} + + #proj_branch_info = gerrit.ls_projects() + cmd = "ssh -p %s %s gerrit ls-projects %s > proj_list" % (os.getenv('GERRIT_SSHPORT', 29418), + os.getenv('GERRIT_HOSTNAME', "review.tizen.org"), + arg) + subprocess.check_output(cmd, shell=True) + with open("proj_list", "r") as f: + for l in f: + branches_available = l.strip().split(' ') + gerrit_project = branches_available[-1] + i = -1 + for b in branches: + i=i+1 + # skip if no commits for that branch + if branches_available[i][0] == '-': + continue + if b not in branch_project_map: + branch_project_map[b] = [] + branch_project_map[b].append(gerrit_project) + + + final_map = {"branches": list(branches), "branch_projects": branch_project_map} + + target_filename = os.path.join(target_dir, "mapping.json") + + with open(target_filename, "w") as f: + f.write(json.dumps(final_map)) + +########################################################### +# test +#prjdir = os.path.join('.') +#target_dir = ".dashboard/branch_project_mapping" +#generate_mapping(prjdir, target_dir) +#exit(0) + +########################################################### +# main +WORKSPACE = os.getenv('WORKSPACE', '.') +MAPPING_PRJ = os.getenv("MAPPING_PRJ", "scm/git-obs-mapping") + +print "proj = ", MAPPING_PRJ + +prjdir = os.path.join(WORKSPACE, MAPPING_PRJ) + +clone_gitproject(MAPPING_PRJ, prjdir) +target_dir = ".dashboard/branch_project_mapping" + +generate_mapping(prjdir, target_dir) + +sync_dest = os.path.join(os.getenv("IMG_SYNC_DEST_BASE"), "snapshots", target_dir) +# sync to the download server. +sync(target_dir, sync_dest) diff --git a/job_update_git_obs_mapping_for_dashboard.py b/job_update_git_obs_mapping_for_dashboard.py new file mode 100755 index 0000000..4471b29 --- /dev/null +++ b/job_update_git_obs_mapping_for_dashboard.py @@ -0,0 +1,88 @@ +#!/usr/bin/python + +import xml.etree.ElementTree as ET +import sys +import os +import re +import json +import subprocess + +from common.utils import sync +from common.gerrit import Gerrit, get_gerrit_event +from common.git import clone_gitproject +from common.git_obs_mapping import GitObsMapping + +#================================================================================ +# generate_mapping() +#================================================================================ +def generate_mapping(prjdir, target_dir): + mapping = GitObsMapping(prjdir) + + if not os.path.exists(target_dir): + os.makedirs(target_dir) + + #mapping.print_to_file(os.path.join(target_dir, "mapping.json")) + + branches = mapping.get_branches() + #mapping.print_pretty() + + arg = ' -b '+' -b '.join(branches) + + git_obs_mapping_info = {} + + #proj_branch_info = gerrit.ls_projects() + cmd = "ssh -p %s %s gerrit ls-projects %s > proj_list" % (os.getenv('GERRIT_SSHPORT', 29418), + os.getenv('GERRIT_HOSTNAME', "review.tizen.org"), + arg) + subprocess.check_output(cmd, shell=True) + with open("proj_list", "r") as f: + for l in f: + branches_available = l.strip().split(' ') + gerrit_project = branches_available[-1] + git_obs_mapping_info[gerrit_project] = {} + i = -1 + for b in branches: + i=i+1 + if branches_available[i][0] == '-': + continue + m = mapping.get_git_obs_map(gerrit_project, b) + if m is not None: + for p in m: + package = os.path.basename(gerrit_project) + if "OBS_package" in m[p]: + package = m[p]["OBS_package"] + git_obs_mapping_info[gerrit_project][p] = package + + target_filename = os.path.join(target_dir, "mapping.json") + + with open(target_filename, "w") as f: + f.write(json.dumps(git_obs_mapping_info)) + +########################################################### +# test code +#target_dir = ".dashboard/git_obs_mapping" +#generate_mapping('.', target_dir) +#exit(0) + +########################################################### +# main +GERRIT_PROJECT = os.getenv('GERRIT_PROJECT') +WORKSPACE = os.getenv('WORKSPACE') +GERRIT_HOSTNAME = os.getenv('GERRIT_HOSTNAME') +GERRIT_USERNAME = os.getenv('GERRIT_USERNAME') +GERRIT_SSHPORT = os.getenv('GERRIT_SSHPORT') +GERRIT_SILENT_MODE = int(os.getenv('GERRIT_SILENT_MODE')) + +events = get_gerrit_event() +print(events) + +prjdir = os.path.join(WORKSPACE, "git-obs-mapping") + +clone_gitproject(GERRIT_PROJECT, prjdir) +target_dir = ".dashboard/git_obs_mapping" + +generate_mapping(prjdir, target_dir) + +sync_dest = os.path.join(os.getenv("IMG_SYNC_DEST_BASE"), "snapshots", target_dir) +# sync to the download server. +sync(target_dir, sync_dest) diff --git a/packaging/jenkins-scripts.spec b/packaging/jenkins-scripts.spec index be0e60e..f4a9f4e 100644 --- a/packaging/jenkins-scripts.spec +++ b/packaging/jenkins-scripts.spec @@ -125,6 +125,7 @@ fi %{destdir}/common/check_scm_meta_git.py %{destdir}/common/git_diff_parse.py %{destdir}/common/buildmonitor_db.py +%{destdir}/common/git_obs_mapping.py %{destdir}/job_re.py %{destdir}/job_create_snapshot.py %{destdir}/job_release_snapshot.py @@ -141,6 +142,8 @@ fi %{destdir}/job_repa.py %{destdir}/job_trbs_test_result_receiver.py %{destdir}/job_update_scm_meta_git_for_dashboard.py +%{destdir}/job_update_git_obs_mapping_for_dashboard.py +%{destdir}/job_update_git_branch_project_mapping_for_dashboard.py %{destdir}/job_litmus_jira_issue_receiver.py %{destdir}/job_litmus_tct_file_receiver.py %dir %{destdir}/templates