From d6891193df0108574d266d7513ddd42767240678 Mon Sep 17 00:00:00 2001 From: Junghyun Kim Date: Thu, 22 Jun 2017 10:52:24 +0900 Subject: [PATCH] Alternative implementation of git-domain mapping PROBLEM: SPIN gerrit does not maintain scm/meta/git repository. So, it is not possible to git-domain-user mapping using scm/meta/git SOLUTION: Gather data directly from gerrit. Change-Id: I8513b5a0bd8255fb9e4741061536d48574a78293 Signed-off-by: Junghyun Kim --- debian/jenkins-scripts.install | 2 +- job_git_domain.py | 225 +++++++++++++++++++++++++++++++ job_update_scm_meta_git_for_dashboard.py | 180 ------------------------- packaging/jenkins-scripts.spec | 2 +- 4 files changed, 227 insertions(+), 182 deletions(-) create mode 100644 job_git_domain.py delete mode 100644 job_update_scm_meta_git_for_dashboard.py diff --git a/debian/jenkins-scripts.install b/debian/jenkins-scripts.install index 9a255c1..ab438e3 100644 --- a/debian/jenkins-scripts.install +++ b/debian/jenkins-scripts.install @@ -17,7 +17,7 @@ debian/tmp/job_ref_import_rpm_obs.py /var/lib/jenkins/jenkins-scripts/ debian/tmp/job_ref_purge_prj_obs.py /var/lib/jenkins/jenkins-scripts/ debian/tmp/job_ref_precheck_project_obs.py /var/lib/jenkins/jenkins-scripts/ debian/tmp/job_test_trigger_info_update.py /var/lib/jenkins/jenkins-scripts/ -debian/tmp/job_update_scm_meta_git_for_dashboard.py /var/lib/jenkins/jenkins-scripts/ +debian/tmp/job_git_domain.py /var/lib/jenkins/jenkins-scripts/ debian/tmp/job_update_git_obs_mapping_for_dashboard.py /var/lib/jenkins/jenkins-scripts/ debian/tmp/job_update_git_branch_project_mapping_for_dashboard.py /var/lib/jenkins/jenkins-scripts/ debian/tmp/job_litmus_jira_issue_receiver.py /var/lib/jenkins/jenkins-scripts/ diff --git a/job_git_domain.py b/job_git_domain.py new file mode 100644 index 0000000..68a3afd --- /dev/null +++ b/job_git_domain.py @@ -0,0 +1,225 @@ +#!/usr/bin/env python +# vim: ai ts=4 sts=4 et sw=4 +# +# Copyright (C) 2016 Samsung +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; version 2 +# of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# + +import subprocess +import os +import json + +from common.utils import sync + +#================================================================================ +# get_domain_group +#================================================================================ +def get_domain_group(g): + sp = g.split(' - ') + if len(sp) != 2: + return '', '', '' + + domain = sp[0]; + subdomain = "" + group = sp[1]; + + sp = domain.split(' / ') + if len(sp) == 2: + domain = sp[0] + subdomain = sp[1] + + return domain, subdomain, group + +#================================================================================ +# get_groups +#================================================================================ +def get_groups(): + cmd = "ssh -p %s %s gerrit ls-groups" % \ + (os.getenv('GERRIT_SSHPORT', 29418), os.getenv('GERRIT_HOSTNAME', "review.tizen.org")) + popen = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + (stdoutdata, stderrdata) = popen.communicate() + + return stdoutdata.strip().split('\n') + +#================================================================================ +# get_projects +#================================================================================ +def get_projects(): + cmd = "ssh -p %s %s gerrit ls-projects --format json -t -b master -b refs/meta/config " % \ + (os.getenv('GERRIT_SSHPORT', 29418), os.getenv('GERRIT_HOSTNAME', "review.tizen.org")) + popen = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + (stdoutdata, stderrdata) = popen.communicate() + + return json.loads(stdoutdata) + +#================================================================================ +# get_group_members() +#================================================================================ +def get_group_members(g): + g = g.replace(' ', '\ ') + cmd = "ssh -p %s %s gerrit ls-members \"%s\""%\ + (os.getenv('GERRIT_SSHPORT', 29418), os.getenv('GERRIT_HOSTNAME', "review.tizen.org"), g) + popen = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + (stdoutdata, stderrdata) = popen.communicate() + + members = [] + for l in stdoutdata.strip().split('\n'): + sp = l.strip().split('\t') + if sp[3] != "email": + members.append(sp[3]) + return members + +#================================================================================ +# make_acl_to_domain() +#================================================================================ +def make_acl_to_domain(domain_user_map): + acl_to_domain = {} + for domain in domain_user_map: + dc = domain.replace("&","and").replace(" ","_").lower() + domain_converted = "scm/acls/domain_"+dc + acl_to_domain[domain_converted] = (domain, '') + for subdomain in domain_user_map[domain]: + sdc = subdomain.replace("&","and").replace(" ","_").lower() + domain_converted = "scm/acls/domain_"+dc+"/"+sdc + acl_to_domain[domain_converted] = (domain, subdomain) + return acl_to_domain + +#================================================================================ +# generate_domain_user_map() +#================================================================================ +def generate_domain_user_map(): + domain_user_map = {} + git_user_map = {} + git_domain_map = {} + + groups = get_groups() + + for g in groups: + domain, subdomain, group = get_domain_group(g) + + if len(domain) == 0: + continue + members = get_group_members(g) + + if domain not in domain_user_map: + domain_user_map[domain] = {} + if subdomain not in domain_user_map[domain]: + domain_user_map[domain][subdomain] = {} + if group not in domain_user_map[domain][subdomain]: + domain_user_map[domain][subdomain][group] = [] + + domain_user_map[domain][subdomain][group].extend(members) + + acl_to_domain = make_acl_to_domain(domain_user_map) + + j = get_projects() + for p in j: + try: + (domain, subdomain) = acl_to_domain[j[p]["parent"]] + if "projects" not in domain_user_map[domain][subdomain]: + domain_user_map[domain][subdomain]["projects"] = [] + domain_user_map[domain][subdomain]["projects"].append(p) + + for g in domain_user_map[domain][subdomain]: + if g == "projects": + continue + if p not in git_user_map: + git_user_map[p] = {} + git_domain_map[p] = {} + git_user_map[p][g] = domain_user_map[domain][subdomain][g] + git_domain_map[p] = [domain, subdomain] + except KeyError: + print "Failed to find acl_to_domain. project=%s, acl=%s" % (p, j[p]["parent"]) + + return domain_user_map, git_user_map, git_domain_map; + +#================================================================================ +# make_domain_user_map_array() +#================================================================================ +def make_domain_user_map_array(domain_user_map): + domain_user_map_array = [] + for d in domain_user_map: + for sd in domain_user_map[d]: + A = [] + M = [] + I = [] + R = [] + if "Architects" in domain_user_map[d][sd]: + A = domain_user_map[d][sd]["Architects"] + if "Maintainers" in domain_user_map[d][sd]: + M = domain_user_map[d][sd]["Maintainers"] + if "Integrators" in domain_user_map[d][sd]: + I = domain_user_map[d][sd]["Integrators"] + if "Reviewers" in domain_user_map[d][sd]: + R = domain_user_map[d][sd]["Reviewers"] + + if "projects" in domain_user_map[d][sd]: + for g in domain_user_map[d][sd]["projects"]: + di = {"domain":d, "subdomain":sd, "p":g, "A":A, "M":M, "I":I, "R":R} + domain_user_map_array.append(di) + else: + print "projects not found in domain (%s/%s)" % (d, sd) + + return domain_user_map_array; + +#================================================================================ +# print_to_file() +#================================================================================ +def print_to_file(filename, dict): + with open(filename, "w") as f: + f.write(json.dumps(dict)) + +#================================================================================ +# generate_mapping() +#================================================================================ +def generate_mapping(target_dir): + + if not os.path.exists(target_dir): + os.makedirs(target_dir) + + domain_user_map, git_user_map, git_domain_map = generate_domain_user_map() + f = "/".join([target_dir,"domain_user_map.json"]) + print_to_file(f, domain_user_map) + + f = "/".join([target_dir,"git_user_map.json"]) + print_to_file(f, git_user_map) + + f = "/".join([target_dir,"git_domain_map.json"]) + print_to_file(f, git_domain_map) + + domain_user_map_array = make_domain_user_map_array(domain_user_map) + f = "/".join([target_dir,"domain_user_map_array.json"]) + print_to_file(f, domain_user_map_array) + +########################################################### +# test + +#target_dir = ".dashboard/git_domain" +# +#generate_mapping(target_dir) +#exit(0) + +########################################################### +# main +target_dir = ".dashboard/git_domain" + +generate_mapping(target_dir) + +sync_dest = os.path.join(os.getenv("IMG_SYNC_DEST_BASE"), "snapshots", target_dir) +# sync to the download server. +ret = sync(target_dir, sync_dest) +if ret != 0: + raise Exception("rsync failed. ret_code=%d"%ret) diff --git a/job_update_scm_meta_git_for_dashboard.py b/job_update_scm_meta_git_for_dashboard.py deleted file mode 100644 index 9e88f31..0000000 --- a/job_update_scm_meta_git_for_dashboard.py +++ /dev/null @@ -1,180 +0,0 @@ -#!/usr/bin/python -import json -import os -import copy -from common.utils import sync -from common.gerrit import Gerrit, get_gerrit_event -from common.git import clone_gitproject - -#================================================================================ -# class DomainUserMap -#================================================================================ -class DomainUserMap: - def __init__(self, filename): - self.f = open(filename, "r") - self.domain_user_map = {} - self.parse_domain(self.f) - - def __del__(self): - self.f.close() - - def parse_email(self, line): - lpos = line.find("<"); - rpos = line.rfind(">"); - return line[lpos+1 : rpos].strip() - - def append_person(self, level, email): - if level not in self.domain_user_map[self.cur_domain]: - self.domain_user_map[self.cur_domain][level] = [] - - self.domain_user_map[self.cur_domain][level].append(email) - - def parse_parent_domain(self, parent_domain): - # propagate parent domain's info - self.domain_user_map[self.cur_domain] = copy.deepcopy(self.domain_user_map[parent_domain]) - - - def parse_domain_line(self, line): - line = line.strip() - if len(line) == 0: - pass - elif line[0] == 'D': - # domain - self.cur_domain = line[3:] - if self.cur_domain in self.domain_user_map: - raise Exception("Already declared domain name: "+self.cur_domain) - self.domain_user_map[self.cur_domain] = {} - elif line[0] == 'N': - # parent domain - self.parse_parent_domain(line[3:]) - elif line[0] == 'A': - # architect - self.append_person("A", self.parse_email(line)) - elif line[0] == 'M': - self.append_person("M", self.parse_email(line)) - # maintainer - elif line[0] == 'I': - # integrator - self.append_person("I", self.parse_email(line)) - elif line[0] == 'R': - # reviewer - self.append_person("R", self.parse_email(line)) - else: - print "undefined : ", line, - - def parse_domain(self, f): - while True: - line = f.readline() - if not line: break; - self.parse_domain_line(line) - - def dump(self): - print self.domain_user_map - - def print_to_file(self, filename): - with open(filename, "w") as f: - f.write(json.dumps(self.domain_user_map)) - -#================================================================================ -# class GitDomainMap -#================================================================================ -class GitDomainMap: - def __init__(self, filename): - self.f = open(filename, "r") - self.git_domain_map = {} - self.parse_git_tree(self.f) - - def __del__(self): - self.f.close() - - def parse_line(self, line): - line = line.strip() - if len(line) == 0: - pass - elif line[0] == 'G': - # git path - self.cur_project = line[3:] - elif line[0] == 'D': - # domain - domain = line[3:] - if self.cur_project in self.git_domain_map: - raise Exception("A project " + self.cur_project + "is matched to more than two domains:(" + self.cur_project + "," + self.git_domain_map[self.cur_project] + ")!!") - self.git_domain_map[self.cur_project] = domain - elif line[0] == 'S': - # submit type - pass - elif line[0] == 'T': - # package type - pass - elif line[0] == 'O': - # owner - pass - elif line[0] == 'B': - # branch - pass - elif line[0] == 'L': - # license - pass - elif line[0] == 'C': - # comments - pass - else: - print "undefined : ", line, - - def parse_git_tree(self, f): - while True: - line = f.readline() - if not line: break; - self.parse_line(line) - - def dump(self): - print self.git_domain_map - - def print_to_file(self, filename): - with open(filename, "w") as f: - f.write(json.dumps(self.git_domain_map)) - -#================================================================================ -# generate_mappings -#================================================================================ -def generate_mappings(prjdir, target_dir): - d = DomainUserMap(os.path.join(prjdir, "domains")) - g = GitDomainMap(os.path.join(prjdir,"git-trees")) - - domain_user_map_filename = "domain_user_map.json" - git_domain_filename = "git_domain_map.json" - - if not os.path.exists(target_dir): - os.makedirs(target_dir) - - d.print_to_file(os.path.join(target_dir, domain_user_map_filename)); - g.print_to_file(os.path.join(target_dir, git_domain_filename)); - -#target_dir = ".dashboard" - -#generate_mappings(".", target_dir) -#exit(0) -#================================================================================ -# main -#================================================================================ -GERRIT_PROJECT = os.getenv('GERRIT_PROJECT') -WORKSPACE = os.getenv('WORKSPACE') -GERRIT_HOSTNAME = os.getenv('GERRIT_HOSTNAME') -GERRIT_USERNAME = os.getenv('GERRIT_USERNAME') -GERRIT_SSHPORT = os.getenv('GERRIT_SSHPORT') -GERRIT_SILENT_MODE = int(os.getenv('GERRIT_SILENT_MODE')) - -events = get_gerrit_event() -print(events) - -prjdir = os.path.join(WORKSPACE, "meta-git") - -clone_gitproject(GERRIT_PROJECT, prjdir) - -target_dir = ".dashboard/domain" - -generate_mappings("meta-git", target_dir) - -sync_dest = os.path.join(os.getenv("IMG_SYNC_DEST_BASE"), "snapshots", target_dir) -# sync to the download server. -sync(target_dir, sync_dest) diff --git a/packaging/jenkins-scripts.spec b/packaging/jenkins-scripts.spec index 3707901..0bd17ef 100644 --- a/packaging/jenkins-scripts.spec +++ b/packaging/jenkins-scripts.spec @@ -144,7 +144,7 @@ fi %{destdir}/job_ref_purge_prj_obs.py %{destdir}/job_ref_precheck_project_obs.py %{destdir}/job_test_trigger_info_update.py -%{destdir}/job_update_scm_meta_git_for_dashboard.py +%{destdir}/job_git_domain.py %{destdir}/job_update_git_obs_mapping_for_dashboard.py %{destdir}/job_add_dotnet_launching_performance_test.py %{destdir}/job_update_git_branch_project_mapping_for_dashboard.py -- 2.7.4