From 2b6a204cfbc0f7001661d9fae1ecabb38d68933b Mon Sep 17 00:00:00 2001 From: hyokeun Date: Thu, 22 Feb 2018 12:03:07 +0900 Subject: [PATCH] GBS - shrink reason data to id index Change-Id: I5e4cd853de2eff0c96ef3304898c504600f9665f --- common/buildmonitor_extention.py | 97 ++++++++++++++++++++++++++++++++-------- job_buildmonitor_gbs.py | 24 ++++++++-- 2 files changed, 100 insertions(+), 21 deletions(-) diff --git a/common/buildmonitor_extention.py b/common/buildmonitor_extention.py index e0d78b6..8ff9df5 100644 --- a/common/buildmonitor_extention.py +++ b/common/buildmonitor_extention.py @@ -17,6 +17,7 @@ import os import sys import base64 import json +import copy from datetime import datetime from urllib import quote_plus from common.buildtrigger import trigger_next, remote_jenkins_build_job @@ -520,19 +521,39 @@ class BuildMonitorExtention(object): if not self.is_connect(): return - query = "SELECT id FROM build_repository_name WHERE repository = %s" - query_data = (repo_name,) + if type(repo_name) == str: + repo_name = [repo_name] - return get_value_from_query_data(query, query_data) + param_count_str = ','.join(["%s"] * len(repo_name)) + query = "SELECT id, repository FROM build_repository_name WHERE repository IN (%s)" % param_count_str + query_data = set(repo_name) + rows = get_multi_values_from_query_data(query, query_data) + + ret_data = {} + for x in rows: + ret_data[str(x[1])] = int(x[0]) + + return ret_data def update_build_repository_name(self, repo_name): if not self.is_connect(): return - query = "INSERT INTO build_repository_name (repository) SELECT * FROM (SELECT %s) AS tmp \ - WHERE NOT EXISTS (SELECT repository FROM build_repository_name WHERE repository = %s) LIMIT 1" - do_query(query, (repo_name, repo_name)) + if type(repo_name) == str: + repo_name = [repo_name] + + rows = self.get_build_repository_name_id(repo_name) + + new_repo_name_list = copy.deepcopy(repo_name) + for x in rows: + new_repo_name_list.remove(x) + + query = "INSERT IGNORE INTO build_repository_name (repository) VALUES(%s)" + query_list = [] + for x in new_repo_name_list: + query_list.append((x,)) + do_many_query(query, query_list) return self.get_build_repository_name_id(repo_name) @@ -541,19 +562,39 @@ class BuildMonitorExtention(object): if not self.is_connect(): return - query = "SELECT id FROM build_arch_name WHERE repository = %s" - query_data = (arch_name,) + if type(arch_name) == str: + arch_name = [arch_name] - return get_value_from_query_data(query, query_data) + param_count_str = ','.join(["%s"] * len(arch_name)) + query = "SELECT id, arch FROM build_arch_name WHERE arch IN (%s)" % param_count_str + query_data = set(arch_name) + rows = get_multi_values_from_query_data(query, query_data) + + ret_data = {} + for x in rows: + ret_data[str(x[1])] = int(x[0]) + + return ret_data def update_build_arch_name(self, arch_name): if not self.is_connect(): return - query = "INSERT INTO build_arch_name (arch) SELECT * FROM (SELECT %s) AS tmp \ - WHERE NOT EXISTS (SELECT arch FROM build_arch_name WHERE arch = %s) LIMIT 1" - do_query(query, (arch_name, arch_name)) + if type(arch_name) == str: + arch_name = [arch_name] + + rows = self.get_build_arch_name_id(arch_name) + + new_arch_name_list = copy.deepcopy(arch_name) + for x in rows: + new_arch_name_list.remove(x) + + query = "INSERT IGNORE INTO build_arch_name (arch) VALUES(%s)" + query_list = [] + for x in new_arch_name_list: + query_list.append((x,)) + do_many_query(query, query_list) return self.get_build_arch_name_id(arch_name) @@ -562,19 +603,39 @@ class BuildMonitorExtention(object): if not self.is_connect(): return - query = "SELECT id FROM device_name WHERE device_name = %s" - query_data = (device_name,) + if type(device_name) == str: + device_name = [device_name] - return get_value_from_query_data(query, query_data) + param_count_str = ','.join(["%s"] * len(device_name)) + query = "SELECT id, device_name FROM device_name WHERE device_name IN (%s)" % param_count_str + query_data = set(device_name) + rows = get_multi_values_from_query_data(query, query_data) + + ret_data = {} + for x in rows: + ret_data[str(x[1])] = int(x[0]) + + return ret_data def update_device_name(self, device_name): if not self.is_connect(): return - query = "INSERT INTO device_name (device_name) SELECT * FROM (SELECT %s) AS tmp \ - WHERE NOT EXISTS (SELECT device_name FROM device_name WHERE device_name = %s) LIMIT 1" - do_query(query, (device_name, device_name)) + if type(device_name) == str: + device_name = [device_name] + + rows = self.get_device_name_id(device_name) + + new_device_name_list = copy.deepcopy(device_name) + for x in rows: + new_device_name_list.remove(x) + + query = "INSERT IGNORE INTO device_name (device_name) VALUES(%s)" + query_list = [] + for x in new_device_name_list: + query_list.append((x,)) + do_many_query(query, query_list) return self.get_device_name_id(device_name) diff --git a/job_buildmonitor_gbs.py b/job_buildmonitor_gbs.py index 1d0099a..f1a4d3f 100644 --- a/job_buildmonitor_gbs.py +++ b/job_buildmonitor_gbs.py @@ -52,11 +52,29 @@ class LocalError(Exception): def update_reason_lookup(gbs_inst, data): for k, v in data.iteritems(): if k == "repo": - data[k] = int(gbs_inst.update_build_repository_name(data[k])) + data[k] = int(gbs_inst.update_build_repository_name(data[k])[data[k]]) elif k == "arch": - data[k] = int(gbs_inst.update_build_arch_name(data[k])) + data[k] = int(gbs_inst.update_build_arch_name(data[k])[data[k]]) elif k == "ks_name": - data[k] = int(gbs_inst.update_device_name(data[k])) + data[k] = int(gbs_inst.update_device_name(data[k])[data[k]]) + elif k == "ks_name_list": + ret_dict = gbs_inst.update_device_name(data[k]) + data[k] = ",".join(str(e) for e in ret_dict.values()) + elif k == "repo_arch_list": + repo_arch_map = {} + for x in v: + if x["repo"] not in repo_arch_map: + repo_arch_map[x["repo"]] = [] + repo_arch_map[x["repo"]].append(x["arch"]) + ret_repos = gbs_inst.update_build_repository_name(repo_arch_map.keys()) + index_str = "" + for x in repo_arch_map: + if index_str is not None and index_str != "": + index_str += "|" + index_str += str(ret_repos[x]) + "-" + ret_archs = gbs_inst.update_build_arch_name(repo_arch_map[x]) + index_str += ",".join(str(e) for e in ret_archs.values()) + data[k] = index_str def _set_stage(gbs_inst, stage, content): -- 2.7.4