From 9c614b15413f4ff81c7fa84defac3c9461382166 Mon Sep 17 00:00:00 2001 From: SungHun Hwang Date: Thu, 16 Mar 2017 09:35:04 +0900 Subject: [PATCH] [BuildMonitor] apply bulk insert for [build_snapshot_package] table [job_buildmonitor.py] 1. put the information of build pkgs into query_list 2. call do_many_query func [buildmonitor_db.py] 1. add 'do_many_query' func Change-Id: I9966eec4870288060a6945a545bb5104e087182a Signed-off-by: SungHun Hwang --- common/buildmonitor_db.py | 9 ++++++++ job_buildmonitor.py | 53 +++++++++++++++++++++++++++++------------------ 2 files changed, 42 insertions(+), 20 deletions(-) diff --git a/common/buildmonitor_db.py b/common/buildmonitor_db.py index e364b41..3c4df8c 100644 --- a/common/buildmonitor_db.py +++ b/common/buildmonitor_db.py @@ -60,6 +60,15 @@ def get_value_from_query_data(query, data): except MySQLdb.IntegrityError as err: print 'Error %d: %s' % (err.args[0], err.args[1]) +def do_many_query(query, data): + try: + cursor = conn.cursor() + cursor.executemany(query, data) + conn.commit() + cursor.close() + except MySQLdb.IntegrityError as err: + print 'Error %d: %s' % (err.args[0], err.args[1]) + def do_query(query, data): try: # skip print bcz of too many logs diff --git a/job_buildmonitor.py b/job_buildmonitor.py index d1594eb..b9d7115 100644 --- a/job_buildmonitor.py +++ b/job_buildmonitor.py @@ -458,16 +458,20 @@ def create_snapshot_packages_for_build_snapshot_package(project, bm_repo, bm_arc curr_build_snapshot_id = buildmonitor_db.get_value_from_query_data(query, query_data) #print '[%s] curr_build_snapshot_id(%s)\n' % (__file__, curr_build_snapshot_id) + # bulk insert + query = "INSERT INTO build_snapshot_package (build_snapshot_id, repository, " \ + "arch, package_name, created_date, package_size, package_url) " \ + "VALUES(%s, %s, %s, %s, %s, %s, %s)" + + query_list = [] for each_pkg_name, each_pkg_mdate, each_pkg_size in zip(bm_pkg_name_lst, bm_pkg_mdate_lst, bm_pkg_size_lst): timestamp = datetime.datetime.fromtimestamp(each_pkg_mdate) - #print '[%s] each_pkg_name(%s), timestamp(%s), each_pkg_size(%s)\n' \ + #print '[%s] each_pkg_name(%s), timestamp(%s), each_pkg_size(%s)' \ # % (__file__, each_pkg_name, timestamp, each_pkg_size) - query = "INSERT INTO build_snapshot_package (build_snapshot_id, repository, " \ - "arch, package_name, created_date, package_size, package_url) " \ - "VALUES(%s, %s, %s, %s, %s, %s, %s)" - query_data = (curr_build_snapshot_id, bm_repo, bm_arch, each_pkg_name, - timestamp, each_pkg_size, bm_pkg_url) - buildmonitor_db.do_query(query, query_data) + query_list.append((curr_build_snapshot_id, bm_repo, bm_arch, + each_pkg_name, timestamp, each_pkg_size, bm_pkg_url)) + + buildmonitor_db.do_many_query(query, query_list) def end_pre_create_snapshot_for_sr_stage(project, bm_git_tag, bm_start_datetime, bm_end_datetime): @@ -1086,17 +1090,22 @@ def create_snapshot_packages_for_post_build_snapshot_package(bm_snapshot_name, curr_build_snapshot_id = buildmonitor_db.get_value_from_query_data(query, query_data) print '[%s] curr_build_snapshot_id(%s)\n' % (__file__, curr_build_snapshot_id) + + # bulk insert + query = "INSERT INTO build_snapshot_package (build_snapshot_id, repository, arch, " \ + "package_name, created_date, package_size, package_url) " \ + "VALUES(%s, %s, %s, %s, %s, %s, %s)" + + query_list = [] for each_pkg_name, each_pkg_mdate, each_pkg_size in zip(bm_pkg_name_lst, bm_pkg_mdate_lst, bm_pkg_size_lst): timestamp = datetime.datetime.fromtimestamp(each_pkg_mdate) # post build #print '[%s] curr_build_snapshot_id(%s), each_pkg_name(%s), timestamp(%s), each_pkg_size(%s)\n' \ # % (__file__, curr_build_snapshot_id, each_pkg_name, timestamp, each_pkg_size) - query = "INSERT INTO build_snapshot_package (build_snapshot_id, repository, arch, " \ - "package_name, created_date, package_size, package_url) " \ - "VALUES(%s, %s, %s, %s, %s, %s, %s)" - query_data = (curr_build_snapshot_id, bm_repo, bm_arch, - each_pkg_name, timestamp, each_pkg_size, bm_pkg_url) - buildmonitor_db.do_query(query, query_data) + query_list.append((curr_build_snapshot_id, bm_repo, bm_arch, + each_pkg_name, timestamp, each_pkg_size, bm_pkg_url)) + + buildmonitor_db.do_many_query(query, query_list) def end_create_snapshot_create_images_for_sr_stage(bm_snapshot_name, bm_start_datetime, @@ -1576,16 +1585,20 @@ def TRBS_create_snapshot_packages_for_build_snapshot_package(project, bm_repo, b curr_build_snapshot_id = buildmonitor_db.get_value_from_query_data(query, query_data) #print '[%s] curr_build_snapshot_id(%s)\n' % (__file__, curr_build_snapshot_id) + # bulk insert + query = "INSERT INTO build_snapshot_package (build_snapshot_id, repository, " \ + "arch, package_name, created_date, package_size, package_url) " \ + "VALUES(%s, %s, %s, %s, %s, %s, %s)" + + query_list = [] for each_pkg_name, each_pkg_mdate, each_pkg_size in zip(bm_pkg_name_lst, bm_pkg_mdate_lst, bm_pkg_size_lst): timestamp = datetime.datetime.fromtimestamp(each_pkg_mdate) - #print '[%s] each_pkg_name(%s), timestamp(%s), each_pkg_size(%s)\n' \ + #print '[%s] each_pkg_name(%s), timestamp(%s), each_pkg_size(%s)' \ # % (__file__, each_pkg_name, timestamp, each_pkg_size) - query = "INSERT INTO build_snapshot_package (build_snapshot_id, repository, " \ - "arch, package_name, created_date, package_size, package_url) " \ - "VALUES(%s, %s, %s, %s, %s, %s, %s)" - query_data = (curr_build_snapshot_id, bm_repo, bm_arch, each_pkg_name, - timestamp, each_pkg_size, bm_pkg_url) - buildmonitor_db.do_query(query, query_data) + query_list.append((curr_build_snapshot_id, bm_repo, bm_arch, + each_pkg_name, timestamp, each_pkg_size, bm_pkg_url)) + + buildmonitor_db.do_many_query(query, query_list) def TRBS_end_pre_create_snapshot_for_sr_stage(project, bm_git_tag, bm_start_datetime, bm_end_datetime): -- 2.7.4