[BuildMonitor] apply bulk insert for [build_snapshot_package] table

[job_buildmonitor.py]
  1. put the information of build pkgs into query_list
  2. call do_many_query func

[buildmonitor_db.py]
  1. add 'do_many_query' func

Change-Id: I9966eec4870288060a6945a545bb5104e087182a
Signed-off-by: SungHun Hwang <sh0924.hwang@samsung.com>

diff --git a/common/buildmonitor_db.py b/common/buildmonitor_db.py
index e364b41..3c4df8c 100644 (file)
--- a/common/buildmonitor_db.py
+++ b/common/buildmonitor_db.py
@@ -60,6 +60,15 @@ def get_value_from_query_data(query, data):
     except MySQLdb.IntegrityError as err:
         print 'Error %d: %s' % (err.args[0], err.args[1])
 
+def do_many_query(query, data):
+    try:
+        cursor = conn.cursor()
+        cursor.executemany(query, data)
+        conn.commit()
+        cursor.close()
+    except MySQLdb.IntegrityError as err:
+        print 'Error %d: %s' % (err.args[0], err.args[1])
+
 def do_query(query, data):
     try:
         # skip print bcz of too many logs

diff --git a/job_buildmonitor.py b/job_buildmonitor.py
index d1594eb..b9d7115 100644 (file)
--- a/job_buildmonitor.py
+++ b/job_buildmonitor.py
@@ -458,16 +458,20 @@ def create_snapshot_packages_for_build_snapshot_package(project, bm_repo, bm_arc
     curr_build_snapshot_id = buildmonitor_db.get_value_from_query_data(query, query_data)
     #print '[%s] curr_build_snapshot_id(%s)\n' % (__file__, curr_build_snapshot_id)
 
+    # bulk insert
+    query = "INSERT INTO build_snapshot_package (build_snapshot_id, repository, " \
+            "arch, package_name, created_date, package_size, package_url) " \
+            "VALUES(%s, %s, %s, %s, %s, %s, %s)"
+
+    query_list = []
     for each_pkg_name, each_pkg_mdate, each_pkg_size in zip(bm_pkg_name_lst, bm_pkg_mdate_lst, bm_pkg_size_lst):
         timestamp = datetime.datetime.fromtimestamp(each_pkg_mdate)
-        #print '[%s] each_pkg_name(%s), timestamp(%s), each_pkg_size(%s)\n' \
+        #print '[%s] each_pkg_name(%s), timestamp(%s), each_pkg_size(%s)' \
         #      % (__file__, each_pkg_name, timestamp, each_pkg_size)
-        query = "INSERT INTO build_snapshot_package (build_snapshot_id, repository, " \
-                "arch, package_name, created_date, package_size, package_url) " \
-                "VALUES(%s, %s, %s, %s, %s, %s, %s)"
-        query_data = (curr_build_snapshot_id, bm_repo, bm_arch, each_pkg_name,
-                      timestamp, each_pkg_size, bm_pkg_url)
-        buildmonitor_db.do_query(query, query_data)
+        query_list.append((curr_build_snapshot_id, bm_repo, bm_arch,
+                           each_pkg_name, timestamp, each_pkg_size, bm_pkg_url))
+
+    buildmonitor_db.do_many_query(query, query_list)
 
 def end_pre_create_snapshot_for_sr_stage(project, bm_git_tag,
                                          bm_start_datetime, bm_end_datetime):
@@ -1086,17 +1090,22 @@ def create_snapshot_packages_for_post_build_snapshot_package(bm_snapshot_name,
     curr_build_snapshot_id = buildmonitor_db.get_value_from_query_data(query, query_data)
     print '[%s] curr_build_snapshot_id(%s)\n' % (__file__, curr_build_snapshot_id)
 
+
+    # bulk insert
+    query = "INSERT INTO build_snapshot_package (build_snapshot_id, repository, arch, " \
+            "package_name, created_date, package_size, package_url) " \
+            "VALUES(%s, %s, %s, %s, %s, %s, %s)"
+
+    query_list = []
     for each_pkg_name, each_pkg_mdate, each_pkg_size in zip(bm_pkg_name_lst, bm_pkg_mdate_lst, bm_pkg_size_lst):
         timestamp = datetime.datetime.fromtimestamp(each_pkg_mdate)
         # post build
         #print '[%s] curr_build_snapshot_id(%s), each_pkg_name(%s), timestamp(%s), each_pkg_size(%s)\n' \
         #      % (__file__, curr_build_snapshot_id, each_pkg_name, timestamp, each_pkg_size)
-        query = "INSERT INTO build_snapshot_package (build_snapshot_id, repository, arch, " \
-                "package_name, created_date, package_size, package_url) " \
-                "VALUES(%s, %s, %s, %s, %s, %s, %s)"
-        query_data = (curr_build_snapshot_id, bm_repo, bm_arch,
-                      each_pkg_name, timestamp, each_pkg_size, bm_pkg_url)
-        buildmonitor_db.do_query(query, query_data)
+        query_list.append((curr_build_snapshot_id, bm_repo, bm_arch,
+                           each_pkg_name, timestamp, each_pkg_size, bm_pkg_url))
+
+    buildmonitor_db.do_many_query(query, query_list)
 
 def end_create_snapshot_create_images_for_sr_stage(bm_snapshot_name,
                                                    bm_start_datetime,
@@ -1576,16 +1585,20 @@ def TRBS_create_snapshot_packages_for_build_snapshot_package(project, bm_repo, b
     curr_build_snapshot_id = buildmonitor_db.get_value_from_query_data(query, query_data)
     #print '[%s] curr_build_snapshot_id(%s)\n' % (__file__, curr_build_snapshot_id)
 
+    # bulk insert
+    query = "INSERT INTO build_snapshot_package (build_snapshot_id, repository, " \
+            "arch, package_name, created_date, package_size, package_url) " \
+            "VALUES(%s, %s, %s, %s, %s, %s, %s)"
+
+    query_list = []
     for each_pkg_name, each_pkg_mdate, each_pkg_size in zip(bm_pkg_name_lst, bm_pkg_mdate_lst, bm_pkg_size_lst):
         timestamp = datetime.datetime.fromtimestamp(each_pkg_mdate)
-        #print '[%s] each_pkg_name(%s), timestamp(%s), each_pkg_size(%s)\n' \
+        #print '[%s] each_pkg_name(%s), timestamp(%s), each_pkg_size(%s)' \
         #      % (__file__, each_pkg_name, timestamp, each_pkg_size)
-        query = "INSERT INTO build_snapshot_package (build_snapshot_id, repository, " \
-                "arch, package_name, created_date, package_size, package_url) " \
-                "VALUES(%s, %s, %s, %s, %s, %s, %s)"
-        query_data = (curr_build_snapshot_id, bm_repo, bm_arch, each_pkg_name,
-                      timestamp, each_pkg_size, bm_pkg_url)
-        buildmonitor_db.do_query(query, query_data)
+        query_list.append((curr_build_snapshot_id, bm_repo, bm_arch,
+                           each_pkg_name, timestamp, each_pkg_size, bm_pkg_url))
+
+    buildmonitor_db.do_many_query(query, query_list)
 
 def TRBS_end_pre_create_snapshot_for_sr_stage(project, bm_git_tag,
                                          bm_start_datetime, bm_end_datetime):