[BuildMonitor] support the trbs project and enable the multi env

1. support the trbs project
2. enable the multi env and rearrange the job order
3. fixed the bm_img_size error at job_imager.py

Change-Id: I90b09d7e233f5f9a704992e2cceb536b6406889c
Signed-off-by: SungHun Hwang <sh0924.hwang@samsung.com>

diff --git a/job_buildmonitor.py b/job_buildmonitor.py
index 16e988c..0f5aa40 100644 (file)
--- a/job_buildmonitor.py
+++ b/job_buildmonitor.py
@@ -43,6 +43,8 @@ TABLE  : sr_status
 import os
 import sys
 import datetime
+
+from time import sleep
 from common import buildmonitor_db
 from common.buildtrigger import trigger_info, trigger_next
 
@@ -165,6 +167,10 @@ def package_build_for_sr_detail_sr_stage(git_tag, bm_start_datetime,
     curr_sr_status_id = buildmonitor_db.get_value_from_query_data(query, query_data)
     #print '[%s] curr_sr_status_id(%s)\n' % (__file__, curr_sr_status_id)
 
+    # Wait 60 seconds to find the info_project_id
+    print '[%s] wait 60 sec\n' % (__file__)
+    sleep(60)
+
     ### for sr_status_detail & sr_stage
     # bm_src_project_lst is needed for multi profile
     print '[%s] bm_src_project_lst(%s)\n' % (__file__, bm_src_project_lst)
@@ -175,18 +181,25 @@ def package_build_for_sr_detail_sr_stage(git_tag, bm_start_datetime,
         query = "SELECT id FROM info_project WHERE project_name = %s"
         query_data = (bm_src_project,)
         info_project_id = buildmonitor_db.get_value_from_query_data(query, query_data)
-        ###================== [16_1110][TEMP] build_project
-        print '[%s][HSH] TEMP!!\n' % (__file__)
-        query = "INSERT INTO build_project (info_project_id, status) VALUES(%s, %s)"
-        query_data = (info_project_id, 'C')
-        buildmonitor_db.do_query(query, query_data)
-        ###================== [16_1110][TEMP] build_project
         query = "SELECT id FROM build_project WHERE info_project_id = %s"
         query_data = (info_project_id,)
         pre_build_project_id = buildmonitor_db.get_value_from_query_data(query, query_data)
         print '[%s] info_project_id(%s), pre_build_project_id(%s)\n' \
                % (__file__, info_project_id, pre_build_project_id)
 
+        ###================== [16_1110][TEMP] build_project
+        if pre_build_project_id == 9876:
+            print '[%s][HSH] cannot find build_project_id!! Adding NOW!!\n' % (__file__)
+            query = "INSERT INTO build_project (info_project_id, status) VALUES(%s, %s)"
+            query_data = (info_project_id, 'C')
+            buildmonitor_db.do_query(query, query_data)
+            query = "SELECT id FROM build_project WHERE info_project_id = %s"
+            query_data = (info_project_id,)
+            pre_build_project_id = buildmonitor_db.get_value_from_query_data(query, query_data)
+            print '[%s][After] info_project_id(%s), pre_build_project_id(%s)\n' \
+                   % (__file__, info_project_id, pre_build_project_id)
+        ###================== [16_1110][TEMP] build_project
+
         # get sr_status_detail_id for checking group submit
         query = "SELECT id FROM sr_status_detail WHERE sr_status_id = %s " \
                 "AND pre_build_project_id = %s"
@@ -333,11 +346,18 @@ def start_pre_create_snapshot_for_build_snapshot(project, bm_start_datetime):
     #print '[%s] pre_info_project_id(%s), pre_build_project_id(%s)\n' \
     #      % (__file__, pre_info_project_id, pre_build_project_id)
 
+    ### to check duplicate build_snapshot_id
+    query = "SELECT id FROM build_snapshot WHERE build_project_id = %s"
+    query_data = (pre_build_project_id,)
+    pre_build_snapshot_id = buildmonitor_db.get_value_from_query_data(query, query_data)
     ### for build_snapshot
-    query = "INSERT INTO build_snapshot (build_project_id, start_time, status) " \
-            "VALUES(%s, %s, %s)"
-    query_data = (pre_build_project_id, bm_start_datetime, 'R')
-    buildmonitor_db.do_query(query, query_data)
+    if pre_build_snapshot_id == 9876:
+        query = "INSERT INTO build_snapshot (build_project_id, start_time, status) " \
+                "VALUES(%s, %s, %s)"
+        query_data = (pre_build_project_id, bm_start_datetime, 'R')
+        buildmonitor_db.do_query(query, query_data)
+    else:
+        print '[%s] duplicated build_snapshot_id!! skip insert\n' % (__file__)
 
 def create_snapshot_packages_for_build_snapshot_package(project, bm_repo, bm_arch,
                                                         bm_pkg_url,
@@ -407,7 +427,8 @@ def end_pre_create_snapshot_for_sr_stage(project, bm_git_tag,
     buildmonitor_db.do_query(query, query_data)
 
 def end_pre_create_snapshot_for_build_snapshot(project, bm_snapshot_name,
-                                               bm_snapshot_url, bm_end_datetime):
+                                               bm_snapshot_url, bm_snapshot_num,
+                                               bm_end_datetime):
     print '[%s] enter end_pre_create_snapshot_for_build_snapshot\n' % (__file__)
 
     # get pre_info_project_id -> get pre_build_project_id
@@ -422,9 +443,9 @@ def end_pre_create_snapshot_for_build_snapshot(project, bm_snapshot_name,
 
     ### for build_snapshot
     query = "UPDATE build_snapshot SET snapshot_name = %s, snapshot_url = %s, " \
-            "end_time = %s, status = %s WHERE build_project_id = %s"
-    query_data = (bm_snapshot_name, bm_snapshot_url, bm_end_datetime,
-                  'S', pre_build_project_id)
+            "snapshot_num = %s, end_time = %s, status = %s WHERE build_project_id = %s"
+    query_data = (bm_snapshot_name, bm_snapshot_url, bm_snapshot_num,
+                  bm_end_datetime, 'S', pre_build_project_id)
     buildmonitor_db.do_query(query, query_data)
 
 #===============================================================================
@@ -435,7 +456,9 @@ def update_fail_create_image_for_sr_stage(fields, bm_start_datetime):
 
     bm_prj_name = fields["project"]
     bm_split_data = fields["project"].split(':')[-1]
-    bm_git_tag = 'submit/tizen/' + bm_split_data
+    bm_branch_name = fields["project"].split(':')[-2]
+    #bm_git_tag = 'submit/tizen/' + bm_split_data
+    bm_git_tag = 'submit/' + bm_branch_name + '/' + bm_split_data
     bm_url = os.path.join(fields.get('url_pub_base', ''),
                            fields['repo_path'], fields['images_path'])
     #print '[%s] bm_prj_name(%s), bm_split_data(%s), bm_git_tag(%s), bm_url(%s)\n' \
@@ -444,8 +467,8 @@ def update_fail_create_image_for_sr_stage(fields, bm_start_datetime):
     bm_end_datetime = datetime.datetime.now()
 
     #if 'prerelease' in bm_url:
-    if not ((bm_url.find("prerelease")) == -1):
-
+    #if not ((bm_url.find("prerelease")) == -1):
+    if not ((bm_url.find("prerelease")) == -1) or not ((bm_url.find("trbs")) == -1):
         # [pre_build] get current_sr_status_id
         query = "SELECT id FROM sr_status WHERE sr = %s"
         query_data = (bm_git_tag,)
@@ -524,7 +547,8 @@ def create_image_for_build_image(fields, bm_start_datetime,
 def end_create_image_for_sr_stage(bm_start_datetime, project):
     print '[%s] enter end_create_image_for_sr_stage project(%s)\n' % (__file__, project)
 
-    if project.startswith("home:prerelease:"):
+    #if project.startswith("home:prerelease:"):
+    if project.startswith("home:prerelease:") or project.startswith("home:trbs:"):
         bm_end_datetime = datetime.datetime.now()
         bm_git_tag = project.split(":")[-3] + '/' + project.split(":")[-2] + '/' + project.split(":")[-1]
         #print '[%s] bm_git_tag(%s)\n' % (__file__, bm_git_tag)
@@ -922,7 +946,15 @@ def main():
     Script entry point.
     """
     bm_connect_db()
-    content = trigger_info(os.getenv('TRIGGER_INFO'))
+    #content = trigger_info(os.getenv('TRIGGER_INFO'))
+    trigger_info_data = os.getenv('TRIGGER_INFO')
+    if trigger_info_data:
+        content = trigger_info(os.getenv('TRIGGER_INFO'))
+    else:
+        print '[%s] trigger_info_data is empty!!(%s)\n' \
+              % (__file__, trigger_info_data)
+        return 1
+
     bm_stage = content.get("bm_stage")
 
     #=======================================================
@@ -970,9 +1002,11 @@ def main():
         # get vars
         project = content.get("project")
         bm_git_tag = content.get("bm_git_tag")
-        bm_start_datetime = bm_end_datetime = datetime.datetime.now()
+        bm_start_datetime = content.get("bm_start_datetime")
+        bm_end_datetime = content.get("bm_end_datetime")
         bm_snapshot_name = content.get("bm_snapshot_name")
         bm_snapshot_url = content.get("bm_snapshot_url")
+        bm_snapshot_num = content.get("bm_snapshot_num")
 
         # func call
         end_pre_create_snapshot_for_sr_stage(project, bm_git_tag,
@@ -981,6 +1015,7 @@ def main():
         end_pre_create_snapshot_for_build_snapshot(project,
                                                    bm_snapshot_name,
                                                    bm_snapshot_url,
+                                                   bm_snapshot_num,
                                                    bm_end_datetime)
 
     elif bm_stage == 'Pre_Snap_packages':
@@ -1017,13 +1052,14 @@ def main():
         bm_start_datetime  = content.get("bm_start_datetime")
         bm_end_datetime = content.get("bm_end_datetime")
         build_id = content.get("build_id")
+        bm_img_size = content.get("bm_img_size")
         #print '[HSH][%s] fields(%s), bm_start_datetime(%s), build_id(%s)\n' \
         #      % (__file__, fields, bm_start_datetime, build_id)
 
         # func call
         if image_status == 'success':
             create_image_for_build_image(fields, bm_start_datetime,
-                                         bm_end_datetime, build_id)
+                                         bm_end_datetime, build_id, bm_img_size)
         elif image_status == 'failed':
             update_fail_create_image_for_sr_stage(fields, bm_start_datetime)
     #=======================================================
@@ -1116,15 +1152,11 @@ def main():
     #    print 'no bm_stage\n'
 
     #bm_disconnect_db()
-    '''
-
 
 if __name__ == '__main__':
     try:
-        #sys.exit(main(os.path.basename(sys.argv[1])))
         sys.exit(main())
     except LocalError, err:
         print err
-        #print '[HSH] except!!\n'
         #bm_disconnect_db()
         sys.exit(1)

diff --git a/job_create_snapshot.py b/job_create_snapshot.py
index 9ba2eb1..53b5c33 100755 (executable)
--- a/job_create_snapshot.py
+++ b/job_create_snapshot.py
@@ -117,13 +117,14 @@ def make_repo(project, backenddb, base_path, live_repo_base):
         #buildmonitor.start_create_snapshot_for_post_build_snapshot(project,
         #                                                           bm_snapshot_name,
         #                                                           bm_start_datetime)
-        bm_data = {"bm_stage" : 'Post_Snap_snapshot',
+        bm_stage = 'Post_Snap_snapshot'
+        bm_data = {"bm_stage" : bm_stage,
                    "bm_stage_func" : 'start_create_snapshot_for_post_build_snapshot',
                    "project" : project,
                    "bm_snapshot_name" : bm_snapshot_name,
                    "bm_start_datetime" : str(bm_start_datetime)
                   }
-        trigger_next("BUILD-MONITOR", bm_data)
+        trigger_next("BUILD-MONITOR-2-%s" % bm_stage, bm_data)
 
     # Convert live repo to download structure
     for repo in targets:
@@ -147,12 +148,12 @@ def make_repo(project, backenddb, base_path, live_repo_base):
             if buildmonitor_enabled:
                 print '[%s][LocalError] bm_project(%s)\n' % (__file__, bm_project)
                 #buildmonitor.update_fail_create_snapshot_for_sr_stage(bm_project)
-                bm_data = {"bm_stage" : 'Post_Snap_Fail',
+                bm_stage = 'Post_Snap_Fail'
+                bm_data = {"bm_stage" : bm_stage,
                            "bm_stage_func" : 'update_fail_create_snapshot_for_sr_stage',
                            "bm_project" : bm_project
                           }
-                trigger_next("BUILD-MONITOR", bm_data)
-
+                trigger_next("BUILD-MONITOR-5-%s" % bm_stage, bm_data)
             raise LocalError("Image configuration not found in %s" %
                               snapshot.path)
 
@@ -201,7 +202,8 @@ def make_repo(project, backenddb, base_path, live_repo_base):
             #                                                                      repo['Architectures'][0],
             #                                                                      base_path,
             #                                                                      snapshot)
-            bm_data = {"bm_stage" : 'Post_Snap_packages',
+            bm_stage = 'Post_Snap_packages'
+            bm_data = {"bm_stage" : bm_stage,
                        "bm_stage_func" : 'create_snapshot_packages_for_post_build_snapshot_package',
                        "project" : project,
                        "bm_repo" : bm_repo,
@@ -211,7 +213,7 @@ def make_repo(project, backenddb, base_path, live_repo_base):
                        "bm_pkg_mdate_lst" : bm_pkg_mdate_lst,
                        "bm_pkg_size_lst" : bm_pkg_size_lst,
                       }
-            trigger_next("BUILD-MONITOR", bm_data)
+            trigger_next("BUILD-MONITOR-3-%s" % bm_stage, bm_data)
 
         # Generate image info to builddata/ dir
         repomaker.gen_image_info()
@@ -259,12 +261,13 @@ def main():
         global bm_project # for update_fail_xxx
         bm_project = project
         #buildmonitor.start_create_snapshot_for_sr_stage(project, bm_start_datetime)
-        bm_data = {"bm_stage" : 'Post_Snap_Start',
+        bm_stage = 'Post_Snap_Start'
+        bm_data = {"bm_stage" : bm_stage,
                    "bm_stage_func" : 'start_create_snapshot_for_sr_stage',
                    "project" : project,
                    "bm_start_datetime" : str(bm_start_datetime)
                   }
-        trigger_next("BUILD-MONITOR", bm_data)
+        trigger_next("BUILD-MONITOR-1-%s" % bm_stage, bm_data)
 
     build = BuildService(obs_api, obs_user, obs_passwd)
 
@@ -322,7 +325,8 @@ def main():
             #                                                         bm_end_datetime,
             #                                                         bm_snapshot_url,
             #                                                         repo_data['build_id'])
-            bm_data = {"bm_stage" : 'Post_Snap_End',
+            bm_stage = 'Post_Snap_End'
+            bm_data = {"bm_stage" : bm_stage,
                        "bm_stage_func" : 'end_create_snapshot_create_images_for_sr_stage',
                        "project" : project,
                        "bm_start_datetime" : str(bm_start_datetime),
@@ -330,7 +334,7 @@ def main():
                        "bm_snapshot_url" : bm_snapshot_url,
                        "repo_data_build_id" : repo_data['build_id']
                       }
-            trigger_next("BUILD-MONITOR", bm_data)
+            trigger_next("BUILD-MONITOR-4-%s" % bm_stage, bm_data)
 
 if __name__ == '__main__':
     try:

diff --git a/job_imager.py b/job_imager.py
index e4b69ea..c415cfa 100755 (executable)
--- a/job_imager.py
+++ b/job_imager.py
@@ -337,27 +337,28 @@ def main():
         trigger_next("IMAGE-TESTING", fields)
 
         if buildmonitor_enabled:
-            bm_base_path = os.getenv('PATH_REPO_BASE')
-            bm_img_url = os.path.join(bm_base_path, fields['repo_path'], fields['images_path'])
-            bm_img_path = bm_img_url + '/' + build_id + '_' + bm_device_name + '.tar.gz'
+            bm_img_url = os.path.join(sync_src, fields['images_path'])
+            bm_img_path = bm_img_url + '/' + build_id + '_' + fields['name'] + '.tar.gz'
             bm_img_size = os.path.getsize(bm_img_path)
             #print '[%s] bm_img_path(%s), bm_img_size(%s)\n' \
             #      % (__file__, bm_img_path, bm_img_size)
 
             bm_end_datetime = datetime.datetime.now()
-            bm_data = {"bm_stage": 'Image',
+            bm_stage = 'Image'
+            bm_data = {"bm_stage" : bm_stage,
                        "status" : status,
                        "fields" : fields,
                        "bm_start_datetime" : str(bm_start_datetime),
                        "bm_end_datetime" : str(bm_end_datetime),
-                       "build_id" : build_id,
+                       "build_id": build_id,
                        "bm_img_size" : bm_img_size
                       }
             trigger_next("BUILD-MONITOR", bm_data)
     else:
         if buildmonitor_enabled:
             bm_end_datetime = datetime.datetime.now()
-            bm_data = {"bm_stage": 'Image',
+            bm_stage = 'Image'
+            bm_data = {"bm_stage" : bm_stage,
                        "status" : status,
                        "fields" : fields,
                        "bm_start_datetime": str(bm_start_datetime),

diff --git a/job_post_image.py b/job_post_image.py
index 2225305..59679dc 100755 (executable)
--- a/job_post_image.py
+++ b/job_post_image.py
@@ -79,16 +79,17 @@ def main():
 
         if buildmonitor_enabled:
             saveinfo = build.get_info(project)
-                if len(saveinfo['images']) == int(saveinfo['images_count']):
-                    print '[%s] last image(%s)\n' \
-                          % (__file__, len(saveinfo['images']))
-                    #buildmonitor.end_create_image_for_sr_stage(bm_start_datetime,
-                    #                                           project)
-                    bm_data = {"bm_stage" : 'post_image',
-                               "bm_start_datetime" : str(bm_start_datetime),
-                               "project" : project,
-                              }
-                    trigger_next("BUILD-MONITOR", bm_data)
+            if len(saveinfo['images']) == saveinfo['images_count']:
+                print '[%s] last image(%s)\n' \
+                      % (__file__, len(saveinfo['images']))
+                #buildmonitor.end_create_image_for_sr_stage(bm_start_datetime,
+                #                                           project)
+                bm_stage = 'Post_Image'
+                bm_data = {"bm_stage" : bm_stage,
+                           "bm_start_datetime" : str(bm_start_datetime),
+                           "project" : project,
+                          }
+                trigger_next("BUILD-MONITOR", bm_data)
 
 if __name__ == "__main__":
     sys.exit(main())

diff --git a/job_pre_release_obs.py b/job_pre_release_obs.py
index b8342c0..70c77c2 100755 (executable)
--- a/job_pre_release_obs.py
+++ b/job_pre_release_obs.py
@@ -195,10 +195,10 @@ def make_repo(project, repo, backenddb, base_url, base_path,
                     print '[%s][LocalError] bm_git_tag(%s)\n' % (__file__, bm_git_tag)
                     #buildmonitor.update_fail_status_for_sr_stage(project, bm_git_tag)
                     bm_stage = 'Pre_Snap_Fail'
-                    bm_data = {"bm_stage": bm_stage,
+                    bm_data = {"bm_stage" : bm_stage,
                                "bm_git_tag" : bm_git_tag,
                               }
-                    trigger_next("BUILD-MONITOR-%s" % bm_stage, bm_data)
+                    trigger_next("BUILD-MONITOR-4-%s" % bm_stage, bm_data)
 
                 raise LocalError("Image configuration not found in %s" %
                         snapshot.path)
@@ -257,8 +257,7 @@ def make_repo(project, repo, backenddb, base_url, base_path,
                        "bm_pkg_mdate_lst" : bm_pkg_mdate_lst,
                        "bm_pkg_size_lst" : bm_pkg_size_lst
                       }
-            #trigger_next("BUILD-MONITOR", bm_data)
-            trigger_next("BUILD-MONITOR-%s" % bm_stage, bm_data)
+            trigger_next("BUILD-MONITOR-2-%s" % bm_stage, bm_data)
 
         # Generate image info to builddata/ dir
         repomaker.gen_image_info(images_ks)
@@ -392,14 +391,13 @@ def main(action):
             bm_git_tag = info['git_tag']
             #buildmonitor.start_pre_create_snapshot_for_sr_stage(project, bm_git_tag, bm_start_datetime)
             #buildmonitor.start_pre_create_snapshot_for_build_snapshot(project, bm_start_datetime)
-            bm_stage = 'Pre_Snap_Start'
+            bm_stage = 'Pre_Snap_Start' # (+Pre_Snap_snapshot)
             bm_data = {"bm_stage" : bm_stage,
                        "project" : project,
                        "bm_git_tag" : bm_git_tag,
                        "bm_start_datetime": str(bm_start_datetime)
                         }
-            #trigger_next("BUILD-MONITOR", bm_data)
-            trigger_next("BUILD-MONITOR-%s" % bm_stage, bm_data)
+            trigger_next("BUILD-MONITOR-1-%s" % bm_stage, bm_data)
 
         buildstatus = build.getbuildstatus(project)
         print 'buildstatus=%s' %(buildstatus)
@@ -439,7 +437,7 @@ def main(action):
         #                                                        bm_snapshot_url,
         #                                                        bm_end_datetime)
         bm_stage = 'Pre_Snap_End'
-        bm_data = {"bm_stage": bm_stage,
+        bm_data = {"bm_stage" : bm_stage,
                    "project" : project,
                    "bm_git_tag": bm_git_tag,
                    "bm_start_datetime": str(bm_start_datetime),
@@ -447,8 +445,7 @@ def main(action):
                    "bm_snapshot_name" : bm_snapshot_name,
                    "bm_snapshot_url" : bm_snapshot_url,
                     }
-        #trigger_next("BUILD-MONITOR", bm_data)
-        trigger_next("BUILD-MONITOR-%s" % bm_stage, bm_data)
+        trigger_next("BUILD-MONITOR-3-%s" % bm_stage, bm_data)
 
 if __name__ == '__main__':
     try:

diff --git a/job_request.py b/job_request.py
index 4feffbf..50239cf 100755 (executable)
--- a/job_request.py
+++ b/job_request.py
@@ -436,7 +436,8 @@ def main():
         #                                                   bm_start_datetime,
         #                                                   bm_end_datetime,
         #                                                   data["GIT_TAG"])
-        bm_data = {"bm_stage" : 'SR_Accept',
+        bm_stage = 'SR_Accept'
+        bm_data = {"bm_stage" : bm_stage,
                    "event_fields" : event_fields,
                    "bm_start_datetime" : str(bm_start_datetime),
                    "bm_end_datetime" : str(bm_end_datetime),

diff --git a/job_submit.py b/job_submit.py
index 25ffcb6..34093ce 100755 (executable)
--- a/job_submit.py
+++ b/job_submit.py
@@ -691,7 +691,8 @@ def main(build_type):
         #buildmonitor.package_build_for_sr_detail_sr_stage(git_tag, bm_start_datetime,
         #                                                  bm_end_datetime,
         #                                                  submit_info['pre_created'])
-        bm_data = {"bm_stage": 'Submit',
+        bm_stage = 'Submit'
+        bm_data = {"bm_stage" : bm_stage,
                    "commit_date" : commit_date,
                    "commit_msg" : commit_msg,
                    "submit_date" : submit_date,