From ceef57ba0afc1ab04f4928555ac651533ed4a498 Mon Sep 17 00:00:00 2001 From: Junghyun Kim Date: Wed, 8 Mar 2017 11:23:07 +0900 Subject: [PATCH] NEWJOB: litmus_jira_issue_receiver and litmus_tct_file_receiver These scripts are written by Jaekyu Park. Change-Id: I3e77263ee617cf8ea76c28912f6ff5ae7bf63ed5 Signed-off-by: Junghyun Kim --- job_litmus_jira_issue_receiver.py | 159 ++++++++++++++++++++++++++++++++++++ job_litmus_tct_file_receiver.py | 164 ++++++++++++++++++++++++++++++++++++++ packaging/jenkins-scripts.spec | 2 + 3 files changed, 325 insertions(+) create mode 100755 job_litmus_jira_issue_receiver.py create mode 100755 job_litmus_tct_file_receiver.py diff --git a/job_litmus_jira_issue_receiver.py b/job_litmus_jira_issue_receiver.py new file mode 100755 index 0000000..7eadf84 --- /dev/null +++ b/job_litmus_jira_issue_receiver.py @@ -0,0 +1,159 @@ +#!/usr/bin/python +#-*-coding:utf-8-*- + +import os +import sys +import shutil +import subprocess +import json +import pprint +import time +import datetime +import base64 +import MySQLdb +import MySQLdb.cursors + + +def unicode_to_str(obj): + """convert unicode object to str""" + + if isinstance(obj, list): + return [unicode_to_str(element) for element in obj] + elif isinstance(obj, dict): + return {unicode_to_str(key) : unicode_to_str(value) for key, value \ + in obj.iteritems()} + elif isinstance(obj, unicode): + return obj.encode('utf-8') + else: + return obj + + + +if __name__ == "__main__": + + + CONFIG = { + 'file_base' : 'config.base', + 'BASE' : {}, + 'file_mysql' : 'config.mysql.receiver', + 'MYSQL' : {} } + + + RAW = {} + MYSQL = { 'query' : {}, + 'fetchall' : {} } + RESULT = {} + + + + print '=====[ LITMUS Info receiving...]=====\n' + + + if os.path.isfile( CONFIG['file_base'] ): + + f = open( CONFIG['file_base'], 'r') + CONFIG['BASE'] = json.loads(f.read()) + f.close() + print '[CONFIG] base - load config file.\n' + + else: + + CONFIG['BASE']['location'] = 'external' + print '[CONFIG] base - load default.\n' + + + + + + if CONFIG['BASE']['location'] != 'internal': + + # load config data + CONFIG['MYSQL']['host'] = os.getenv("BUILDMONITOR_IP") + CONFIG['MYSQL']['user'] = os.getenv("BUILDMONITOR_USER") + CONFIG['MYSQL']['password'] = os.getenv("BUILDMONITOR_PASS") + CONFIG['MYSQL']['database'] = os.getenv("BUILDMONITOR_NAME") + print '[CONFIG] mysql - load getenv.\n' + + else: + + if not os.path.isfile( CONFIG['file_mysql'] ): + sys.exit("[ERROR] %s was not found!\n" % CONFIG['file_mysql']) + + f = open( CONFIG['file_mysql'], 'r') + CONFIG['MYSQL'] = json.loads(f.read()) + f.close() + print '[CONFIG] mysql - load config file.\n' + + + + + + # load received data + if CONFIG['BASE']['location'] != 'internal': + + trbs_data = os.getenv('TRBS_DATA').replace(' ', '+') + + if trbs_data: + RAW = unicode_to_str(json.loads(base64.b64decode(trbs_data))) + else: + sys.exit( "[Error] TRBS_DATA is empty!\n" ) + + print '[RAW] load TRBS_DATA.\n' + + else: + RAW = {} + RAW['info'] = { 'sr' : '', + 'snapshot_num' : 0 } + + + + + # Connect MySQL + MYSQL['connection'] = MySQLdb.connect( host =CONFIG['MYSQL']['host'], + user =CONFIG['MYSQL']['user'], + passwd=CONFIG['MYSQL']['password'], + db =CONFIG['MYSQL']['database'], + cursorclass=MySQLdb.cursors.DictCursor) + MYSQL['cursor'] = MYSQL['connection'].cursor() + print '[INFO] connect MySQL' + + + + for table, value in RAW.items(): + + if type( value ) == list: + + for item in value: + + try: + + for k, v in item.items(): + if type(v) == str: + item[k] = v.replace("'", '"') + + columns_string = "(`" + "`,`".join(item.keys())+"`)" + values_string = "('" + "','".join(map(str, item.values())) + "')" + sql = "INSERT INTO %s %s VALUES %s" % (table, columns_string, values_string) + + MYSQL['cursor'].execute( sql ) + MYSQL['connection'].commit() + + if 'Id' in item.keys(): + print "[Write] "+table+" id =", item['Id'] + else: + print "[Write] "+table+" id =", item['id'] + + except MySQLdb.IntegrityError: + #if 'Id' in item.keys(): + # print "[WARNING] Duplicate "+table+" id =", item['Id'] + #else: + # print "[WARNING] Duplicate "+table+" id =", item['id'] + print "[ERROR]", sql + + + + MYSQL['cursor'].close() + MYSQL['connection'].close() + + + exit() diff --git a/job_litmus_tct_file_receiver.py b/job_litmus_tct_file_receiver.py new file mode 100755 index 0000000..5926e2c --- /dev/null +++ b/job_litmus_tct_file_receiver.py @@ -0,0 +1,164 @@ +#!/usr/bin/python +#-*-coding:utf-8-*- + +import os +import sys +import shutil +import subprocess +import json +import pprint +import time +import datetime +import base64 +import MySQLdb +import MySQLdb.cursors + + +def unicode_to_str(obj): + """convert unicode object to str""" + + if isinstance(obj, list): + return [unicode_to_str(element) for element in obj] + elif isinstance(obj, dict): + return {unicode_to_str(key) : unicode_to_str(value) for key, value \ + in obj.iteritems()} + elif isinstance(obj, unicode): + return obj.encode('utf-8') + else: + return obj + + + +if __name__ == "__main__": + + + CONFIG = { + 'file_base' : 'config.base', + 'BASE' : {}, + 'file_mysql' : 'config.mysql.receiver', + 'MYSQL' : {} } + + + RAW = {} + MYSQL = { 'query' : {}, + 'fetchall' : {} } + RESULT = {} + + + + print '=====[ LITMUS File receiving...]=====\n' + + + if os.path.isfile( CONFIG['file_base'] ): + + f = open( CONFIG['file_base'], 'r') + CONFIG['BASE'] = json.loads(f.read()) + f.close() + print '[CONFIG] base - load config file.\n' + + else: + + CONFIG['BASE']['location'] = 'external' + print '[CONFIG] base - load default.\n' + + + + + + if CONFIG['BASE']['location'] != 'internal': + + # load config data + CONFIG['MYSQL']['host'] = os.getenv("BUILDMONITOR_IP") + CONFIG['MYSQL']['user'] = os.getenv("BUILDMONITOR_USER") + CONFIG['MYSQL']['password'] = os.getenv("BUILDMONITOR_PASS") + CONFIG['MYSQL']['database'] = os.getenv("BUILDMONITOR_NAME") + print '[CONFIG] mysql - load getenv.\n' + + else: + + if not os.path.isfile( CONFIG['file_mysql'] ): + sys.exit("[ERROR] %s was not found!\n" % CONFIG['file_mysql']) + + f = open( CONFIG['file_mysql'], 'r') + CONFIG['MYSQL'] = json.loads(f.read()) + f.close() + print '[CONFIG] mysql - load config file.\n' + + + + + + # load received data + if CONFIG['BASE']['location'] != 'internal': + + trbs_data = os.getenv('file0') + + if trbs_data: + f = open( 'file0', 'r') + RAW = unicode_to_str(json.loads(f.read())) + f.close() + else: + sys.exit( "[Error] TRBS_DATA is empty!\n" ) + + print '[RAW] load TRBS_DATA.\n' + + else: + RAW = {} + RAW['info'] = { 'sr' : '', + 'snapshot_num' : 0 } + + + + + # Connect MySQL + MYSQL['connection'] = MySQLdb.connect( host =CONFIG['MYSQL']['host'], + user =CONFIG['MYSQL']['user'], + passwd=CONFIG['MYSQL']['password'], + db =CONFIG['MYSQL']['database'], + cursorclass=MySQLdb.cursors.DictCursor) + MYSQL['cursor'] = MYSQL['connection'].cursor() + print '[INFO] connect MySQL' + + + + for table, value in RAW.items(): + + if type( value ) == list: + + print "\n" + print "[INFO] " + table + " amount is", len( value ) + + for item in value: + + try: + + for k, v in item.items(): + if type(v) == str: + item[k] = v.replace("'", '"') + + columns_string = "(`" + "`,`".join(item.keys())+"`)" + values_string = "('" + "','".join(map(str, item.values())) + "')" + sql = "INSERT INTO %s %s VALUES %s" % (table, columns_string, values_string) + + MYSQL['cursor'].execute( sql ) + MYSQL['connection'].commit() + + if 'Id' in item.keys(): + print "[Write] "+table+" id =", item['Id'] + else: + print "[Write] "+table+" id =", item['id'] + + except MySQLdb.IntegrityError: + #if 'Id' in item.keys(): + # print "[WARNING] Duplicate "+table+" id =", item['Id'] + #else: + # print "[WARNING] Duplicate "+table+" id =", item['id'] + print "[ERROR]", sql + + + + MYSQL['cursor'].close() + MYSQL['connection'].close() + + + exit() diff --git a/packaging/jenkins-scripts.spec b/packaging/jenkins-scripts.spec index df1ef10..2d7f7a8 100644 --- a/packaging/jenkins-scripts.spec +++ b/packaging/jenkins-scripts.spec @@ -141,6 +141,8 @@ fi %{destdir}/job_repa.py %{destdir}/job_trbs_test_result_receiver.py %{destdir}/job_update_scm_meta_git_for_dashboard.py +%{destdir}/job_litmus_jira_issue_receiver.py +%{destdir}/job_litmus_tct_file_receiver.py %dir %{destdir}/templates %{destdir}/templates/index.html %{destdir}/job_update_local_git.py -- 2.7.4