From a974becc1ecc0633c75747005b0f009339046daa Mon Sep 17 00:00:00 2001 From: SungHun Hwang Date: Mon, 29 May 2017 10:09:57 +0900 Subject: [PATCH] fix the trbs test information error PROBLEM: The test information of TRBS cannot display on dashboard in public SOLUTION: fix the test job code Change-Id: Ia905f4da8243d29fc91f42e5dcc2411076a7d9ec Signed-off-by: SungHun Hwang --- codebase.py | 103 +++++ job_trbs_test_result_receiver.py | 950 ++++++++++++++++++++++++++++++--------- 2 files changed, 830 insertions(+), 223 deletions(-) create mode 100755 codebase.py mode change 100644 => 100755 job_trbs_test_result_receiver.py diff --git a/codebase.py b/codebase.py new file mode 100755 index 0000000..4559262 --- /dev/null +++ b/codebase.py @@ -0,0 +1,103 @@ +#!/usr/bin/python +#-*-coding:utf-8-*- + +import os +import sys +import json +import base64 +import pprint +import xml.etree.ElementTree as ET +import urllib2 + + +def loadJSON( jsonfile ): + + VALUE = {} + + if not os.path.isfile( jsonfile ): + sys.exit("[ERROR] file(%s) was not found!" % jsonfile) + + f = open( jsonfile, 'r') + VALUE = json.loads(f.read()) + f.close() + print ('[INFO] load JSON file - file(%s)' % jsonfile) + + return VALUE + + + + +def unicode_to_str(obj): + """convert unicode object to str""" + + if isinstance(obj, list): + return [unicode_to_str(element) for element in obj] + elif isinstance(obj, dict): + return {unicode_to_str(key) : unicode_to_str(value) for key, value \ + in obj.iteritems()} + elif isinstance(obj, unicode): + return obj.encode('utf-8') + else: + return obj + + + +def loadXML( url, proxy="ON", error_continue=True ): + + try: + if( proxy == "OFF" ): + + proxy_handler = urllib2.ProxyHandler({}) + opener = urllib2.build_opener(proxy_handler) + doc = ET.parse( opener.open(url) ) + + else: + doc = ET.parse( urllib2.urlopen(url).read() ) + + root = doc.getroot() + + except urllib2.HTTPError, e: + print ("[ERROR] HTTP error: %d - %s" % (e.code, url)) + if( error_continue == False ): + exit(1) + else: + return None + + except urllib2.URLError, e: + print ("[ERROR] Network error: %s - %s" % (e.reason.args[1], url)) + if( error_continue == False ): + exit(1) + else: + return None + + return root + + + + + + +def default(obj): + import datetime + + if isinstance(obj, datetime.datetime): + return obj.strftime('%Y-%m-%d %H:%M:%S') + + raise TypeError('Not sure how to serialize %s' % (obj,)) + + + + + + + +def makeInsertSQL( table, item ): + + for k, v in item.items(): + if type(v) == str: + item[k] = v.replace("'", '"') + + columns_string = "(`" + "`,`".join(item.keys())+"`)" + values_string = "('" + "','".join(map(str, item.values())) + "')" + + return ("INSERT INTO %s %s VALUES %s" % (table, columns_string, values_string)) diff --git a/job_trbs_test_result_receiver.py b/job_trbs_test_result_receiver.py old mode 100644 new mode 100755 index 73d81fb..d9dcae5 --- a/job_trbs_test_result_receiver.py +++ b/job_trbs_test_result_receiver.py @@ -3,351 +3,855 @@ import os import sys -import shutil -import subprocess -import json -import pprint -import time -import datetime -import base64 import MySQLdb import MySQLdb.cursors +import xml.etree.ElementTree as ET +import urllib2 +import re +from codebase import * -#class MySQLCursorDict(mysql.connector.cursor.MySQLCursor): - #def _row_to_python(self, rowdata, desc=None): - #row = super(MySQLCursorDict, self)._row_to_python(rowdata, desc) - #if row: - #return dict(zip(self.column_names, row)) - #return None +if __name__ == "__main__": + + CONFIG = { + 'FILE' : { + 'BASE' : "config.base", + 'MYSQL' : "config.mysql.test", + 'RAW' : "trbs.txt" + }, + + 'BASE' : {}, + 'MYSQL' : {}, + 'RAW' : {} + } + + + + print "=====[ TRBS Test Write ]=====\n" -def unicode_to_str(obj): - """convert unicode object to str""" +# For Debug +# CONFIG['BASE'] = loadJSON( CONFIG['FILE']['BASE'] ) +# CONFIG['MYSQL'] = loadJSON( CONFIG['FILE']['MYSQL'] ) - if isinstance(obj, list): - return [unicode_to_str(element) for element in obj] - elif isinstance(obj, dict): - return {unicode_to_str(key) : unicode_to_str(value) for key, value \ - in obj.iteritems()} - elif isinstance(obj, unicode): - return obj.encode('utf-8') + + #pprint.pprint( CONFIG['RAW'] ) + #exit() + + temp = os.getenv('file0') + if temp: + paramfile = "file0" + if not os.path.isfile( paramfile ): + paramfile = "../file0" + if not os.path.isfile( paramfile ): + print "[ERROR] file(%s) was not found!" % paramfile + exit(1) + + f = open( paramfile, 'r') + CONFIG['RAW'] = unicode_to_str(json.loads(f.read())) + f.close() else: - return obj + print "[ERROR] TRBS_DATA is empty!" + exit(1) + print '[INFO] Jenkins TRBS_DATA loaded' + pprint.pprint( CONFIG['RAW'] ) + #exit(0) -if __name__ == "__main__": + #CONFIG['RAW'] = loadJSON( CONFIG['FILE']['RAW'] ) - ITEM = { - 'sr' : '', - 'num' : '', - 'config_mysql' : 'config_mysql2.json' } + #pprint.pprint( CONFIG['RAW'] ) + #exit() - RESULT = { 'raw' : {} } + if( not 'BASE' in CONFIG['RAW'].keys() ): + print "[ERROR] BASE value empty!" + exit(1) + DINFO = { + 'sr' : CONFIG['RAW']['BASE']['sr'], + 'snapshot_num' : CONFIG['RAW']['BASE']['snapshot_num'] + } - print '=====[BuildMonitor DB connecting...]=====\n' + print "[INFO] CONFIG : sr =", DINFO['sr'], ", snapshot_num =", DINFO['snapshot_num'] - # load config data - db_ip = os.getenv("BUILDMONITOR_IP") - db_user = os.getenv("BUILDMONITOR_USER") - db_pass = os.getenv("BUILDMONITOR_PASS") - db_name = os.getenv("BUILDMONITOR_NAME") - print '%s %s %s' % (db_ip, db_user, db_name) - #if not os.path.isfile( ITEM['config_mysql'] ): - #ITEM['config_mysql'] = "jenkins-scripts/" + ITEM['config_mysql'] - #if not os.path.isfile( ITEM['config_mysql'] ): - #sys.exit("ERROR: %s was not found!" % ITEM['config_mysql']) -# -# - #f = open( ITEM['config_mysql'], 'r') - #mconfig = json.loads(f.read()) - #f.close() - trbs_data = os.getenv('TRBS_DATA').replace(' ', '+') + MYSQL = {} - if trbs_data: - RESULT['raw']['trbs_data'] = unicode_to_str(json.loads(base64.b64decode(trbs_data))) - else: - sys.exit( "Error: trbs_data is empty!" ) + # connect mysql +# if CONFIG['BASE']['MODE'] == "debug": +# MYSQL['connection'] = MySQLdb.connect( host =CONFIG['MYSQL']['host'], +# user =CONFIG['MYSQL']['user'], +# passwd=CONFIG['MYSQL']['password'], +# db =CONFIG['MYSQL']['database'], +# cursorclass=MySQLdb.cursors.DictCursor) +# print "[DEBUG] MYSQL : Connect host =", CONFIG['MYSQL']['host'], ", database =", CONFIG['MYSQL']['database'] - print RESULT['raw']['trbs_data'] + MYSQL['connection'] = MySQLdb.connect( host =os.getenv("BUILDMONITOR_IP"), + user =os.getenv("BUILDMONITOR_USER"), + passwd=os.getenv("BUILDMONITOR_PASS"), + db =os.getenv("BUILDMONITOR_NAME"), + cursorclass=MySQLdb.cursors.DictCursor) + print "[INFO] MYSQL : Connect host =", os.getenv("BUILDMONITOR_IP"), ", database =", os.getenv("BUILDMONITOR_NAME") - ITEM['sr'] = RESULT['raw']['trbs_data']['info']['sr'] - ITEM['num'] = RESULT['raw']['trbs_data']['info']['snapshot_num'] - #dbcon = mysql.connector.connect( user=mconfig['user'], password=mconfig['password'], host=mconfig['host'], database=mconfig['database'] ) - #dbcon = mysql.connector.connect( user=db_user, password=db_pass, host=db_ip, database=db_name ) - dbcon = MySQLdb.connect(host=db_ip, user=db_user, passwd=db_pass, db=db_name, cursorclass=MySQLdb.cursors.DictCursor) - #query = dbcon.cursor(cursor_class=MySQLCursorDict) - #query = dbcon.cursor(dictionary=True) - query = dbcon.cursor() + MYSQL['cursor'] = MYSQL['connection'].cursor() + SQL = {} + SQL['SR'] = {} + # select sr + SQL['SR']['select_srinfo'] = ( + "SELECT * " + "FROM sr_status s, sr_status_detail d " + "WHERE s.sr= %(sr)s AND d.sr_status_id = s.id ORDER BY d.id DESC" + ) - # SR의 ID값 확인 - select_sr = "SELECT * FROM sr_status WHERE sr = %s" - query.execute( select_sr, [ITEM['sr']] ) - RESULT['raw']['sr_status'] = query.fetchall() +# print "SQL[sr][select_srinfo] : ", SQL['SR']['select_srinfo'] +# print "DINFO : ", DINFO - print RESULT['raw']['sr_status'] + MYSQL['cursor'].execute( SQL['SR']['select_srinfo'], DINFO ) + TEMP = MYSQL['cursor'].fetchall() - if len( RESULT['raw']['sr_status'] ) != 1: - sys.exit("ERROR: %s was not found in database(sr_status)!" % ITEM['sr']) - RESULT['sr_id'] = RESULT['raw']['sr_status'][0]['id'] - print "sr_id is", RESULT['sr_id'] + # 결과가 1개 이거나 아니면 latest 결과가 정답일 것이라 전제 + if len( TEMP ) > 0: + DINFO['sr_status_id'] = TEMP[0]['sr_status_id'] + DINFO['status'] = TEMP[0]['status'] + DINFO['trbs_build_project_id'] = TEMP[0]['pre_build_project_id'] + print ("[INFO] MYSQL : sr_status_id = '%(sr_status_id)s', trbs_build_project_id = %(trbs_build_project_id)s" % DINFO) + else: + print ("[ERROR] MYSQL : Can not search sr_status where sr = %(sr)s" % DINFO) + exit(1) - # build_project_id 값 확인 - select_sr_detail = ( "SELECT id, sr_status_id, trbs_build_project_id, post_build_project_id, obs_request_no, obs_request_comment, obs_request_date, obs_request_status " - "FROM sr_status_detail " - "WHERE sr_status_id = %s " - "ORDER BY id DESC" ) - print select_sr_detail - query.execute( select_sr_detail, [RESULT['sr_id']] ) - RESULT['raw']['sr_detail'] = query.fetchall() - print RESULT['raw']['sr_detail'] - if len( RESULT['raw']['sr_detail'] ) < 1: - sys.exit("ERROR: %s was not found in database(sr_status_detail)!" % ITEM['sr']) - RESULT['build_project_id'] = RESULT['raw']['sr_detail'][0]['trbs_build_project_id'] - print "build_project_id is", RESULT['build_project_id'] + #print( DINFO['sr'] ) + #pprint.pprint( CONFIG['RAW']['REF']['environment'] ) + #exit() + #print( CONFIG['RAW']['REF']['environment'] - # snapshot 정보 확인 - select_snapshot = ( "SELECT build_snapshot.id, snapshot_name, snapshot_num, snapshot_url, start_time, end_time, build_snapshot.status, info_project_id " - "FROM build_snapshot " - "LEFT JOIN build_project ON build_project.id = build_snapshot.build_project_id " - "WHERE build_project_id = %s ORDER BY snapshot_num DESC" ) + SQL['REF'] = {} + # select ref + SQL['REF']['select'] = ( + "SELECT * " + "FROM trbs_ref " + "WHERE build_id= %(build_id)s AND tct_profile = %(tct_profile)s ORDER BY id DESC" + ) + MYSQL['cursor'].execute( SQL['REF']['select'], CONFIG['RAW']['REF']['environment'] ) + TEMP = MYSQL['cursor'].fetchall() - query.execute( select_snapshot, [RESULT['build_project_id']] ) - temp = query.fetchall() - print temp + # 결과가 1개 이거나 아니면 latest 결과가 정답일 것이라 전제 + if len( TEMP ) > 0: + DINFO['ref_id'] = TEMP[0]['id'] - if len(temp) < 1: - sys.exit("ERROR: snapshot_num is empty!") + print ("[INFO] MYSQL : Reference id = %(id)s, build_id = %(build_id)s" % TEMP[0]) + + else: + print "[WARN] MYSQL : Can not search Reference TCT - ", CONFIG['RAW']['REF']['environment']['build_id'] + + #pprint.pprint( CONFIG['RAW']['REF'] ) + + VALUES = { + 'total' : CONFIG['RAW']['REF']['COUNT']['total_case'], + 'pass' : CONFIG['RAW']['REF']['COUNT']['pass_case'], + 'fail' : CONFIG['RAW']['REF']['COUNT']['fail_case'], + 'block' : CONFIG['RAW']['REF']['COUNT']['block_case'], + 'na' : CONFIG['RAW']['REF']['COUNT']['na_case'], + 'pcnt' : round( (float(CONFIG['RAW']['REF']['COUNT']['pass_case'])/float(CONFIG['RAW']['REF']['COUNT']['total_case']))*100, 2 ), + 'plan_name' : CONFIG['RAW']['REF']['plan_name'], + 'time_start' : CONFIG['RAW']['REF']['start_at'], + 'time_end' : CONFIG['RAW']['REF']['end_at'], + 'build_id' : CONFIG['RAW']['REF']['environment']['build_id'], + 'device_id' : CONFIG['RAW']['REF']['environment']['device_id'], + 'device_model' : CONFIG['RAW']['REF']['environment']['device_model'], + 'device_name' : CONFIG['RAW']['REF']['environment']['device_name'], + 'device_host' : CONFIG['RAW']['REF']['environment']['device_host'], + 'manufacturer' : CONFIG['RAW']['REF']['environment']['manufacturer'], + 'tct_profile' : CONFIG['RAW']['REF']['environment']['tct_profile'], + 'tct_version' : CONFIG['RAW']['REF']['environment']['tct_version'], + 'url' : CONFIG['RAW']['REF']['url'] + } + + #pprint.pprint( VALUES ) + + table = "trbs_ref" + + try: + sql = makeInsertSQL( table, VALUES ) + MYSQL['cursor'].execute( sql ) + MYSQL['connection'].commit() + + print "[INFO] MYSQL : INSERT to", table, ", build_id =", VALUES['build_id'] + + + except MySQLdb.Error, e: + try: + print "[ERROR] MYSQL : [%d]: %s (%s)" % (e.args[0], e.args[1], sql) + except IndexError: + print "[ERROR] MYSQL : %s (%s)" % (str(e), sql) - RESULT['raw']['snapshot'] = temp - RESULT['snapshot_num'] = RESULT['raw']['snapshot'][0]['snapshot_num'] - print "snapshot_num is", RESULT['snapshot_num'] + MYSQL['cursor'].execute( SQL['REF']['select'], CONFIG['RAW']['REF']['environment'] ) + TEMP = MYSQL['cursor'].fetchall() + # 결과가 1개 이거나 아니면 latest 결과가 정답일 것이라 전제 + if len( TEMP ) > 0: + DINFO['ref_id'] = TEMP[0]['id'] + print ("[INFO] MYSQL : Reference id = %(id)s, build_id = %(build_id)s" % TEMP[0]) + else: + print "[ERROR] MYSQL : Can not process reference TCT" + exit(1) - if ITEM['num'] != RESULT['snapshot_num']: - sys.exit("Error: snapshot_num different!!") - for key, value in RESULT['raw']['trbs_data'].items(): - if key == 'trbs_tct': + for suite in CONFIG['RAW']['REF']['SUITE'].keys(): - insert_tct = ( "INSERT INTO trbs_tct " - "( id, sr_status_id, ref_id, sr_name, snapshot_num, profile, plan_name, total, pass, fail, " - "block, na, pcnt, time_start, time_end, build_id, device_id, device_model, device_name, " - "device_host, manufacturer, tct_profile, tct_version, url) " - "VALUES ( %(id)s, %(sr_status_id)s, %(ref_id)s, %(sr_name)s, %(snapshot_num)s, %(profile)s, %(plan_name)s, %(total)s, %(pass)s, %(fail)s, " - "%(block)s, %(na)s, %(pcnt)s, %(time_start)s, %(time_end)s, %(build_id)s, %(device_id)s, %(device_model)s, %(device_name)s, " - "%(device_host)s, %(manufacturer)s, %(tct_profile)s, %(tct_version)s, %(url)s )" ) + #pprint.pprint( CONFIG['RAW']['REF']['SUITE'][suite] ) + #pprint.pprint( DINFO ) + #exit() - try: - value['sr_status_id'] = RESULT['sr_id'] + VALUES = { + 'trbs_ref_id' : DINFO['ref_id'], + 'name' : suite, + 'total' : CONFIG['RAW']['REF']['SUITE'][suite]['total_case'], + 'pass' : CONFIG['RAW']['REF']['SUITE'][suite]['pass_case'], + 'pass_rate' : CONFIG['RAW']['REF']['SUITE'][suite]['pass_rate'], + 'fail' : CONFIG['RAW']['REF']['SUITE'][suite]['fail_case'], + 'fail_rate' : CONFIG['RAW']['REF']['SUITE'][suite]['fail_rate'], + 'block' : CONFIG['RAW']['REF']['SUITE'][suite]['block_case'], + 'block_rate' : CONFIG['RAW']['REF']['SUITE'][suite]['block_rate'], + 'na' : CONFIG['RAW']['REF']['SUITE'][suite]['na_case'], + 'na_rate' : CONFIG['RAW']['REF']['SUITE'][suite]['na_rate'] + } + + + + SQL['REF']['select_suite'] = ( + "SELECT * " + "FROM trbs_ref_suite " + "WHERE trbs_ref_id = %(trbs_ref_id)s AND name = %(name)s" + ) + + MYSQL['cursor'].execute( SQL['REF']['select_suite'], VALUES ) + TEMP = MYSQL['cursor'].fetchall() + + if len( TEMP ) > 0: + print ("[WARN] MYSQL : Duplicate REF Suite, id = %(id)s, name = %(name)s" % TEMP[0]) + + else: + + + table = "trbs_ref_suite" - query.execute( insert_tct, value ) - dbcon.commit() - print "[Write: trbs_tct] id =", value['id'] - except MySQLdb.IntegrityError: - print "[Error] Duplicate entry (" + key + ")-", value + try: + sql = makeInsertSQL( table, VALUES ) - elif key == 'trbs_tct_suite': + #print sql + #exit() - insert_tct_suite = ( "INSERT INTO trbs_tct_suite " - "( id, trbs_tct_id, name, total, pass, pass_rate, fail, fail_rate, block, block_rate, na, na_rate ) " - "VALUES ( %(id)s, %(trbs_tct_id)s, %(name)s, %(total)s, %(pass)s, %(pass_rate)s, %(fail)s, %(fail_rate)s, %(block)s, %(block_rate)s, %(na)s, %(na_rate)s )" ) + MYSQL['cursor'].execute( sql ) + MYSQL['connection'].commit() - for temp in value: + print "[INFO] MYSQL : INSERT to", table, ", trbs_ref_id =", DINFO['ref_id'], ", name =", suite + except MySQLdb.Error, e: try: - query.execute( insert_tct_suite, temp ) - dbcon.commit() - print "[Write: trbs_tct_suite] id =", temp['id'] - except MySQLdb.IntegrityError: - print "[Error] Duplicate entry (" + key + ")-", temp + print "[ERROR] MYSQL : [%d]: %s (%s)" % (e.args[0], e.args[1], sql) + except IndexError: + print "[ERROR] MYSQL : %s (%s)" % (str(e), sql) + + + + + + + + - elif key == 'trbs_ref': + SQL['TCT'] = {} + # select tct + SQL['TCT']['select'] = ( + "SELECT * " + "FROM trbs_tct " + "WHERE sr_name = %(sr)s AND snapshot_num = %(snapshot_num)s ORDER BY id DESC" + ) - insert_ref = ( "INSERT INTO trbs_ref " - "( id, plan_name, total, pass, fail, block, na, pcnt, time_start, time_end, build_id, device_id, " - "device_model, device_name, device_host, manufacturer, tct_profile, tct_version, url ) " - "VALUES( %(id)s, %(plan_name)s, %(total)s, %(pass)s, %(fail)s, %(block)s, %(na)s, %(pcnt)s, %(time_start)s, %(time_end)s, %(build_id)s, %(device_id)s, " - "%(device_model)s, %(device_name)s, %(device_host)s, %(manufacturer)s, %(tct_profile)s, %(tct_version)s, %(url)s )" ) + MYSQL['cursor'].execute( SQL['TCT']['select'], DINFO ) + TEMP = MYSQL['cursor'].fetchall() + if len( TEMP ) > 0: + DINFO['trbs_tct_id'] = TEMP[0]['id'] + print ("[WARN] MYSQL : Duplicate TCT, id = %(id)s, sr_name = %(sr_name)s" % TEMP[0]) + + else: + + #pprint.pprint( CONFIG['RAW']['TCT'] ) + #pprint.pprint( DINFO ) + + VALUES = { + 'sr_status_id' : DINFO['sr_status_id'], + 'ref_id' : DINFO['ref_id'], + 'sr_name' : DINFO['sr'], + 'snapshot_num' : DINFO['snapshot_num'], + 'profile' : CONFIG['RAW']['TCT']['environment']['tct_profile'], + 'total' : CONFIG['RAW']['TCT']['COUNT']['total_case'], + 'pass' : CONFIG['RAW']['TCT']['COUNT']['pass_case'], + 'fail' : CONFIG['RAW']['TCT']['COUNT']['fail_case'], + 'block' : CONFIG['RAW']['TCT']['COUNT']['block_case'], + 'na' : CONFIG['RAW']['TCT']['COUNT']['na_case'], + 'pcnt' : round( (float(CONFIG['RAW']['TCT']['COUNT']['pass_case'])/float(CONFIG['RAW']['TCT']['COUNT']['total_case']))*100, 2 ), + 'plan_name' : CONFIG['RAW']['TCT']['plan_name'], + 'time_start' : CONFIG['RAW']['TCT']['start_at'], + 'time_end' : CONFIG['RAW']['TCT']['end_at'], + 'build_id' : CONFIG['RAW']['TCT']['environment']['build_id'], + 'device_id' : CONFIG['RAW']['TCT']['environment']['device_id'], + 'device_model' : CONFIG['RAW']['TCT']['environment']['device_model'], + 'device_name' : CONFIG['RAW']['TCT']['environment']['device_name'], + 'device_host' : CONFIG['RAW']['TCT']['environment']['device_host'], + 'manufacturer' : CONFIG['RAW']['TCT']['environment']['manufacturer'], + 'tct_profile' : CONFIG['RAW']['TCT']['environment']['tct_profile'], + 'tct_version' : CONFIG['RAW']['TCT']['environment']['tct_version'], + 'url' : CONFIG['RAW']['TCT']['url'] + } + + + #pprint.pprint( VALUES ) + + table = "trbs_tct" + + try: + sql = makeInsertSQL( table, VALUES ) + + #print sql + #exit() + + MYSQL['cursor'].execute( sql ) + MYSQL['connection'].commit() + + print "[INFO] MYSQL : INSERT to", table, ", build_id =", VALUES['build_id'] + + except MySQLdb.Error, e: try: - query.execute( insert_ref, value ) - dbcon.commit() - print "[Write: trbs_ref] id =", value['id'] - except MySQLdb.IntegrityError: - print "[Error] Duplicate entry (" + key + ")-", value + print "[ERROR] MYSQL : [%d]: %s (%s)" % (e.args[0], e.args[1], sql) + except IndexError: + print "[ERROR] MYSQL : %s (%s)" % (str(e), sql) + + + + MYSQL['cursor'].execute( SQL['TCT']['select'], DINFO ) + TEMP = MYSQL['cursor'].fetchall() + + # 결과가 1개 이거나 아니면 latest 결과가 정답일 것이라 전제 + if len( TEMP ) > 0: + DINFO['trbs_tct_id'] = TEMP[0]['id'] + print ("[INFO] MYSQL : TCT id = %(id)s, build_id = %(build_id)s" % TEMP[0]) + else: + print "[ERROR] MYSQL : Can not process TCT" + exit(1) + + + - elif key == 'trbs_ref_suite': - insert_ref_suite = ( "INSERT INTO trbs_ref_suite " - "( id, trbs_ref_id, name, total, pass, pass_rate, fail, fail_rate, block, block_rate, na, na_rate ) " - "VALUES ( %(id)s, %(trbs_ref_id)s, %(name)s, %(total)s, %(pass)s, %(pass_rate)s, %(fail)s, %(fail_rate)s, %(block)s, %(block_rate)s, %(na)s, %(na_rate)s )" ) - for temp in value: + + #pprint.pprint( CONFIG['RAW']['TCT'] ) + #exit() + + for suite in CONFIG['RAW']['TCT']['SUITE'].keys(): + + #pprint.pprint( CONFIG['RAW']['TCT']['SUITE'][suite] ) + #pprint.pprint( DINFO ) + #exit() + + VALUES = { + 'trbs_tct_id' : DINFO['trbs_tct_id'], + 'name' : suite, + 'total' : CONFIG['RAW']['TCT']['SUITE'][suite]['total_case'], + 'pass' : CONFIG['RAW']['TCT']['SUITE'][suite]['pass_case'], + 'pass_rate' : CONFIG['RAW']['TCT']['SUITE'][suite]['pass_rate'], + 'fail' : CONFIG['RAW']['TCT']['SUITE'][suite]['fail_case'], + 'fail_rate' : CONFIG['RAW']['TCT']['SUITE'][suite]['fail_rate'], + 'block' : CONFIG['RAW']['TCT']['SUITE'][suite]['block_case'], + 'block_rate' : CONFIG['RAW']['TCT']['SUITE'][suite]['block_rate'], + 'na' : CONFIG['RAW']['TCT']['SUITE'][suite]['na_case'], + 'na_rate' : CONFIG['RAW']['TCT']['SUITE'][suite]['na_rate'] + } + + + + SQL['TCT']['select_suite'] = ( + "SELECT * " + "FROM trbs_tct_suite " + "WHERE trbs_tct_id = %(trbs_tct_id)s AND name = %(name)s" + ) + + MYSQL['cursor'].execute( SQL['TCT']['select_suite'], VALUES ) + TEMP = MYSQL['cursor'].fetchall() + + if len( TEMP ) > 0: + print ("[WARN] MYSQL : Duplicate TCT Suite, id = %(id)s, name = %(name)s" % TEMP[0]) + + else: + + + table = "trbs_tct_suite" + + try: + sql = makeInsertSQL( table, VALUES ) + + #print sql + #exit() + + MYSQL['cursor'].execute( sql ) + MYSQL['connection'].commit() + + print "[INFO] MYSQL : INSERT to", table, ", trbs_tct_id =", DINFO['trbs_tct_id'], ", name =", suite + + except MySQLdb.Error, e: try: - query.execute( insert_ref_suite, temp ) - dbcon.commit() - print "[Write: trbs_ref_suite] id =", temp['id'] - except MySQLdb.IntegrityError: - print "[Error] Duplicate entry (" + key + ")-", temp + print "[ERROR] MYSQL : [%d]: %s (%s)" % (e.args[0], e.args[1], sql) + except IndexError: + print "[ERROR] MYSQL : %s (%s)" % (str(e), sql) + + + + + + + + + + + + + + + + + + + + + + + SQL['SMOKES'] = {} + # select smokes + SQL['SMOKES']['select'] = ( + "SELECT * " + "FROM trbs_smoke " + "WHERE sr_name = %(sr)s AND snapshot_num = %(snapshot_num)s ORDER BY id DESC" + ) + + MYSQL['cursor'].execute( SQL['SMOKES']['select'], DINFO ) + TEMP = MYSQL['cursor'].fetchall() + + if len( TEMP ) > 0: + DINFO['trbs_smoke_id'] = TEMP[0]['id'] + print ("[WARN] MYSQL : Duplicate SMOKE, id = %(id)s, sr_name = %(sr_name)s" % TEMP[0]) + + else: + #pprint.pprint( CONFIG['RAW']['SMOKES'] ) + #exit() - elif key == 'trbs_smoke': + VALUES = { + 'sr_status_id' : DINFO['sr_status_id'], + 'sr_name' : DINFO['sr'], + 'snapshot_num' : DINFO['snapshot_num'], + 'profile' : CONFIG['RAW']['TCT']['environment']['tct_profile'], + 'total' : CONFIG['RAW']['SMOKES']['COUNT']['total_case'], + 'pass' : CONFIG['RAW']['SMOKES']['COUNT']['pass_case'], + 'fail' : CONFIG['RAW']['SMOKES']['COUNT']['fail_case'], + 'pcnt' : round( (float(CONFIG['RAW']['SMOKES']['COUNT']['pass_case'])/float(CONFIG['RAW']['SMOKES']['COUNT']['total_case']))*100, 2 ), + 'time_start' : CONFIG['RAW']['TCT']['start_at'], + 'time_end' : CONFIG['RAW']['TCT']['start_at'], + 'url' : CONFIG['RAW']['TCT']['url'] + } - insert_smoke = ( "INSERT INTO trbs_smoke " - "( id, sr_status_id, sr_name, snapshot_num, profile, pass, fail, total, pcnt, url, time_start, time_end ) " - "VALUES ( %(id)s, %(sr_status_id)s, %(sr_name)s, %(snapshot_num)s, %(profile)s, %(pass)s, %(fail)s, %(total)s, %(pcnt)s, %(url)s, %(time_start)s, %(time_end)s ) " ) + #pprint.pprint( VALUES ) + #exit() + + table = "trbs_smoke" + + try: + sql = makeInsertSQL( table, VALUES ) + + #print sql + #exit() + + MYSQL['cursor'].execute( sql ) + MYSQL['connection'].commit() + + print "[INFO] MYSQL : INSERT to", table + + except MySQLdb.Error, e: try: - value['sr_status_id'] = RESULT['sr_id'] + print "[ERROR] MYSQL : [%d]: %s (%s)" % (e.args[0], e.args[1], sql) + except IndexError: + print "[ERROR] MYSQL : %s (%s)" % (str(e), sql) + + + + MYSQL['cursor'].execute( SQL['SMOKES']['select'], DINFO ) + TEMP = MYSQL['cursor'].fetchall() + + # 결과가 1개 이거나 아니면 latest 결과가 정답일 것이라 전제 + if len( TEMP ) > 0: + DINFO['trbs_smoke_id'] = TEMP[0]['id'] + print ("[INFO] MYSQL : SMOKE id = %(id)s" % TEMP[0]) + else: + print "[ERROR] MYSQL : Can not process SMOKE" + exit(1) + + + + #pprint.pprint( CONFIG['RAW']['SMOKES'] ) + #exit() + + for suite in CONFIG['RAW']['SMOKES']['SUITE'].keys(): + + #pprint.pprint( CONFIG['RAW']['SMOKES']['SUITE'][suite] ) + #pprint.pprint( DINFO ) + #exit() + + VALUES = { + 'trbs_smoke_id' : DINFO['trbs_smoke_id'], + 'name' : suite, + 'pass' : CONFIG['RAW']['SMOKES']['SUITE'][suite]['COUNT']['pass_case'], + 'fail' : CONFIG['RAW']['SMOKES']['SUITE'][suite]['COUNT']['fail_case'], + } + + #pprint.pprint( VALUES ) + #exit() + + SQL['SMOKES']['select_suite'] = ( + "SELECT * " + "FROM trbs_smoke_suite " + "WHERE trbs_smoke_id = %(trbs_smoke_id)s AND name = %(name)s" + ) + + MYSQL['cursor'].execute( SQL['SMOKES']['select_suite'], VALUES ) + TEMP = MYSQL['cursor'].fetchall() + + if len( TEMP ) > 0: + DINFO['trbs_smoke_suite_id'] = TEMP[0]['id'] + print ("[WARN] MYSQL : Duplicate SMOKE Suite, id = %(id)s, name = %(name)s" % TEMP[0]) + + else: - query.execute( insert_smoke, value ) - dbcon.commit() - print "[Write: trbs_smoke] id =", value['id'] - except MySQLdb.IntegrityError: - print "[Error] Duplicate entry (" + key + ")-", value + table = "trbs_smoke_suite" - elif key == 'trbs_smoke_suite': + try: + sql = makeInsertSQL( table, VALUES ) + + #print sql + #exit() + + MYSQL['cursor'].execute( sql ) + MYSQL['connection'].commit() - insert_smoke_suite = ( "INSERT INTO trbs_smoke_suite " - "( id, trbs_smoke_id, name, pass, fail ) " - "VALUES( %(id)s, %(trbs_smoke_id)s, %(name)s, %(pass)s, %(fail)s )" ) + print "[INFO] MYSQL : INSERT to", table, ", trbs_smoke_id =", DINFO['trbs_smoke_id'], ", name =", suite - for temp in value: + except MySQLdb.Error, e: try: - query.execute( insert_smoke_suite, temp ) - dbcon.commit() - print "[Write: trbs_smoke_suite] id =", temp['id'] - except MySQLdb.IntegrityError: - print "[Error] Duplicate entry (" + key + ")-", temp + print "[ERROR] MYSQL : [%d]: %s (%s)" % (e.args[0], e.args[1], sql) + except IndexError: + print "[ERROR] MYSQL : %s (%s)" % (str(e), sql) + + + + MYSQL['cursor'].execute( SQL['SMOKES']['select_suite'], VALUES ) + TEMP = MYSQL['cursor'].fetchall() + + if len( TEMP ) == 0: + print "[ERROR] MYSQL : Can not process SMOKE Suite" + else: + DINFO['trbs_smoke_suite_id'] = TEMP[0]['id'] + + + for testcase in CONFIG['RAW']['SMOKES']['SUITE'][suite]['TESTCASE'].keys(): + + VALUES = { + 'trbs_smoke_suite_id' : DINFO['trbs_smoke_suite_id'], + 'name' : testcase, + 'result' : CONFIG['RAW']['SMOKES']['SUITE'][suite]['TESTCASE'][testcase]['passed'] + } + + #pprint.pprint( VALUES ) + #exit() - elif key == 'trbs_smoke_suite_test': + SQL['SMOKES']['select_suite_test'] = ( + "SELECT * " + "FROM trbs_smoke_suite_test " + "WHERE trbs_smoke_suite_id = %(trbs_smoke_suite_id)s AND name = %(name)s" + ) - insert_smoke_suite_test = ( "INSERT INTO trbs_smoke_suite_test (id, trbs_smoke_suite_id, name, result) " - "VALUES ( %(id)s, %(trbs_smoke_suite_id)s, %(name)s, %(result)s )" ) + MYSQL['cursor'].execute( SQL['SMOKES']['select_suite_test'], VALUES ) + TEMP = MYSQL['cursor'].fetchall() + + if len( TEMP ) > 0: + print ("[WARN] MYSQL : Duplicate SMOKE Suite Test, id = %(id)s, name = %(name)s" % TEMP[0]) + else: + + table = "trbs_smoke_suite_test" - for temp in value: try: + sql = makeInsertSQL( table, VALUES ) + + #print sql + #exit() + + MYSQL['cursor'].execute( sql ) + MYSQL['connection'].commit() + + print "[INFO] MYSQL : INSERT to", table, ", trbs_smoke_suite_id =", DINFO['trbs_smoke_suite_id'], ", name =", testcase + + except MySQLdb.Error, e: + try: + print "[ERROR] MYSQL : [%d]: %s (%s)" % (e.args[0], e.args[1], sql) + except IndexError: + print "[ERROR] MYSQL : %s (%s)" % (str(e), sql) + + + + - query.execute( insert_smoke_suite_test, temp ) - dbcon.commit() - print "[Write: trbs_smoke_suite_test] id =", temp['id'] - except MySQLdb.IntegrityError: - print "[Error] Duplicate entry (" + key + ")-", temp - elif key == 'trbs_perf': - insert_perf = ( "INSERT INTO trbs_perf " - "( id, sr_status_id, sr_name, snapshot_num, profile, device_core, device_dvfs, device_freq, device_kernel, " - "device_platform, total, pass, fail, pcnt, time_start, time_end, url ) " - "VALUES ( %(id)s, %(sr_status_id)s, %(sr_name)s, %(snapshot_num)s, %(profile)s, %(device_core)s, %(device_dvfs)s, %(device_freq)s, %(device_kernel)s, " - "%(device_platform)s, %(total)s, %(pass)s, %(fail)s, %(pcnt)s, %(time_start)s, %(time_end)s, %(url)s )" ) + + + + + + + + SQL['PERFS'] = {} + # select perfs + SQL['PERFS']['select'] = ( + "SELECT * " + "FROM trbs_perf " + "WHERE sr_name = %(sr)s AND snapshot_num = %(snapshot_num)s ORDER BY id DESC" + ) + + MYSQL['cursor'].execute( SQL['PERFS']['select'], DINFO ) + TEMP = MYSQL['cursor'].fetchall() + + if len( TEMP ) > 0: + DINFO['trbs_perf_id'] = TEMP[0]['id'] + print ("[WARN] MYSQL : Duplicate PERF, id = %(id)s, sr_name = %(sr_name)s" % TEMP[0]) + + else: + + #pprint.pprint( CONFIG['RAW']['PERFS'] ) + #exit() + + VALUES = { + 'sr_status_id' : DINFO['sr_status_id'], + 'sr_name' : DINFO['sr'], + 'snapshot_num' : DINFO['snapshot_num'], + 'profile' : CONFIG['RAW']['TCT']['environment']['tct_profile'], + 'device_core' : CONFIG['RAW']['PERFS']['DEVICE']['Core_Nums'], + 'device_dvfs' : CONFIG['RAW']['PERFS']['DEVICE']['Def._DVFS_governor'], + 'device_freq' : CONFIG['RAW']['PERFS']['DEVICE']['Frequency_range'], + 'device_kernel' : CONFIG['RAW']['PERFS']['DEVICE']['Kernel_ver.'], + 'device_platform' : CONFIG['RAW']['PERFS']['DEVICE']['Platform_ver.'], + 'total' : CONFIG['RAW']['PERFS']['COUNT']['total_case'], + 'pass' : CONFIG['RAW']['PERFS']['COUNT']['pass_case'], + 'fail' : CONFIG['RAW']['PERFS']['COUNT']['fail_case'], + 'pcnt' : round( (float(CONFIG['RAW']['PERFS']['COUNT']['pass_case'])/float(CONFIG['RAW']['PERFS']['COUNT']['total_case']))*100, 2 ), + 'time_start' : CONFIG['RAW']['TCT']['start_at'], + 'time_end' : CONFIG['RAW']['TCT']['start_at'], + 'url' : CONFIG['RAW']['TCT']['url'] + } + + #pprint.pprint( VALUES ) + #exit() + + table = "trbs_perf" + + try: + sql = makeInsertSQL( table, VALUES ) + + #print sql + #exit() + + MYSQL['cursor'].execute( sql ) + MYSQL['connection'].commit() + + print "[INFO] MYSQL : INSERT to", table + + except MySQLdb.Error, e: try: - value['sr_status_id'] = RESULT['sr_id'] + print "[ERROR] MYSQL : [%d]: %s (%s)" % (e.args[0], e.args[1], sql) + except IndexError: + print "[ERROR] MYSQL : %s (%s)" % (str(e), sql) - query.execute( insert_perf, value ) - dbcon.commit() - print "[Write: trbs_perf] id =", value['id'] - except MySQLdb.IntegrityError: - print "[Error] Duplicate entry (" + key + ")-", value - elif key == 'trbs_perf_suite': + MYSQL['cursor'].execute( SQL['PERFS']['select'], DINFO ) + TEMP = MYSQL['cursor'].fetchall() - insert_perf_suite = ( "INSERT INTO trbs_perf_suite " - "( id, trbs_perf_id, name, total, pass, fail ) " - "VALUES ( %(id)s, %(trbs_perf_id)s, %(name)s, %(total)s, %(pass)s, %(fail)s )" ) + # 결과가 1개 이거나 아니면 latest 결과가 정답일 것이라 전제 + if len( TEMP ) > 0: + DINFO['trbs_perf_id'] = TEMP[0]['id'] + print ("[INFO] MYSQL : PERF id = %(id)s" % TEMP[0]) + else: + print "[ERROR] MYSQL : Can not process PERF" + exit(1) - for temp in value: - try: - query.execute( insert_perf_suite, temp ) - dbcon.commit() - print "[Write: trbs_perf_suite] id =", temp['id'] - except MySQLdb.IntegrityError: - print "[Error] Duplicate entry (" + key + ")-", temp - elif key == 'trbs_perf_suite_test': + #pprint.pprint( CONFIG['RAW']['PERFS'] ) + #exit() - insert_perf_suite_test = ( "INSERT INTO trbs_perf_suite_test " - "( `id`, `trbs_perf_suite_id`, `name`, `actualperf`, `expectedperf`, `range`, `result`, `criteria`, `unit` ) " - "VALUES ( %(id)s, %(trbs_perf_suite_id)s, %(name)s, %(actualperf)s, %(expectedperf)s, %(range)s, %(result)s, %(criteria)s, %(unit)s )" ) + for suite in CONFIG['RAW']['PERFS']['SUITE'].keys(): - for temp in value: - try: - query.execute( insert_perf_suite_test, temp ) - dbcon.commit() - print "[Write: trbs_perf_suite_test] id =", temp['id'] - except MySQLdb.IntegrityError: - print "[Error] Duplicate entry (" + key + ")-", temp + #pprint.pprint( CONFIG['RAW']['PERFS']['SUITE'][suite] ) + #pprint.pprint( DINFO ) + #exit() + + VALUES = { + 'trbs_perf_id' : DINFO['trbs_perf_id'], + 'name' : suite, + 'pass' : CONFIG['RAW']['PERFS']['SUITE'][suite]['COUNT']['pass_case'], + 'fail' : CONFIG['RAW']['PERFS']['SUITE'][suite]['COUNT']['fail_case'], + 'total' : CONFIG['RAW']['PERFS']['SUITE'][suite]['COUNT']['total_case'] + } + + #pprint.pprint( VALUES ) + #exit() + + SQL['PERFS']['select_suite'] = ( + "SELECT * " + "FROM trbs_perf_suite " + "WHERE trbs_perf_id = %(trbs_perf_id)s AND name = %(name)s" + ) + MYSQL['cursor'].execute( SQL['PERFS']['select_suite'], VALUES ) + TEMP = MYSQL['cursor'].fetchall() + + if len( TEMP ) > 0: + DINFO['trbs_perf_suite_id'] = TEMP[0]['id'] + print ("[WARN] MYSQL : Duplicate PERF Suite, id = %(id)s, name = %(name)s" % TEMP[0]) else: - if key == 'info': - pass + table = "trbs_perf_suite" + + try: + sql = makeInsertSQL( table, VALUES ) + + #print sql + #exit() + + MYSQL['cursor'].execute( sql ) + MYSQL['connection'].commit() + + print "[INFO] MYSQL : INSERT to", table, ", trbs_perf_id =", DINFO['trbs_perf_id'], ", name =", suite + + except MySQLdb.Error, e: + try: + print "[ERROR] MYSQL : [%d]: %s (%s)" % (e.args[0], e.args[1], sql) + except IndexError: + print "[ERROR] MYSQL : %s (%s)" % (str(e), sql) + + + + MYSQL['cursor'].execute( SQL['PERFS']['select_suite'], VALUES ) + TEMP = MYSQL['cursor'].fetchall() + + + if len( TEMP ) == 0: + print "[ERROR] MYSQL : Can not process PERF Suite" + else: + DINFO['trbs_perf_suite_id'] = TEMP[0]['id'] + + + for testcase in CONFIG['RAW']['PERFS']['SUITE'][suite]['TESTCASE'].keys(): + + VALUES = { + 'trbs_perf_suite_id' : DINFO['trbs_perf_suite_id'], + 'name' : testcase, + 'actualperf' : CONFIG['RAW']['PERFS']['SUITE'][suite]['TESTCASE'][testcase]['ActualPerf'], + 'criteria' : CONFIG['RAW']['PERFS']['SUITE'][suite]['TESTCASE'][testcase]['Criteria'], + 'expectedperf' : CONFIG['RAW']['PERFS']['SUITE'][suite]['TESTCASE'][testcase]['ExpectedPerf'], + 'range' : CONFIG['RAW']['PERFS']['SUITE'][suite]['TESTCASE'][testcase]['Range'], + 'result' : CONFIG['RAW']['PERFS']['SUITE'][suite]['TESTCASE'][testcase]['Result'], + 'unit' : CONFIG['RAW']['PERFS']['SUITE'][suite]['TESTCASE'][testcase]['Unit'] + } + + #pprint.pprint( VALUES ) + #exit() + + + SQL['PERFS']['select_suite_test'] = ( + "SELECT * " + "FROM trbs_perf_suite_test " + "WHERE trbs_perf_suite_id = %(trbs_perf_suite_id)s AND name = %(name)s" + ) + + MYSQL['cursor'].execute( SQL['PERFS']['select_suite_test'], VALUES ) + TEMP = MYSQL['cursor'].fetchall() + + if len( TEMP ) > 0: + print ("[WARN] MYSQL : Duplicate PERF Suite Test, id = %(id)s, name = %(name)s" % TEMP[0]) else: - print "[Error] wrong data (", key, ") -", value + table = "trbs_perf_suite_test" + + try: + sql = makeInsertSQL( table, VALUES ) + + #print sql + #exit() + + MYSQL['cursor'].execute( sql ) + MYSQL['connection'].commit() + + print "[INFO] MYSQL : INSERT to", table, ", trbs_perf_suite_id =", DINFO['trbs_perf_suite_id'], ", name =", testcase - trbs_test_stage_id = 101 - cur_time = datetime.datetime.now() - insert_sr_stage_sql = "INSERT INTO sr_stage (sr_status_id, info_stage_id, stage_start_time, stage_end_time, stage_status, build_project_id) VALUES ( %s, %s, %s, %s, %s, %s)"; - insert_sr_stage_data = (RESULT['sr_id'], trbs_test_stage_id, cur_time, cur_time, 'S', RESULT['build_project_id'] ) - try: - query.execute(insert_sr_stage_sql, insert_sr_stage_data) - dbcon.commit(); - print "[Insert: sr_stage]" + str(insert_sr_stage_sql) + "\n" + str(insert_sr_stage_data) - except MySQLdb.IntegrityError: - print "[Error] " + str(insert_sr_stage_sql) + "\n" + str(insert_sr_stage_data) + except MySQLdb.Error, e: + try: + print "[ERROR] MYSQL : [%d]: %s (%s)" % (e.args[0], e.args[1], sql) + except IndexError: + print "[ERROR] MYSQL : %s (%s)" % (str(e), sql) - # update sr_stage status. - query.close() - dbcon.close() - exit() + exit(0) -- 2.7.4