From: Hyoun Jiil Date: Thu, 1 Aug 2013 08:24:57 +0000 (+0900) Subject: [Title] fix analyze and statistics X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=64e606a61778ff5918863cee40ece1e96736902c;p=sdk%2Ftools%2Fsdk-build.git [Title] fix analyze and statistics [Type] Enhancement [Module] Toolchain / [Priority] Minor [Jira#] [Redmine#] [Problem] [Cause] [Solution] [TestCase] Change-Id: Ibbaf7fabc11ca0fea213907c8917e24160047f51 --- diff --git a/test/analyze.sh b/test/analyze.sh index 5567a25..3d38e6c 100755 --- a/test/analyze.sh +++ b/test/analyze.sh @@ -1,2 +1,3 @@ #!/bin/sh -x -./performance_analyzer -d "Mysql:database=rsa;host=172.21.17.46;port=3306" -u "root" -p "qmscore" -w "start_time >= '`date '+%Y-%m-%d'`' AND distribution_id = 9 AND status = 'FINISHED'" -n common-eplugin -c performance_log/`date '+%Y-%m-%d'`.sample.csv +dir=`dirname $0` +$dir/performance_analyzer -d "Mysql:database=rsa;host=172.21.17.46;port=3306" -u "root" -p "qmscore" -w "start_time >= '`date '+%Y-%m-%d'`' AND distribution_id = 9 AND status = 'FINISHED'" -n common-eplugin -c performance_log/`date '+%Y-%m-%d'`.sample.csv diff --git a/test/statistics b/test/statistics index d26e94b..8d763d5 100755 --- a/test/statistics +++ b/test/statistics @@ -34,7 +34,7 @@ require 'dbi' class CommonJob attr_accessor :id, :project, :user, :supported_os, :distribution, :parent_job, :remote_build_server, :jtype, :status, :user_group - attr_accessor :t_BASE, :t_S, :t_IS, :t_IE, :t_WS, :t_RUS, :t_RUE, :t_RDS, :t_RDE, :t_US, :t_UE, :t_E + attr_accessor :t_BASE, :t_S, :t_IS, :t_IE, :t_WS, :t_RUS, :t_RUE, :t_RDS, :t_RDE, :t_US, :t_UE, :t_E, :p_d, :p_w, :p_m attr_accessor :d_BASE, :d_waitI, :d_INIT, :d_waitW, :d_RU, :d_WORK, :d_RD, :d_waitU, :d_U, :d_FINISH, :d_total, :sub_total, :sub_work_total attr_accessor :sub_job def initialize(option, projects, supported_os, distributions, groups) @@ -52,15 +52,9 @@ class CommonJob t = Time.now offset = t.getlocal.gmt_offset @t_S = option[:start_time].to_time - offset if not option[:start_time].nil? - @t_IS = option[:INIT_START].to_time - offset if not option[:INIT_START].nil? - @t_IE = option[:INIT_END].to_time - offset if not option[:INIT_END].nil? - @t_WS = option[:WORK_START].to_time - offset if not option[:WORK_START].nil? - @t_RUS = option[:REMOTE_UPLOAD_START].to_time - offset if not option[:REMOTE_UPLOAD_START].nil? - @t_RUE = option[:REMOTE_UPLOAD_END].to_time - offset if not option[:REMOTE_UPLOAD_END].nil? - @t_RDS = option[:REMOTE_DOWNLOAD_START].to_time - offset if not option[:REMOTE_DOWNLOAD_START].nil? - @t_RDE = option[:REMOTE_DOWNLOAD_END].to_time - offset if not option[:REMOTE_DOWNLOAD_END].nil? - @t_US = option[:UPLOAD_START].to_time - offset if not option[:UPLOAD_START].nil? - @t_UE = option[:UPLOAD_END].to_time - offset if not option[:UPLOAD_END].nil? + @p_d = @t_S.strftime("%y-%m-%d day") if not @t_S.nil? + @p_w = @t_S.strftime("%y-%W week") if not @t_S.nil? + @p_m = @t_S.strftime("%y-%m month") if not @t_S.nil? @t_E = option[:end_time].to_time - offset if not option[:end_time].nil? @sub_job = [] @d_BASE = 0 @@ -77,38 +71,14 @@ class CommonJob @sub_total = 0 @sub_work_total = 0 end - def gen_duration - @sub_job.each {|x| x.gen_duration} - @d_BASE = @t_S - @t_BASE if not @t_S.nil? and not @t_BASE.nil? - @d_waitI = @t_IS - @t_S if not @t_IS.nil? and not @t_S.nil? - @d_INIT = @t_IE - @t_IS if not @t_IE.nil? and not @t_IS.nil? - @d_waitW = @t_WS - @t_IE if not @t_WS.nil? and not @t_IE.nil? - @d_RU = @t_RUE - @t_RUS if not @t_RUE.nil? and not @t_RUS.nil? - @d_RD = @t_RDE - @t_RDS if not @t_RDE.nil? and not @t_RDS.nil? - @d_U = @t_UE - @t_US if not @t_UE.nil? and not @t_US.nil? - @d_waitU = @t_US - @t_RDE if not @t_US.nil? and not @t_RDE.nil? - work_end = ([@t_E, @t_US, @t_RDS].compact.nil?)? [@t_E, @t_US, @t_RDS] : [@t_E, @t_US, @t_RDS].compact - work_start = ([@t_WS, @t_RUE].compact.nil?)? [@t_WS, @t_RUE] : [@t_WS, @t_RUE].compact - @d_WORK = work_end.min - work_start.max if not work_end.empty? and not work_start.empty? - @d_FINISH = @t_E - work_end.max if not work_end.empty? - @d_total = @t_E - @t_S if not @t_E.nil? and not @t_S.nil? - @sub_job.each do |x| - @sub_total = @sub_total + x.d_total if not x.d_total.nil? - @sub_total = @sub_total + x.sub_total if not x.sub_total.nil? - end - @sub_job.each do |x| - @sub_work_total = @sub_work_total + x.d_WORK if not x.d_WORK.nil? - @sub_work_total = @sub_work_total + x.sub_work_total if not x.sub_work_total.nil? - end - end - def print_cvs(pre_pending = 0, post_pending = 2) + def print_csv(pre_pending = 0, post_pending = 2) result = [] work = 0 work = @d_WORK if @jtype != "MULTIBUILD" result.push (", " * pre_pending) + @id.to_s + (", " * post_pending) + ", #{@status.to_s}, #{@distribution.to_s}, #{@project.to_s}, #{@supported_os.to_s}, " + [@d_BASE, @d_waitI, @d_INIT, @d_waitW, @d_RU, @d_WORK, @d_RD, @d_waitU, @d_U, @d_FINISH, @d_total, @sub_work_total, @sub_total].map{|x| x.to_s }.join(", ") + ", #{((@d_U + work + @sub_work_total)/(@d_total + @sub_total) * 100).round 2 if (@d_total + @sub_total) > 0}%" pre_pending = pre_pending +1 post_pending = post_pending -1 - result = result + sub_job.map{|x| x.print_cvs pre_pending, post_pending } + result = result + sub_job.map{|x| x.print_csv pre_pending, post_pending } return result end def print_multi @@ -133,6 +103,15 @@ def option_parse options[:statistics] = true options[:slv] = slv end + options[:period] = "d" + opts.on('-P', '--period ', 'd/w/m(day/week/month)') do |period| + if period.strip == "w" or period.strip == "d" or period.strip == "m" then + options[:period] = period.strip + else + puts "ERROR period #{period} is not supported! use d/w/m" + exit + end + end opts.on('-d', '--dsn ', 'data server name') do |dsn| options[:dsn] = dsn end @@ -145,11 +124,11 @@ def option_parse opts.on('-w', '--where ', 'ex) jobs.id >= 79000') do |where| options[:where] = "WHERE " + where end - opts.on('-c', '--csv ', 'save csv file') do |cvs| - options[:cvs] = cvs + opts.on('-c', '--csv ', 'save csv file') do |csv| + options[:csv] = csv end opts.on('-h', '--help', 'print this message') do |help| - puts "useage: performance_analizer -d \"Mysql:database=rsa;host=172.21.17.46;port=3306\" -u \"root\" -p \"password\" -w \"start_time >= '2013-07-21'\" -c result.cvs" + puts "useage: performance_analizer -d \"Mysql:database=rsa;host=172.21.17.46;port=3306\" -u \"root\" -p \"password\" -w \"start_time >= '2013-07-21'\" -c result.csv" puts opts.help exit end @@ -185,20 +164,7 @@ conn.select_all "SELECT user_groups.user_id,groups.name FROM user_groups, groups #puts "#{row[:user_id]} => #{row[:name]}" end -performance_log = "" -if not option[:statistics] then - performance_log = " , (SELECT stamp_time FROM rsa.job_timestamps WHERE job_timestamps.job_id = jobs.id AND stamp_name = 'INIT_START') INIT_START -, (SELECT stamp_time FROM rsa.job_timestamps WHERE job_timestamps.job_id = jobs.id AND stamp_name = 'INIT_END') INIT_END -, (SELECT stamp_time FROM rsa.job_timestamps WHERE job_timestamps.job_id = jobs.id AND stamp_name = 'WORK_START') WORK_START -, (SELECT stamp_time FROM rsa.job_timestamps WHERE job_timestamps.job_id = jobs.id AND stamp_name = 'REMOTE_UPLOAD_START') REMOTE_UPLOAD_START -, (SELECT stamp_time FROM rsa.job_timestamps WHERE job_timestamps.job_id = jobs.id AND stamp_name = 'REMOTE_UPLOAD_END') REMOTE_UPLOAD_END -, (SELECT stamp_time FROM rsa.job_timestamps WHERE job_timestamps.job_id = jobs.id AND stamp_name = 'REMOTE_DOWNLOAD_START') REMOTE_DOWNLOAD_START -, (SELECT stamp_time FROM rsa.job_timestamps WHERE job_timestamps.job_id = jobs.id AND stamp_name = 'REMOTE_DOWNLOAD_END') REMOTE_DOWNLOAD_END -, (SELECT stamp_time FROM rsa.job_timestamps WHERE job_timestamps.job_id = jobs.id AND stamp_name = 'UPLOAD_START') UPLOAD_START -, (SELECT stamp_time FROM rsa.job_timestamps WHERE job_timestamps.job_id = jobs.id AND stamp_name = 'UPLOAD_END') UPLOAD_END " -end - -conn.select_all "SELECT * #{performance_log} +conn.select_all "SELECT * FROM rsa.jobs #{option[:where]} ;" do |row| @@ -215,14 +181,12 @@ FROM rsa.jobs set = false multiJobs.each do |x| if x.id == row[:parent_job_id] then - new.t_BASE = x.t_S x.sub_job.push new set = true break else x.sub_job.each do |y| if y.id == row[:parent_job_id] then - new.t_BASE = x.t_S y.sub_job.push new set = true break @@ -233,7 +197,6 @@ FROM rsa.jobs if not set then singleJobs.each do |x| if x.id == row[:parent_job_id] then - new.t_BASE = x.t_S x.sub_job.push new set = true break @@ -243,7 +206,6 @@ FROM rsa.jobs if not set then registerJobs.each do |x| if x.id == row[:parent_job_id] then - new.t_BASE = x.t_S x.sub_job.push new set = true break @@ -253,7 +215,6 @@ FROM rsa.jobs if not set then orphanJobs.each do |x| if x.id == row[:parent_job_id] then - new.t_BASE = x.t_S x.sub_job.push new set = true break @@ -263,41 +224,29 @@ FROM rsa.jobs end end end -if option[:statistics] then - multiJobs.each do |x| - x.project = x.sub_job.map{|s| s.project.to_s}.uniq.join(" ") - x.supported_os = x.sub_job.map{|s| s.supported_os.to_s}.uniq.join(" ") + +multiJobs.each do |x| + x.project = x.sub_job.map{|s| s.project.to_s}.uniq.join(" ") + x.supported_os = x.sub_job.map{|s| s.supported_os.to_s}.uniq.join(" ") +end +def print_array(job_array, lvl, period) + case period + when "m" then periods = job_array.map{|x| x.p_m }.uniq + when "w" then periods = job_array.map{|x| x.p_w }.uniq + when "d" then periods = job_array.map{|x| x.p_d }.uniq end - puts "=== multi build ===" - days = multiJobs.map{|x| x.t_S.to_s.split(" ")[0]}.uniq - days.each do |d| - day_job = multiJobs.select{|j| j.t_S.to_s.start_with? d} - puts " #{d}: success #{day_job.select{|j| j.status.to_s == "FINISHED"}.count}, fail #{day_job.select{|j| j.status.to_s != "FINISHED"}.count}" - if option[:slv] != 1.to_s then - day_job.map{|x| x.user_group}.uniq.each do |u| - group_job= day_job.select{|j| j.user_group.to_s.eql? u} - puts " #{u}: success #{group_job.select{|j| j.status.to_s == "FINISHED"}.count}, fail #{group_job.select{|j| j.status.to_s != "FINISHED"}.count}" - if option[:slv] != 2.to_s then - group_job.map{|x| "#{x.project}"}.uniq.each do |p| - project_job = group_job.select{|j| j.project == p} - puts " #{p}: success #{project_job.select{|j| j.status.to_s == "FINISHED"}.count}, fail #{project_job.select{|j| j.status.to_s != "FINISHED"}.count}" - end - puts "" - end - end - puts "" + periods.each do |p| + case period + when "m" then period_job = job_array.select{|x| x.p_m.eql? p } + when "w" then period_job = job_array.select{|x| x.p_w.eql? p } + when "d" then period_job = job_array.select{|x| x.p_d.eql? p } end - end - puts "=== single build ===" - days = singleJobs.map{|x| x.t_S.to_s.split(" ")[0]}.uniq - days.each do |d| - day_job = singleJobs.select{|j| j.t_S.to_s.start_with? d} - puts " #{d}: success #{day_job.select{|j| j.status.to_s == "FINISHED"}.count}, fail #{day_job.select{|j| j.status.to_s != "FINISHED"}.count}" - if option[:slv] != 1.to_s then - day_job.map{|x| x.user_group}.uniq.each do |u| - group_job= day_job.select{|j| j.user_group.to_s.eql? u} + puts " #{p}: success #{period_job.select{|j| j.status.to_s == "FINISHED"}.count}, fail #{period_job.select{|j| j.status.to_s != "FINISHED"}.count}" + if lvl != 1.to_s then + period_job.map{|x| x.user_group}.uniq.each do |u| + group_job= period_job.select{|j| j.user_group.to_s.eql? u} puts " #{u}: success #{group_job.select{|j| j.status.to_s == "FINISHED"}.count}, fail #{group_job.select{|j| j.status.to_s != "FINISHED"}.count}" - if option[:slv] != 2.to_s then + if lvl != 2.to_s then group_job.map{|x| "#{x.project}"}.uniq.each do |p| project_job = group_job.select{|j| j.project == p} puts " #{p}: success #{project_job.select{|j| j.status.to_s == "FINISHED"}.count}, fail #{project_job.select{|j| j.status.to_s != "FINISHED"}.count}" @@ -305,92 +254,53 @@ if option[:statistics] then puts "" end end - puts "" + puts "" end end - puts "=== register build ===" - days = registerJobs.map{|x| x.t_S.to_s.split(" ")[0]}.uniq - days.each do |d| - day_job = registerJobs.select{|j| j.t_S.to_s.start_with? d} - puts " #{d}: success #{day_job.select{|j| j.status.to_s == "FINISHED"}.count}, fail #{day_job.select{|j| j.status.to_s != "FINISHED"}.count}" - if option[:slv] != 1.to_s then - day_job.map{|x| x.user_group}.uniq.each do |u| - group_job= day_job.select{|j| j.user_group.to_s.eql? u} - puts " #{u}: success #{group_job.select{|j| j.status.to_s == "FINISHED"}.count}, fail #{group_job.select{|j| j.status.to_s != "FINISHED"}.count}" - if option[:slv] != 2.to_s then - group_job.map{|x| "#{x.project}"}.uniq.each do |p| - project_job = group_job.select{|j| j.project == p} - puts " #{p}: success #{project_job.select{|j| j.status.to_s == "FINISHED"}.count}, fail #{project_job.select{|j| j.status.to_s != "FINISHED"}.count}" - end - puts "" - end - end - puts "" - end - end -else - multiJobs.each do |x| - x.project = x.sub_job.map{|s| s.project.to_s}.uniq.join(" ") - x.supported_os = x.sub_job.map{|s| s.supported_os.to_s}.uniq.join(" ") - x.t_US = x.sub_job.map{|y| y.t_E}.max if x.t_US.nil? and not x.sub_job.map{|y| y.t_E}.include? nil - x.t_UE = x.t_E if x.t_UE.nil? - x.gen_duration - end - singleJobs.each do |x| - x.gen_duration - end - registerJobs.each do |x| - #x.d_WORK = x.d_U - x.gen_duration - end - - if option[:cvs].nil? then - if not multiJobs.empty? then - puts "multi job" - puts "time, status, branch, project, os, upload, sub_work, total, sub_total, (upload + sub_work)/(total + sub_total)%" - multiJobs.each do |m| - m.print_multi - end - end - - if not singleJobs.empty? then - puts "single job" - puts "time, status, branch, project, os, work, upload, sub_work, total, sub_total, (work + upload + sub_work)/(total + sub_total)%" - singleJobs.each do |s| - s.print_single - end - end +end - if not registerJobs.empty? then - puts "register job" - puts "time, status, branch, project, os, upload, sub_work, total, sub_total, (upload + sub_work)/(total + sub_total)%" - registerJobs.each do |r| - r.print_register - end +def print_csv(job_array, period, csv, message) + case period + when "m" then periods = job_array.map{|x| x.p_m }.uniq + when "w" then periods = job_array.map{|x| x.p_w }.uniq + when "d" then periods = job_array.map{|x| x.p_d }.uniq + end + File.open(csv, "a+") do |f| + case period + when "m" then pe = 'month' + when "w" then pe = 'week' + when "d" then pe = 'day' end - else - File.open(option[:cvs],"w") do |f| - if not multiJobs.empty? then - f.puts "multi job" - f.puts "multi, sub, reverse, status, branch, projects, os, start, wait, init, wait, remote upload, work, remote donwload, wait, upload, finish, total, sub work total, sub total, performance (w/t)%" - multiJobs.each do |m| - f.puts m.print_cvs - end + f.puts "message" + f.puts "#{pe}, group, projects, success, fail" + periods.each do |p| + case period + when "m" then period_job = job_array.select{|x| x.p_m.eql? p } + when "w" then period_job = job_array.select{|x| x.p_w.eql? p } + when "d" then period_job = job_array.select{|x| x.p_d.eql? p } end - if not singleJobs.empty? then - f.puts "single job" - f.puts "job, reverse, status, branch, projects, os, start, wait, init, wait, remote upload, work, remote donwload, wait, upload, finish, total, sub work total, sub total, performance (w/t)%" - singleJobs.each do |m| - f.puts m.print_cvs(0,1) - end - end - if not registerJobs.empty? then - f.puts "register job" - f.puts "register, reverse, status, branch, projects, os, start, wait, init, wait, remote upload, work, remote donwload, wait, upload, finish, total, sub work total, sub total, performance (w/t)%" - registerJobs.each do |m| - f.puts m.print_cvs(0,1) + f.puts "#{p},,,#{period_job.select{|j| j.status.to_s == "FINISHED"}.count},#{period_job.select{|j| j.status.to_s != "FINISHED"}.count}" + period_job.map{|x| x.user_group}.uniq.each do |u| + group_job= period_job.select{|j| j.user_group.to_s.eql? u} + f.puts "#{p},#{u},,#{group_job.select{|j| j.status.to_s == "FINISHED"}.count},#{group_job.select{|j| j.status.to_s != "FINISHED"}.count}" + group_job.map{|x| "#{x.project}"}.uniq.each do |pro| + project_job = group_job.select{|j| j.project == pro} + f.puts "#{p},#{u},#{pro},#{project_job.select{|j| j.status.to_s == "FINISHED"}.count},#{project_job.select{|j| j.status.to_s != "FINISHED"}.count}" end end end end end + +if option[:csv].nil? or option[:csv].empty? then + puts 'Multi build jobs' + print_array(multiJobs, option[:slv], option[:period]) + puts 'Single build jobs' + print_array(singleJobs, option[:slv], option[:period]) + puts 'Register build jobs' + print_array(registerJobs, option[:slv], option[:period]) +else + print_csv(multiJobs, option[:period], option[:csv], "Multi build jobs") + print_csv(singleJobs, option[:period], option[:csv], "Single build jobs") + print_csv(registerJobs, option[:period], option[:csv], "Register build jobs") +end