From: Hyoun Jiil Date: Tue, 30 Jul 2013 07:03:18 +0000 (+0900) Subject: [Title] add dibs-test and performance analyzer X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=5cff9e5b74b3883c4fc604e97ddc822f4a6a7b27;p=sdk%2Ftools%2Fsdk-build.git [Title] add dibs-test and performance analyzer [Type] Enhancement [Module] Toolchain / [Priority] Major [Jira#] [Redmine#] [Problem] [Cause] [Solution] [TestCase] Change-Id: Ia9591b099af0fb5c3af8ed26dd6facb21a63bb91 --- diff --git a/package/build.linux b/package/build.linux index 4e6e1fd..2ec4d59 100755 --- a/package/build.linux +++ b/package/build.linux @@ -15,6 +15,9 @@ build() # install install() { + BIN_DIR=$SRCDIR/package/tizen-dibs-test.package.${TARGET_OS}/data/tools/dibs + mkdir -p $BIN_DIR + BIN_DIR=$SRCDIR/package/dibs.package.${TARGET_OS}/data/tools/dibs/ mkdir -p $BIN_DIR cp -f $SRCDIR/pkg-* $BIN_DIR/ @@ -24,4 +27,11 @@ install() cp -f $SRCDIR/upgrade $BIN_DIR/ cp -f $SRCDIR/web-svr $BIN_DIR/ echo $VERSION > $BIN_DIR/VERSION + + TEST_DIR=$SRCDIR/package/dibs-test.package.${TARGET_OS}/data/tools/dibs/test + mkdir -p $TEST_DIR + cp -f $SRCDIR/performance_analyzer $TEST_DIR/ + cp -f $SRCDIR/statistics $TEST_DIR/ + cp -f $SRCDIR/test/performance_run.sh $TEST_DIR/ + cp -f $SRCDIR/test/analyze.sh $TEST_DIR/ } diff --git a/package/changelog b/package/changelog index 0461468..fe079c0 100644 --- a/package/changelog +++ b/package/changelog @@ -1,3 +1,6 @@ +* 2.1.41 +- add tizen-dibs-test and dibs-test package +== hyoun jiil 2013-07-28 * 2.1.40 - Fixed some minor bugs == Sungmin Kim 2013-06-04 diff --git a/package/pkginfo.manifest b/package/pkginfo.manifest index c550b41..0a98a42 100644 --- a/package/pkginfo.manifest +++ b/package/pkginfo.manifest @@ -1,8 +1,24 @@ Source : dibs -Version :2.1.40 +Version :2.1.41 Maintainer : taejun ha, jiil hyoun , donghyuk yang , donghee yang , sungmin kim +Jiil Hyoun +Donghyuk Yang +DongHee Yang + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Contributors: +- S-Core Co., Ltd +=end + +require 'time' +require 'optparse' +require 'dbi' + +class CommonJob + attr_accessor :id, :project, :user, :supported_os, :distribution, :parent_job, :remote_build_server, :jtype, :status, :user_group + attr_accessor :t_BASE, :t_S, :t_IS, :t_IE, :t_WS, :t_RUS, :t_RUE, :t_RDS, :t_RDE, :t_US, :t_UE, :t_E + attr_accessor :d_BASE, :d_waitI, :d_INIT, :d_waitW, :d_RU, :d_WORK, :d_RD, :d_waitU, :d_U, :d_FINISH, :d_total, :sub_total, :sub_work_total + attr_accessor :sub_job + def initialize(option, projects, supported_os, distributions) + @id = option[:id] + @project = projects[option[:project_id]] + @project = "" if @project.nil? + @supported_os = supported_os[option[:supported_os_id]] + @distribution = distributions[option[:distribution_id]] + @parent_job = option[:parent_job] + @remote_build_server = option[:remote_build_server] + @jtype = option[:jtype] + @status = option[:status] + #set time stamp + @t_BASE = nil + t = Time.now + offset = t.getlocal.gmt_offset + @t_S = option[:start_time].to_time - offset if not option[:start_time].nil? + @t_IS = option[:INIT_START].to_time - offset if not option[:INIT_START].nil? + @t_IE = option[:INIT_END].to_time - offset if not option[:INIT_END].nil? + @t_WS = option[:WORK_START].to_time - offset if not option[:WORK_START].nil? + @t_RUS = option[:REMOTE_UPLOAD_START].to_time - offset if not option[:REMOTE_UPLOAD_START].nil? + @t_RUE = option[:REMOTE_UPLOAD_END].to_time - offset if not option[:REMOTE_UPLOAD_END].nil? + @t_RDS = option[:REMOTE_DOWNLOAD_START].to_time - offset if not option[:REMOTE_DOWNLOAD_START].nil? + @t_RDE = option[:REMOTE_DOWNLOAD_END].to_time - offset if not option[:REMOTE_DOWNLOAD_END].nil? + @t_US = option[:UPLOAD_START].to_time - offset if not option[:UPLOAD_START].nil? + @t_UE = option[:UPLOAD_END].to_time - offset if not option[:UPLOAD_END].nil? + @t_E = option[:end_time].to_time - offset if not option[:end_time].nil? + @sub_job = [] + @d_BASE = 0 + @d_waitI = 0 + @d_INIT = 0 + @d_waitW = 0 + @d_RU = 0 + @d_RD = 0 + @d_U = 0 + @d_waitU = 0 + @d_WORK = 0 + @d_FINISH = 0 + @d_total = 0 + @sub_total = 0 + @sub_work_total = 0 + end + def gen_duration + @sub_job.each {|x| x.gen_duration} + @d_BASE = @t_S - @t_BASE if not @t_S.nil? and not @t_BASE.nil? + @d_waitI = @t_IS - @t_S if not @t_IS.nil? and not @t_S.nil? + @d_INIT = @t_IE - @t_IS if not @t_IE.nil? and not @t_IS.nil? + @d_waitW = @t_WS - @t_IE if not @t_WS.nil? and not @t_IE.nil? + @d_RU = @t_RUE - @t_RUS if not @t_RUE.nil? and not @t_RUS.nil? + @d_RD = @t_RDE - @t_RDS if not @t_RDE.nil? and not @t_RDS.nil? + @d_U = @t_UE - @t_US if not @t_UE.nil? and not @t_US.nil? + @d_waitU = @t_US - @t_RDE if not @t_US.nil? and not @t_RDE.nil? + work_end = [@t_E, @t_US, @t_RDS].compact + work_start = [@t_WS, @t_RUE].compact + @d_WORK = work_end.min - work_start.max if not work_end.empty? and not work_start.empty? + @d_FINISH = @t_E - work_end.max if not work_end.empty? and @t_E.nil? + @d_total = @t_E - @t_S if not @t_E.nil? and not @t_S.nil? + @sub_job.each do |x| + @sub_total = @sub_total + x.d_total if not x.d_total.nil? + @sub_total = @sub_total + x.sub_total if not x.sub_total.nil? + end + @sub_job.each do |x| + @sub_work_total = @sub_work_total + x.d_WORK if not x.d_WORK.nil? + @sub_work_total = @sub_work_total + x.sub_work_total if not x.sub_work_total.nil? + end + end + def print_cvs(pre_pending = 0, post_pending = 2) + result = [] + work = 0 + work = @d_WORK if @jtype != "MULTIBUILD" + result.push (", " * pre_pending) + @id.to_s + (", " * post_pending) + ", #{@status.to_s}, #{@distribution.to_s}, #{@project.to_s}, #{@supported_os.to_s}, " + [@d_BASE, @d_waitI, @d_INIT, @d_waitW, @d_RU, @d_WORK, @d_RD, @d_waitU, @d_U, @d_FINISH, @d_total, @sub_work_total, @sub_total].map{|x| x.to_s }.join(", ") + ", #{((@d_U + work + @sub_work_total)/(@d_total + @sub_total) * 100).round 2 if (@d_total + @sub_total) > 0}%" + pre_pending = pre_pending +1 + post_pending = post_pending -1 + result = result + sub_job.map{|x| x.print_cvs pre_pending, post_pending } + return result + end + def print_multi + puts "#{@t_S.to_s}, #{@status.to_s}, #{@distribution.to_s}, #{@project.to_s}, #{@supported_os.to_s}, " + [ @d_U, @sub_work_total, @d_total, @sub_total].map{|x| x.round.to_s }.join(", ") + ", #{((@d_U + @sub_work_total)/(@d_total + @sub_total) * 100).round 2 if (@d_total + @sub_total) > 0}%" + end + def print_single + puts "#{@t_S.to_s}, #{@status.to_s}, #{@distribution.to_s}, #{@project.to_s}, #{@supported_os.to_s}, " + [@d_WORK, @d_U, @sub_work_total, @d_total, @sub_total].map{|x| x.round.to_s }.join(", ") + ", #{((@d_WORK + @d_U + @sub_work_total)/(@d_total + @sub_total) * 100).round 2 if (@d_total + @sub_total) > 0}%" + end + def print_register + puts "#{@t_S.to_s}, #{@status.to_s}, #{@distribution.to_s}, #{@project.to_s}, #{@supported_os.to_s}, " + [@d_WORK, @sub_work_total, @d_total, @sub_total].map{|x| x.round.to_s }.join(", ") + ", #{((@d_WORK + @sub_work_total)/(@d_total + @sub_total) * 100).round 2 if (@d_total + @sub_total) > 0}%" + end +end + +def option_parse + options = {} + banner = "test" + + optparse = OptionParser.new(nil, 32, ' '*8) do |opts| + opts.banner = banner + opts.on('-d', '--dsn ', 'data server name') do |dsn| + options[:dsn] = dsn + end + opts.on('-u', '--user ', 'user name') do |user| + options[:user] = user + end + opts.on('-p', '--password ', 'user password') do |pass| + options[:pass] = pass + end + options[:name] = [] + opts.on('-n', '--name ', 'project name for print seperate by \",\"') do |name| + options[:name] = name.strip.split(",") + end + opts.on('-w', '--where ', 'ex) jobs.id >= 79000') do |where| + options[:where] = "WHERE " + where + end + opts.on('-c', '--csv ', 'save csv file') do |cvs| + options[:cvs] = cvs + end + opts.on('-h', '--help', 'print this message') do |help| + puts "useage: performance_analizer -d \"Mysql:database=rsa;host=172.21.17.46;port=3306\" -u \"root\" -p \"password\" -w \"start_time >= '2013-07-21'\" -c result.cvs" + puts opts.help + exit + end + end + optparse.parse! + return options +end + +option = option_parse + +multiJobs = [] +registerJobs = [] +singleJobs = [] +orphanJobs = [] + +projects = {} +supported_os = {} +distributions = {} +puts "dsn : #{option[:dsn]}" +puts "user : #{option[:user]}" +puts "pass : #{option[:pass]}" +puts "where : #{option[:where]}" +conn = DBI.connect "DBI:" + option[:dsn], option[:user], option[:pass] + +conn.select_all "SELECT id,name FROM projects ;" do |row| + projects[row[:id]] = row[:name] +end +conn.select_all "SELECT id,name FROM supported_os ;" do |row| + supported_os[row[:id]] = row[:name] +end +conn.select_all "SELECT id,name FROM distributions ;" do |row| + distributions[row[:id]] = row[:name] +end + +performance_log = " , (SELECT stamp_time FROM rsa.job_timestamps WHERE job_timestamps.job_id = jobs.id AND stamp_name = 'INIT_START') INIT_START +, (SELECT stamp_time FROM rsa.job_timestamps WHERE job_timestamps.job_id = jobs.id AND stamp_name = 'INIT_END') INIT_END +, (SELECT stamp_time FROM rsa.job_timestamps WHERE job_timestamps.job_id = jobs.id AND stamp_name = 'WORK_START') WORK_START +, (SELECT stamp_time FROM rsa.job_timestamps WHERE job_timestamps.job_id = jobs.id AND stamp_name = 'REMOTE_UPLOAD_START') REMOTE_UPLOAD_START +, (SELECT stamp_time FROM rsa.job_timestamps WHERE job_timestamps.job_id = jobs.id AND stamp_name = 'REMOTE_UPLOAD_END') REMOTE_UPLOAD_END +, (SELECT stamp_time FROM rsa.job_timestamps WHERE job_timestamps.job_id = jobs.id AND stamp_name = 'REMOTE_DOWNLOAD_START') REMOTE_DOWNLOAD_START +, (SELECT stamp_time FROM rsa.job_timestamps WHERE job_timestamps.job_id = jobs.id AND stamp_name = 'REMOTE_DOWNLOAD_END') REMOTE_DOWNLOAD_END +, (SELECT stamp_time FROM rsa.job_timestamps WHERE job_timestamps.job_id = jobs.id AND stamp_name = 'UPLOAD_START') UPLOAD_START +, (SELECT stamp_time FROM rsa.job_timestamps WHERE job_timestamps.job_id = jobs.id AND stamp_name = 'UPLOAD_END') UPLOAD_END " + +def set_stamp(stamp) + result = {} + stamp.each do |x| + case x[:stamp_name] + when "INIT_START" then result[:INIT_START] = x[:stamp_time] + when "INIT_END" then result[:INIT_END] = x[:stamp_time] + when "WORK_START" then result[:WORK_START] = x[:stamp_time] + when "REMOTE_UPLOAD_START" then result[:REMOTE_UPLOAD_START] = x[:stamp_time] + when "REMOTE_UPLOAD_END" then result[:REMOTE_UPLOAD_END] = x[:stamp_time] + when "REMOTE_DOWNLOAD_START" then result[:REMOTE_DOWNLOAD_START] = x[:stamp_time] + when "REMOTE_DOWNLOAD_END" then result[:REMOTE_DOWNLOAD_END] = x[:stamp_time] + when "UPLOAD_START" then result[:UPLOAD_START] = x[:stamp_time] + when "UPLOAD_END" then result[:UPLOAD_END] = x[:stamp_time] + end + puts x + end + return result +end +puts 'run' +conn.select_all "SELECT * #{performance_log} +FROM rsa.jobs +#{option[:where]} +;" do |row| + print "." + #stamp_hash = {} + #conn.select_all("SELECT stamp_name, stamp_time FROM rsa.job_timestamps WHERE job_id = '#{row[:id]}'").each do |stamp| + # stamp_hash[stamp[:stamp_name].to_sym] = stamp[:stamp_time] + #end + + #puts stamp_hash + new = CommonJob.new(row, projects, supported_os, distributions) + case row[:jtype] + when "MULTIBUILD" then + multiJobs.push new + when "REGISTER" + registerJobs.push new + when "BUILD" + if row[:parent_job_id].nil? + singleJobs.push new + else + set = false + multiJobs.each do |x| + if x.id == row[:parent_job_id] then + new.t_BASE = x.t_S + x.sub_job.push new + set = true + break + else + x.sub_job.each do |y| + if y.id == row[:parent_job_id] then + new.t_BASE = x.t_S + y.sub_job.push new + set = true + break + end + end + end + end + if not set then + singleJobs.each do |x| + if x.id == row[:parent_job_id] then + new.t_BASE = x.t_S + x.sub_job.push new + set = true + break + end + end + end + if not set then + registerJobs.each do |x| + if x.id == row[:parent_job_id] then + new.t_BASE = x.t_S + x.sub_job.push new + set = true + break + end + end + end + if not set then + orphanJobs.each do |x| + if x.id == row[:parent_job_id] then + new.t_BASE = x.t_S + x.sub_job.push new + set = true + break + end + end + end + end + end +end +conn.disconnect if conn +puts "" +multiJobs.each do |x| + x.project = x.sub_job.map{|s| s.project.to_s}.uniq.join(" ") + x.supported_os = x.sub_job.map{|s| s.supported_os.to_s}.uniq.join(" ") + x.t_US = x.sub_job.map{|y| y.t_E}.max if x.t_US.nil? and not x.sub_job.map{|y| y.t_E}.include? nil + x.t_UE = x.t_E if x.t_UE.nil? + x.gen_duration +end +singleJobs.each do |x| + x.gen_duration +end +registerJobs.each do |x| + x.gen_duration +end + +def name_extractor(jobs, names) + return jobs if names.empty? + return jobs.select{|j| (names - j.project.split(" ")).empty? } +end + +if option[:cvs].nil? then + mjobs = name_extractor(multiJobs,option[:name]) + if not mjobs.empty? then + puts "multi job" + puts "time, status, branch, project, os, upload, sub_work, total, sub_total, (upload + sub_work)/(total + sub_total)%" + mjobs.each do |m| + m.print_multi + end + end + + sjobs = name_extractor(singleJobs,option[:name]) + if not sjobs.empty? then + puts "single job" + puts "time, status, branch, project, os, work, upload, sub_work, total, sub_total, (work + upload + sub_work)/(total + sub_total)%" + sjobs.each do |s| + s.print_single + end + end + + rjobs = name_extractor(registerJobs,option[:name]) + if not rjobs.empty? then + puts "register job" + puts "time, status, branch, project, os, upload, sub_work, total, sub_total, (upload + sub_work)/(total + sub_total)%" + rjobs.each do |r| + r.print_register + end + end +else + File.open(option[:cvs],"w") do |f| + mjobs = name_extractor(multiJobs,option[:name]) + if not mjobs.empty? then + f.puts "multi job" + f.puts "multi, sub, reverse, status, branch, projects, os, start, wait, init, wait, remote upload, work, remote donwload, wait, upload, finish, total, sub work total, sub total, performance (w/t)%" + mjobs.each do |m| + f.puts m.print_cvs + end + end + sjobs = name_extractor(singleJobs,option[:name]) + if not sjobs.empty? then + f.puts "single job" + f.puts "job, reverse, status, branch, projects, os, start, wait, init, wait, remote upload, work, remote donwload, wait, upload, finish, total, sub work total, sub total, performance (w/t)%" + sjobs.each do |m| + f.puts m.print_cvs(0,1) + end + end + rjobs = name_extractor(registerJobs,option[:name]) + if not rjobs.empty? then + f.puts "register job" + f.puts "register, reverse, status, branch, projects, os, start, wait, init, wait, remote upload, work, remote donwload, wait, upload, finish, total, sub work total, sub total, performance (w/t)%" + rjobs.each do |m| + f.puts m.print_cvs(0,1) + end + end + end +end diff --git a/src/build_server/RemoteBuilder.rb b/src/build_server/RemoteBuilder.rb index 413686a..a960a1e 100644 --- a/src/build_server/RemoteBuilder.rb +++ b/src/build_server/RemoteBuilder.rb @@ -151,7 +151,7 @@ class RemoteBuilder client.receive_data() client.terminate end - rescue e + rescue => e @log.error(e, Log::LV_USER) end end diff --git a/statistics b/statistics new file mode 100755 index 0000000..d26e94b --- /dev/null +++ b/statistics @@ -0,0 +1,396 @@ +#!/usr/bin/ruby + +=begin + + statistics + +Copyright (c) 2000 - 2011 Samsung Electronics Co., Ltd. All rights reserved. + +Contact: +Taejun Ha +Jiil Hyoun +Donghyuk Yang +DongHee Yang + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Contributors: +- S-Core Co., Ltd +=end + +require 'time' +require 'optparse' +require 'dbi' + +class CommonJob + attr_accessor :id, :project, :user, :supported_os, :distribution, :parent_job, :remote_build_server, :jtype, :status, :user_group + attr_accessor :t_BASE, :t_S, :t_IS, :t_IE, :t_WS, :t_RUS, :t_RUE, :t_RDS, :t_RDE, :t_US, :t_UE, :t_E + attr_accessor :d_BASE, :d_waitI, :d_INIT, :d_waitW, :d_RU, :d_WORK, :d_RD, :d_waitU, :d_U, :d_FINISH, :d_total, :sub_total, :sub_work_total + attr_accessor :sub_job + def initialize(option, projects, supported_os, distributions, groups) + @id = option[:id] + @project = projects[option[:project_id]] + @supported_os = supported_os[option[:supported_os_id]] + @distribution = distributions[option[:distribution_id]] + @user_group = groups[option[:user_id]] + @parent_job = option[:parent_job] + @remote_build_server = option[:remote_build_server] + @jtype = option[:jtype] + @status = option[:status] + #set time stamp + @t_BASE = nil + t = Time.now + offset = t.getlocal.gmt_offset + @t_S = option[:start_time].to_time - offset if not option[:start_time].nil? + @t_IS = option[:INIT_START].to_time - offset if not option[:INIT_START].nil? + @t_IE = option[:INIT_END].to_time - offset if not option[:INIT_END].nil? + @t_WS = option[:WORK_START].to_time - offset if not option[:WORK_START].nil? + @t_RUS = option[:REMOTE_UPLOAD_START].to_time - offset if not option[:REMOTE_UPLOAD_START].nil? + @t_RUE = option[:REMOTE_UPLOAD_END].to_time - offset if not option[:REMOTE_UPLOAD_END].nil? + @t_RDS = option[:REMOTE_DOWNLOAD_START].to_time - offset if not option[:REMOTE_DOWNLOAD_START].nil? + @t_RDE = option[:REMOTE_DOWNLOAD_END].to_time - offset if not option[:REMOTE_DOWNLOAD_END].nil? + @t_US = option[:UPLOAD_START].to_time - offset if not option[:UPLOAD_START].nil? + @t_UE = option[:UPLOAD_END].to_time - offset if not option[:UPLOAD_END].nil? + @t_E = option[:end_time].to_time - offset if not option[:end_time].nil? + @sub_job = [] + @d_BASE = 0 + @d_waitI = 0 + @d_INIT = 0 + @d_waitW = 0 + @d_RU = 0 + @d_RD = 0 + @d_U = 0 + @d_waitU = 0 + @d_WORK = 0 + @d_FINISH = 0 + @d_total = 0 + @sub_total = 0 + @sub_work_total = 0 + end + def gen_duration + @sub_job.each {|x| x.gen_duration} + @d_BASE = @t_S - @t_BASE if not @t_S.nil? and not @t_BASE.nil? + @d_waitI = @t_IS - @t_S if not @t_IS.nil? and not @t_S.nil? + @d_INIT = @t_IE - @t_IS if not @t_IE.nil? and not @t_IS.nil? + @d_waitW = @t_WS - @t_IE if not @t_WS.nil? and not @t_IE.nil? + @d_RU = @t_RUE - @t_RUS if not @t_RUE.nil? and not @t_RUS.nil? + @d_RD = @t_RDE - @t_RDS if not @t_RDE.nil? and not @t_RDS.nil? + @d_U = @t_UE - @t_US if not @t_UE.nil? and not @t_US.nil? + @d_waitU = @t_US - @t_RDE if not @t_US.nil? and not @t_RDE.nil? + work_end = ([@t_E, @t_US, @t_RDS].compact.nil?)? [@t_E, @t_US, @t_RDS] : [@t_E, @t_US, @t_RDS].compact + work_start = ([@t_WS, @t_RUE].compact.nil?)? [@t_WS, @t_RUE] : [@t_WS, @t_RUE].compact + @d_WORK = work_end.min - work_start.max if not work_end.empty? and not work_start.empty? + @d_FINISH = @t_E - work_end.max if not work_end.empty? + @d_total = @t_E - @t_S if not @t_E.nil? and not @t_S.nil? + @sub_job.each do |x| + @sub_total = @sub_total + x.d_total if not x.d_total.nil? + @sub_total = @sub_total + x.sub_total if not x.sub_total.nil? + end + @sub_job.each do |x| + @sub_work_total = @sub_work_total + x.d_WORK if not x.d_WORK.nil? + @sub_work_total = @sub_work_total + x.sub_work_total if not x.sub_work_total.nil? + end + end + def print_cvs(pre_pending = 0, post_pending = 2) + result = [] + work = 0 + work = @d_WORK if @jtype != "MULTIBUILD" + result.push (", " * pre_pending) + @id.to_s + (", " * post_pending) + ", #{@status.to_s}, #{@distribution.to_s}, #{@project.to_s}, #{@supported_os.to_s}, " + [@d_BASE, @d_waitI, @d_INIT, @d_waitW, @d_RU, @d_WORK, @d_RD, @d_waitU, @d_U, @d_FINISH, @d_total, @sub_work_total, @sub_total].map{|x| x.to_s }.join(", ") + ", #{((@d_U + work + @sub_work_total)/(@d_total + @sub_total) * 100).round 2 if (@d_total + @sub_total) > 0}%" + pre_pending = pre_pending +1 + post_pending = post_pending -1 + result = result + sub_job.map{|x| x.print_cvs pre_pending, post_pending } + return result + end + def print_multi + puts "#{@t_S.to_s}, #{@status.to_s}, #{@distribution.to_s}, #{@project.to_s}, #{@supported_os.to_s}, " + [ @d_U, @sub_work_total, @d_total, @sub_total].map{|x| x.round.to_s }.join(", ") + ", #{((@d_U + @sub_work_total)/(@d_total + @sub_total) * 100).round 2 if (@d_total + @sub_total) > 0}%" + end + def print_single + puts "#{@t_S.to_s}, #{@status.to_s}, #{@distribution.to_s}, #{@project.to_s}, #{@supported_os.to_s}, " + [@d_WORK, @d_U, @sub_work_total, @d_total, @sub_total].map{|x| x.round.to_s }.join(", ") + ", #{((@d_WORK + @d_U + @sub_work_total)/(@d_total + @sub_total) * 100).round 2 if (@d_total + @sub_total) > 0}%" + end + def print_register + puts "#{@t_S.to_s}, #{@status.to_s}, #{@distribution.to_s}, #{@project.to_s}, #{@supported_os.to_s}, " + [@d_WORK, @sub_work_total, @d_total, @sub_total].map{|x| x.round.to_s }.join(", ") + ", #{((@d_WORK + @sub_work_total)/(@d_total + @sub_total) * 100).round 2 if (@d_total + @sub_total) > 0}%" + end +end + +def option_parse + options = {} + banner = "test" + + optparse = OptionParser.new(nil, 32, ' '*8) do |opts| + opts.banner = banner + options[:statistics] = false + opts.on('-s', '--statistics ', 'statistics') do |slv| + options[:statistics] = true + options[:slv] = slv + end + opts.on('-d', '--dsn ', 'data server name') do |dsn| + options[:dsn] = dsn + end + opts.on('-u', '--user ', 'user name') do |name| + options[:name] = name + end + opts.on('-p', '--password ', 'user password') do |pass| + options[:pass] = pass + end + opts.on('-w', '--where ', 'ex) jobs.id >= 79000') do |where| + options[:where] = "WHERE " + where + end + opts.on('-c', '--csv ', 'save csv file') do |cvs| + options[:cvs] = cvs + end + opts.on('-h', '--help', 'print this message') do |help| + puts "useage: performance_analizer -d \"Mysql:database=rsa;host=172.21.17.46;port=3306\" -u \"root\" -p \"password\" -w \"start_time >= '2013-07-21'\" -c result.cvs" + puts opts.help + exit + end + end + optparse.parse! + return options +end + +option = option_parse + +multiJobs = [] +registerJobs = [] +singleJobs = [] +orphanJobs = [] + +projects = {} +supported_os = {} +distributions = {} +groups = {} +conn = DBI.connect "DBI:" + option[:dsn], option[:name], option[:pass] + +conn.select_all "SELECT id,name FROM projects ;" do |row| + projects[row[:id]] = row[:name] +end +conn.select_all "SELECT id,name FROM supported_os ;" do |row| + supported_os[row[:id]] = row[:name] +end +conn.select_all "SELECT id,name FROM distributions ;" do |row| + distributions[row[:id]] = row[:name] +end +conn.select_all "SELECT user_groups.user_id,groups.name FROM user_groups, groups WHERE user_groups.group_id = groups.id;" do |row| + groups[row[:user_id]] = row[:name] + #puts "#{row[:user_id]} => #{row[:name]}" +end + +performance_log = "" +if not option[:statistics] then + performance_log = " , (SELECT stamp_time FROM rsa.job_timestamps WHERE job_timestamps.job_id = jobs.id AND stamp_name = 'INIT_START') INIT_START +, (SELECT stamp_time FROM rsa.job_timestamps WHERE job_timestamps.job_id = jobs.id AND stamp_name = 'INIT_END') INIT_END +, (SELECT stamp_time FROM rsa.job_timestamps WHERE job_timestamps.job_id = jobs.id AND stamp_name = 'WORK_START') WORK_START +, (SELECT stamp_time FROM rsa.job_timestamps WHERE job_timestamps.job_id = jobs.id AND stamp_name = 'REMOTE_UPLOAD_START') REMOTE_UPLOAD_START +, (SELECT stamp_time FROM rsa.job_timestamps WHERE job_timestamps.job_id = jobs.id AND stamp_name = 'REMOTE_UPLOAD_END') REMOTE_UPLOAD_END +, (SELECT stamp_time FROM rsa.job_timestamps WHERE job_timestamps.job_id = jobs.id AND stamp_name = 'REMOTE_DOWNLOAD_START') REMOTE_DOWNLOAD_START +, (SELECT stamp_time FROM rsa.job_timestamps WHERE job_timestamps.job_id = jobs.id AND stamp_name = 'REMOTE_DOWNLOAD_END') REMOTE_DOWNLOAD_END +, (SELECT stamp_time FROM rsa.job_timestamps WHERE job_timestamps.job_id = jobs.id AND stamp_name = 'UPLOAD_START') UPLOAD_START +, (SELECT stamp_time FROM rsa.job_timestamps WHERE job_timestamps.job_id = jobs.id AND stamp_name = 'UPLOAD_END') UPLOAD_END " +end + +conn.select_all "SELECT * #{performance_log} +FROM rsa.jobs +#{option[:where]} +;" do |row| + new = CommonJob.new(row, projects, supported_os, distributions, groups) + case row[:jtype] + when "MULTIBUILD" then + multiJobs.push new + when "REGISTER" + registerJobs.push new + when "BUILD" + if row[:parent_job_id].nil? + singleJobs.push new + else + set = false + multiJobs.each do |x| + if x.id == row[:parent_job_id] then + new.t_BASE = x.t_S + x.sub_job.push new + set = true + break + else + x.sub_job.each do |y| + if y.id == row[:parent_job_id] then + new.t_BASE = x.t_S + y.sub_job.push new + set = true + break + end + end + end + end + if not set then + singleJobs.each do |x| + if x.id == row[:parent_job_id] then + new.t_BASE = x.t_S + x.sub_job.push new + set = true + break + end + end + end + if not set then + registerJobs.each do |x| + if x.id == row[:parent_job_id] then + new.t_BASE = x.t_S + x.sub_job.push new + set = true + break + end + end + end + if not set then + orphanJobs.each do |x| + if x.id == row[:parent_job_id] then + new.t_BASE = x.t_S + x.sub_job.push new + set = true + break + end + end + end + end + end +end +if option[:statistics] then + multiJobs.each do |x| + x.project = x.sub_job.map{|s| s.project.to_s}.uniq.join(" ") + x.supported_os = x.sub_job.map{|s| s.supported_os.to_s}.uniq.join(" ") + end + puts "=== multi build ===" + days = multiJobs.map{|x| x.t_S.to_s.split(" ")[0]}.uniq + days.each do |d| + day_job = multiJobs.select{|j| j.t_S.to_s.start_with? d} + puts " #{d}: success #{day_job.select{|j| j.status.to_s == "FINISHED"}.count}, fail #{day_job.select{|j| j.status.to_s != "FINISHED"}.count}" + if option[:slv] != 1.to_s then + day_job.map{|x| x.user_group}.uniq.each do |u| + group_job= day_job.select{|j| j.user_group.to_s.eql? u} + puts " #{u}: success #{group_job.select{|j| j.status.to_s == "FINISHED"}.count}, fail #{group_job.select{|j| j.status.to_s != "FINISHED"}.count}" + if option[:slv] != 2.to_s then + group_job.map{|x| "#{x.project}"}.uniq.each do |p| + project_job = group_job.select{|j| j.project == p} + puts " #{p}: success #{project_job.select{|j| j.status.to_s == "FINISHED"}.count}, fail #{project_job.select{|j| j.status.to_s != "FINISHED"}.count}" + end + puts "" + end + end + puts "" + end + end + puts "=== single build ===" + days = singleJobs.map{|x| x.t_S.to_s.split(" ")[0]}.uniq + days.each do |d| + day_job = singleJobs.select{|j| j.t_S.to_s.start_with? d} + puts " #{d}: success #{day_job.select{|j| j.status.to_s == "FINISHED"}.count}, fail #{day_job.select{|j| j.status.to_s != "FINISHED"}.count}" + if option[:slv] != 1.to_s then + day_job.map{|x| x.user_group}.uniq.each do |u| + group_job= day_job.select{|j| j.user_group.to_s.eql? u} + puts " #{u}: success #{group_job.select{|j| j.status.to_s == "FINISHED"}.count}, fail #{group_job.select{|j| j.status.to_s != "FINISHED"}.count}" + if option[:slv] != 2.to_s then + group_job.map{|x| "#{x.project}"}.uniq.each do |p| + project_job = group_job.select{|j| j.project == p} + puts " #{p}: success #{project_job.select{|j| j.status.to_s == "FINISHED"}.count}, fail #{project_job.select{|j| j.status.to_s != "FINISHED"}.count}" + end + puts "" + end + end + puts "" + end + end + puts "=== register build ===" + days = registerJobs.map{|x| x.t_S.to_s.split(" ")[0]}.uniq + days.each do |d| + day_job = registerJobs.select{|j| j.t_S.to_s.start_with? d} + puts " #{d}: success #{day_job.select{|j| j.status.to_s == "FINISHED"}.count}, fail #{day_job.select{|j| j.status.to_s != "FINISHED"}.count}" + if option[:slv] != 1.to_s then + day_job.map{|x| x.user_group}.uniq.each do |u| + group_job= day_job.select{|j| j.user_group.to_s.eql? u} + puts " #{u}: success #{group_job.select{|j| j.status.to_s == "FINISHED"}.count}, fail #{group_job.select{|j| j.status.to_s != "FINISHED"}.count}" + if option[:slv] != 2.to_s then + group_job.map{|x| "#{x.project}"}.uniq.each do |p| + project_job = group_job.select{|j| j.project == p} + puts " #{p}: success #{project_job.select{|j| j.status.to_s == "FINISHED"}.count}, fail #{project_job.select{|j| j.status.to_s != "FINISHED"}.count}" + end + puts "" + end + end + puts "" + end + end +else + multiJobs.each do |x| + x.project = x.sub_job.map{|s| s.project.to_s}.uniq.join(" ") + x.supported_os = x.sub_job.map{|s| s.supported_os.to_s}.uniq.join(" ") + x.t_US = x.sub_job.map{|y| y.t_E}.max if x.t_US.nil? and not x.sub_job.map{|y| y.t_E}.include? nil + x.t_UE = x.t_E if x.t_UE.nil? + x.gen_duration + end + singleJobs.each do |x| + x.gen_duration + end + registerJobs.each do |x| + #x.d_WORK = x.d_U + x.gen_duration + end + + if option[:cvs].nil? then + if not multiJobs.empty? then + puts "multi job" + puts "time, status, branch, project, os, upload, sub_work, total, sub_total, (upload + sub_work)/(total + sub_total)%" + multiJobs.each do |m| + m.print_multi + end + end + + if not singleJobs.empty? then + puts "single job" + puts "time, status, branch, project, os, work, upload, sub_work, total, sub_total, (work + upload + sub_work)/(total + sub_total)%" + singleJobs.each do |s| + s.print_single + end + end + + if not registerJobs.empty? then + puts "register job" + puts "time, status, branch, project, os, upload, sub_work, total, sub_total, (upload + sub_work)/(total + sub_total)%" + registerJobs.each do |r| + r.print_register + end + end + else + File.open(option[:cvs],"w") do |f| + if not multiJobs.empty? then + f.puts "multi job" + f.puts "multi, sub, reverse, status, branch, projects, os, start, wait, init, wait, remote upload, work, remote donwload, wait, upload, finish, total, sub work total, sub total, performance (w/t)%" + multiJobs.each do |m| + f.puts m.print_cvs + end + end + if not singleJobs.empty? then + f.puts "single job" + f.puts "job, reverse, status, branch, projects, os, start, wait, init, wait, remote upload, work, remote donwload, wait, upload, finish, total, sub work total, sub total, performance (w/t)%" + singleJobs.each do |m| + f.puts m.print_cvs(0,1) + end + end + if not registerJobs.empty? then + f.puts "register job" + f.puts "register, reverse, status, branch, projects, os, start, wait, init, wait, remote upload, work, remote donwload, wait, upload, finish, total, sub work total, sub total, performance (w/t)%" + registerJobs.each do |m| + f.puts m.print_cvs(0,1) + end + end + end + end +end diff --git a/test/analyze.sh b/test/analyze.sh new file mode 100755 index 0000000..ae4ce82 --- /dev/null +++ b/test/analyze.sh @@ -0,0 +1,2 @@ +#!/bin/sh -x + ../performance_analyzer -d "Mysql:database=rsa;host=172.21.17.46;port=3306" -u "root" -p "qmscore" -w "start_time >= '`date '+%Y-%m-%d'`' AND distribution_id = 9 AND status = 'FINISHED'" -n common-eplugin -c result.`date '+%Y-%m-%d'`.csv diff --git a/test/performance_run.sh b/test/performance_run.sh new file mode 100755 index 0000000..1e86bd2 --- /dev/null +++ b/test/performance_run.sh @@ -0,0 +1,5 @@ +#!/bin/sh +#full build +#../build-cli build -N "assignmenttracing-eplugin,certificate-generator,cli,codecoverage-eplugin,common-eplugin,dynamic-analysis-ide-eplugin,dynamic-analyzer,emulator-kernel,emulator-manager,emulator-storages,event-injector,eventinjector-eplugin,gcc-4.5,gdb-7.5,image-creator,js-assignmenttracing-eplugin,jsdt-extension-eplugin,meta-emulator-tools,meta-sdk-dev-tools,meta-tizen-components,meta-tizen-documents,meta-tizen-ide,meta-tizen-installtype,native-eplugin,native-ext-eplugin,native-ui-builder,native-ui-effect-builder,nativeappcommon-eplugin,nativecommon-eplugin,nativeplatform-eplugin,nativeplatform-sample,pde-build,product,profiler-eplugin,qemu,rpm-tools,sbi-plugins,sdb,sdk-build,smart-build-interface,ui-customizer,unittest-eplugin,unittest-framework,version-manager,vgabios,web-sample,web-simulator,web-ui-builder-eplugin,web-ui-fw,webapp-eplugin,websimulator-eplugin" -d 172.21.17.46:3336 -u "admin@user" -w 1 -D test --rebuild -o "ubuntu-32,ubuntu-64,windows-32,windows-64,macos-64" +# reverse build +../build-cli build -N "common-eplugin" -d 172.21.17.46:3336 -u "admin@user" -w 1 -D test --rebuild -o "ubuntu-32,ubuntu-64,windows-32,windows-64,macos-64"