From 6b14f67520a9ce9f217dcb75e34c2b6702a4a97f Mon Sep 17 00:00:00 2001 From: donghee yang Date: Wed, 3 Apr 2013 11:06:37 +0900 Subject: [PATCH] [Title] Modified build-cli to use RemoteBuilder --- build-cli | 91 ++------- src/build_server/BuildJob.rb | 2 +- src/build_server/RemoteBuilder.rb | 178 +++++++++++------- src/common/BuildComm.rb | 1 + src/common/log.rb | 38 ++++ .../build-server.basic1/build-cli-05.testcase | 2 +- .../build-server.basic1/build-cli-06.testcase | 2 +- .../build-server.basic2/build-svr-20.testcase | 2 + 8 files changed, 166 insertions(+), 150 deletions(-) diff --git a/build-cli b/build-cli index 901b94e..ecbfc9c 100755 --- a/build-cli +++ b/build-cli @@ -39,6 +39,7 @@ require "FileTransferViaFTP" require "FileTransferViaDirect" require "JobLog" require "BuildServerException" +require "RemoteBuilder" #option parsing @@ -187,87 +188,21 @@ begin exit 1 end - # Request build - begin - client = BuildCommClient.create( addr[0], addr[1], nil, 0 ) - if client.nil? then - puts "Connection to server failed!" - exit 1 - end - - cmd = "BUILD|GIT|#{option[:project]}|#{option[:passwd]}|#{option[:os]}|#{option[:async]}|#{option[:noreverse]}|#{option[:dist]}|#{option[:user]}|#{option[:rebuild]}" - - client.send(cmd) - - result = client.read_lines do |line| - if line.strip.start_with?("=JOB_START") then - job_id = line.strip.split(",")[1] - elsif line.strip.start_with?("=JOB_STATUS") then - data = line.strip.split(",") - job_status = data[1] - job_error = data[2] - else - # print log - puts line - end - end - - if not result then - puts "Error: Communication failed! #{client.get_error_msg()}" - elsif job_id.nil? then - puts job_error - result = false - end - rescue => e - puts "ERROR: #{e}" - exit 1 - ensure - client.terminate if not client.nil? - end + builder = RemoteBuilder.new(addr[0], addr[1], nil, nil, nil, nil) + options = RemoteBuilder.init_options() + options[:projects] = option[:project] + options[:passwd] = option[:passwd] + options[:os] = option[:os] + options[:noreverse] = (option[:noreverse] == "YES") ? true : false + options[:dist] = option[:dist] + options[:user] = option[:user] + options[:rebuild] = (option[:rebuild] == "YES") ? true : false + options[:async] = (option[:async] == "YES") ? true : false + options[:monitor_level] = Log::LV_USER + result = builder.build(options) if not result then exit(-1) end - # Query log in case sync - job_status = "" - job_error = "" - if result and option[:async].eql? "NO" then - begin - result = JobLog.monitor(addr[0], addr[1], job_id) do |line,status,errmsg| - job_status = status - job_error = errmsg - if line.nil? then - next - end - - category, level, contents = JobLog.parse_log(line) - if level < Log::LV_USER then next end - - if category == "INFO" then - puts "Info: #{contents}" - elsif category == "WARN" then - puts "Warn: #{contents}" - elsif category == "ERROR" then - puts "Error: #{contents}" - else - next - end - end - - # Check job status - if not job_status.eql? "FINISHED" then - result = false - end - rescue BuildServerException => e - puts e.err_message() - result = false - rescue => e - puts "ERROR: #{e}" - result = false - end - end - - if not result then exit(1) end - when "query" result = Utils.parse_server_addr(option[:domain]) if result.nil? then diff --git a/src/build_server/BuildJob.rb b/src/build_server/BuildJob.rb index 882f876..f1e9a01 100644 --- a/src/build_server/BuildJob.rb +++ b/src/build_server/BuildJob.rb @@ -732,7 +732,7 @@ class BuildJob < CommonJob # create builder if @is_remote_job then - builder = RemoteBuilder.new("JB#{@id}", @remote_server, @server.ftp_addr, @server.ftp_port, @server.ftp_username, @server.ftp_passwd) + builder = RemoteBuilder.create_from_server(@remote_server, @server.ftp_addr, @server.ftp_port, @server.ftp_username, @server.ftp_passwd) @log.info( "JobBuilder##{@id} is created", Log::LV_USER) @log.info( " - Remote Server : #{@remote_server.ip}:#{@remote_server.port}" ) if not @server.ftp_addr.nil? then diff --git a/src/build_server/RemoteBuilder.rb b/src/build_server/RemoteBuilder.rb index f10c73e..bb960cb 100644 --- a/src/build_server/RemoteBuilder.rb +++ b/src/build_server/RemoteBuilder.rb @@ -38,23 +38,28 @@ require "BuildServerException.rb" class RemoteBuilder - attr_accessor :id, :log + attr_accessor :log # initialize - def initialize( id, server,ftp_addr, ftp_port, ftp_username, ftp_passwd) - @id = id - @server = server - @addr = server.ip - @port = server.port + def initialize( addr, port ,ftp_addr, ftp_port, ftp_username, ftp_passwd) + @addr = addr + @port = port @ftp_addr = ftp_addr @ftp_port = ftp_port @ftp_username = ftp_username @ftp_passwd = ftp_passwd - @log = DummyLog.new + + @log = StandardOutLogPrinter.new @job = nil end + def RemoteBuilder.create_from_server(rserver, ftp_addr, ftp_port, ftp_username, ftp_passwd) + + return RemoteBuilder.new(rserver.ip, rserver.port, ftp_addr, ftp_port, ftp_username, ftp_passwd) + end + + # build_job def build_job( job, local_pkgs ) # set job @@ -63,7 +68,7 @@ class RemoteBuilder @log = job.log # build - options = init_options() + options = RemoteBuilder.init_options() options[:git_repos] = @job.get_project().repository options[:git_commit] = @job.git_commit options[:src_path] = @job.source_path @@ -74,7 +79,7 @@ class RemoteBuilder options[:user] = "admin@user" options[:is_rev_build] = @job.is_rev_build_check_job() options[:dock] = Utils.create_uniq_name() - + options[:save_local] = @job.is_rev_build_check_job() ? false : true ret = build( options ) # reset job @@ -88,59 +93,68 @@ class RemoteBuilder # build def build( options ) - @log.info( "Start to build on remote server...", Log::LV_USER ) + @log.info( "Start to build on remote server...", Log::LV_USER ) if not @job.nil? - # send local packages - begin - @server.add_file_transfer() - options[:local_pkgs].each do |pkg_path| - @log.info( "Sending file... : #{pkg_path}", Log::LV_USER ) - result = send_file_to_remote( pkg_path, options[:dock] ) - if not result then - @log.error( "File transfering failed!", Log::LV_USER ) - @server.remove_file_transfer() - return false + # send local packages if build is invoked from JOB + if not @job.nil? and not options[:local_pkgs].empty? then + begin + @job.get_remote_server().add_file_transfer() + options[:local_pkgs].each do |pkg_path| + @log.info( "Sending file... : #{pkg_path}", Log::LV_USER ) + result = send_file_to_remote( pkg_path, options[:dock] ) + if not result then + @log.error( "File transfering failed!", Log::LV_USER ) + @job.get_remote_server().remove_file_transfer() + return false + end end + ensure + @job.get_remote_server().remove_file_transfer() end - ensure - @server.remove_file_transfer() end # send build request - @log.info( "Sending build request to remote server...", Log::LV_USER ) + @log.info( "Sending build request to remote server...", Log::LV_USER ) if not @job.nil? - result = send_build_request( options ) + result, job_id, job_error = send_build_request( options ) if not result then - @log.error( "Building job request on remote server failed!", Log::LV_USER ) + @log.error( job_error, Log::LV_USER ) if not job_error.nil? + @log.error( "Building job request on remote server failed!", Log::LV_USER ) if not @job.nil? return false end - result, result_files = send_monitor_request(options[:is_rev_build]) + if options[:async] then return result end + + result, result_files = send_monitor_request(job_id, options[:monitor_level]) if not result then @log.error( "Building job on remote server failed!", Log::LV_USER ) #Cancel remote job - begin - client = BuildCommClient.create( @addr, @port, @log ) - if not client.nil? then - client.send "CANCEL|#{@job.remote_id}|#{@job.get_project.passwd}|admin@user" - client.receive_data() - client.terminate + if not @job.nil? then + begin + client = BuildCommClient.create( @addr, @port, @log ) + if not client.nil? then + client.send "CANCEL|#{@job.remote_id}|#{@job.get_project.passwd}|admin@user" + client.receive_data() + client.terminate + end + rescue e + @log.error(e, Log::LV_USER) end - rescue e - @log.error(e, Log::LV_USER) end return false end - # receive binary package - result_files.each do |file_name| - @log.info( "Receiving file from remote server : #{file_name}", Log::LV_USER ) - result = receive_file_from_remote( "#{options[:src_path]}/#{file_name}", options[:dock] ) - if not result then - @log.error( "File transfering failed! : #{file_name}", Log::LV_USER ) - return false + # receive binary package if not reverse build + if options[:save_local] then + result_files.each do |file_name| + @log.info( "Receiving file from remote server : #{file_name}", Log::LV_USER ) + result = receive_file_from_remote( "#{options[:src_path]}/#{file_name}", options[:dock] ) + if not result then + @log.error( "File transfering failed! : #{file_name}", Log::LV_USER ) + return false + end end end @@ -149,8 +163,10 @@ class RemoteBuilder protected - def init_options() + def RemoteBuilder.init_options() options = {} + options[:projects] = nil + options[:passwd] = nil options[:git_repos] = nil options[:git_commit] = nil options[:src_path] = nil @@ -160,6 +176,10 @@ class RemoteBuilder options[:dist] = nil options[:user] = "admin@user" options[:is_rev_build] = false + options[:rebuild] = false + options[:save_local] = false + options[:async] = false + options[:monitor_level] = Log::LV_NORMAL return options end @@ -204,31 +224,45 @@ class RemoteBuilder protected def send_build_request( options ) result = true + job_id = nil + job_status = "" + job_error = nil + begin client = BuildCommClient.create( @addr, @port, @log ) if client.nil? then raise BuildServerException.new("ERR018"), "#{@addr}:#{@port}" end - # send - commit = options[:git_commit].nil? ? "":options[:git_commit] - pkg_list = options[:local_pkgs].map { |path| File.basename(path) }.join(",") - # format: BUILD|GIT|repository|passwd|os|async| - # no_reverse|dist_name| - # user_email|rebuild|internal|rev-build| - # commit|pkgs|dock_num - msg = "BUILD|GIT|#{options[:git_repos]}||#{options[:os]}|NO|"+ - "#{options[:no_reverse] ? "YES" : "NO"}|#{options[:dist]}|"+ - "#{options[:user]}|NO|YES|#{options[:is_rev_build] ? "YES":"NO"}|"+ - "#{commit}|#{pkg_list}|#{options[:dock]}" + if not options[:git_repos].nil? then + commit = options[:git_commit].nil? ? "" : options[:git_commit] + pkg_list = options[:local_pkgs].map { |path| File.basename(path) }.join(",") + # format: BUILD|GIT|repository|passwd|os|async| + # no_reverse|dist_name| + # user_email|rebuild|internal|rev-build| + # commit|pkgs|dock_num + msg = "BUILD|GIT|#{options[:git_repos]}||#{options[:os]}|NO|"+ + "#{options[:no_reverse] ? "YES" : "NO"}|#{options[:dist]}|"+ + "#{options[:user]}|NO|YES|#{options[:is_rev_build] ? "YES":"NO"}|"+ + "#{commit}|#{pkg_list}|#{options[:dock]}" + else + # format: BUILD|GIT|project_list|passwd| + # os|async| + # no_reverse|dist_name| + # user_email| + # rebuild + msg = "BUILD|GIT|#{options[:projects]}|#{options[:passwd]}|"+ + "#{options[:os]}|"+ + "#{options[:async] ? "YES" : "NO"}|"+ + "#{options[:no_reverse] ? "YES" : "NO"}|#{options[:dist]}|"+ + "#{options[:user]}|"+ + "#{options[:rebuild] ? "YES" : "NO"}" + end if not client.send( msg ) then raise BuildServerException.new("ERR020"), "#{client.get_error_msg()}" end - job_id = nil - job_status = "" - job_error = "" result = client.read_lines do |line| if line.strip.start_with?("=JOB_START") then job_id = line.strip.split(",")[1] @@ -243,13 +277,14 @@ class RemoteBuilder if not result then raise BuildServerException.new("ERR019"), "#{client.get_error_msg()}" - elsif job_id.nil? then - raise BuildServerException.new("ERR019"), "#{job_error}" + end + if job_id.nil? then + result = false end - @job.remote_id = job_id - @job.remote_status = job_status - @job.remote_error_msg = job_error + @job.remote_id = job_id if not @job.nil? + @job.remote_status = job_status if not @job.nil? + @job.remote_error_msg = job_error if not @job.nil? rescue BuildServerException => e @log.error( e.err_message(""), Log::LV_USER ) result = false @@ -260,29 +295,32 @@ class RemoteBuilder client.terminate if not client.nil? end - return result + return result, job_id, job_error end # send monitor request protected - def send_monitor_request(is_rev_build) + def send_monitor_request(job_id, min_level) result_files = [] result = true + job_status = "JUST_CREATED" begin - result = JobLog.monitor(@addr, @port,@job.remote_id) do |line,status,errmsg| - @job.remote_status = status - @job.remote_error_msg = errmsg + result = JobLog.monitor(@addr, @port, job_id) do |line,status,errmsg| + job_status = status + @job.remote_status = status if not @job.nil? + @job.remote_error_msg = errmsg if not @job.nil? if line.nil? then next end category, level, contents = JobLog.parse_log(line) + if level < min_level then next end if category == "INFO" then @log.info( contents, level ) # gather result files if not reverse build - if not is_rev_build and line =~ /Creating package file \.\.\. (.*)/ then + if line =~ /Creating package file \.\.\. (.*)/ then file_name = $1 result_files.push file_name end @@ -303,8 +341,10 @@ class RemoteBuilder result = false end - if @job.remote_status != "FINISHED" then - @log.error( "Remote job is finished with \"#{@job.remote_status}\"! #{@job.remote_error_msg}", Log::LV_USER) + if job_status != "FINISHED" then + if not @job.nil? then + @log.error( "Remote job is finished with \"#{@job.remote_status}\"! #{@job.remote_error_msg}", Log::LV_USER) + end result = false end diff --git a/src/common/BuildComm.rb b/src/common/BuildComm.rb index 4bb603c..8e4fe0e 100644 --- a/src/common/BuildComm.rb +++ b/src/common/BuildComm.rb @@ -1,4 +1,5 @@ require "socket" +require "timeout" =begin diff --git a/src/common/log.rb b/src/common/log.rb index 7592eab..a8ee767 100644 --- a/src/common/log.rb +++ b/src/common/log.rb @@ -133,3 +133,41 @@ class StandardOutLog < Log end end end + + +class StandardOutLogPrinter < Log + + def initialize() + super(nil) + end + + def info(msg, lv=LV_NORMAL) + if lv >= @level then + puts "Info: #{msg}" + @cnt = @cnt + 1 + end + end + + def warn(msg, lv=LV_NORMAL) + if lv >= @level then + puts "Warn: #{msg}" + @cnt = @cnt + 1 + end + end + + + def error(msg, lv=LV_NORMAL) + if lv >= @level then + puts "Error: #{msg}" + @cnt = @cnt + 1 + end + end + + + def output(msg, lv=LV_NORMAL) + if lv >= @level then + puts msg + @cnt = @cnt + 1 + end + end +end diff --git a/test/build-server.basic1/build-cli-05.testcase b/test/build-server.basic1/build-cli-05.testcase index 0072e13..27e3a9d 100644 --- a/test/build-server.basic1/build-cli-05.testcase +++ b/test/build-server.basic1/build-cli-05.testcase @@ -3,4 +3,4 @@ ../../build-cli build -N testa -d 127.0.0.1:11113 -o ubuntu-32 #POST-EXEC #EXPECT -Connection to server failed! +Error: Connection failed!: 127.0.0.1:11113 diff --git a/test/build-server.basic1/build-cli-06.testcase b/test/build-server.basic1/build-cli-06.testcase index 1682338..901aafc 100644 --- a/test/build-server.basic1/build-cli-06.testcase +++ b/test/build-server.basic1/build-cli-06.testcase @@ -3,4 +3,4 @@ ../../build-cli build -N testa -d 111.11q.111.111:1111 -o ubuntu-32 #POST-EXEC #EXPECT -Connection to server failed! +Error: Connection failed!: 111.11q.111.111:1111 diff --git a/test/build-server.basic2/build-svr-20.testcase b/test/build-server.basic2/build-svr-20.testcase index 1396b9c..d325842 100644 --- a/test/build-server.basic2/build-svr-20.testcase +++ b/test/build-server.basic2/build-svr-20.testcase @@ -4,6 +4,7 @@ mkdir buildsvr01 cd buildsvr01; ../../../build-svr create -n testserver3 -t ftp://ftpuser:ftpuser@172.21.111.124 cd buildsvr01; ../../../build-svr add-dist -n testserver3 -D BASE -u `pwd`/../pkgsvr01/unstable -d 127.0.0.1:3333 ../../build-svr add-os -n testserver3 -o ubuntu-32 +../../build-svr add-prj -n testserver3 -N bin -P bin -o ubuntu-32 mkdir -p bin cp ../bin/bin_0.0.0_ubuntu-32.zip bin/bin_0.0.0_wrongosname.zip ../../build-svr start -n testserver3 -p 2223 & @@ -17,6 +18,7 @@ sleep 1 rm -rf buildsvr01 rm -rf bin/bin/bin_0.0.0_wrongosname.zip #EXPECT +Info: Added new job Info: Initializing job... Error: Unsupported OS "wrongosname" is used! Error: Job is stopped by ERROR -- 2.34.1