From 241778005ffbaa938ed7bf0416ba7613ad218c57 Mon Sep 17 00:00:00 2001 From: donghee yang Date: Tue, 5 Mar 2013 17:35:18 +0900 Subject: [PATCH] [Title] Fixed to update the status of remote server asynchronously --- src/build_server/BuildJob.rb | 6 + src/build_server/BuildServer.rb | 34 +++--- src/build_server/JobManager.rb | 33 ++++-- src/build_server/RemoteBuildServer.rb | 103 +++++++++++++++--- src/build_server/SocketJobRequestListener.rb | 2 +- src/common/BuildComm.rb | 3 +- .../build-server.basic1/build-cli-11.testcase | 2 +- 7 files changed, 138 insertions(+), 45 deletions(-) diff --git a/src/build_server/BuildJob.rb b/src/build_server/BuildJob.rb index fff67de..a762b8c 100644 --- a/src/build_server/BuildJob.rb +++ b/src/build_server/BuildJob.rb @@ -137,6 +137,12 @@ class BuildJob < CommonJob def set_remote_job(server) @is_remote_job = true @remote_server=server + server.add_working_job(self) + end + + + def is_remote_job() + return (not @remote_server.nil?) end diff --git a/src/build_server/BuildServer.rb b/src/build_server/BuildServer.rb index eae94f2..4bbd5d7 100644 --- a/src/build_server/BuildServer.rb +++ b/src/build_server/BuildServer.rb @@ -73,7 +73,7 @@ class BuildServer @req_listener = [] @finish = false # status - @status = "RUNNING" + @status = "INITIALIZING" # host_os @host_os = HOST_OS # log @@ -244,21 +244,20 @@ class BuildServer @pkg_sync = PackageServerSynchronizer.new(self) @pkg_sync.start + # update friend server status + @log.info "Initializing Remote Servers..." + @remote_servers = get_remote_servers() + @remote_servers.each do |server| + server.create_system_monitor() + end + # main loop @log.info "Entering main loop..." + @status = "RUNNING" begin if @test_time > 0 then start_time = Time.now end while( not @finish ) - # update friend server status - @remote_servers = get_remote_servers() - @remote_servers.each do |server| - # update state - get_db_connection() do |db| - server.update_state(db) - end - end - # handle jobs @jobmgr.handle() @@ -310,19 +309,22 @@ class BuildServer def get_remote_servers() get_db_connection() do |db| - return RemoteBuildServer.load_all(db) + return RemoteBuildServer.load_all(db, self) end end # add new remote friend server def add_remote_server( ip, port ) - + if @status == "RUNNING" then return false end + newsvr = nil get_db_connection() do |db| - rs = RemoteBuildServer.load(ip, port, db) + rs = RemoteBuildServer.load(ip, port, db, self) if not rs.nil? then return false end - RemoteBuildServer.new(ip, port, "").save(db) + newsvr = RemoteBuildServer.new(ip, port, "", self) + newsvr.save(db) end + @remote_servers.push newsvr return true end @@ -330,6 +332,8 @@ class BuildServer # remove remote friend server def remove_remote_server( ip, port ) + if @status == "RUNNING" then return false end + get_db_connection() do |db| rs = RemoteBuildServer.load(ip, port, db) if rs.nil? then return false end @@ -747,7 +751,7 @@ class BuildServer begin while true BuildCommServer.send(conn, - "=STATUS,OK,#{host_os},#{@jobmgr.max_working_jobs}") + "=STATUS,#{@status},#{@host_os},#{@jobmgr.max_working_jobs}") sleep 5 end rescue => e diff --git a/src/build_server/JobManager.rb b/src/build_server/JobManager.rb index 8c4d47e..e5afdd5 100644 --- a/src/build_server/JobManager.rb +++ b/src/build_server/JobManager.rb @@ -272,15 +272,15 @@ class JobManager if job.status == "ERROR" save_job_status(job) @server.log.info "Job \"#{job.id}\" is stopped by ERROR" - @reverse_build_jobs.delete job + dispose_job(job) elsif job.status == "FINISHED" save_job_status(job) @server.log.info "Job \"#{job.id}\" is removed by FINISH status" - @reverse_build_jobs.delete job + dispose_job(job) elsif job.status == "CANCELED" save_job_status(job) @server.log.info "Job \"#{job.id}\" is removed by CANCELED status" - @reverse_build_jobs.delete job + dispose_job(job) end # if "JUST_CREATED", initialize it @@ -296,15 +296,15 @@ class JobManager if job.status == "ERROR" save_job_status(job) @server.log.info "Job \"#{job.id}\" is stopped by ERROR" - @internal_jobs.delete job + dispose_job(job) elsif job.status == "FINISHED" save_job_status(job) @server.log.info "Job \"#{job.id}\" is removed by FINISH status" - @internal_jobs.delete job + dispose_job(job) elsif job.status == "CANCELED" save_job_status(job) @server.log.info "Job \"#{job.id}\" is removed by CANCELED status" - @internal_jobs.delete job + dispose_job(job) end # if "JUST_CREATED", initialize it @@ -320,15 +320,15 @@ class JobManager if job.status == "ERROR" save_job_status(job) @server.log.info "Job \"#{job.id}\" is stopped by ERROR" - @jobs.delete job + dispose_job(job) elsif job.status == "FINISHED" save_job_status(job) @server.log.info "Job \"#{job.id}\" is removed by FINISH status" - @jobs.delete job + dispose_job(job) elsif job.status == "CANCELED" save_job_status(job) @server.log.info "Job \"#{job.id}\" is removed by CANCELED status" - @jobs.delete job + dispose_job(job) end # if "JUST_CREATED", initialize it @@ -343,7 +343,7 @@ class JobManager job.status = "ERROR" save_job_status(job) - @jobs.delete( job ) + dispose_job( job ) @server.log.info "Job \"#{job.id}\" is disconnected by user. Removed!" end end @@ -367,6 +367,19 @@ class JobManager end + def dispose_job(job) + # if remote job, remove it from remote server's queue + if job.type == "BUILD" and job.is_remote_job then + job.get_remote_server.remove_working_job(job) + job.get_remote_server.update_db() + end + + # remove from queue + @jobs.delete_if {|j| j.id == job.id} + @internal_jobs.delete_if {|j| j.id == job.id} + @reverse_build_jobs.delete_if {|j| j.id == job.id} + end + # select the job whith no build-dependency problem def get_available_job # select reverse build job with round-robin method diff --git a/src/build_server/RemoteBuildServer.rb b/src/build_server/RemoteBuildServer.rb index b038be1..345c8b6 100644 --- a/src/build_server/RemoteBuildServer.rb +++ b/src/build_server/RemoteBuildServer.rb @@ -39,7 +39,7 @@ class RemoteBuildServer attr_accessor :jobmgr, :distmgr # initialize - def initialize(ip, port, desc) + def initialize(ip, port, desc, parent) @id = -1 @ip = ip @port = port @@ -48,14 +48,14 @@ class RemoteBuildServer @host_os = Utils::HOST_OS @max_working_jobs = 2 @working_jobs = [] - @working_job_count = 0 @waiting_jobs = [] - @waiting_job_count = 0 @path = "" @file_transfer_cnt_mutex = Mutex.new @file_transfer_cnt = 0 @jobmgr = nil @distmgr = nil + @monitor_thread = nil + @parent = parent end @@ -71,6 +71,70 @@ class RemoteBuildServer end + # create monitor to check remote build server status + def create_system_monitor() + @parent.log.info " * Creating system monitor for #{@ip}:#{@port}..." + @monitor_thread = Thread.new do + while true + begin + client = BuildCommClient.create( @ip, @port ) + if not client.nil? and client.send("MONITOR|SYSTEM") then + result = client.read_lines(10,10) do |l| + tok = l.split(",").map { |x| x.strip } + status = tok[1] + host_os = tok[2] + max_working_jobs = tok[3].to_i + if status != @status or max_working_jobs != @max_working_jobs then + @status = status + @max_working_jobs = max_working_jobs + @host_os = host_os + update_db() + end + end + end + rescue => e + @parent.log.error "Remote server connection failed! : #{e.message}" + ensure + if not client.nil? then client.terminate end + end + + @status = "DISCONNECTED" + update_db() + + # after 1 min, try to reconnect + sleep 60 + end + end + + @parent.log.info "Created system monitor for #{@ip}:#{@port}" + end + + + def update_db() + @parent.get_db_connection() do |db| + working_job_count = @working_jobs.count + waiting_job_count = @waiting_jobs.count + + db.do "UPDATE remote_build_servers SET + status = '#{@status}', + supported_os_id = (SELECT supported_os.id FROM supported_os WHERE supported_os.name = '#{@host_os}'), + max_job_count = #{@max_working_jobs}, + working_job_count = #{working_job_count}, + waiting_job_count = #{waiting_job_count} WHERE id = #{@id}" + end + end + + + def add_working_job(job) + @working_jobs.push job + end + + + def remove_working_job(job) + @working_jobs.delete job + end + + # query remote server info & update server state def update_state(db) @@ -132,33 +196,33 @@ class RemoteBuildServer if @status == "DISCONNECTED" then db.do "UPDATE remote_build_servers SET status = 'DISCONNECTED', max_job_count = 0, working_job_count = 0, waiting_job_count = 0 WHERE id = #{@id}" else - @working_job_count = @working_jobs.count - @waiting_job_count = @waiting_jobs.count + working_job_count = @working_jobs.count + waiting_job_count = @waiting_jobs.count db.do "UPDATE remote_build_servers SET status = '#{@status}', supported_os_id = (SELECT supported_os.id FROM supported_os WHERE supported_os.name = '#{@host_os}'), max_job_count = #{@max_working_jobs}, - working_job_count = #{@working_job_count}, - waiting_job_count = #{@waiting_job_count} WHERE id = #{@id}" + working_job_count = #{working_job_count}, + waiting_job_count = #{waiting_job_count} WHERE id = #{@id}" end end # return available working slot def get_number_of_empty_room - return @max_working_jobs - @working_job_count + return @max_working_jobs - @working_jobs.count end # check there are working jobs def has_working_jobs - return (@working_job_count > 0) + return (@working_jobs.count > 0) end # check there are waiting jobs def has_waiting_jobs - return (@waiting_job_count > 0) + return (@waiting_jobs.count > 0) end @@ -198,39 +262,44 @@ class RemoteBuildServer end - def self.load(ip, port, db) + def self.load(ip, port, db, parent) saddr="#{ip}:#{port}" row = db.select_one("SELECT remote_build_servers.*,supported_os.name as host_os_name FROM remote_build_servers, supported_os WHERE svr_addr='#{saddr}' and remote_build_servers.supported_os_id = supported_os.id") if not row.nil? then - return load_row(row) + return load_row(row, parent) end return nil end - def self.load_all(db) + def self.load_all(db, parent) result = [] rows = db.select_all("SELECT *,'' as host_os_name FROM remote_build_servers WHERE supported_os_id IS NULL UNION ALL SELECT remote_build_servers.*, supported_os.name as host_os_name FROM remote_build_servers, supported_os WHERE remote_build_servers.supported_os_id = supported_os.id") rows.each do |row| - result.push load_row(row) + result.push load_row(row, parent) end return result end - def self.load_row(row) + def self.load_row(row, parent) svr_ip,svr_port=row['svr_addr'].strip.split(":") - new_obj = new(svr_ip, svr_port, row['description'] ) + new_obj = new(svr_ip, svr_port, row['description'], parent ) new_obj.set_id( row['id'] ) new_obj.status = row['status'] - new_obj.max_working_jobs =row['max_job_count'] + if row['max_job_count'].nil? then + new_obj.max_working_jobs = 0 + else + new_obj.max_working_jobs = row['max_job_count'].to_i + end new_obj.working_job_count =row['working_job_count'] new_obj.waiting_job_count =row['waiting_job_count'] new_obj.host_os = row['host_os_name'] + return new_obj end diff --git a/src/build_server/SocketJobRequestListener.rb b/src/build_server/SocketJobRequestListener.rb index 30bc7b2..0bd69aa 100644 --- a/src/build_server/SocketJobRequestListener.rb +++ b/src/build_server/SocketJobRequestListener.rb @@ -556,7 +556,7 @@ class SocketJobRequestListener when "FRIEND" # print GIT projects @parent_server.remote_servers.each do |server| - BuildCommServer.send(req,"#{server.status},#{server.host_os},#{server.waiting_job_count},#{server.working_job_count},#{server.max_working_jobs},#{server.get_file_transfer_cnt}") + BuildCommServer.send(req,"#{server.status},#{server.host_os},#{server.waiting_jobs.count},#{server.working_jobs.count},#{server.max_working_jobs},#{server.get_file_transfer_cnt}") end BuildCommServer.send_end(req) BuildCommServer.disconnect(req) diff --git a/src/common/BuildComm.rb b/src/common/BuildComm.rb index 6b866ed..e0308e8 100644 --- a/src/common/BuildComm.rb +++ b/src/common/BuildComm.rb @@ -459,6 +459,7 @@ class BuildCommClient # get contents result = true while true + line = nil if not data_timeout.nil? then timeout( data_timeout ) do line = @socket.gets() @@ -492,7 +493,7 @@ class BuildCommClient begin l = nil - timeout(FIRST_REPONSE_TIMEOUT) do + timeout(FIRST_RESPONSE_TIMEOUT) do l = @socket.gets() end diff --git a/test/build-server.basic1/build-cli-11.testcase b/test/build-server.basic1/build-cli-11.testcase index beb3e82..83311d4 100644 --- a/test/build-server.basic1/build-cli-11.testcase +++ b/test/build-server.basic1/build-cli-11.testcase @@ -4,7 +4,7 @@ echo "if there doe not exist server to build, error" ../../build-cli build -N testa -d 127.0.0.1:2223 -o windows-32 #POST-EXEC #EXPECT -Info: Added new job "5" for windows-32! +Info: Added new job Info: Initializing job... Error: No servers that are able to build your packages. Error: Host-OS (windows-32) is not supported in build server. -- 2.34.1