From: donghee yang Date: Mon, 27 Aug 2012 09:04:17 +0000 (+0900) Subject: [Title] Modified to use independent slot(dock) for transfering files X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=24ec7fd9172b3e8dc272586490bd0a30360f1dd3;p=sdk%2Ftools%2Fsdk-build.git [Title] Modified to use independent slot(dock) for transfering files --- diff --git a/src/build_server/BuildComm.rb b/src/build_server/BuildComm.rb index 5b0b0d4..dabae88 100644 --- a/src/build_server/BuildComm.rb +++ b/src/build_server/BuildComm.rb @@ -228,7 +228,9 @@ class BuildCommServer for attempt in ATTEMPTS dst_filepath = FileTransfer.getfile(ip, port, username, passwd, filepath, dst_file, @log) if not dst_filepath.nil? then break - else "Server is the #{attempt} download attempt fails" end + else + @log.warn "Server is the #{attempt} download attempt fails" + end end if dst_filepath.nil? then req.puts "ERROR" diff --git a/src/build_server/BuildJob.rb b/src/build_server/BuildJob.rb index b8dea94..d5695a5 100644 --- a/src/build_server/BuildJob.rb +++ b/src/build_server/BuildJob.rb @@ -91,7 +91,10 @@ class BuildJob # job type @is_rev_build_check_job = false @is_remote_job = false + + # for internal(transferred) job @is_internal_job = false + @dock_num = "0" @external_pkgs = [] @force_rebuild = false @@ -147,8 +150,9 @@ class BuildJob end - def set_internal_job() + def set_internal_job( dock_num ) @is_internal_job = true + @dock_num = dock_num end @@ -173,8 +177,8 @@ class BuildJob # add external packages to overwrite before build - def add_external_package( file_path ) - @external_pkgs.push file_path + def add_external_package( file_name ) + @external_pkgs.push "#{@job_root}/external_pkgs/#{file_name}" end @@ -539,14 +543,14 @@ class BuildJob # checking build dependency if not @is_remote_job and not @is_internal_job and not check_build_dependency() then - if @is_internal_job then copy_log_file(@server.outgoing_path) end + if @is_internal_job then copy_result_files_to_master() end @status = "ERROR" return false end # clean build if not build() then - if @is_internal_job then copy_log_file(@server.outgoing_path) end + if @is_internal_job then copy_result_files_to_master() end @status = "ERROR" return false @@ -562,8 +566,7 @@ class BuildJob # copy result files to outgoing path if @is_internal_job then - copy_log_file(@server.outgoing_path) - copy_result_files(@server.outgoing_path) + copy_result_files_to_master() elsif not @parent.nil? and not @is_rev_build_check_job then copy_result_files(@parent.source_path) end @@ -1072,13 +1075,15 @@ class BuildJob end - def copy_log_file(dst_path) - @log.info( "Copying log to #{dst_path}", Log::LV_USER) - + # copy binary package files and log file to outgoing dir + def copy_result_files_to_master() + outgoing_dir = "#{@server.incoming_path}/#{@dock_num}" + + @log.info( "Copying log to #{outgoing_dir}", Log::LV_USER) file = "#{@source_path}/../log" - FileUtils.copy_file(file,"#{dst_path}/remote_log") - - return true + FileUtils.copy_file(file, "#{outgoing_dir}/remote_log") + + return copy_result_files( outgoing_dir ) end diff --git a/src/build_server/GitBuildJob.rb b/src/build_server/GitBuildJob.rb index dca1705..94afb7e 100644 --- a/src/build_server/GitBuildJob.rb +++ b/src/build_server/GitBuildJob.rb @@ -108,10 +108,8 @@ class GitBuildJob < BuildJob # verify def init - # mkdir - if not File.exist? @job_root then - FileUtils.mkdir_p @job_root - end + # mkdir job root + if not File.exist? @job_root then FileUtils.mkdir_p @job_root end # create logger if @log.nil? then @@ -120,6 +118,19 @@ class GitBuildJob < BuildJob @log.info( "Initializing job...", Log::LV_USER) + # if internal job, copy external_pkgs + if @is_internal_job then + @log.info( "Copying external dependent pkgs...", Log::LV_USER) + ext_pkgs_dir = "#{@job_root}/external_pkgs" + + incoming_dir = "#{@server.incoming_path}/#{@dock_num}" + if File.exist? incoming_dir then + FileUtils.mv "#{incoming_dir}", "#{ext_pkgs_dir}" + end + + FileUtils.mkdir_p incoming_dir + end + # download source code @git_commit = @project.get_source_code(@git_repos, @git_branch, @git_commit, @source_path, @log) if @git_commit.nil? then diff --git a/src/build_server/JobClean.rb b/src/build_server/JobClean.rb index 846e84d..4a6f585 100644 --- a/src/build_server/JobClean.rb +++ b/src/build_server/JobClean.rb @@ -69,6 +69,9 @@ class JobCleanAction < Action if File.exist? "#{@job_path}/temp" then FileUtils.rm_rf "#{@job_path}/temp" end + if File.exist? "#{@job_path}/external_pkgs" then + FileUtils.rm_rf "#{@job_path}/external_pkgs" + end # remove line for the job $access_listfile.synchronize { diff --git a/src/build_server/RemoteBuilder.rb b/src/build_server/RemoteBuilder.rb index 647da7e..951e72d 100644 --- a/src/build_server/RemoteBuilder.rb +++ b/src/build_server/RemoteBuilder.rb @@ -52,10 +52,14 @@ class RemoteBuilder # build def build( git_repos, source_path, os, is_rev_build, srcinfo, local_pkgs ) @log.info( "Start to build on remote server...", Log::LV_USER ) + + # create unique dock number + dock = Utils.create_uniq_name() + # send local packages local_pkgs.each do |pkg_path| @log.info( "Sending file... : #{pkg_path}", Log::LV_USER ) - result = send_file_to_remote( pkg_path ) + result = send_file_to_remote( pkg_path, dock ) if not result then @log.error( "File transfering failed!", Log::LV_USER ) return false @@ -65,10 +69,10 @@ class RemoteBuilder # send build request @log.info( "Sending build request to remote server...", Log::LV_USER ) result, result_files = send_build_request(git_repos, os, - is_rev_build, srcinfo, local_pkgs) + is_rev_build, srcinfo, local_pkgs, dock) @log.info( "Receiving log file from remote server...", Log::LV_USER ) - if not receive_file_from_remote( "#{source_path}/../remote_log" ) then + if not receive_file_from_remote( "#{source_path}/../remote_log", dock ) then @log.warn( "File transfering failed! : remote_log", Log::LV_USER ) end @@ -80,7 +84,7 @@ class RemoteBuilder # receive binary package result_files.each do |file_name| @log.info( "Receiving file from remote server : #{file_name}", Log::LV_USER ) - result = receive_file_from_remote( "#{source_path}/#{file_name}" ) + result = receive_file_from_remote( "#{source_path}/#{file_name}", dock ) if not result then @log.error( "File transfering failed! : #{file_name}", Log::LV_USER ) return false @@ -94,7 +98,7 @@ class RemoteBuilder # upload binary packages that is need to be overwrite # before remote package protected - def send_file_to_remote(file_path) + def send_file_to_remote(file_path, dock = "0") # create client client = BuildCommClient.create( @addr, @port, @log ) if client.nil? then @@ -105,7 +109,8 @@ class RemoteBuilder # upload file result = true file_name = file_path.split("/")[-1] - if client.send("UPLOAD") then + msg = "UPLOAD|#{dock}" + if client.send( msg ) then result=client.send_file( @ftp_addr, @ftp_port, @ftp_username, @ftp_passwd, file_path ) if not result then @log.error( "File uploading failed...#{file_name}", Log::LV_USER) @@ -121,7 +126,7 @@ class RemoteBuilder # send build request protected - def send_build_request(git_repos, os, is_rev_build, commit,local_pkgs) + def send_build_request(git_repos, os, is_rev_build, commit, local_pkgs, dock = "0") result_files = [] client = BuildCommClient.create( @addr, @port, @log ) @@ -134,13 +139,14 @@ class RemoteBuilder local_pkg_names = local_pkgs.map { |path| File.basename(path) } # send - # format: BUILD|GIT|repository|passwd|os|async|internal|rev-build|commit|pkgs - # value : BUILD|GIT|repository| |os|NO |YES |rev-build|commit|pkgs + # format: BUILD|GIT|repository|passwd|os|async|internal|rev-build|commit|pkgs|dock_num + # value : BUILD|GIT|repository| |os|NO |YES |rev-build|commit|pkgs|dock_num result = true - commit=commit.nil? ? "":commit - pkg_list=local_pkg_names.join(",") - rev= is_rev_build ? "YES":"NO" - if client.send("BUILD|GIT|#{git_repos}||#{os}|NO|YES|#{rev}|#{commit}|#{pkg_list}") then + commit = commit.nil? ? "":commit + pkg_list = local_pkg_names.join(",") + rev = is_rev_build ? "YES":"NO" + msg = "BUILD|GIT|#{git_repos}||#{os}|NO|YES|#{rev}|#{commit}|#{pkg_list}|#{dock}" + if client.send( msg ) then result = client.read_lines do |l| # write log first @log.output( l.strip, Log::LV_USER) @@ -170,7 +176,7 @@ class RemoteBuilder # receive binary package of remote server protected - def receive_file_from_remote(file_path) + def receive_file_from_remote(file_path, dock = "0") # create client client = BuildCommClient.create( @addr, @port, @log ) if client.nil? then @@ -181,7 +187,8 @@ class RemoteBuilder # download file result = true file_name = file_path.split("/")[-1] - if client.send("DOWNLOAD|#{file_name}") then + msg = "DOWNLOAD|#{dock}|#{file_name}" + if client.send( msg ) then result=client.receive_file( @ftp_addr, @ftp_port, @ftp_username, @ftp_passwd, file_path ) if not result then @log.error( "File downloading failed...#{file_name}", Log::LV_USER) diff --git a/src/build_server/SocketJobRequestListener.rb b/src/build_server/SocketJobRequestListener.rb index 259ccfc..66355d5 100644 --- a/src/build_server/SocketJobRequestListener.rb +++ b/src/build_server/SocketJobRequestListener.rb @@ -152,7 +152,7 @@ class SocketJobRequestListener end # Case1. BUILD|GIT|project_name|passwd|os_list|async - # Case2. BUILD|GIT|git_repos||os|async|internal|rev_build|commit|pkgs + # Case2. BUILD|GIT|git_repos||os|async|internal|rev_build|commit|pkgs|dock_num # parse project_name_list = tok[2].split(",") @@ -163,6 +163,7 @@ class SocketJobRequestListener rev_job = tok[7].eql? "YES" git_commit = (not tok[8].nil? and not tok[8].empty?) ? tok[8] : nil pkg_files = (not tok[9].nil? and not tok[9].empty?) ? tok[9].split(",") : [] + dock_num = (not tok[10].nil? and not tok[10].empty?) ? tok[10].strip : "0" # check supported os if not internal job if not is_internal then @@ -201,7 +202,7 @@ class SocketJobRequestListener git_repos = project_name_list[0] os = os_list[0] - new_job = create_new_internal_job(git_repos, os, git_commit, pkg_files ) + new_job = create_new_internal_job(git_repos, os, git_commit, pkg_files, dock_num ) if rev_job then new_job.set_rev_build_check_job(nil) end # single job @@ -609,43 +610,62 @@ class SocketJobRequestListener # "UPLOAD" def handle_cmd_upload( line, req ) + @log.info "Received File transfer REQ : #{line}" tok = line.split("|").map { |x| x.strip } - if tok.count < 1 then + if tok.count < 2 then @log.info "Received Wrong REQ: #{line}" raise "Invalid request format is used: #{line}" end + dock_num = tok[1].strip + BuildCommServer.send_begin(req) begin - @comm_server.receive_file(req, @parent_server.incoming_path) + incoming_dir = "#{@parent_server.incoming_path}/#{dock_num}" + if not File.exist? incoming_dir then FileUtils.mkdir_p incoming_dir end + @comm_server.receive_file( req, incoming_dir ) rescue => e - puts "Failed to receive file" - puts e.message - puts e.backtrace.inspect + @log.error "Failed to receive file" + @log.error e.message + @log.error e.backtrace.inspect end BuildCommServer.send_end(req) end # "DOWNLOAD" + # format = DOWNLOAD|dock_num|file_name def handle_cmd_download( line, req ) + @log.info "Received File transfer REQ : #{line}" tok = line.split("|").map { |x| x.strip } - if tok.count < 1 then + if tok.count < 3 then @log.info "Received Wrong REQ: #{line}" raise "Invalid request format is used: #{line}" end - file_name = tok[1] + dock_num = tok[1].strip + file_name = tok[2] @log.info "Received a request for download file : #{file_name}" + outgoing_dir = "#{@parent_server.incoming_path}/#{dock_num}" BuildCommServer.send_begin(req) begin - @comm_server.send_file(req, "#{@parent_server.outgoing_path}/#{file_name}") + @log.info "Sending requested file...: #{file_name}" + @comm_server.send_file(req, "#{outgoing_dir}/#{file_name}") + # remove file if "dock" defined + if dock_num != "0" then + @log.info "Removing requested file...: #{file_name}" + FileUtils.rm_rf "#{outgoing_dir}/#{file_name}" + if Utils.directory_emtpy?(outgoing_dir) then + FileUtils.rm_rf "#{outgoing_dir}" + end + end rescue => e - puts "transfer failed" - puts e.message - puts e.backtrace.inspect + @log.error "transfer failed" + @log.error e.message + @log.error e.backtrace.inspect end + BuildCommServer.send_end(req) end @@ -781,16 +801,17 @@ class SocketJobRequestListener private - def create_new_internal_job( git_repos, os, git_commit, pkg_files ) + def create_new_internal_job( git_repos, os, git_commit, pkg_files, dock_num ) prj = @parent_server.prjmgr.get_git_project( git_repos ) if prj.nil? then prj = @parent_server.prjmgr.create_unnamed_git_project( git_repos ) end new_job = prj.create_new_job(os) - new_job.set_internal_job() + new_job.set_internal_job( dock_num ) new_job.set_git_commit(git_commit) + incoming_dir = "#{@parent_server.incoming_path}/#{dock_num}" pkg_files.each { |file| - new_job.add_external_package( "#{@parent_server.incoming_path}/#{file}" ) + new_job.add_external_package( file ) } return new_job diff --git a/src/builder/Builder.rb b/src/builder/Builder.rb index fc42014..dc528d2 100644 --- a/src/builder/Builder.rb +++ b/src/builder/Builder.rb @@ -220,6 +220,9 @@ class Builder else local_dep_pkgs.each do |l| @log.info( "Installing local pacakge...#{l}", Log::LV_USER) + if not File.exist? l then + @log.error( "File not found!: #{l}", Log::LV_USER ) + end cl.install_local_pkg(l,true,false,File.dirname(l)) end end diff --git a/src/common/utils.rb b/src/common/utils.rb index 377641c..f48ad89 100644 --- a/src/common/utils.rb +++ b/src/common/utils.rb @@ -437,6 +437,10 @@ class Utils return ( not arg.end_with? seperator and not arg.split( seperator ).select{|x| x.empty?}.length > 0 ) end + def Utils.directory_emtpy?(dir_path) + return (Dir.entries(dir_path).join == "...") + end + if defined?(HOST_OS).nil? then HOST_OS = Utils.identify_current_OS() diff --git a/src/pkg_server/SocketRegisterListener.rb b/src/pkg_server/SocketRegisterListener.rb index 640047f..9d97550 100644 --- a/src/pkg_server/SocketRegisterListener.rb +++ b/src/pkg_server/SocketRegisterListener.rb @@ -107,15 +107,15 @@ class SocketRegisterListener # "UPLOAD" def handle_cmd_upload( line, req ) @log.info "Received File transfer REQ : #{line}" + BuildCommServer.send_begin(req) begin - @comm_server.receive_file(req, @parent_server.incoming_path) + incoming_dir = "#{@parent_server.incoming_path}" + @comm_server.receive_file(req, incoming_dir) rescue => e @log.error "Failed to transfer file" @log.error e.message @log.error e.backtrace.inspect - BuildCommServer.send_end(req) - return end BuildCommServer.send_end(req) end