From d69a7f77c381a8083f15f726b80d5bd349a98fa6 Mon Sep 17 00:00:00 2001 From: donghee yang Date: Thu, 11 Oct 2012 16:57:14 +0900 Subject: [PATCH] [Title] Added new "direct file transfer module" --- build-cli | 124 ++---------- build-svr | 25 +-- src/build_server/BuildClientOptionParser.rb | 5 +- src/build_server/BuildJob.rb | 14 +- src/build_server/BuildServerController.rb | 37 ++-- src/build_server/BuildServerOptionParser.rb | 5 +- src/build_server/MultiBuildJob.rb | 2 +- src/build_server/RegisterPackageJob.rb | 2 +- src/build_server/RemoteBuilder.rb | 17 +- src/build_server/SocketJobRequestListener.rb | 14 +- src/common/BuildComm.rb | 186 ++++++++++++------ src/common/FileTransferViaDirect.rb | 107 ++++++++++ ...{fileTransfer.rb => FileTransferViaFTP.rb} | 4 +- src/pkg_server/SocketRegisterListener.rb | 1 - src/pkg_server/client.rb | 18 +- .../build-server.basic1/build-cli-01.testcase | 2 +- test/build-server.basic1/buildsvr.init | 2 +- .../build-server.basic2/build-svr-02.testcase | 2 +- test/build-server.multi-svr1/buildsvr2.init | 2 +- 19 files changed, 338 insertions(+), 231 deletions(-) create mode 100644 src/common/FileTransferViaDirect.rb rename src/common/{fileTransfer.rb => FileTransferViaFTP.rb} (99%) diff --git a/build-cli b/build-cli index 1146087..d7bc25c 100755 --- a/build-cli +++ b/build-cli @@ -35,7 +35,8 @@ $LOAD_PATH.unshift File.dirname(__FILE__)+"/src/build_server" require "utils" require "BuildClientOptionParser" require "BuildComm" - +require "FileTransferViaFTP" +require "FileTransferViaDirect" #option parsing @@ -270,17 +271,22 @@ begin bs_ip = result[0] bs_port = result[1] - ftp_result = Utils.parse_ftpserver_url(option[:fdomain]) - if ftp_result.nil? or ftp_result.length != 4 then - puts "FTP server url is incorrect. (#{option[:fdomain]})" - puts "Tune as following format." - puts " ftp://:@
" - exit 1 - end - ip = ftp_result[0] - port = ftp_result[1] - username = ftp_result[2] - passwd = ftp_result[3] + if not option[:fdomain].nil? then + ftp_result = Utils.parse_ftpserver_url(option[:fdomain]) + if ftp_result.nil? or ftp_result.length != 4 then + puts "FTP server url is incorrect. (#{option[:fdomain]})" + puts "Tune as following format." + puts " ftp://:@
" + exit 1 + end + ip = ftp_result[0] + port = ftp_result[1] + username = ftp_result[2] + passwd = ftp_result[3] + transporter = FileTransferFTP.new( nil, ip, port, username, passwd ) + else + transporter = FileTransferDirect.new( nil ) + end # upload client = BuildCommClient.create( bs_ip, bs_port, nil, 0 ) @@ -291,7 +297,7 @@ begin dock = Utils.create_uniq_name() msg = "UPLOAD|#{dock}" client.send( msg ) - result = client.send_file(ip, port, username, passwd, option[:package]) + result = client.send_file(option[:package], transporter) client.terminate if not result then puts "Uploading file failed!.. #{option[:package]}" @@ -308,98 +314,6 @@ begin client.print_stream client.terminate - # for test - when "upload" - # check file exist - if not File.exist? option[:file] then - puts "The file does not exist!.. #{option[:file]}" - exit(-1) - end - - result = Utils.parse_server_addr(option[:domain]) - if result.nil? then - puts "Server address is incorrect. (#{option[:domain]})" - puts "Tune as following format." - puts " :" - exit 1 - end - - # FTP INFO - client = BuildCommClient.create( result[0], result[1], nil, 0 ) - if client.nil? then - puts "Can't access server #{result[0]}:#{result[1]}" - exit(-1) - end - client.send "QUERY|FTP" - result0 = client.receive_data() - if result0.nil? then - client.terminate - exit(-1) - end - result0 = result0[0].split(",").map { |x| x.strip } - ip = result0[0] - username = result0[1] - passwd = result0[2] - client.terminate - - client = BuildCommClient.create( result[0], result[1], nil, 0 ) - if client.nil? then - puts "Can't access server #{result[0]}:#{result[1]}" - exit(-1) - end - client.send("UPLOAD") - result = client.send_file(ip, username, passwd, option[:file]) - client.terminate - if not result then - puts "Uploading file failed!.. #{option[:file]}" - exit(-1) - else - puts "Uploading file succeeded!" - end - - when "download" - result = Utils.parse_server_addr(option[:domain]) - if result.nil? then - puts "Server address is incorrect. (#{option[:domain]})" - puts "Tune as following format." - puts " :" - exit 1 - end - - # FTP INFO - client = BuildCommClient.create( result[0], result[1], nil, 0 ) - if client.nil? then - puts "Can't access server #{result[0]}:#{result[1]}" - exit(-1) - end - client.send "QUERY|FTP" - result0 = client.receive_data() - if result0.nil? then - client.terminate - exit(-1) - end - result0 = result0[0].split(",").map { |x| x.strip } - ip = result0[0] - username = result0[1] - passwd = result0[2] - client.terminate - - # download - client = BuildCommClient.create( result[0], result[1], nil, 0 ) - if client.nil? then - puts "Can't access server #{result[0]}:#{result[1]}" - exit(-1) - end - file_name = option[:file] - client.send("DOWNLOAD|#{file_name}") - result = client.receive_file(ip, username, passwd, "./#{file_name}") - client.terminate - if not result then - puts "Downloading file failed!.. #{option[:file]}" - exit(-1) - else - puts "Downloading file succeeded!" - end else raise RuntimeError, "input option incorrect : #{option[:cmd]}" end diff --git a/build-svr b/build-svr index 6a86283..3e5a57c 100755 --- a/build-svr +++ b/build-svr @@ -53,17 +53,20 @@ end begin case option[:cmd] when "create" - ftp_result = Utils.parse_ftpserver_url(option[:fdomain]) - if ftp_result.nil? or ftp_result.length != 4 then - puts "FTP server url is incorrect. (#{option[:fdomain]})" - puts "Tune as following format." - puts " ftp://:@
:" - exit 1 - end - ftpsvr_addr = ftp_result[0] - ftpsvr_port = ftp_result[1] - ftpsvr_username = ftp_result[2] - ftpsvr_passwd = ftp_result[3] + ftpsvr_addr = nil; ftpsvr_port = nil; ftpsvr_username = nil; ftpsvr_passwd = nil + if not option[:fdomain].nil? then + ftp_result = Utils.parse_ftpserver_url(option[:fdomain]) + if ftp_result.nil? or ftp_result.length != 4 then + puts "FTP server url is incorrect. (#{option[:fdomain]})" + puts "Tune as following format." + puts " ftp://:@
:" + exit 1 + end + ftpsvr_addr = ftp_result[0] + ftpsvr_port = ftp_result[1] + ftpsvr_username = ftp_result[2] + ftpsvr_passwd = ftp_result[3] + end pkgsvr_url = pkgsvr_addr = pkgsvr_port = nil if not option[:domain].nil? and not option[:url].nil? then svr_result = Utils.parse_server_addr(option[:domain]) diff --git a/src/build_server/BuildClientOptionParser.rb b/src/build_server/BuildClientOptionParser.rb index 1943390..5aa74f7 100644 --- a/src/build_server/BuildClientOptionParser.rb +++ b/src/build_server/BuildClientOptionParser.rb @@ -38,7 +38,7 @@ class BuildClientUsage QUERY_PROJECT="build-cli query-project -d " QUERY_JOB="build-cli query-job -d " CANCEL="build-cli cancel -j -d [-w ]" - REGISTER="build-cli register -P -d -t [-w ] [-D ]" + REGISTER="build-cli register -P -d [-t ] [-w ] [-D ]" end @@ -84,8 +84,7 @@ def option_error_check( options ) end when "register" then if options[:package].nil? or options[:package].empty? or - options[:domain].nil? or options[:domain].empty? or - options[:fdomain].nil? or options[:fdomain].empty? then + options[:domain].nil? or options[:domain].empty? then raise ArgumentError, "Usage: " + BuildClientUsage::REGISTER end diff --git a/src/build_server/BuildJob.rb b/src/build_server/BuildJob.rb index ed0e900..e48b84b 100644 --- a/src/build_server/BuildJob.rb +++ b/src/build_server/BuildJob.rb @@ -742,7 +742,9 @@ class BuildJob < CommonJob builder = RemoteBuilder.new("JB#{@id}", @remote_server, @server.ftp_addr, @server.ftp_port, @server.ftp_username, @server.ftp_passwd) @log.info( "JobBuilder##{@id} is created", Log::LV_USER) @log.info( " - Remote Server : #{@remote_server.ip}:#{@remote_server.port}" ) - @log.info( " - FTP Server : #{@server.ftp_addr}" ) + if not @server.ftp_addr.nil? then + @log.info( " - FTP Server : #{@server.ftp_addr}" ) + end else builder = Builder.create( "JB#{@id}", @pkgsvr_url, nil, "#{@buildroot_dir}", @server.build_cache_dir ) @@ -838,7 +840,9 @@ class BuildJob < CommonJob builder = RemoteBuilder.new("JB#{@id}", @remote_server, @server.ftp_addr, @server.ftp_port, @server.ftp_username, @server.ftp_passwd) @log.info( "JobBuilder##{@id} is created", Log::LV_USER) @log.info( " - Remote Server : #{@remote_server.ip}:#{@remote_server.port}" ) - @log.info( " - FTP Server : #{@server.ftp_addr}" ) + if not @server.ftp_addr.nil? then + @log.info( " - FTP Server : #{@server.ftp_addr}" ) + end else builder = Builder.create( "JB#{@id}", @pkgsvr_url, nil, "#{@buildroot_dir}/#{@os}", @server.build_cache_dir ) @@ -916,7 +920,9 @@ class BuildJob < CommonJob builder = RemoteBuilder.new("JB#{@id}", @remote_server, @server.ftp_addr, @server.ftp_port, @server.ftp_username, @server.ftp_passwd) @log.info( "JobBuilder##{@id} is created", Log::LV_USER) @log.info( " - Remote Server : #{@remote_server.ip}:#{@remote_server.port}" ) - @log.info( " - FTP Server : #{@server.ftp_addr}" ) + if not @server.ftp_addr.nil? then + @log.info( " - FTP Server : #{@server.ftp_addr}" ) + end else builder = Builder.create( "JB#{@id}", @pkgsvr_url, nil, "#{@buildroot_dir}/#{@os}", @server.build_cache_dir ) @@ -1001,7 +1007,7 @@ class BuildJob < CommonJob # upload u_client = Client.new( @pkgsvr_url, nil, @log ) - snapshot = u_client.upload( @pkgsvr_ip, @pkgsvr_port, @server.ftp_addr, @server.ftp_port, @server.ftp_username, @server.ftp_passwd, binpkg_path_list) + snapshot = u_client.upload( @pkgsvr_ip, @pkgsvr_port, binpkg_path_list, @server.ftp_addr, @server.ftp_port, @server.ftp_username, @server.ftp_passwd) if snapshot.nil? then @log.info( "Upload failed...", Log::LV_USER) diff --git a/src/build_server/BuildServerController.rb b/src/build_server/BuildServerController.rb index 9a6d05d..f935964 100644 --- a/src/build_server/BuildServerController.rb +++ b/src/build_server/BuildServerController.rb @@ -34,7 +34,7 @@ class BuildServerController @@instance_map = {} # create - def self.create_server (id, path, ftpsvr_addr, ftpsvr_port, ftpsvr_username, ftpsvr_passwd, pkgsvr_url=nil, pkgsvr_addr=nil, pkgsvr_port=3333 ) + def self.create_server (id, path, ftpsvr_addr=nil, ftpsvr_port=nil, ftpsvr_username=nil, ftpsvr_passwd=nil, pkgsvr_url=nil, pkgsvr_addr=nil, pkgsvr_port=3333 ) # check server config root check_build_server_root @@ -775,10 +775,9 @@ class BuildServerController f.puts "TEST_TIME=#{server.test_time}" if server.test_time > 0 f.puts "PASSWORD=#{server.test_time}" if server.password != "0000" f.puts "JOB_KEEP_TIME=#{server.keep_time}" - f.puts "FTP_ADDR=#{server.ftp_addr}" - f.puts "FTP_PORT=#{server.ftp_port}" - f.puts "FTP_USERNAME=#{server.ftp_username}" - f.puts "FTP_PASSWD=#{server.ftp_passwd}" + if not server.ftp_addr.nil? then + f.puts "FTP_URL=ftp://#{server.ftp_username}:#{server.ftp_passwd}@#{server.ftp_addr}:#{server.ftp_port}" + end f.puts "PKG_SYNC_PERIOD=#{server.pkg_sync_period}" f.puts "CHANGELOG_CHECK=#{server.changelog_check}" end @@ -801,10 +800,10 @@ class BuildServerController test_time=0 password="0000" keep_time=86400 - ftp_addr="" - ftp_port="21" - ftp_username="" - ftp_passwd="" + ftp_addr=nil + ftp_port=nil + ftp_username=nil + ftp_passwd=nil pkg_sync_period=600 changelog_check=false @@ -831,14 +830,12 @@ class BuildServerController password = l[idx,length].strip.to_i elsif l.start_with?("JOB_KEEP_TIME=") keep_time = l[idx,length].strip.to_i - elsif l.start_with?("FTP_ADDR=") - ftp_addr = l[idx,length].strip - elsif l.start_with?("FTP_PORT=") - ftp_port = l[idx,length].strip - elsif l.start_with?("FTP_USERNAME=") - ftp_username = l[idx,length].strip - elsif l.start_with?("FTP_PASSWD=") - ftp_passwd = l[idx,length].strip + elsif l.start_with?("FTP_URL=") + ftp_result = Utils.parse_ftpserver_url(l[idx,length].strip) + ftp_addr = ftp_result[0] + ftp_port = ftp_result[1] + ftp_username = ftp_result[2] + ftp_passwd = ftp_result[3] elsif l.start_with?("PKG_SYNC_PERIOD=") pkg_sync_period = l[idx,length].strip.to_i elsif l.start_with?("CHANGELOG_CHECK=") @@ -881,12 +878,6 @@ class BuildServerController # set password obj.keep_time = keep_time - # set ftp infomation - obj.ftp_addr = ftp_addr - obj.ftp_port = ftp_port - obj.ftp_username = ftp_username - obj.ftp_passwd = ftp_passwd - # load from DB obj.load_db() diff --git a/src/build_server/BuildServerOptionParser.rb b/src/build_server/BuildServerOptionParser.rb index 8a59969..04b2f8e 100644 --- a/src/build_server/BuildServerOptionParser.rb +++ b/src/build_server/BuildServerOptionParser.rb @@ -31,7 +31,7 @@ require 'optparse' require 'utils' class BuildServerUsage - CREATE="build-svr create -n -t [-u -d ]" + CREATE="build-svr create -n [-t ] [-u -d ]" REMOVE="build-svr remove -n " START="build-svr start -n -p " STOP="build-svr stop -n " @@ -57,8 +57,7 @@ def option_error_check( options ) case options[:cmd] when "create" - if options[:name].nil? or options[:name].empty? or - options[:fdomain].nil? or options[:fdomain].empty? then + if options[:name].nil? or options[:name].empty? then raise ArgumentError, "Usage: " + BuildServerUsage::CREATE end diff --git a/src/build_server/MultiBuildJob.rb b/src/build_server/MultiBuildJob.rb index 428f4b6..6d9d588 100644 --- a/src/build_server/MultiBuildJob.rb +++ b/src/build_server/MultiBuildJob.rb @@ -405,7 +405,7 @@ class MultiBuildJob < CommonJob # upload u_client = Client.new( @pkgsvr_url, nil, @log ) - snapshot = u_client.upload( @pkgsvr_ip, @pkgsvr_port, @server.ftp_addr, @server.ftp_port, @server.ftp_username, @server.ftp_passwd, binpkg_path_list) + snapshot = u_client.upload( @pkgsvr_ip, @pkgsvr_port, binpkg_path_list, @server.ftp_addr, @server.ftp_port, @server.ftp_username, @server.ftp_passwd) if snapshot.nil? then @log.info( "Upload failed...", Log::LV_USER) diff --git a/src/build_server/RegisterPackageJob.rb b/src/build_server/RegisterPackageJob.rb index e2cd048..89722b4 100644 --- a/src/build_server/RegisterPackageJob.rb +++ b/src/build_server/RegisterPackageJob.rb @@ -483,7 +483,7 @@ class RegisterPackageJob < CommonJob # upload u_client = Client.new( @pkgsvr_url, nil, @log ) - snapshot = u_client.upload( @pkgsvr_ip, @pkgsvr_port, @server.ftp_addr, @server.ftp_port, @server.ftp_username, @server.ftp_passwd, binpkg_path_list) + snapshot = u_client.upload( @pkgsvr_ip, @pkgsvr_port, binpkg_path_list, @server.ftp_addr, @server.ftp_port, @server.ftp_username, @server.ftp_passwd) if snapshot.nil? then @log.info( "Upload failed...", Log::LV_USER) diff --git a/src/build_server/RemoteBuilder.rb b/src/build_server/RemoteBuilder.rb index 1ebccc8..e895c9a 100644 --- a/src/build_server/RemoteBuilder.rb +++ b/src/build_server/RemoteBuilder.rb @@ -32,6 +32,8 @@ $LOAD_PATH.unshift File.dirname(File.dirname(__FILE__))+"/pkg_server" require "utils" require "PackageManifest" require "log" +require "FileTransferViaFTP" +require "FileTransferViaDirect" class RemoteBuilder attr_accessor :id, :log @@ -140,7 +142,13 @@ class RemoteBuilder file_name = file_path.split("/")[-1] msg = "UPLOAD|#{dock}" if client.send( msg ) then - result=client.send_file( @ftp_addr, @ftp_port, @ftp_username, @ftp_passwd, file_path ) + if not @ftp_addr.nil? then + transporter=FileTransferFTP.new(@log, @ftp_addr, @ftp_port, @ftp_username, @ftp_passwd) + else + transporter=FileTransferDirect.new(@log) + end + + result=client.send_file( file_path, transporter ) if not result then @log.error( "File uploading failed...#{file_name}", Log::LV_USER) end @@ -218,7 +226,12 @@ class RemoteBuilder file_name = file_path.split("/")[-1] msg = "DOWNLOAD|#{dock}|#{file_name}" if client.send( msg ) then - result=client.receive_file( @ftp_addr, @ftp_port, @ftp_username, @ftp_passwd, file_path ) + if not @ftp_addr.nil? then + transporter=FileTransferFTP.new(@log, @ftp_addr, @ftp_port, @ftp_username, @ftp_passwd) + else + transporter=FileTransferDirect.new(@log) + end + result=client.receive_file(file_path, transporter) if not result then @log.error( "File downloading failed...#{file_name}", Log::LV_USER) end diff --git a/src/build_server/SocketJobRequestListener.rb b/src/build_server/SocketJobRequestListener.rb index 53648e1..1fda9d8 100644 --- a/src/build_server/SocketJobRequestListener.rb +++ b/src/build_server/SocketJobRequestListener.rb @@ -70,8 +70,12 @@ class SocketJobRequestListener def main() # server open begin - ftp_url = Utils.generate_ftp_url(@parent_server.ftp_addr, @parent_server.ftp_port, - @parent_server.ftp_username, @parent_server.ftp_passwd) + if not @parent_server.ftp_addr.nil? then + ftp_url = Utils.generate_ftp_url(@parent_server.ftp_addr, @parent_server.ftp_port, + @parent_server.ftp_username, @parent_server.ftp_passwd) + else + ftp_url = nil + end cache_dir = "#{@parent_server.transport_path}/.cache" @comm_server = BuildCommServer.create(@parent_server.port, @log, ftp_url, cache_dir) rescue @@ -393,7 +397,11 @@ class SocketJobRequestListener # QUERY, FTP when "FTP" BuildCommServer.send_begin(req) - BuildCommServer.send(req,"#{@parent_server.ftp_addr},#{@parent_server.ftp_username},#{@parent_server.ftp_passwd}") + if not @parent_server.ftp_addr.nil? then + BuildCommServer.send(req,"#{@parent_server.ftp_addr},#{@parent_server.ftp_username},#{@parent_server.ftp_passwd}") + else + BuildCommServer.send(req,"NONE,NONE,NONE") + end BuildCommServer.send_end(req) BuildCommServer.disconnect(req) diff --git a/src/common/BuildComm.rb b/src/common/BuildComm.rb index 950c90c..8226534 100644 --- a/src/common/BuildComm.rb +++ b/src/common/BuildComm.rb @@ -31,9 +31,10 @@ Contributors: $LOAD_PATH.unshift File.dirname(__FILE__) require "log" require 'timeout' -require "fileTransfer" require "net/ftp" require 'thread' +require "FileTransferViaFTP" +require "FileTransferViaDirect" ATTEMPTS = ["first", "second", "third"] @@ -128,22 +129,45 @@ class BuildCommServer end req.puts "READY" - @log.info "Ready to upload file" - - # receive file from client - if not @ftp_url.nil? then - url_contents = Utils.parse_ftpserver_url(@ftp_url) - ip = url_contents[0] - port = url_contents[1] - username = url_contents[2] - passwd = url_contents[3] - @trans = FileTransfer.new(@log,ip,port,username,passwd) - else - @trans = FileTransfer.new(@log) - end + @log.info "Ready to send file" - if not @trans.send_file( src_file, req, false ) then - return false + while line = req.gets() + tok = line.split(",").map { |x| x.strip } + cmd = tok[0].strip + case cmd + when "CHECK_TRANSPORTER" + type = tok[1].strip + case type + when "DIRECT" + transporter = FileTransferDirect.new(@log) + when "FTP" + if not @ftp_url.nil? then + url_contents = Utils.parse_ftpserver_url(@ftp_url) + ip = url_contents[0] + port = url_contents[1] + username = url_contents[2] + passwd = url_contents[3] + transporter = FileTransferFTP.new(@log, ip, port, username, passwd) + else + transporter = FileTransferFTP.new(@log) + end + else + req.puts "ERROR" + @log.error "Unsupported transporter type! : #{type}" + return false + end + + req.puts "TRANSPORTER_OK" + + if not transporter.send_file( src_file, req, false ) then + return false + else + return true + end + + else + @log.warn "Unhandled message: #{line}" + end end rescue => e @@ -159,18 +183,15 @@ class BuildCommServer # NOTE. dst_file can be directory def receive_file(req, dst_file) - # 1. send "READY" - # 2. If "UPLOADED,ip,port,file_path,username,passwd" is received, - # Download the file using my ftp_url. - # If ftp_url is nil, use the url on "UPLOADED" messge instead - # After downloading it, send "SUCC" - begin req.puts "READY" - while l = req.gets() - tok = l.split(",").map { |x| x.strip } + @log.info "Ready to receive file" + + while line = req.gets() + tok = line.split(",").map { |x| x.strip } cmd = tok[0].strip - if cmd == "CHECK_CACHE" then + case cmd + when "CHECK_CACHE" file_name = tok[1] file_size = tok[2].to_i checksum = tok[3] @@ -189,37 +210,44 @@ class BuildCommServer else @log.info "Cached file not found!#{file_name}" req.puts "NOT_CACHED" + end - # receive file from client + when "CHECK_TRANSPORTER" + type = tok[1].strip + case type + when "DIRECT" + transporter = FileTransferDirect.new(@log) + when "FTP" if not @ftp_url.nil? then url_contents = Utils.parse_ftpserver_url(@ftp_url) ip = url_contents[0] port = url_contents[1] username = url_contents[2] passwd = url_contents[3] - @trans = FileTransfer.new(@log, ip, port, username, passwd) + transporter = FileTransferFTP.new(@log, ip, port, username, passwd) else - @trans = FileTransfer.new(@log) + transporter = FileTransferFTP.new(@log) end + else + req.puts "ERROR" + @log.error "Unsupported transporter type! : #{type}" + return false + end - if not @trans.receive_file( dst_file, req, false ) then - return false - end + req.puts "TRANSPORTER_OK" - # add to cache - if not @cache_dir.nil? then - if File.exist? dst_file and File.directory? dst_file then - target_file = File.join(dst_file,File.basename(dst_filepath)) - else - target_file = dst_file - end - add_download_cache(target_file) - end + if not transporter.receive_file( dst_file, req, false ) then + return false + end + + # add to cache + if not @cache_dir.nil? then + add_download_cache(target_file) end break else - @log.warn "Unhandled message: #{l}" + @log.warn "Unhandled message: #{line}" end end rescue => e @@ -456,7 +484,8 @@ class BuildCommClient end - def send_file(ip, port, username, passwd, src_file) + def send_file(src_file, transporter ) + result = true begin l = @socket.gets() if l.nil? then @@ -471,29 +500,42 @@ class BuildCommClient end while line = @socket.gets() - if line.strip == "READY" then - @log.info "Server is ready to receive file" + cmd = line.split(",")[0].strip + case cmd + when "READY" + @log.info "Server is ready!" file_name = File.basename(src_file) file_size = File.size(src_file) checksum = Utils.checksum(src_file) send "CHECK_CACHE,#{file_name},#{file_size},#{checksum}" - elsif line.strip == "CACHED" then + when "CACHED" @log.info "Server already has cached file" break - elsif line.strip == "NOT_CACHED" then - @log.info "Server doest not have cached file" - - # send file to server - @trans = FileTransfer.new(@log, ip, port, username, passwd) - if not @trans.send_file( src_file, @socket, true ) then - return false + when "NOT_CACHED" + @log.info "Server does not have cached file" + send "CHECK_TRANSPORTER,#{transporter.type}" + + when "TRANSPORTER_OK" + if not transporter.send_file( src_file, @socket, true ) then + result = false + else + @log.info "Sending file succeeded!" end - break - elsif line.strip == "=END" then - break + when "TRANSPORTER_FAIL" + @log.warn "Server does not support transporter type: #{transporter.type}" + result = false + + when "ERROR" + result = false + + when "=END" + break + + else + @log.warn "Unhandled message: #{line}" end end rescue => e @@ -503,12 +545,14 @@ class BuildCommClient return false end - return true + return result end # return file - def receive_file(ip, port, username, passwd, dst_file) + def receive_file(dst_file, transporter) + result = true + begin l = @socket.gets() @@ -525,14 +569,26 @@ class BuildCommClient while line = @socket.gets() cmd = line.split(",")[0].strip - if cmd == "READY" then - @trans = FileTransfer.new(@log, ip, port, username, passwd) - if not @trans.receive_file( dst_file, @socket, true ) then - return false + case cmd + when "READY" + @log.info "Server is ready!" + send "CHECK_TRANSPORTER,#{transporter.type}" + + when "TRANSPORTER_OK" + if not transporter.receive_file( dst_file, @socket, true ) then + result = false + else + @log.info "Receiving file succeeded!" end - break - elsif cmd == "=END" then - break + + when "ERROR" + result = false + + when "=END" + break + + else + @log.warn "Unhandled message: #{line}" end end rescue => e @@ -542,7 +598,7 @@ class BuildCommClient return false end - return true + return result end diff --git a/src/common/FileTransferViaDirect.rb b/src/common/FileTransferViaDirect.rb new file mode 100644 index 0000000..55072b3 --- /dev/null +++ b/src/common/FileTransferViaDirect.rb @@ -0,0 +1,107 @@ + +require 'socket' +require 'log' + +class FileTransferDirect + attr_accessor :type + + def initialize(logger) + @type = "DIRECT" + + if not logger.nil? then + @log = logger + else + @log = DummyLog.new + end + end + + + def send_file( src_file, conn, is_client=true ) + + filename = File.basename(src_file) + size = File.size( src_file ) + checksum = Utils.checksum( src_file ) + + if is_client then + conn.puts "RECEIVE_REQ" + end + + while line = conn.gets() + tok = line.split(",") { |x| x.strip } + cmd = tok[0].strip + case cmd + when "SEND_REQ" + conn.puts "FILE_INFO,#{filename},#{size},#{checksum}" + # read file contents + # send via tcp/ip + File.open(src_file, "rb") {|io| + while size > 0 + buf = io.read(size > 1024*1024 ? 1024*1024 : size) + conn.write( buf ) + size -= buf.length + end + } + + @log.info "Upload is succeeded!" + conn.puts "SEND_OK" + + # wait for download result + when "RECEIVE_OK" + @log.info "Received download success message from remote site" + return true + + when "RECEIVE_FAIL" + @log.info "Received download fail message from remote site" + return false + + else + @log.error "Unhandled message: #{line}" + return false + end + end + end + + + def receive_file( dst_file, conn, is_client=false ) + + if is_client then + conn.puts "SEND_REQ" + end + + while line = conn.gets() + tok = line.split(",") { |x| x.strip } + cmd = tok[0].strip + case cmd + when "RECEIVE_REQ" + conn.puts "SEND_REQ" + when "FILE_INFO" + @log.info "Received file info from remote site" + filename = tok[1].strip + size = tok[2].strip.to_i + checksum = tok[3].strip + + if File.directory? dst_file then + dst_file = File.join(dst_file, filename) + end + + File.open( dst_file, "wb" ) { |io| + while size > 0 + buf = conn.read(size > 1024*1024 ? 1024*1024 : size) + io.write( buf ) + size -= buf.length + end + } + + conn.puts "RECEIVE_OK" + + when "SEND_OK" + @log.info "Received success message from remote site" + return true + + else + @log.error "Unhandled message: #{line}" + return false + end + end + end +end diff --git a/src/common/fileTransfer.rb b/src/common/FileTransferViaFTP.rb similarity index 99% rename from src/common/fileTransfer.rb rename to src/common/FileTransferViaFTP.rb index f504c4c..9f0ce06 100644 --- a/src/common/fileTransfer.rb +++ b/src/common/FileTransferViaFTP.rb @@ -2,10 +2,12 @@ require 'socket' require 'log' -class FileTransfer +class FileTransferFTP + attr_accessor :type ATTEMPTS = ["first", "second", "third"] def initialize(logger, ip=nil, port=nil, username=nil, passwd=nil ) + @type = "FTP" @ip = ip @port = port @username = username diff --git a/src/pkg_server/SocketRegisterListener.rb b/src/pkg_server/SocketRegisterListener.rb index 2ad78e3..6ed2882 100644 --- a/src/pkg_server/SocketRegisterListener.rb +++ b/src/pkg_server/SocketRegisterListener.rb @@ -4,7 +4,6 @@ $LOAD_PATH.unshift File.dirname(__FILE__) $LOAD_PATH.unshift File.dirname(File.dirname(__FILE__))+"/common" $LOAD_PATH.unshift File.dirname(File.dirname(__FILE__))+"/build_server" require "packageServerConfig" -require "fileTransfer" require "BuildComm" require "net/ftp" diff --git a/src/pkg_server/client.rb b/src/pkg_server/client.rb index a7e2034..0e2702c 100644 --- a/src/pkg_server/client.rb +++ b/src/pkg_server/client.rb @@ -28,19 +28,23 @@ Contributors: require "fileutils" require "thread" +require "net/ftp" $LOAD_PATH.unshift File.dirname(__FILE__) $LOAD_PATH.unshift File.dirname(File.dirname(__FILE__))+"/common" $LOAD_PATH.unshift File.dirname(File.dirname(__FILE__))+"/build_server" require "downloader" require "installer" -require "fileTransfer" require "packageServerConfig" require "package" require "parser" require "utils" require "log" require "Version" -require "net/ftp" +require "BuildComm" +require "FileTransferViaFTP" +require "FileTransferViaDirect" + + $update_mutex = Mutex.new $get_snapshot_mutex = Mutex.new $filemove_mutex = Mutex.new @@ -329,7 +333,7 @@ class Client public # upload package - def upload(ip, port, ftp_addr, ftp_port, ftp_username, ftp_passwd, binary_path_list) + def upload(ip, port, binary_path_list, ftp_addr=nil, ftp_port=nil, ftp_username=nil, ftp_passwd=nil) # check ip and port if ip.nil? or port.nil? then @@ -365,7 +369,13 @@ class Client end begin - result = client.send_file(ftp_addr, ftp_port, ftp_username, ftp_passwd, bpath) + if not ftp_addr.nil? then + transporter=FileTransferFTP.new(@log, ftp_addr, ftp_port, ftp_username, ftp_passwd) + else + transporter=FileTransferDirect.new(@log) + end + + result = client.send_file(bpath, transporter) rescue => e @log.error "FTP failed to put file (exception)" @log.error "#{e.message}" diff --git a/test/build-server.basic1/build-cli-01.testcase b/test/build-server.basic1/build-cli-01.testcase index 4aa083b..83852c3 100644 --- a/test/build-server.basic1/build-cli-01.testcase +++ b/test/build-server.basic1/build-cli-01.testcase @@ -25,7 +25,7 @@ build-cli query-system -d build-cli query-project -d build-cli query-job -d build-cli cancel -j -d [-w ] -build-cli register -P -d -t [-w ] [-D ] +build-cli register -P -d [-t ] [-w ] [-D ] Options: -N, --project project name diff --git a/test/build-server.basic1/buildsvr.init b/test/build-server.basic1/buildsvr.init index 408c7fd..2dc34b0 100755 --- a/test/build-server.basic1/buildsvr.init +++ b/test/build-server.basic1/buildsvr.init @@ -8,7 +8,7 @@ rm -rf ~/.build_tools/build_server/testserver3 ../../build-svr remove -n testserver3 mkdir buildsvr01 cd buildsvr01 -../../../build-svr create -n testserver3 -u `pwd`/../pkgsvr01/unstable -d 127.0.0.1:3333 -t ftp://ftpuser:ftpuser@127.0.0.1 +../../../build-svr create -n testserver3 -u `pwd`/../pkgsvr01/unstable -d 127.0.0.1:3333 cd .. ../../build-svr add-svr -n testserver3 -d 127.0.0.1:2224 diff --git a/test/build-server.basic2/build-svr-02.testcase b/test/build-server.basic2/build-svr-02.testcase index c2c6aab..1a9bac9 100644 --- a/test/build-server.basic2/build-svr-02.testcase +++ b/test/build-server.basic2/build-svr-02.testcase @@ -30,7 +30,7 @@ set-attr Set build server atribute. get-attr Get build server atribute. Subcommand usage: -build-svr create -n -t [-u -d ] +build-svr create -n [-t ] [-u -d ] build-svr remove -n build-svr start -n -p build-svr stop -n diff --git a/test/build-server.multi-svr1/buildsvr2.init b/test/build-server.multi-svr1/buildsvr2.init index d8e87d8..c5ab069 100755 --- a/test/build-server.multi-svr1/buildsvr2.init +++ b/test/build-server.multi-svr1/buildsvr2.init @@ -4,7 +4,7 @@ rm -rf buildsvr02 ../../build-svr remove -n testserver4 mkdir buildsvr02 cd buildsvr02 -ruby -d ../../../build-svr create -n testserver4 -t ftp://ftpuser:ftpuser@127.0.0.1 +ruby -d ../../../build-svr create -n testserver4 cd .. ruby -d ../../build-svr add-dist -n testserver4 -D unstable -u `pwd`/pkgsvr01/unstable -d 127.0.0.1:3333 -- 2.34.1