From d23a5a5005c54bc8dabb8767be0e50fd3a4c23f5 Mon Sep 17 00:00:00 2001 From: donghee yang Date: Wed, 10 Oct 2012 17:06:58 +0900 Subject: [PATCH] [Title] Refactored the module about file-transfer --- src/{build_server => common}/BuildComm.rb | 221 ++++++---------------- src/common/fileTransfer.rb | 166 ++++++++++++++-- 2 files changed, 204 insertions(+), 183 deletions(-) rename src/{build_server => common}/BuildComm.rb (60%) diff --git a/src/build_server/BuildComm.rb b/src/common/BuildComm.rb similarity index 60% rename from src/build_server/BuildComm.rb rename to src/common/BuildComm.rb index d8a0291..950c90c 100644 --- a/src/build_server/BuildComm.rb +++ b/src/common/BuildComm.rb @@ -28,7 +28,7 @@ Contributors: - S-Core Co., Ltd =end -$LOAD_PATH.unshift File.dirname(__FILE__)+"/src/common" +$LOAD_PATH.unshift File.dirname(__FILE__) require "log" require 'timeout' require "fileTransfer" @@ -120,13 +120,6 @@ class BuildCommServer def send_file(req, src_file) - # 1. send "READY" - # 2. If "FTP,ip,username,passwd" is received, - # Upload the src file using server's ftp_url. - # if then ftp_url is nil, use the url on "FTP" message instead - # After uploading, send "UPLOADED,ftp_filepath" - # 3. If "SUCC" is received, remove the file on FTP server - begin if not File.exist? src_file then @log.error "\"#{src_file}\" file does not exist" @@ -135,57 +128,24 @@ class BuildCommServer end req.puts "READY" - @log.info "Ready to upload file" - while l = req.gets() - tok = l.split(",").map { |x| x.strip } - cmd = tok[0].strip - if cmd == "FTP" then - if tok.count < 5 then - @log.error "Server received wrong REQ : #{l.strip}" - req.puts "ERROR" - return false - end - - # get ftp connection info - if @ftp_url.nil? then - ip = tok[1].strip - port = tok[2].strip - username = tok[3].strip - passwd = tok[4].strip - @log.info "Server received ftp server infomation from client : [#{ip}, #{port}]" - else - url_contents = Utils.parse_ftpserver_url(@ftp_url) - ip = url_contents[0] - port = url_contents[1] - username = url_contents[2] - passwd = url_contents[3] - end + @log.info "Ready to upload file" + + # receive file from client + if not @ftp_url.nil? then + url_contents = Utils.parse_ftpserver_url(@ftp_url) + ip = url_contents[0] + port = url_contents[1] + username = url_contents[2] + passwd = url_contents[3] + @trans = FileTransfer.new(@log,ip,port,username,passwd) + else + @trans = FileTransfer.new(@log) + end + + if not @trans.send_file( src_file, req, false ) then + return false + end - # upload to ftp server - ftp_filepath = nil - @trans = FileTransfer.new(ip, port, username, passwd, @log) - for attempt in ATTEMPTS - ftp_filepath = @trans.putfile( src_file ) - if !ftp_filepath.nil? then break; - else @log.info "Server is the #{attempt} upload attempt fails" end - end - if ftp_filepath.nil? then - req.puts "ERROR" - return false - else - @log.info "Server is the #{attempt} successful attempt to upload file: [#{File.basename(src_file)}]" - end - req.puts "UPLOADED,#{ftp_filepath}" - elsif cmd == "SUCC" then - @log.info "Client downloaded file successfully" - @trans.cleandir( ftp_filepath ) - @log.info "Cleaned temporary dir on FTP server: #{ftp_filepath}" - break - elsif cmd == "ERROR" then - @log.error "Client failed to download file" - return false - end - end rescue => e puts "[BuildCommServer] Exception" @log.error e.message @@ -226,65 +186,38 @@ class BuildCommServer @log.info "Download cache hit! Copied from cache.: #{file_name}" req.puts "CACHED" - break else @log.info "Cached file not found!#{file_name}" req.puts "NOT_CACHED" - end - elsif cmd == "UPLOADED" then - @log.info "Client uploaded file to ftp server successful" - if tok.count < 6 then - @log.error "Server received wrong REQ : #{l.strip}" - req.puts "ERROR" - return false - end - filepath = tok[3].strip - - # get ftp connection info - if @ftp_url.nil? then - ip = tok[1].strip - port = tok[2].strip - username = tok[4].strip - passwd = tok[5].strip - @log.info "Client sent ftp server infomations [#{ip}, #{port}]" - else - url_contents = Utils.parse_ftpserver_url(@ftp_url) - ip = url_contents[0] - port = url_contents[1] - username = url_contents[2] - passwd = url_contents[3] - end - # download from ftp server - dst_filepath = nil - @trans = FileTransfer.new(ip, port, username, passwd, @log) - for attempt in ATTEMPTS - dst_filepath = @trans.getfile( filepath, dst_file ) - if not dst_filepath.nil? then break - else - @log.warn "Server is the #{attempt} download attempt fails" - end - end - if dst_filepath.nil? then - req.puts "ERROR" - return false - else @log.info " Server is the #{attempt} successful attempt to download" end - - # add to cache - if not @cache_dir.nil? then - if File.exist? dst_file and File.directory? dst_file then - target_file = File.join(dst_file,File.basename(dst_filepath)) - else - target_file = dst_file + # receive file from client + if not @ftp_url.nil? then + url_contents = Utils.parse_ftpserver_url(@ftp_url) + ip = url_contents[0] + port = url_contents[1] + username = url_contents[2] + passwd = url_contents[3] + @trans = FileTransfer.new(@log, ip, port, username, passwd) + else + @trans = FileTransfer.new(@log) end - add_download_cache(target_file) - end - req.puts "SUCC" + if not @trans.receive_file( dst_file, req, false ) then + return false + end + + # add to cache + if not @cache_dir.nil? then + if File.exist? dst_file and File.directory? dst_file then + target_file = File.join(dst_file,File.basename(dst_filepath)) + else + target_file = dst_file + end + add_download_cache(target_file) + end + end break - elsif cmd == "ERROR" then - @log.error "Client failed to upload the file" - return false + else @log.warn "Unhandled message: #{l}" end @@ -537,9 +470,6 @@ class BuildCommClient return false end - # 1. If "READY" is received, upload src file to FTP server - # After uploading it, send "UPLOADED,ip,file_path,username,passwd" - # 2. If "SUCC" is received, remove the file on FTP server while line = @socket.gets() if line.strip == "READY" then @log.info "Server is ready to receive file" @@ -547,30 +477,21 @@ class BuildCommClient file_size = File.size(src_file) checksum = Utils.checksum(src_file) send "CHECK_CACHE,#{file_name},#{file_size},#{checksum}" + elsif line.strip == "CACHED" then @log.info "Server already has cached file" + break + elsif line.strip == "NOT_CACHED" then @log.info "Server doest not have cached file" + + # send file to server + @trans = FileTransfer.new(@log, ip, port, username, passwd) + if not @trans.send_file( src_file, @socket, true ) then + return false + end + break - ftp_filepath = nil - @trans = FileTransfer.new(ip, port, username, passwd, @log) - for attempt in ATTEMPTS - ftp_filepath = @trans.putfile( src_file ) - if !ftp_filepath.nil? then break; - else @log.info "Client is the #{attempt} upload attempt fails" end - end - if ftp_filepath.nil? then - send "ERROR" - return false - else @log.info "Client is the #{attempt} successful attempt to upload file" end - send "UPLOADED,#{ip},#{port},#{ftp_filepath},#{username},#{passwd}" - elsif line.strip == "SUCC" then - @log.info "Server downloaded file sucessfully" - @trans.cleandir( ftp_filepath ) - @log.info "Client cleaned temporary dir on ftp server: #{ftp_filepath}" - elsif line.strip == "ERROR" then - @log.error "Server failed to download the file. Please check server log" - return false elsif line.strip == "=END" then break end @@ -602,42 +523,14 @@ class BuildCommClient return false end - # 1. If "READY" is received, send "FTP,ip,port,username,passwd" - # 2. if "UPLOADED,ftp_file_path" is received, - # Download the file - # Send "SUCC" - # 3. If "SUCC" is received, remove the file on FTP server while line = @socket.gets() cmd = line.split(",")[0].strip - #@log.info "[BuildCommClient] Received \"#{cmd}\" message from BuildCommServer" if cmd == "READY" then - send "FTP,#{ip},#{port},#{username},#{passwd}" - @log.info "Client sent ftp server infomation to server : [#{ip}, #{port}]" - elsif cmd == "UPLOADED" then - tok = line.split(",") - if tok.length < 2 then - @log.error "Client received wrong REQ : #{line.strip}" - return false - end - ftp_filepath = tok[1].strip - @log.info "Server uploaded file sucessfully" - dst_filepath = nil - @trans = FileTransfer.new(ip, port, username, passwd, @log) - for attempt in ATTEMPTS - dst_filepath = @trans.getfile( ftp_filepath, dst_file ) - if not dst_filepath.nil? then break - else - @log.warn "Client is the #{attempt} download attempt fails" - end - end - if dst_filepath.nil? then - send "ERROR" + @trans = FileTransfer.new(@log, ip, port, username, passwd) + if not @trans.receive_file( dst_file, @socket, true ) then return false - else @log.info "Client is the #{attempt} successful attempt to download" end - send "SUCC" - elsif cmd == "ERROR" then - @log.error "Server failed to upload file. Check server log" - return false + end + break elsif cmd == "=END" then break end diff --git a/src/common/fileTransfer.rb b/src/common/fileTransfer.rb index 09461cc..f504c4c 100644 --- a/src/common/fileTransfer.rb +++ b/src/common/fileTransfer.rb @@ -3,7 +3,9 @@ require 'socket' require 'log' class FileTransfer - def initialize(ip, port, username, passwd, logger) + ATTEMPTS = ["first", "second", "third"] + + def initialize(logger, ip=nil, port=nil, username=nil, passwd=nil ) @ip = ip @port = port @username = username @@ -16,20 +18,146 @@ class FileTransfer end - def putfile( bpath ) + def send_file( src_file, conn, is_client=true ) + + if is_client then + # check ftp info + if @ip.nil? or @port.nil? or @username.nil? or @passwd.nil? then + @log.error "No FTP information!" + conn.puts "UPLOAD_FAIL" + return false + end + conn.puts "DOWNLOAD_REQ,#{@ip},#{@port},#{@username},#{@passwd}" + end + + ip = @ip; port = @port; username = @username; passwd = @passwd + while line = conn.gets() + tok = line.split(",") { |x| x.strip } + cmd = tok[0].strip + case cmd + when "UPLOAD_REQ" + if @ip.nil? or @port.nil? or @username.nil? or @passwd.nil? then + ip = tok[1].strip + port = tok[2].strip + username = tok[3].strip + passwd = tok[4].strip + @log.info "Using FTP information from remote... [#{ip}, #{port}]" + end + + # upload to ftp + ftp_filepath = nil + for attempt in ATTEMPTS + ftp_filepath = putfile( src_file, ip, port, username, passwd ) + if !ftp_filepath.nil? then + break + else + @log.info "The #{attempt} uploading attempt failed!" + end + end + + if ftp_filepath.nil? then + conn.puts "UPLOAD_FAIL" + return false + else + @log.info "Upload is succeeded at #{attempt}" + conn.puts "UPLOAD_OK,#{ftp_filepath}" + end + + # wait for download result + when "DOWNLOAD_OK" + @log.info "Received download success message from remote site" + # clean + cleandir( ftp_filepath, ip, port, username, passwd) + @log.info "Cleaned temporary dir on FTP server: #{ftp_filepath}" + return true + + when "DOWNLOAD_FAIL" + @log.info "Received download fail message from remote site" + return false + + else + @log.error "Unhandled message: #{line}" + return false + end + end + end + + + def receive_file( dst_file, conn, is_client=false ) + + if is_client then + # check ftp info + if @ip.nil? or @port.nil? or @username.nil? or @passwd.nil? then + @log.error "No FTP information!" + conn.puts "DOWNLOAD_FAIL" + return false + end + conn.puts "UPLOAD_REQ,#{@ip},#{@port},#{@username},#{@passwd}" + end + + ip = @ip; port = @port; username = @username; passwd = @passwd + while line = conn.gets() + tok = line.split(",") { |x| x.strip } + cmd = tok[0].strip + case cmd + when "DOWNLOAD_REQ" + if @ip.nil? or @port.nil? or @username.nil? or @passwd.nil? then + ip = tok[1].strip + port = tok[2].strip + username = tok[3].strip + passwd = tok[4].strip + @log.info "Using FTP information from remote... [#{ip}, #{port}]" + end + + conn.puts "UPLOAD_REQ,#{ip},#{port},#{username},#{passwd}" + when "UPLOAD_OK" + @log.info "Received upload success message from remote site" + filepath = tok[1].strip + # download from ftp + dst_filepath = nil + for attempt in ATTEMPTS + dst_filepath = getfile( filepath, dst_file, ip, port, username, passwd ) + if not dst_filepath.nil? then + break + else + @log.info "The #{attempt} downloading attempt failed!" + end + end + if dst_filepath.nil? then + conn.puts "DOWNLOAD_FAIL" + return false + else + @log.info " Server is the #{attempt} successful attempt to download" + conn.puts "DOWNLOAD_OK" + return true + end + + when "UPLOAD_FAIL" + @log.info "Received upload fail message from remote site" + return false + + else + @log.error "Unhandled message: #{line}" + return false + end + end + end + + + def putfile( bpath, ip, port, username, passwd ) filename = File.basename(bpath) uniqdir = Utils.create_uniq_name ftp_filepath = File.join(uniqdir, filename) begin ftp = Net::FTP.new - if @port.nil? or @port == "" then - ftp.connect(@ip) + if port.nil? or port == "" then + ftp.connect(ip) else - ftp.connect(@ip, @port) + ftp.connect(ip, port) end - @log.info "[FTP log] Connected FTP server (#{@ip}:#{@port})" - ftp.login(@username, @passwd) + @log.info "[FTP log] Connected FTP server (#{ip}:#{port})" + ftp.login(username, passwd) ftp.binary = true ftp.passive = true ftp.mkdir(uniqdir) @@ -53,7 +181,7 @@ class FileTransfer return ftp_filepath end - def getfile( bpath, target ) + def getfile( bpath, target, ip, port, username, passwd ) dirname = File.dirname(bpath) filename = File.basename(bpath) @@ -66,13 +194,13 @@ class FileTransfer begin ftp = Net::FTP.new - if @port.nil? or @port == "" then - ftp.connect(@ip) + if port.nil? or port == "" then + ftp.connect(ip) else - ftp.connect(@ip, @port) + ftp.connect(ip, port) end - @log.info "[FTP log] Connected FTP server (#{@ip}:#{@port})" - ftp.login(@username, @passwd) + @log.info "[FTP log] Connected FTP server (#{ip}:#{port})" + ftp.login(username, passwd) ftp.binary = true ftp.passive = true ftp.chdir(dirname) @@ -94,18 +222,18 @@ class FileTransfer return bpath end - def cleandir(path) + def cleandir(path, ip, port, username, passwd) dirname = File.dirname(path) begin ftp = Net::FTP.new - if @port.nil? or @port == "" then - ftp.connect(@ip) + if port.nil? or port == "" then + ftp.connect(ip) else - ftp.connect(@ip, @port) + ftp.connect(ip, port) end - @log.info "[FTP log] Connected FTP server (#{@ip}:#{@port})" - ftp.login(@username, @passwd) + @log.info "[FTP log] Connected FTP server (#{ip}:#{port})" + ftp.login(username, passwd) old_dir = ftp.pwd ftp.chdir(dirname) list = ftp.ls -- 2.34.1