From: donghee yang <donghee.yang@samsung.com>
Date: Sat, 1 Sep 2012 02:59:34 +0000 (+0900)
Subject: [Title] Modified to use download cache when transfering files between machines
X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=dcc6cf5efefaa0f7b6cb20dcd694a6ceef120312;p=sdk%2Ftools%2Fsdk-build.git

[Title] Modified to use download cache when transfering files between machines
---

diff --git a/package/pkginfo.manifest b/package/pkginfo.manifest
index c82c6cc..b920108 100644
--- a/package/pkginfo.manifest
+++ b/package/pkginfo.manifest
@@ -1,5 +1,5 @@
 Source : dibs
-Version :0.99.27
+Version :0.99.28
 Maintainer : taejun ha<taejun.ha@samsung.com>, jiil hyoun <jiil.hyoun@samsung.com>, donghyuk yang <donghyouk.yang@samsung.com>, donghee yang <donghee.yang@samsung.com>, sungmin kim <dev.sungmin.kim@samsung.com
 
 Package : dibs
diff --git a/src/build_server/BuildComm.rb b/src/build_server/BuildComm.rb
index dabae88..647d755 100644
--- a/src/build_server/BuildComm.rb
+++ b/src/build_server/BuildComm.rb
@@ -33,22 +33,25 @@ require "log"
 require 'timeout'
 require "fileTransfer"
 require "net/ftp"
+require 'thread'
 
 ATTEMPTS = ["first", "second", "third"]
 
 class BuildCommServer
-	VERSION = "1.4.0"
+	VERSION = "1.5.0"
 
 	private_class_method :new
 
-	def initialize(port, log, ftp_url)
+	def initialize(port, log, ftp_url, cache_dir)
 		@port = port
 		@log = log
 		@ftp_url = ftp_url
+		@cache_dir = cache_dir
 		@tcp_server = TCPServer.open( port )
+		@download_cache_mutex = Mutex.new
 	end
 
-    def self.create(port, log, ftp_url=nil)
+    def self.create(port, log, ftp_url=nil, cache_dir=nil)
 		# checking port is  available
 		if port_open? port then
 			raise "Port \"#{@port}\" is already in use."
@@ -56,9 +59,14 @@ class BuildCommServer
 		
 		if log.nil? then
 			log = Log.new(nil)				
-		end				
+		end
+
+		# create cache dir if not nil
+		if not cache_dir.nil? and not File.exist? cache_dir then 
+			FileUtils.mkdir_p cache_dir 
+		end
 
-        return new(port, log, ftp_url)
+        return new(port, log, ftp_url, cache_dir)
     end
 
 
@@ -188,18 +196,41 @@ class BuildCommServer
 	end
 
 
+	# NOTE. dst_file can be directory
 	def receive_file(req, dst_file)
 		# 1. send "READY"
 		# 2. If "UPLOADED,ip,port,file_path,username,passwd" is received,
 		#    Download the file using my ftp_url.
 		#    If ftp_url is nil, use the url on "UPLOADED" messge instead
 		#    After downloading it, send "SUCC"
+
 		begin
             req.puts "READY"
 			while l = req.gets()
 				tok = l.split(",").map { |x| x.strip }
 				cmd = tok[0].strip
-				if cmd == "UPLOADED" then
+				if cmd == "CHECK_CACHE" then
+					file_name = tok[1]
+					file_size = tok[2].to_i
+					checksum = tok[3]
+	
+					# check download cache				
+        			if File.exist? dst_file and File.directory? dst_file then 
+						target_file = File.join(dst_file,file_name)
+        			else 
+						target_file = dst_file 
+					end
+					if not @cache_dir.nil? and 
+						check_download_cache( target_file, file_size, checksum ) then
+
+						@log.info "Download cache hit! Copied from cache.: #{file_name}"
+						req.puts "CACHED"
+						break	
+					else
+						@log.info "Cached file not found!#{file_name}"
+						req.puts "NOT_CACHED"
+					end
+				elsif cmd == "UPLOADED" then
 					@log.info "Client uploaded file to ftp server successful"
 					if tok.count < 6 then
 						@log.error "Server received wrong REQ : #{l.strip}"
@@ -236,11 +267,24 @@ class BuildCommServer
 							req.puts "ERROR"							
 							return false												
 					else @log.info " Server is the #{attempt} successful attempt to download" end
+					
+					# add to cache
+					if not @cache_dir.nil? then
+        				if File.exist? dst_file and File.directory? dst_file then 
+							target_file = File.join(dst_file,File.basename(dst_filepath))
+        				else 
+							target_file = dst_file 
+						end
+						add_download_cache(target_file)
+					end
+
                     req.puts "SUCC"
                     break
 				elsif cmd == "ERROR" then
 					@log.error "Client failed to upload the file"
 					return false				
+				else
+					@log.warn "Unhandled message: #{l}"
 				end
 			end				
 		rescue => e
@@ -273,11 +317,57 @@ class BuildCommServer
 	rescue Timeout::Error
 		false
 	end	
+
+
+	private
+	def check_download_cache(dst_file, file_size, checksum )
+		file_name = File.basename(dst_file)
+		cache_file = "#{@cache_dir}/#{file_name}"
+
+		@download_cache_mutex.synchronize {
+			found = false
+			# check file exist
+			if File.exist? cache_file and 
+				File.size(cache_file) == file_size and
+				Utils.checksum(cache_file) == checksum then
+
+				# if hit , touch and copy
+				FileUtils.touch cache_file
+				FileUtils.copy_file(cache_file, dst_file)
+
+				found = true
+			end
+
+			# refresh cache dir
+			curr_time = Time.now
+			Dir.entries(@cache_dir).each { |fname|
+				if fname == "." or fname == ".." then next end
+				file_path = "#{@cache_dir}/#{fname}"
+				if File.mtime(file_path) + 3600 < curr_time then
+					FileUtils.rm_rf file_path
+				end
+			}
+
+			return found
+		}
+	end
+
+
+	private
+	def add_download_cache(dst_file)
+		file_name = File.basename(dst_file)
+		cache_file = "#{@cache_dir}/#{file_name}"
+		@download_cache_mutex.synchronize {
+			# copy & touch
+			FileUtils.copy_file(dst_file, cache_file)
+			FileUtils.touch cache_file
+		}
+	end
 end
 
 
 class BuildCommClient
-	VERSION = "1.4.0"
+	VERSION = "1.5.0"
 
 	private_class_method :new
 
@@ -451,6 +541,14 @@ class BuildCommClient
             while line = @socket.gets()
                 if line.strip == "READY" then
                 	@log.info "Server is ready to receive file"
+					file_name = File.basename(src_file)
+					file_size = File.size(src_file)
+					checksum = Utils.checksum(src_file)
+                    send "CHECK_CACHE,#{file_name},#{file_size},#{checksum}"
+                elsif line.strip == "CACHED" then
+                    @log.info "Server already has cached file"
+                elsif line.strip == "NOT_CACHED" then
+                	@log.info "Server doest not have cached file"
 					ftp_filepath = nil
 					for attempt in ATTEMPTS
                     	ftp_filepath = FileTransfer.putfile(ip, port, username, passwd, src_file, @log)
diff --git a/src/build_server/SocketJobRequestListener.rb b/src/build_server/SocketJobRequestListener.rb
index 3d106ed..4009c72 100644
--- a/src/build_server/SocketJobRequestListener.rb
+++ b/src/build_server/SocketJobRequestListener.rb
@@ -72,7 +72,8 @@ class SocketJobRequestListener
 		begin
 			ftp_url = Utils.generate_ftp_url(@parent_server.ftp_addr, @parent_server.ftp_port,
 				@parent_server.ftp_username, @parent_server.ftp_passwd)
-			@comm_server = BuildCommServer.create(@parent_server.port, @log, ftp_url)
+			cache_dir = "#{@parent_server.transport_path}/.cache"
+			@comm_server = BuildCommServer.create(@parent_server.port, @log, ftp_url, cache_dir)
 		rescue
 			@log.info "Server creation failed"
 			puts "Server creation failed"
diff --git a/src/common/utils.rb b/src/common/utils.rb
index 8ee5a88..5972b76 100644
--- a/src/common/utils.rb
+++ b/src/common/utils.rb
@@ -525,6 +525,14 @@ class Utils
 	end
 
 
+	def Utils.checksum(file_path)
+		if File.exist? file_path then
+			return `sha256sum #{file_path}`.split(" ")[0]
+		else
+			return nil
+		end
+	end
+
 	if defined?(HOST_OS).nil? then
 		HOST_OS = Utils.identify_current_OS()
 	end