From 2a287791b1256743d7fc4fe968f91c6e3ad3a825 Mon Sep 17 00:00:00 2001 From: "jiil.hyoun" Date: Thu, 14 Mar 2013 14:27:00 +0900 Subject: [PATCH] [Title] apply cache algorithm for client [Type] Enhancement [Module] Toolchain / [Priority] Major [Jira#] [Redmine#] 7956 [Problem] [Cause] [Solution] [TestCase] Change-Id: I67c1a473aea57a620b034608e029ff788b3e937c --- src/common/utils.rb | 9 ++- src/pkg_server/client.rb | 171 ++++++++++++++++++++++++--------------- 2 files changed, 112 insertions(+), 68 deletions(-) diff --git a/src/common/utils.rb b/src/common/utils.rb index d549c83..e7804dd 100644 --- a/src/common/utils.rb +++ b/src/common/utils.rb @@ -625,18 +625,19 @@ class Utils end - def Utils.get_sub_files_info(path_list) # return [PATH,size,time] list + def Utils.get_sub_files_info(path_list) # return [PATH,size,ctime,mtime] list list = [] path_list.each do |path| Find.find(path) do |dep_path| if (File.exist?(dep_path) and not File.directory? dep_path) fsize = File.size(dep_path) - ftime = File.new(dep_path).ctime - list.push [dep_path,fsize,ftime] + ctime = File.new(dep_path).ctime + mtime = File.new(dep_path).mtime + list.push [dep_path,fsize,ctime,mtime] end end end - return list.sort {|a,b| a[2] <=> b[2]} + return list end diff --git a/src/pkg_server/client.rb b/src/pkg_server/client.rb index 447b090..a9a3b61 100644 --- a/src/pkg_server/client.rb +++ b/src/pkg_server/client.rb @@ -47,7 +47,6 @@ require "FileTransferViaDirect" $update_mutex = Mutex.new $get_snapshot_mutex = Mutex.new -$filemove_mutex = Mutex.new class Client # constant @@ -131,33 +130,73 @@ class Client @log.info "Update local package list.. [#{@location}]" read_installed_pkg_list() - #cache_size = 4G - @cache_size = 4000 #(MB) + @configs = {} + #default cache size = 4G + @configs[:CACHE_SIZE] = "4000" end # make cache clean - def clean_cache + private + def remove_old_files_in_cache #get dir info - cache_list = Dir.entries(CONFIG_PATH).select{|dir| not dir.start_with? "." and File.directory? CONFIG_PATH + "/#{dir}/downloads" }.map{|dir| CONFIG_PATH + "/#{dir}/downloads"} - cache_info_list = Utils.get_sub_files_info(cache_list).reverse + cache_list = get_cache_dir_list + cache_info_list = Utils.get_sub_files_info(cache_list).sort {|a,b| a[3] <=> b[3]} # [path,size,ctime,mtime] cache_size = 0 cache_info_list.each {|file| cache_size += file[1]} - extra_size = cache_size - (@cache_size * 1024 * 1024) + read_configs + max_cache_size = get_config_max_cache_size + + extra_size = cache_size - max_cache_size if extra_size < 0 then return end remove_list = [] while extra_size > 0 do - old_package = cache_info_list.pop + old_package = cache_info_list.shift remove_list.push old_package[0] extra_size -= old_package[1] end - FileUtils.rm_rf remove_list + if not remove_list.empty? then + @log.info "Removed oldest package files.. (Now not fully caching)" + @log.info " * #{remove_list.map{|path| File.basename(path)}.join(", ")}" + begin + lock = Utils.file_lock(File.join(CONFIG_PATH,"cache_lock")) + FileUtils.rm_rf remove_list + rescue Exception => e + @log.error( e.message, Log::LV_USER) + ensure + Utils.file_unlock(lock) + end + end + end + + private + def get_cache_dir_list + return Dir.entries(CONFIG_PATH).select{|dir| not dir.start_with? "." and File.directory? CONFIG_PATH + "/#{dir}/downloads" }.map{|dir| CONFIG_PATH + "/#{dir}/downloads"} + end + + private + def get_config_max_cache_size + return (@configs[:CACHE_SIZE] =~ /^([0-9][0-9]*)$/i)? ($1.to_i * 1024 * 1024) : (4 * 1024 * 1024 * 1024) #default size 4G : unit = MB end + private + def read_configs + file_path = CONFIG_PATH + "/config" + if not File.exist? file_path then return end + + File.open file_path, "r" do |f| + f.each_line do |l| + row = l.split(":") + @configs[row[0].strip.to_sym] = row[1..-1].join(":").strip + end + end + end + + public # update package list from server def update() @@ -273,15 +312,22 @@ class Client return nil end url = surl + pkg_path - filename = pkg_path.split('/')[-1] + filename = File.basename pkg_path - if not FileDownLoader.download(url, loc, @log) then - @log.error "File Download Failed!!" - @log.error "* #{url} -> #{loc}" - return nil - end + file_path = get_file_from_cache(filename, pkg_checksum, pkg_size, loc) + if file_path.nil? then + + if not FileDownLoader.download(url, loc, @log) then + @log.error "File Download Failed!!" + @log.error "* #{url} -> #{loc}" + return nil + end - file_path = File.join(loc, filename) + file_path = File.join(loc, filename) + add_file_to_cache(file_path) + else + @log.info "Cached #{pkg_name} package file.. OK" + end file_local_path.push(file_path) end @@ -294,41 +340,48 @@ class Client end private - def remove_downloaded_pkgs(pkg_name, os) - pkg_file_prefix = "#{@download_path}/#{pkg_name}_*_#{os}.zip" - pkg_files = Dir[pkg_file_prefix].sort_by { |f| File.mtime(f) }.reverse - - if not pkg_files.nil? and pkg_files.length >= 2 then - Utils.execute_shell("rm -rf #{pkg_files[1..-1].join(" ")}") - @log.info "Removed old package files.." - @log.info " * #{pkg_files[1..-1].join(", ")}" + def get_file_from_cache(filename, pkg_checksum, pkg_size, location) + downloaded_file_path = nil + begin + lock = Utils.file_lock(File.join(CONFIG_PATH,"cache_lock")) + if Utils.is_linux_like_os( Utils::HOST_OS ) then + cached_filepath = get_cached_filepath(filename, pkg_checksum, pkg_size) + if not cached_filepath.nil? then + FileUtils.cp(cached_filepath, location) + downloaded_file_path = File.join(location, File.basename(cached_filepath)) + end + end + rescue Exception => e + @log.error( e.message, Log::LV_USER) + ensure + Utils.file_unlock(lock) + end + if not downloaded_file_path.nil? and not File.exist? downloaded_file_path then + downloaded_file_path = nil end - clean_cache + return downloaded_file_path end private - def move_downloaded_pkg(filepath, distpath) - if filepath.nil? or filepath == "" then return nil end - filename = filepath.split('/')[-1] - if not File.exist? distpath then FileUtils.mkdir_p "#{distpath}" end - distfile = File.join(distpath, filename) + def add_file_to_cache(filepath) + if filepath.nil? or filepath.empty? then return end + if not File.exist? @download_path then FileUtils.mkdir_p "#{download_path}" end + + filename = File.basename(filepath) + cachefile = File.join(@download_path, filename) + @log.info "Moving \"#{filename}\" to download cache directory" - @log.info " [path: #{distpath}]" - $filemove_mutex.synchronize do - if not File.exist? distfile then - Utils.execute_shell("mv #{filepath} #{distfile}") - else - Utils.execute_shell("rm -f #{filepath}") - return distfile - end + @log.info " [path: #{@download_path}]" + + if not File.exist? cachefile then + FileUtils.cp(filepath, @download_path) end - if File.exist? distfile then return distfile - else + if not File.exist? cachefile then @log.info "Failed to move [#{filenamae}] to " - @log.info " [#{distpath}]" - return nil + @log.info " [#{download_path}]" end + remove_old_files_in_cache end private @@ -348,6 +401,7 @@ class Client cached_filepath = "#{@download_path}/#{pkg_filename}" if File.exist? cached_filepath then + FileUtils.touch cached_filepath checksum = `sha256sum #{cached_filepath}`.split(" ")[0] size = `du -b #{cached_filepath}`.split[0].strip if checksum.eql? pkg_checksum and size.eql? pkg_size then @@ -1413,10 +1467,6 @@ class Client # below code should be changed type = path.split('/')[-2] new_pkg_ver = get_attr_from_pkg(pkg_name, os, "version") - pkg_checksum = get_attr_from_pkg(pkg_name, os, "checksum") - pkg_size = get_attr_from_pkg(pkg_name, os, "size") - pkg_path = get_attr_from_pkg(pkg_name, os, "path") - filename = pkg_path.split('/')[-1] # compare version with installed package versiona compare_result = compare_version_with_installed_pkg(pkg_name, new_pkg_ver) @@ -1446,25 +1496,18 @@ class Client end # install package - cached_filepath = nil - if Utils.is_linux_like_os( Utils::HOST_OS ) then - cached_filepath = get_cached_filepath(filename, pkg_checksum, pkg_size) - end - if not cached_filepath.nil? then - @log.info "Cached #{pkg_name} package file.. OK" - ret = FileInstaller.install(pkg_name, cached_filepath, type, @location, @log) - else - filepath = download(pkg_name, os, false, @tmp_path) - if filepath.nil? then - return false - end - filepath = move_downloaded_pkg(filepath[0], @download_path) - if filepath.nil? then - return false - end - ret = FileInstaller.install(pkg_name, filepath, type, @location, @log) - remove_downloaded_pkgs(pkg_name, os) + #download + tmp_location = File.join(@tmp_path,Utils.create_uniq_name) + filepath = download(pkg_name, os, false, tmp_location) + if filepath.nil? or filepath[0].nil? then + return false end + + #install + ret = FileInstaller.install(pkg_name, filepath[0], type, @location, @log) + + #remove garbage + FileUtils.rm_rf tmp_location return ret end -- 2.34.1