From 5aa74b4c182c01623315bc695c9eda4369a065f1 Mon Sep 17 00:00:00 2001 From: hataejun Date: Wed, 22 Aug 2012 15:09:08 +0900 Subject: [PATCH] [Title] update package server sync lock for pkg list file update time [Type] [Module] [Priority] [Jira#] [Redmine#] [Problem] [Cause] [Solution] [TestCase] --- src/pkg_server/distribution.rb | 215 +++++++++++++++++++++++++++----- src/pkg_server/packageServer.rb | 9 +- test/packageserver01.testcase | 98 +++++++-------- test/packageserver03.testcase | 1 + test/packageserver24.testcase | 7 +- test/pkgsvr.init | 2 +- 6 files changed, 243 insertions(+), 89 deletions(-) diff --git a/src/pkg_server/distribution.rb b/src/pkg_server/distribution.rb index 250d371..e5a0c89 100644 --- a/src/pkg_server/distribution.rb +++ b/src/pkg_server/distribution.rb @@ -32,7 +32,7 @@ require "parser" require "installer" class Distribution - attr_accessor :name, :location, :server_url, :lock_file_name + attr_accessor :name, :location, :server_url, :lock_file_path # constant PKG_LIST_FILE_PREFIX = "pkg_list_" @@ -48,7 +48,7 @@ class Distribution @server_url = server_url @log = pkg_server.log @integrity = pkg_server.integrity - @lock_file_name = "#{location}/#{LOCK_FILE}" + @lock_file_path = "#{location}/#{LOCK_FILE}" @pkg_hash_os = {} @archive_pkg_list = [] @snapshot_hash = [] @@ -199,16 +199,27 @@ class Distribution client = Client.new( @server_url, "#{@location}/binary", @log ) # update os list - client.support_os_list.each do |os| - if not @support_os_list.include? os then - add_os(os) + add_os_list = client.support_os_list - @support_os_list + add_os_list.each do |os| + add_os(os) + pkg_list_update_flag = true + end + + if force then + remove_os_list = @support_os_list - client.support_os_list + remove_os_list.each do |os| + remove_os(os) + pkg_list_update_flag = true end end + update_pkg_list = [] + for os in @support_os_list # error check if client.pkg_hash_os[os].nil? then - raise "Package list can't generated. url is #{@server_url}. os is #{os}" + @log.error("package server does not have os : #{os}", Log::LV_USER) + next end server_pkg_name_list = client.pkg_hash_os[os].keys @@ -217,16 +228,69 @@ class Distribution full_pkg_name_list.each do |pkg_name| ret = sync_package( pkg_name, client, os, force ) - if ret then pkg_list_update_flag = true end + if not ret.nil? then + update_pkg_list.push(ret) + pkg_list_update_flag = true + end end end + # lock + Utils.file_lock(@lock_file_path) + + # reload pkg list from newest pkg list file + reload_binary_pkg_list + + # update pkg_list hash + update_pkg_list.each do |update_option, os, pkg| + # if updated package's os is removed then skip update + if not @support_os_list.include? os then + next + end + + case update_option + when "ADD" + new_pkg = @pkg_hash_os[os][pkg.package_name] + + if (not force) and (not new_pkg.nil?) and + Utils.compare_version(new_pkg.version, pkg.version).eql?(-1) then + # if package is update then skip + next + end + + @pkg_hash_os[os][pkg.package_name] = pkg + when "REMOVE" + if not force then + if @pkg_hash_os[os][pkg.package_name].origin.eql? "local" then + next + end + end + + @pkg_hash_os[os].delete(pkg.package_name) + else + @log.error("Unsupportd update option : #{update_option}", Log::LV_USER) + next + end + end + + # check install dependency integrity + if @integrity.eql? "YES" then + @log.info "integrity check" + check_integrity + else + @log.info "skip integrity check" + end + + # update pkg_list file if pkg_list_update_flag then write_all_pkg_list() return true else return false end + + # uplock + Utils.file_unlock(@lock_file_path) end def sync_archive_pkg @@ -237,23 +301,41 @@ class Distribution # if update list is empty then return false if download_list.empty? then return false end + updated_file_list = [] + download_list.each do |pkg| file = client.download_dep_source(pkg) if file.nil? @log.error("Can't download archive package [#{pkg}]", Log::LV_USER) else - @archive_pkg_list.push pkg + updated_file_list.push pkg end end + # lock + Utils.file_lock(@lock_file_path) + + # reload archive pkg list + @archive_pkg_list = read_archive_pkg_list( "" ) + + # update archive pkg list + updated_file_list.each do |pkg| + if not @archive_pkg_list.include? pkg then + @archive_pkg_list.push pkg + end + end + write_archive_pkg_list() + # uplock + Utils.file_unlock(@lock_file_path) + return true end def add_os(os) if @support_os_list.include? os then - @log.error("[#{os} is already exist ", Log::LV_USER) + @log.error("#{os} is already exist ", Log::LV_USER) end # update os information @@ -267,6 +349,40 @@ class Distribution File.open( "#{@location}/#{PKG_LIST_FILE_PREFIX}#{os}", "w" ) do |f| end end + def remove_os(os) + if not @support_os_list.include? os then + @log.error("Can't remove os : #{os} does not exist ", Log::LV_USER) + end + + # update os information + @support_os_list.delete os + @pkg_hash_os.delete os + + # generate temp file + tmp_file_name = "" + while ( tmp_file_name.empty? ) + tmp_file_name = @location + "/temp/." + Utils.create_uniq_name + + if File.exist? tmp_file_name then + tmp_file_name = "" + end + end + + info_file = File.readlines("#{@location}/#{OS_INFO_FILE}") + File.open(tmp_file_name, "w") do |f| + info_file.each do |line| + if not line.strip.eql? os then + f.puts line + end + end + end + + FileUtils.mv( tmp_file_name, "#{@location}/#{OS_INFO_FILE}", :force => true ) + + # delete pkg_list_#{os} file + File.delete( "#{@location}/#{PKG_LIST_FILE_PREFIX}#{os}" ) + end + def clean( remain_snapshot_list ) file_list = [] used_archive_list = [] @@ -292,7 +408,21 @@ class Distribution # collect remaning file's name from snapshot list for snapshot in remain_snapshot_list - for os in @support_os_list + os_info = "#{@location}/snapshots/#{snapshot}/#{OS_INFO_FILE}" + os_list = [] + # if snapshot has os_info file then using that file + if File.exist? os_info + File.open( os_info, "r" ) do |f| + f.each_line do |l| + os_list.push l.strip + end + end + # if snapshot does not have os_info file then using package server os_info list + else + os_list = @support_os_list + end + + for os in os_list begin info_file = "#{@location}/snapshots/#{snapshot}/#{PKG_LIST_FILE_PREFIX}#{os}" if not File.exist? info_file then @@ -484,7 +614,7 @@ class Distribution end def check_package_integrity(pkg) - error_msg = "[#{pkg.package_name}]'s install dependency not matched in " + error_msg = "[[#{pkg.package_name}] in #{pkg.os}]'s install dependency not matched in " os = pkg.os for dep in pkg.install_dep_list @@ -501,7 +631,7 @@ class Distribution end - error_msg = "[#{pkg.package_name}]'s build dependency not matched in " + error_msg = "[[#{pkg.package_name}] in #{pkg.os}]'s build dependency not matched in " for dep in pkg.build_dep_list if dep.target_os_list.length == 0 then build_dep_os = os @@ -521,7 +651,7 @@ class Distribution end end - error_msg = "[#{pkg.package_name}]'s source dependency not matched in " + error_msg = "[[#{pkg.package_name}] in #{pkg.os}]'s source dependency not matched in " for dep in pkg.source_dep_list if not @archive_pkg_list.include? dep.package_name then raise RuntimeError,(error_msg + dep.to_s) @@ -620,37 +750,34 @@ class Distribution version_cmp = Utils.compare_version( local_pkg.version, server_pkg.version ) if ( version_cmp.eql? 0 ) then # version is same then skip update - return false + return nil end - if ( local_pkg.origin.eql? "local" ) then - if not force then - # local_pkg is generated from local and not force mode then skip update - return false - end + if ( local_pkg.origin.eql? "local" ) and (not force) then + # local_pkg is generated from local and not force mode then skip update + return nil end - sync_package2( server_pkg, client, os, force ) - return true + pkg = sync_package2( server_pkg, client, os, force ) + return ["ADD", os, pkg] # if package exist only server elsif ( not server_pkg.nil? ) then - sync_package2( server_pkg, client, os, force ) - return true + pkg = sync_package2( server_pkg, client, os, force ) + return ["ADD", os, pkg] # if package exist only local elsif ( not local_pkg.nil? ) then # if local pkg is generated from local then skip - if local_pkg.origin.eql? "local" then - return false + if local_pkg.origin.eql? "local" and (not force) then + return nil end # package remove - @pkg_hash_os[os].delete(pkg_name) - return true + return ["REMOVE", os, local_pkg] else raise RuntimeError,"hash merge error!" end - return false + return nil end def sync_package2( pkg, client, os, force ) @@ -662,7 +789,7 @@ class Distribution # file download error check if file_path_list.nil? or file_path_list.empty? then @log.error("Can't download package file [#{pkg_name}]", Log::LV_USER) - return + return nil else file_path = file_path_list[0] end @@ -670,7 +797,7 @@ class Distribution # update pkg class pkg.path = "/binary/#{File.basename(file_path)}" pkg.origin = client.server_addr - @pkg_hash_os[os][pkg_name] = pkg + return pkg end @@ -733,4 +860,32 @@ class Distribution return rdepends_list end + + def reload_binary_pkg_list + if not File.exist?("#{@location}/#{OS_INFO_FILE}") then + return + end + + # get support_os_list + File.open( "#{@location}/#{OS_INFO_FILE}", "r" ) do |f| + f.each_line do |l| + @support_os_list.push l.strip + end + end + + # read package_list file + for os in @support_os_list + @pkg_hash_os[os] = {} + pkg_list_file = "#{@location}/#{PKG_LIST_FILE_PREFIX}#{os}" + + if File.exist? pkg_list_file then + begin + @pkg_hash_os[os] = Parser.read_repo_pkg_list_from( pkg_list_file ) + rescue => e + @log.error( e.message, Log::LV_USER) + @pkg_hash_os[os] = nil + end + end + end + end end diff --git a/src/pkg_server/packageServer.rb b/src/pkg_server/packageServer.rb index 7bc5e04..cf081d8 100644 --- a/src/pkg_server/packageServer.rb +++ b/src/pkg_server/packageServer.rb @@ -127,7 +127,7 @@ class PackageServer distribution = get_distribution( dist_name ) # distribution lock - @lock_file = Utils.file_lock(distribution.lock_file_name) + @lock_file = Utils.file_lock(distribution.lock_file_path) updated_os_list = [] registed_package_list = [] @@ -252,7 +252,7 @@ class PackageServer @log.info "generating snapshot" distribution = get_distribution( dist_name ) - @lock_file = Utils.file_lock(distribution.lock_file_name) + @lock_file = Utils.file_lock(distribution.lock_file_path) snapshot_name = distribution.generate_snapshot( snpashot_name, base_snapshot, true) @@ -270,7 +270,6 @@ class PackageServer return end - @lock_file = Utils.file_lock(distribution.lock_file_name) ret1 = distribution.sync(mode) ret2 = distribution.sync_archive_pkg if ret1 or ret2 then @@ -278,8 +277,6 @@ class PackageServer end @log.output( "package server [#{@id}]'s distribution [#{dist_name}] has been synchronized.", Log::LV_USER ) - - Utils.file_unlock(@lock_file) end def add_distribution( dist_name, server_url, clone ) @@ -320,7 +317,7 @@ class PackageServer dist = get_distribution(dist_name) # distribution lock - @lock_file = Utils.file_lock(dist.lock_file_name) + @lock_file = Utils.file_lock(dist.lock_file_path) dist.add_os(os) diff --git a/test/packageserver01.testcase b/test/packageserver01.testcase index 70108cb..d8a261f 100644 --- a/test/packageserver01.testcase +++ b/test/packageserver01.testcase @@ -8,56 +8,56 @@ Package-server administer service command-line tool. Usage: pkg-svr [OPTS] or pkg-svr -h Subcommands: - create Create a package-server. - add-dist Add a distribution to package-server. - register Register a package in package-server. - remove Remove a package-server. - remove-dist Remove a distribution to package-server. - remove-snapshot Remove a snapshot in package-server. - gen-snapshot Generate a snapshot in package-server. - sync Synchronize the package-server from parent package server. - start Start the package-server. - stop Stop the package-server. - clean Delete unneeded package files in package-server. - list Show all pack +create Create a package-server. +add-dist Add a distribution to package-server. +register Register a package in package-server. +remove Remove a package-server. +remove-dist Remove a distribution to package-server. +remove-snapshot Remove a snapshot in package-server. +gen-snapshot Generate a snapshot in package-server. +sync Synchronize the package-server from parent package server. +start Start the package-server. +stop Stop the package-server. +clean Delete unneeded package files in package-server. +list Show all pack Subcommand usage: - pkg-svr create -n -d [-u ] [-l ] - pkg-svr add-dist -n -d [-u ] [--clone] - pkg-svr add-os -n -d -o - pkg-svr register -n -d -P [--gen] [--test] - pkg-svr link -n -d --origin-pkg-name --origin-pkg-os --link-os-list - pkg-svr remove -n - pkg-svr remove-dist -n -d - pkg-svr remove-pkg -n -d -P [-o ] - pkg-svr remove-snapshot -n -d -s - pkg-svr gen-snapshot -n -d -s [-b ] - pkg-svr sync -n -d [--force] - pkg-svr clean -n -d [-s ] - pkg-svr start -n -p - pkg-svr stop -n -p - pkg-svr list [-n ] +pkg-svr create -n -d [-u ] [-l ] +pkg-svr add-dist -n -d [-u ] [--clone] +pkg-svr add-os -n -d -o +pkg-svr register -n -d -P [--gen] [--test] +pkg-svr link -n -d --origin-pkg-name --origin-pkg-os --link-os-list +pkg-svr remove -n +pkg-svr remove-dist -n -d +pkg-svr remove-pkg -n -d -P [-o ] +pkg-svr remove-snapshot -n -d -s +pkg-svr gen-snapshot -n -d -s [-b ] +pkg-svr sync -n -d [--force] +pkg-svr clean -n -d [-s ] +pkg-svr start -n -p +pkg-svr stop -n -p +pkg-svr list [-n ] Options: - -n, --name package server name - -d, --dist package server distribution - -u, --url remote server url: http://127.0.0.1/dibs/unstable - -o, --os target operating system - -P, --pkgs package file path list - -s, --snapshot a snapshot name or snapshot list - -b, --base base snapshot name - -l, --loc server location - -p, --port port number - --recursive remove all depends packages - --clone clone mode - --force force update pkg file - --test upload for test - --gen generate snapshot - --origin-pkg-name - origin package name - --origin-pkg-os - origin package os - --link-os-list - target os list to link origin file - -h, --help display manual - -v, --version display version +-n, --name package server name +-d, --dist package server distribution +-u, --url remote server url: http://127.0.0.1/dibs/unstable +-o, --os target operating system +-P, --pkgs package file path list +-s, --snapshot a snapshot name or snapshot list +-b, --base base snapshot name +-l, --loc server location +-p, --port port number + --recursive remove all depends packages + --clone clone mode + --force force update pkg file + --test upload for test + --gen generate snapshot + --origin-pkg-name + origin package name + --origin-pkg-os + origin package os + --link-os-list + target os list to link origin file +-h, --help display manual +-v, --version display version diff --git a/test/packageserver03.testcase b/test/packageserver03.testcase index 2067eea..11eb774 100644 --- a/test/packageserver03.testcase +++ b/test/packageserver03.testcase @@ -4,4 +4,5 @@ ../pkg-svr create -n temp_remote -d unstable -u http://172.21.111.177/tmppkgsvr/tmp #POST-EXEC #EXPECT +snapshot is generated : package server [temp_remote] created successfully diff --git a/test/packageserver24.testcase b/test/packageserver24.testcase index 92ca5a3..2657bab 100644 --- a/test/packageserver24.testcase +++ b/test/packageserver24.testcase @@ -1,7 +1,8 @@ #PRE-EXEC #EXEC -../pkg-svr add-os -n temp_local -d unstable -o ubuntu-10.04-32 +../pkg-svr add-os -n temp_local -d unstable -o ubuntu-32 #POST-EXEC -../pkg-svr add-os -n temp_local -d unstable -o windows-7-32 +../pkg-svr add-os -n temp_local -d unstable -o windows-32 #EXPECT -package server add os [ubuntu-10.04-32] successfully +snapshot is generated : +package server add os [ubuntu-32] successfully diff --git a/test/pkgsvr.init b/test/pkgsvr.init index f251058..96b4f3f 100755 --- a/test/pkgsvr.init +++ b/test/pkgsvr.init @@ -4,4 +4,4 @@ rm -rf `pwd`/pkgsvr01 ruby -d ../pkg-svr create -n pkgsvr01 -d unstable ruby -d ../pkg-svr add-os -n pkgsvr01 -d unstable -o ubuntu-32 ruby -d ../pkg-svr add-os -n pkgsvr01 -d unstable -o windows-32 -ruby -d ../pkg-svr start -n pkgsvr01 +ruby -d ../pkg-svr start -n pkgsvr01 -p 3333 -- 2.34.1