[Title] update package server sync lock for pkg list file update time
authorhataejun <taejun.ha@samsung.com>
Wed, 22 Aug 2012 06:09:08 +0000 (15:09 +0900)
committerhataejun <taejun.ha@samsung.com>
Wed, 22 Aug 2012 06:09:08 +0000 (15:09 +0900)
[Type]
[Module]
[Priority]
[Jira#]
[Redmine#]
[Problem]
[Cause]
[Solution]
[TestCase]

src/pkg_server/distribution.rb
src/pkg_server/packageServer.rb
test/packageserver01.testcase
test/packageserver03.testcase
test/packageserver24.testcase
test/pkgsvr.init

index 250d371588d8b83dd830e01a0282076f8afea42e..e5a0c89430b98284c69a6d534830b7966b28a9dc 100644 (file)
@@ -32,7 +32,7 @@ require "parser"
 require "installer"
 
 class Distribution 
-       attr_accessor :name, :location, :server_url, :lock_file_name
+       attr_accessor :name, :location, :server_url, :lock_file_path
 
     # constant
     PKG_LIST_FILE_PREFIX = "pkg_list_" 
@@ -48,7 +48,7 @@ class Distribution
                @server_url = server_url
                @log = pkg_server.log
                @integrity = pkg_server.integrity 
-               @lock_file_name = "#{location}/#{LOCK_FILE}"
+               @lock_file_path = "#{location}/#{LOCK_FILE}"
                @pkg_hash_os = {}  
                @archive_pkg_list = [] 
                @snapshot_hash = [] 
@@ -199,16 +199,27 @@ class Distribution
                client = Client.new( @server_url, "#{@location}/binary", @log )
 
                # update os list
-               client.support_os_list.each do |os|
-                       if not @support_os_list.include? os then
-                               add_os(os)
+               add_os_list = client.support_os_list - @support_os_list
+               add_os_list.each do |os|
+                       add_os(os)
+                       pkg_list_update_flag = true
+               end
+
+               if force then
+                       remove_os_list = @support_os_list - client.support_os_list 
+                       remove_os_list.each do |os|
+                               remove_os(os)
+                               pkg_list_update_flag = true
                        end
                end
 
+               update_pkg_list = []
+
         for os in @support_os_list 
                        # error check 
                        if client.pkg_hash_os[os].nil? then 
-                               raise "Package list can't generated. url is #{@server_url}. os is #{os}"
+                               @log.error("package server does not have os : #{os}", Log::LV_USER)
+                               next
                        end 
 
                        server_pkg_name_list = client.pkg_hash_os[os].keys
@@ -217,16 +228,69 @@ class Distribution
             
                        full_pkg_name_list.each do |pkg_name| 
                                ret = sync_package( pkg_name, client, os, force )
-                               if ret then pkg_list_update_flag = true end
+                               if not ret.nil? then 
+                                       update_pkg_list.push(ret)
+                                       pkg_list_update_flag = true 
+                               end
                        end
                end
                
+               # lock
+               Utils.file_lock(@lock_file_path)
+
+               # reload pkg list from newest pkg list file
+               reload_binary_pkg_list
+
+               # update pkg_list hash
+               update_pkg_list.each do |update_option, os, pkg|
+                       # if updated package's os is removed then skip update
+                       if not @support_os_list.include? os then
+                               next
+                       end
+
+                       case update_option
+                       when "ADD"
+                               new_pkg = @pkg_hash_os[os][pkg.package_name]
+                               
+                               if (not force) and (not new_pkg.nil?) and 
+                                               Utils.compare_version(new_pkg.version, pkg.version).eql?(-1) then
+                                       # if package is update then skip
+                                       next
+                               end
+                                       
+                               @pkg_hash_os[os][pkg.package_name] = pkg
+                       when "REMOVE" 
+                               if not force then
+                                       if @pkg_hash_os[os][pkg.package_name].origin.eql? "local" then 
+                                               next
+                                       end
+                               end
+
+                               @pkg_hash_os[os].delete(pkg.package_name)
+                       else
+                               @log.error("Unsupportd update option : #{update_option}", Log::LV_USER)
+                               next
+                       end
+               end
+
+               # check install dependency integrity
+               if @integrity.eql? "YES" then
+                       @log.info "integrity check"
+                       check_integrity 
+               else
+                       @log.info "skip integrity check"
+               end 
+
+               # update pkg_list file
                if pkg_list_update_flag then
                        write_all_pkg_list()
                        return true
                else
                        return false
                end
+
+               # uplock 
+               Utils.file_unlock(@lock_file_path)
        end
 
     def sync_archive_pkg
@@ -237,23 +301,41 @@ class Distribution
                # if update list is empty then return false
                if download_list.empty? then return false end 
 
+               updated_file_list = []
+
         download_list.each do |pkg|
                        file = client.download_dep_source(pkg)
                if file.nil?
                        @log.error("Can't download archive package [#{pkg}]", Log::LV_USER)
                        else
-                               @archive_pkg_list.push pkg
+                               updated_file_list.push pkg
                end
         end
 
+               # lock
+               Utils.file_lock(@lock_file_path)
+
+               # reload archive pkg list
+               @archive_pkg_list = read_archive_pkg_list( "" )
+
+               # update archive pkg list
+               updated_file_list.each do |pkg|
+                       if not @archive_pkg_list.include? pkg then
+                               @archive_pkg_list.push pkg
+                       end
+               end
+
         write_archive_pkg_list()
 
+               # uplock 
+               Utils.file_unlock(@lock_file_path)
+
                return true
     end
 
        def add_os(os)
                if @support_os_list.include? os then
-                       @log.error("[#{os} is already exist ", Log::LV_USER)
+                       @log.error("#{os} is already exist ", Log::LV_USER)
                end
 
                # update os information
@@ -267,6 +349,40 @@ class Distribution
                File.open( "#{@location}/#{PKG_LIST_FILE_PREFIX}#{os}", "w" ) do |f| end
        end
 
+       def remove_os(os)
+               if not @support_os_list.include? os then
+                       @log.error("Can't remove os : #{os} does not exist ", Log::LV_USER)
+               end
+
+               # update os information
+               @support_os_list.delete os
+               @pkg_hash_os.delete os
+
+               # generate temp file
+               tmp_file_name = ""
+               while ( tmp_file_name.empty? )
+                       tmp_file_name = @location + "/temp/." + Utils.create_uniq_name
+
+                       if File.exist? tmp_file_name then 
+                               tmp_file_name = ""
+                       end 
+               end
+
+               info_file = File.readlines("#{@location}/#{OS_INFO_FILE}")
+               File.open(tmp_file_name, "w") do |f|
+                       info_file.each do |line|
+                               if not line.strip.eql? os then 
+                                       f.puts line 
+                               end
+                       end
+               end
+
+               FileUtils.mv( tmp_file_name, "#{@location}/#{OS_INFO_FILE}", :force => true )
+
+               # delete pkg_list_#{os} file 
+               File.delete( "#{@location}/#{PKG_LIST_FILE_PREFIX}#{os}" ) 
+       end
+
        def clean( remain_snapshot_list )
                file_list = []
                used_archive_list = []
@@ -292,7 +408,21 @@ class Distribution
 
                # collect remaning file's name from snapshot list
                for snapshot in remain_snapshot_list  
-                       for os in @support_os_list
+                       os_info = "#{@location}/snapshots/#{snapshot}/#{OS_INFO_FILE}"
+                       os_list = []
+                       # if snapshot has os_info file then using that file
+                       if File.exist? os_info
+                               File.open( os_info, "r" ) do |f|
+                                       f.each_line do |l|
+                                               os_list.push l.strip
+                                       end
+                               end
+                       # if snapshot does not have os_info file then using package server os_info list
+                       else
+                               os_list = @support_os_list
+                       end
+
+                       for os in os_list
                                begin
                                        info_file = "#{@location}/snapshots/#{snapshot}/#{PKG_LIST_FILE_PREFIX}#{os}" 
                                        if not File.exist? info_file then
@@ -484,7 +614,7 @@ class Distribution
        end
 
        def check_package_integrity(pkg)
-               error_msg = "[#{pkg.package_name}]'s install dependency not matched in "  
+               error_msg = "[[#{pkg.package_name}] in #{pkg.os}]'s install dependency not matched in "  
                os = pkg.os
 
                for dep in pkg.install_dep_list  
@@ -501,7 +631,7 @@ class Distribution
 
                end 
 
-               error_msg = "[#{pkg.package_name}]'s build dependency not matched in "  
+               error_msg = "[[#{pkg.package_name}] in #{pkg.os}]'s build dependency not matched in "  
                for dep in pkg.build_dep_list 
                        if dep.target_os_list.length == 0 then
                                build_dep_os = os  
@@ -521,7 +651,7 @@ class Distribution
                        end 
                end 
                
-               error_msg = "[#{pkg.package_name}]'s source dependency not matched in "  
+               error_msg = "[[#{pkg.package_name}] in #{pkg.os}]'s source dependency not matched in "  
                for dep in pkg.source_dep_list
                        if not @archive_pkg_list.include? dep.package_name then
                                raise RuntimeError,(error_msg + dep.to_s) 
@@ -620,37 +750,34 @@ class Distribution
                        version_cmp = Utils.compare_version( local_pkg.version, server_pkg.version )
                        if ( version_cmp.eql? 0 ) then
                                # version is same then skip update
-                               return false
+                               return nil
                        end
 
-                       if ( local_pkg.origin.eql? "local" ) then 
-                               if not force then
-                                       # local_pkg is generated from local and not force mode then skip update
-                                       return false
-                               end
+                       if ( local_pkg.origin.eql? "local" ) and (not force) then 
+                               # local_pkg is generated from local and not force mode then skip update
+                               return nil
                        end 
                                
-                       sync_package2( server_pkg, client, os, force )
-                       return true
+                       pkg = sync_package2( server_pkg, client, os, force )
+                       return ["ADD", os, pkg]
                # if package exist only server
                elsif ( not server_pkg.nil? ) then
-                       sync_package2( server_pkg, client, os, force )
-                       return true
+                       pkg = sync_package2( server_pkg, client, os, force )
+                       return ["ADD", os, pkg]
                # if package exist only local
                elsif ( not local_pkg.nil? ) then  
                        # if local pkg is generated from local then skip
-                       if local_pkg.origin.eql? "local" then  
-                               return false
+                       if local_pkg.origin.eql? "local" and (not force) then  
+                               return nil
                        end
                        
                        # package remove
-                       @pkg_hash_os[os].delete(pkg_name)
-                       return true
+                       return ["REMOVE", os, local_pkg]
                else
                        raise RuntimeError,"hash merge error!"
                end
 
-               return false
+               return nil
        end 
 
        def sync_package2( pkg, client, os, force ) 
@@ -662,7 +789,7 @@ class Distribution
                # file download error check 
                if file_path_list.nil? or file_path_list.empty? then   
                        @log.error("Can't download package file [#{pkg_name}]", Log::LV_USER)
-                       return
+                       return nil
                else
                        file_path = file_path_list[0]
                end 
@@ -670,7 +797,7 @@ class Distribution
                # update pkg class  
                pkg.path = "/binary/#{File.basename(file_path)}"
                pkg.origin = client.server_addr
-               @pkg_hash_os[os][pkg_name] = pkg 
+               return pkg 
 
        end
 
@@ -733,4 +860,32 @@ class Distribution
 
                return rdepends_list
        end  
+
+       def reload_binary_pkg_list
+               if not File.exist?("#{@location}/#{OS_INFO_FILE}") then
+                       return
+               end 
+
+               # get support_os_list 
+               File.open( "#{@location}/#{OS_INFO_FILE}", "r" ) do |f|
+                       f.each_line do |l|
+                               @support_os_list.push l.strip
+                       end
+               end
+
+               # read package_list file
+        for os in @support_os_list 
+                       @pkg_hash_os[os] = {}
+                       pkg_list_file = "#{@location}/#{PKG_LIST_FILE_PREFIX}#{os}" 
+
+                       if File.exist? pkg_list_file then 
+                               begin
+                                       @pkg_hash_os[os] = Parser.read_repo_pkg_list_from( pkg_list_file )
+                               rescue => e
+                                       @log.error( e.message, Log::LV_USER)
+                                       @pkg_hash_os[os] = nil
+                               end
+                       end
+               end
+       end
 end
index 7bc5e040ea6845417fc5c90583811226d21c9f16..cf081d865ca7cf333895410e050b810020a902da 100644 (file)
@@ -127,7 +127,7 @@ class PackageServer
                distribution = get_distribution( dist_name )
 
                # distribution lock 
-               @lock_file = Utils.file_lock(distribution.lock_file_name)
+               @lock_file = Utils.file_lock(distribution.lock_file_path)
                
                updated_os_list = []
                registed_package_list = []
@@ -252,7 +252,7 @@ class PackageServer
                @log.info "generating snapshot"
                distribution = get_distribution( dist_name )
 
-               @lock_file = Utils.file_lock(distribution.lock_file_name)
+               @lock_file = Utils.file_lock(distribution.lock_file_path)
 
                snapshot_name = distribution.generate_snapshot( snpashot_name, base_snapshot, true)
                
@@ -270,7 +270,6 @@ class PackageServer
                        return
                end 
 
-               @lock_file = Utils.file_lock(distribution.lock_file_name)
                ret1 = distribution.sync(mode)
                ret2 = distribution.sync_archive_pkg
                if ret1 or ret2 then 
@@ -278,8 +277,6 @@ class PackageServer
                end
                
                @log.output( "package server [#{@id}]'s distribution [#{dist_name}] has been synchronized.", Log::LV_USER )
-
-               Utils.file_unlock(@lock_file)
        end
 
        def add_distribution( dist_name, server_url, clone )
@@ -320,7 +317,7 @@ class PackageServer
                dist = get_distribution(dist_name)
                
                # distribution lock 
-               @lock_file = Utils.file_lock(dist.lock_file_name)
+               @lock_file = Utils.file_lock(dist.lock_file_path)
                
                dist.add_os(os)
 
index 70108cbfd6ad5b7815a7e1f90ba86232f0e1707c..d8a261fbe4a4c68f2e5853e6eafd1c4ad2853528 100644 (file)
@@ -8,56 +8,56 @@ Package-server administer service command-line tool.
 Usage: pkg-svr <SUBCOMMAND> [OPTS] or pkg-svr -h
 
 Subcommands:
-       create          Create a package-server.
-       add-dist    Add a distribution to package-server.
-       register    Register a package in package-server.
-       remove        Remove a package-server.
-       remove-dist    Remove a distribution to package-server.
-       remove-snapshot Remove a snapshot in package-server.
-       gen-snapshot    Generate a snapshot in package-server.
-       sync        Synchronize the package-server from parent package server.
-       start        Start the package-server.
-       stop        Stop the package-server.
-       clean        Delete unneeded package files in package-server.
-       list        Show all pack
+create          Create a package-server.
+add-dist    Add a distribution to package-server.
+register    Register a package in package-server.
+remove        Remove a package-server.
+remove-dist    Remove a distribution to package-server.
+remove-snapshot Remove a snapshot in package-server.
+gen-snapshot    Generate a snapshot in package-server.
+sync        Synchronize the package-server from parent package server.
+start        Start the package-server.
+stop        Stop the package-server.
+clean        Delete unneeded package files in package-server.
+list        Show all pack
 
 Subcommand usage:
-       pkg-svr create -n <server name> -d <distribution> [-u <remote server url>] [-l <location>] 
-       pkg-svr add-dist -n <server name> -d <distribution> [-u <remote_server_url>] [--clone] 
-       pkg-svr add-os -n <server name> -d <distribution> -o <os> 
-       pkg-svr register -n <server name> -d <distribution> -P <package file list> [--gen] [--test] 
-       pkg-svr link -n <server name> -d <distribution> --origin-pkg-name <origin pkg name> --origin-pkg-os <origin pkg os> --link-os-list <link os list>
-       pkg-svr remove -n <server name> 
-       pkg-svr remove-dist -n <server name> -d <distribution>
-       pkg-svr remove-pkg -n <server name> -d <distribution> -P <package name list> [-o <os>] 
-       pkg-svr remove-snapshot -n <server name> -d <distribution> -s <snapshot list>
-       pkg-svr gen-snapshot -n <server name> -d <distribution> -s <snapshot name> [-b <base snapshot name>] 
-       pkg-svr sync -n <server name> -d <distribution> [--force] 
-       pkg-svr clean -n <server name> -d <distribution> [-s <snapshot list>] 
-       pkg-svr start -n <server name> -p <port>
-       pkg-svr stop -n <server name> -p <port>
-       pkg-svr list [-n <server name>] 
+pkg-svr create -n <server name> -d <distribution> [-u <remote server url>] [-l <location>] 
+pkg-svr add-dist -n <server name> -d <distribution> [-u <remote_server_url>] [--clone] 
+pkg-svr add-os -n <server name> -d <distribution> -o <os> 
+pkg-svr register -n <server name> -d <distribution> -P <package file list> [--gen] [--test] 
+pkg-svr link -n <server name> -d <distribution> --origin-pkg-name <origin pkg name> --origin-pkg-os <origin pkg os> --link-os-list <link os list>
+pkg-svr remove -n <server name> 
+pkg-svr remove-dist -n <server name> -d <distribution>
+pkg-svr remove-pkg -n <server name> -d <distribution> -P <package name list> [-o <os>] 
+pkg-svr remove-snapshot -n <server name> -d <distribution> -s <snapshot list>
+pkg-svr gen-snapshot -n <server name> -d <distribution> -s <snapshot name> [-b <base snapshot name>] 
+pkg-svr sync -n <server name> -d <distribution> [--force] 
+pkg-svr clean -n <server name> -d <distribution> [-s <snapshot list>] 
+pkg-svr start -n <server name> -p <port>
+pkg-svr stop -n <server name> -p <port>
+pkg-svr list [-n <server name>] 
 
 Options:
-        -n, --name <server name>         package server name
-        -d, --dist <distribution>        package server distribution
-        -u, --url <server url>           remote server url: http://127.0.0.1/dibs/unstable
-        -o, --os <operating system>      target operating system
-        -P, --pkgs <package file list>   package file path list
-        -s, --snapshot <snapshot>        a snapshot name or snapshot list
-        -b, --base <base snapshot>       base snapshot name
-        -l, --loc <location>             server location
-        -p, --port <port>                port number
-            --recursive                  remove all depends packages
-            --clone                      clone mode
-            --force                      force update pkg file
-            --test                       upload for test
-            --gen                        generate snapshot
-            --origin-pkg-name <origin_pkg_name>
-                                         origin package name
-            --origin-pkg-os <origin_pkg_os>
-                                         origin package os
-            --link-os-list <link_os_list>
-                                         target os list to link origin file
-        -h, --help                       display manual
-        -v, --version                    display version
+-n, --name <server name>         package server name
+-d, --dist <distribution>        package server distribution
+-u, --url <server url>           remote server url: http://127.0.0.1/dibs/unstable
+-o, --os <operating system>      target operating system
+-P, --pkgs <package file list>   package file path list
+-s, --snapshot <snapshot>        a snapshot name or snapshot list
+-b, --base <base snapshot>       base snapshot name
+-l, --loc <location>             server location
+-p, --port <port>                port number
+    --recursive                  remove all depends packages
+    --clone                      clone mode
+    --force                      force update pkg file
+    --test                       upload for test
+    --gen                        generate snapshot
+    --origin-pkg-name <origin_pkg_name>
+                                 origin package name
+    --origin-pkg-os <origin_pkg_os>
+                                 origin package os
+    --link-os-list <link_os_list>
+                                 target os list to link origin file
+-h, --help                       display manual
+-v, --version                    display version
index 2067eeafac634ef56e9979a19f2c21f51ee9110e..11eb7742407682f32228ad5376189137959396d4 100644 (file)
@@ -4,4 +4,5 @@
 ../pkg-svr create -n temp_remote -d unstable -u http://172.21.111.177/tmppkgsvr/tmp
 #POST-EXEC
 #EXPECT
+snapshot is generated : 
 package server [temp_remote] created successfully
index 92ca5a35013e9c7aa1e8952455aebd6ce68669de..2657bab772a3516d30871e195a3b3a5cbdc066c9 100644 (file)
@@ -1,7 +1,8 @@
 #PRE-EXEC
 #EXEC
-../pkg-svr add-os -n temp_local -d unstable -o ubuntu-10.04-32
+../pkg-svr add-os -n temp_local -d unstable -o ubuntu-32
 #POST-EXEC 
-../pkg-svr add-os -n temp_local -d unstable -o windows-7-32
+../pkg-svr add-os -n temp_local -d unstable -o windows-32
 #EXPECT
-package server add os [ubuntu-10.04-32] successfully
+snapshot is generated : 
+package server add os [ubuntu-32] successfully
index f251058a884dd1644b76310e6769f5a9855048ae..96b4f3f7d7b94d42b3a4f0a7e9337403061812d7 100755 (executable)
@@ -4,4 +4,4 @@ rm -rf `pwd`/pkgsvr01
 ruby -d ../pkg-svr create -n pkgsvr01 -d unstable
 ruby -d ../pkg-svr add-os -n pkgsvr01 -d unstable -o ubuntu-32
 ruby -d ../pkg-svr add-os -n pkgsvr01 -d unstable -o windows-32
-ruby -d ../pkg-svr start -n pkgsvr01
+ruby -d ../pkg-svr start -n pkgsvr01 -p 3333