From cf04f963304019bcad8c7aa191e05d301ef8507e Mon Sep 17 00:00:00 2001 From: donghee yang Date: Tue, 14 Aug 2012 09:49:57 +0900 Subject: [PATCH] [Title] Synchronized with private --- package/pkginfo.manifest | 2 +- pkg-cli | 4 +- src/build_server/BuildComm.rb | 19 +- src/build_server/BuildJob.rb | 106 +++-- src/build_server/BuildServer.rb | 3 - src/build_server/GitBuildJob.rb | 4 + src/build_server/JobClean.rb | 20 +- src/build_server/JobLog.rb | 19 +- src/build_server/JobManager.rb | 65 +++- src/build_server/MultiBuildJob.rb | 120 +++--- src/build_server/RegisterPackageJob.rb | 9 + src/build_server/RemoteBuilder.rb | 3 +- src/build_server/ReverseBuildChecker.rb | 35 +- src/build_server/SocketJobRequestListener.rb | 10 +- src/builder/Builder.rb | 87 +++-- src/common/ScheduledActionHandler.rb | 4 +- src/common/fileTransfer.rb | 56 ++- src/pkg_server/client.rb | 383 ++++++++++++------- src/pkg_server/clientOptParser.rb | 4 +- src/pkg_server/distribution.rb | 4 +- src/pkg_server/downloader.rb | 18 +- src/pkg_server/installer.rb | 53 ++- 22 files changed, 640 insertions(+), 388 deletions(-) diff --git a/package/pkginfo.manifest b/package/pkginfo.manifest index e2e31d7..d0a8434 100644 --- a/package/pkginfo.manifest +++ b/package/pkginfo.manifest @@ -1,5 +1,5 @@ Source : dibs -Version : 0.99.18 +Version : 0.99.20 Maintainer : taejun ha, jiil hyoun , donghyuk yang , donghee yang , sungmin kim 1 then - compat_found = false - p.os_list.each do |o| - #check other package already in package server - ver_svr = @pkgsvr_client.get_attr_from_pkg( p.package_name, o, "version") - if not ver_svr.nil? and p.version.eql? ver_svr then - # get package file name - if compatable_packages[p.package_name].nil? then - compatable_packages[p.package_name] = [o] - else - compatable_packages[p.package_name].push o + if p.os_list.count <= 1 then return [] end + + compat_found = false + p.os_list.each do |o| + # check parent pkgs first + if not @parent.nil? then + parent_pkgs = Dir.glob("#{@parent.source_path}/#{p.package_name}_*_*.zip") + parent_pkgs.each do |lp| + lpname = Utils.get_package_name_from_package_file( lp ) + lver = Utils.get_version_from_package_file(lp) + los = Utils.get_os_from_package_file( lp ) + if lpname == p.package_name and o == los and lver == p.version then + compat_pkgs.push [p.package_name,o,lp] + compat_found = true + break end - compat_found = true end end - if not compat_found then return {} end - else - return {} + if compat_found then break end + + # check other package already in package server + ver_svr = @pkgsvr_client.get_attr_from_pkg( p.package_name, o, "version") + if not ver_svr.nil? and p.version.eql? ver_svr then + compat_pkgs.push [p.package_name,o,nil] + compat_found = true + break + end end + + # if there is no compat pkgs for one pkg, then must build + if not compat_found then return [] end end - return compatable_packages + + return compat_pkgs end @@ -672,7 +701,7 @@ class BuildJob @log.info( " - FTP Server : #{@server.ftp_addr}" ) else builder = Builder.create( "JB#{@id}", @pkgserver_url, @log.path, - "#{@buildroot_dir}/#{@os}", @server.build_cache_dir ) + "#{@buildroot_dir}", @server.build_cache_dir ) if builder.nil? @log.error( "Creating job builder failed", Log::LV_USER) return false @@ -703,18 +732,37 @@ class BuildJob end #compatable os support - comp_pkgs = check_compatable_packages - if comp_pkgs.size > 0 and not @is_rev_build_check_job then + compat_ok = true + compat_pkgs = check_compatable_packages + if compat_pkgs.size > 0 and not @is_rev_build_check_job then # bring package from server for reverse check - comp_pkgs.each do |pkg_name,os_list| - loc = @pkgsvr_client.download(pkg_name, os_list[0], false) - loc.each do |location| - ext = File.extname(location) - base_package_name= File.basename( location, "#{os_list[0]}#{ext}" ) - FileUtils.mv location, "#{@source_path}/#{base_package_name}#{@os}#{ext}" + compat_pkgs.each do |p| + pkg_name = p[0]; cos = p[1]; local_path = p[2] + + if not local_path.nil? then + ext = File.extname(local_path) + base_package_name= File.basename(local_path, "#{cos}#{ext}") + @log.info( "Copying compatible package:#{local_path}", Log::LV_USER) + FileUtils.cp local_path, "#{@source_path}/#{base_package_name}#{@os}#{ext}" + else + @log.info( "Downloading compatible package:#{pkg_name}(#{cos})", Log::LV_USER) + loc = @pkgsvr_client.download(pkg_name, cos, false) + if loc.nil? or loc.count != 1 then + @log.warn( "Downloading compatible package failed!:#{pkg_name}(#{cos})", Log::LV_USER) + compat_ok = false + break + end + ext = File.extname(loc[0]) + base_package_name= File.basename(loc[0], "#{cos}#{ext}") + FileUtils.mv loc[0], "#{@source_path}/#{base_package_name}#{@os}#{ext}" end end else + compat_ok = false + end + + # if compat check failed + if not compat_ok then # build if @is_remote_job then result = builder.build(@project.name, @project.passwd, @source_path, @os, diff --git a/src/build_server/BuildServer.rb b/src/build_server/BuildServer.rb index 54648c7..4e6e142 100644 --- a/src/build_server/BuildServer.rb +++ b/src/build_server/BuildServer.rb @@ -237,9 +237,6 @@ class BuildServer # calculate empty rooms # if sub job, his parent should be excluded local_empty_rooms = @jobmgr.get_number_of_empty_room - if job.is_sub_job? then - local_empty_rooms += 1 - end if local_empty_rooms > 0 and can_build?(job) then candidates.push self diff --git a/src/build_server/GitBuildJob.rb b/src/build_server/GitBuildJob.rb index 72bfe96..12c7566 100644 --- a/src/build_server/GitBuildJob.rb +++ b/src/build_server/GitBuildJob.rb @@ -54,7 +54,10 @@ class GitBuildJob < BuildJob @log.error( "Job is CANCELED" , Log::LV_USER) @server.cleaner.clean_afterwards(@id) else + @log.info( "Job is FINISHED successfully!" , Log::LV_USER) + # if succeeded, register source info and copy pkginfo.manifest + @log.info( "Updating the source info for project \"#{@project.name}\"" , Log::LV_USER) @project.add_source_info( @pkginfo.get_version(), @git_commit) @project.copy_package_info( @pkginfo.get_version(), "#{@source_path}/package/pkginfo.manifest") @@ -152,6 +155,7 @@ class GitBuildJob < BuildJob # check availabiltiy if not @server.check_job_availability( self ) then @log.error( "No servers that are able to build your packages.", Log::LV_USER) + @log.error( "Host-OS (#{@os}) is not supported in build server.", Log::LV_USER) @status = "ERROR" @server.log.info "Adding the job \"#{@id}\" is canceled" end diff --git a/src/build_server/JobClean.rb b/src/build_server/JobClean.rb index 5bc4abd..6ece8d1 100644 --- a/src/build_server/JobClean.rb +++ b/src/build_server/JobClean.rb @@ -38,12 +38,13 @@ $access_listfile = Mutex.new class JobCleanAction < Action - def initialize( time, job_path, list_file ) + def initialize( time, job_path, list_file, server ) super(time,0) @job_path = job_path @job_id = @job_path.split("/")[-1] @list_file = list_file + @server = server end @@ -57,7 +58,10 @@ class JobCleanAction < Action def execute - + + # Start to clean job + @server.log.info "Executing clean action for the job #{@job_id}" + # remove directories if File.exist? "#{@job_path}/buildroot" then FileUtils.rm_rf "#{@job_path}/buildroot" @@ -128,7 +132,7 @@ class JobCleaner job_path = "#{jobs_path}/#{id}" time = Time.mktime(year.to_i, month.to_i, day.to_i, hour.to_i, min.to_i, sec.to_i) @server.log.info "Registered clean-action for the job in list : #{id}" - @handler.register(JobCleanAction.new(time,job_path,@list_file)) + @handler.register(JobCleanAction.new(time,job_path,@list_file, @server)) # add clean list clean_list.push id @@ -146,7 +150,7 @@ class JobCleaner job_path = "#{jobs_path}/#{id}" time = Time.now @server.log.info "Registered clean-action for old job : #{id}" - @handler.register(JobCleanAction.new(time,job_path,@list_file)) + @handler.register(JobCleanAction.new(time,job_path,@list_file, @server)) end end @@ -160,7 +164,9 @@ class JobCleaner curr = Time.now time = Time.now + @server.keep_time job_path = "#{@server.path}/jobs/#{job_id}" - @handler.register(JobCleanAction.new(time, job_path, @list_file)) + @handler.register(JobCleanAction.new(time, job_path, @list_file, @server)) + + @server.log.info "Registered delayed clean-action for the job #{job_id}" end @@ -168,6 +174,8 @@ class JobCleaner def clean(job_id) time = Time.now job_path = "#{@server.path}/jobs/#{job_id}" - @handler.register(JobCleanAction.new(time, job_path, @list_file)) + @handler.register(JobCleanAction.new(time, job_path, @list_file, @server)) + + @server.log.info "Registered clean-action for the job #{job_id}" end end diff --git a/src/build_server/JobLog.rb b/src/build_server/JobLog.rb index 0989b87..e005207 100644 --- a/src/build_server/JobLog.rb +++ b/src/build_server/JobLog.rb @@ -104,21 +104,16 @@ class JobLog < Log BuildCommServer.send( @second_out, msg ) end rescue - close() + # close second_out + @second_out.close + @second_out = nil + error "Connection closed by remote client" - if not @parent_job.nil? then - @parent_job.status="ERROR" - # terminate job - @parent_job.terminate - # exit thread if independent worker thread - if @parent_job.thread == Thread.current then - error "Thread wiil be terminated" - @parent_job.thread=nil - Thread.exit - end + # cancel parent job + if not @parent_job.nil? and @parent_job.cancel_state == "NONE" then + @parent_job.cancel_state = "INIT" end - end end diff --git a/src/build_server/JobManager.rb b/src/build_server/JobManager.rb index 4d63136..529d16e 100644 --- a/src/build_server/JobManager.rb +++ b/src/build_server/JobManager.rb @@ -37,7 +37,8 @@ require "RegisterPackageJob.rb" require "packageServer.rb" class JobManager - attr_accessor :max_working_jobs, :jobs, :internal_jobs + attr_accessor :max_working_jobs, :jobs, :internal_jobs, :reverse_build_jobs + attr_accessor :internal_job_schedule # initialize def initialize( parent ) @@ -123,6 +124,7 @@ class JobManager end @parent.log.info "Checking the job \"#{job.id}\" was finished!" } + @parent.log.info "Job \"#{job.id}\" entered INITIALIZING status" end @@ -152,13 +154,13 @@ class JobManager def cancel_job( job) job.cancel_state = "WORKING" + @parent.log.info "Creating thread for canceling the job \"#{job.id}\"" Thread.new { # thread terminate if not job.thread.nil? then #terminate job thread job.thread.terminate job.thread = nil - job.terminate end # job cacncel @@ -166,25 +168,31 @@ class JobManager # cancel finished job.status = "CANCELED" + + # call terminate process for job + job.terminate } end # handle def handle() # for cancel jobs - (@jobs + @internal_jobs + @reverse_build_jobs).select{|j| j.cancel_state == "INIT" }.each do |job| + (@jobs + @internal_jobs + @reverse_build_jobs).select {|j| j.cancel_state == "INIT" }.each do |job| cancel_job( job ) end # for reverse build jobs - for job in @reverse_build_jobs + job_list = @reverse_build_jobs + for job in job_list # if "ERROR", "FINISHED", "CANCELED" remove it from list if job.status == "ERROR" @parent.log.info "Job \"#{job.id}\" is stopped by ERROR" @reverse_build_jobs.delete job elsif job.status == "FINISHED" + @parent.log.info "Job \"#{job.id}\" is removed by FINISH status" @reverse_build_jobs.delete job elsif job.status == "CANCELED" + @parent.log.info "Job \"#{job.id}\" is removed by CANCELED status" @reverse_build_jobs.delete job end @@ -195,14 +203,17 @@ class JobManager end # for internal jobs - for job in @internal_jobs + job_list = @internal_jobs + for job in job_list # if "ERROR", "FINISHED", "CANCELED" remove it from list if job.status == "ERROR" @parent.log.info "Job \"#{job.id}\" is stopped by ERROR" @internal_jobs.delete job elsif job.status == "FINISHED" + @parent.log.info "Job \"#{job.id}\" is removed by FINISH status" @internal_jobs.delete job elsif job.status == "CANCELED" + @parent.log.info "Job \"#{job.id}\" is removed by CANCELED status" @internal_jobs.delete job end @@ -213,14 +224,17 @@ class JobManager end # for normal job - for job in @jobs + job_list = @jobs + for job in job_list # if "ERROR", "FINISHED", "CANCELED" remove it from list if job.status == "ERROR" @parent.log.info "Job \"#{job.id}\" is stopped by ERROR" @jobs.delete job elsif job.status == "FINISHED" + @parent.log.info "Job \"#{job.id}\" is removed by FINISH status" @jobs.delete job elsif job.status == "CANCELED" + @parent.log.info "Job \"#{job.id}\" is removed by CANCELED status" @jobs.delete job end @@ -272,24 +286,41 @@ class JobManager if not selected_job.nil? then return selected_job end # if no reverse build job exist! - if not @internal_job_schedule.locked? and @internal_jobs.count > 0 then - return get_available_job_in_list(@internal_jobs, true) - else - return get_available_job_in_list(@jobs, false) - end + @internal_job_schedule.synchronize { + # internal job first + ret = nil + if @internal_jobs.count > 0 then + ret = get_available_job_in_list(@internal_jobs, true) + end + + # not found, select normal job + if ret.nil? then + ret = get_available_job_in_list(@jobs, false) + end + + return ret + } end # return "max_working_jobs_cnt - current_working_jobs_cnt" def get_number_of_empty_room working_cnt = 0 + parent_list = [] for job in @jobs + @internal_jobs + @reverse_build_jobs if job.status == "WORKING" then working_cnt = working_cnt + 1 end + + # must exclude parent job + if not job.get_parent_job().nil? then + parent_list.push job.get_parent_job() + end end - return @max_working_jobs - working_cnt + parent_list.uniq! + + return @max_working_jobs - working_cnt + parent_list.count end @@ -375,6 +406,8 @@ class JobManager # gather all working jobs and full-build jobs check_dep_jobs = [] for job in jobs + if job.cancel_state != "NONE" then next end + if job.status == "WORKING" or job.status == "REMOTE_WORKING" or job.status == "PENDING" then check_dep_jobs.push job elsif ( check_dep_wait and job.status == "WAITING") then @@ -384,13 +417,19 @@ class JobManager # for waiting jobs for job in jobs + if job.cancel_state != "NONE" then next end if job.status != "WAITING" then next end # check build dependency against working job pre_jobs = [] for cjob in check_dep_jobs if job == cjob then next end - if (cjob.status == "WORKING" or cjob.status == "REMOTE_WORKING" or cjob.status == "PENDING" ) and + # In case that "WORKING/REMOTE_WORKING" job has build dependency on me + if (cjob.status == "WORKING" or cjob.status == "REMOTE_WORKING" ) and + (job.has_build_dependency?( cjob ) or job.is_compatible_with?( cjob)) then + pre_jobs.push cjob + # In case that "PENDING" job is depends on me (not depended ) + elsif cjob.status == "PENDING" and (not job.does_depend_on? cjob) and (job.has_build_dependency?( cjob ) or job.is_compatible_with?( cjob)) then pre_jobs.push cjob elsif check_dep_wait and cjob.status == "WAITING" and diff --git a/src/build_server/MultiBuildJob.rb b/src/build_server/MultiBuildJob.rb index c7cda6a..4aa5b50 100644 --- a/src/build_server/MultiBuildJob.rb +++ b/src/build_server/MultiBuildJob.rb @@ -69,6 +69,16 @@ class MultiBuildJob end + def get_buildroot() + return @buildroot_dir + end + + + def get_parent_job() + return nil + end + + # execute def execute(sync=false) @log.info( "Invoking a thread for MULTI-BUILD Job #{@id}", Log::LV_USER) @@ -148,12 +158,35 @@ class MultiBuildJob #terminate def terminate() + # report error if @status == "ERROR" then + # register delayed clean action for sub jobs + for job in @sub_jobs + @server.cleaner.clean_afterwards(job.id) + end + + # register delayed clean action for me @log.error( "Job is stopped by ERROR" , Log::LV_USER) @server.cleaner.clean_afterwards(@id) + + elsif @status == "CANCELED" then + # register delayed clean action for sub jobs + for job in @sub_jobs + @server.cleaner.clean_afterwards(job.id) + end + + # register delayed clean action for me + @log.error( "Job is stopped by CANCEL" , Log::LV_USER) + @server.cleaner.clean_afterwards(@id) + else - # clean up + # terminate all sub jobs + for job in @sub_jobs + if not job.log.nil? then job.terminate() end + end + + # register direct clean action for me @server.cleaner.clean(@id) end @@ -326,68 +359,69 @@ class MultiBuildJob end # add to internal job - @server.jobmgr.stop_internal_job_schedule() - for job in @sub_jobs - # init finished, add internal_jobs - @server.jobmgr.add_internal_job(job) - @log.info( "Added new job \"#{job.get_project().name}\" for #{job.os}! (#{job.id})", - Log::LV_USER) - if not @server.job_log_url.empty? then - @log.info( " * Log URL : #{@server.job_log_url}/#{job.id}/log", Log::LV_USER) + @server.jobmgr.internal_job_schedule.synchronize { + for job in @sub_jobs + # init finished, add internal_jobs + @server.jobmgr.add_internal_job(job) + @log.info( "Added new job \"#{job.get_project().name}\" for #{job.os}! (#{job.id})", + Log::LV_USER) + if not @server.job_log_url.empty? then + @log.info( " * Log URL : #{@server.job_log_url}/#{job.id}/log", Log::LV_USER) + end end - end - @server.jobmgr.resume_internal_job_schedule() + } # show job status changes all_jobs_finished = false - error_exist = false - while not all_jobs_finished and not error_exist + stop_status = "FINISHED" + while not all_jobs_finished all_jobs_finished = true for job in @sub_jobs - # check status chanaged, if then print if job_status_map[ job.id ] != job.status then - @log.info("Sub-Job \"#{job.get_project().name}\" for #{job.os} has entered \"#{job.status}\" state. (#{job.id})", Log::LV_USER) + @log.info(" * Sub-Job \"#{job.get_project().name}(#{job.os})\" has entered \"#{job.status}\" state. (#{job.id})", Log::LV_USER) job_status_map[ job.id ] = job.status end - if job.status != "ERROR" and job.status != "FINISHED" and job.cancel_state == "NONE" then + # check all jobs are finished + if job.status != "ERROR" and job.status != "FINISHED" and job.status != "CANCELED" then all_jobs_finished = false end - if job.status == "ERROR" then - error_exist = true + # check there is some error or cancel + if stop_status == "FINISHED" and + (job.status == "ERROR" or job.status == "CANCELED") then + + # cancel all other un-finished jobs + @sub_jobs.each do |sub| + if sub.status != "ERROR" and sub.status != "FINISHED" and + sub.status != "CANCELED" and sub.cancel_state == "NONE" then + @log.info(" * Sub-Job \"#{sub.get_project().name}(#{sub.os})\" has entered \"CANCELING\" state. (#{sub.id})", Log::LV_USER) + sub.cancel_state = "INIT" + end + end + + stop_status = job.status break end end + + # sleep 1 end - if @cancel_state != "NONE" then - @status = "CANCELED" - if not @log.nil? then - @log.info( "JOB is canceled by cancel operation !!", Log::LV_USER) - end - else - # check error - if error_exist then - @sub_jobs.each do |sub| - if sub.status != "ERROR" and sub.status != "FINISHED" and sub.cancel_state == "NONE" then - sub.cancel_state = "INIT" - end - end - @status = "ERROR" - return - end - - # upload - if not upload() then - @status = "ERROR" - return - end + if stop_status == "ERROR" or stop_status == "CANCELED" then + @status = stop_status + return + end - # INFO. don't change this string - @log.info( "Job is completed!", Log::LV_USER) - @status = "FINISHED" + # upload + if not upload() then + @status = "ERROR" + return end + + # INFO. don't change this string + @log.info( "Job is completed!", Log::LV_USER) + @status = "FINISHED" end diff --git a/src/build_server/RegisterPackageJob.rb b/src/build_server/RegisterPackageJob.rb index 28c0b1c..ef300ae 100644 --- a/src/build_server/RegisterPackageJob.rb +++ b/src/build_server/RegisterPackageJob.rb @@ -91,6 +91,15 @@ class RegisterPackageJob return @project end + + def get_buildroot() + return @buildroot_dir + end + + def get_parent_job() + return nil + end + # execute def execute(sync=false) @log.info( "Invoking a thread for REGISTER Job #{@id}", Log::LV_USER) diff --git a/src/build_server/RemoteBuilder.rb b/src/build_server/RemoteBuilder.rb index 13ca041..0ba969a 100644 --- a/src/build_server/RemoteBuilder.rb +++ b/src/build_server/RemoteBuilder.rb @@ -133,7 +133,8 @@ class RemoteBuilder if client.send("BUILD|GIT|#{project_name}|#{project_passwd}|#{os}|NO|YES|#{rev}|#{commit}|#{pkg_list}") then result = client.read_lines do |l| # check build result - if l.include? "Job is stopped by ERROR" then + if l.include? "Job is stopped by ERROR" or + l.include? "Error:" then result = false break end diff --git a/src/build_server/ReverseBuildChecker.rb b/src/build_server/ReverseBuildChecker.rb index f8415b5..bd2d344 100644 --- a/src/build_server/ReverseBuildChecker.rb +++ b/src/build_server/ReverseBuildChecker.rb @@ -87,13 +87,13 @@ class ReverseBuildChecker # if job on resolve process, its unresolved project #of pending ancestor must be excluded. - if not job.pending_ancestor.nil? then + if job.type == "BUILD" and not job.pending_ancestor.nil? then found = false job.pending_ancestor.rev_fail_projects.each { |fp| f_prj = fp[0] f_os = fp[1] - if rev_prj == f_prj and rev_os == os then + if rev_prj == f_prj and rev_os == f_os then found = true break end @@ -128,10 +128,11 @@ class ReverseBuildChecker end # for all reverse job - for new_job in rev_build_jobs + for rev_job in rev_build_jobs # add to job manager - job.server.jobmgr.add_reverse_build_job(new_job) - log.info( " * Added new job for reverse-build ... #{rev_prj.name}(#{rev_os}) (#{new_job.id})", Log::LV_USER) + job.server.jobmgr.add_reverse_build_job(rev_job) + log.info( " * Added new job for reverse-build ... \ + #{rev_job.get_project().name}(#{rev_job.os}) (#{rev_job.id})", Log::LV_USER) end # wait for job finish @@ -153,11 +154,16 @@ class ReverseBuildChecker failure_list.push [ rev_prj, rev_os ] end + # if "exist on error" cancel all other jobs if exit_on_error then - # cancel all other jobs cancel_other_jobs = true + for j in rev_build_jobs + if j.status != "ERROR" and j.status != "FINISHED" and + j.status != "CANCELED" and j.cancel_state == "NONE" then - # exit + j.cancel_state = "INIT" + end + end break end when "FINISHED" @@ -167,17 +173,22 @@ class ReverseBuildChecker success_list.push rev_job end else - if exit_on_error and cancel_other_jobs then - rev_job.cancel_state = "INIT" - else - rev_build_finished = false - end + rev_build_finished = false end end sleep 1 end + # clean up all reverse build jobs + for rev_job in rev_build_jobs + if rev_job.status == "ERROR" or rev_job.status == "CANCELED" then + rev_job.server.cleaner.clean_afterwards(rev_job.id) + else + rev_job.server.cleaner.clean(rev_job.id) + end + end + return failure_list end diff --git a/src/build_server/SocketJobRequestListener.rb b/src/build_server/SocketJobRequestListener.rb index 2dcd400..b04494f 100644 --- a/src/build_server/SocketJobRequestListener.rb +++ b/src/build_server/SocketJobRequestListener.rb @@ -175,6 +175,14 @@ class SocketJobRequestListener if os_list.nil? or os_list.empty? then raise "Unsupported OS name is used!" end + + # check project type + if prj.type == "BINARY" then + BuildCommServer.send_begin(req) + req.puts "Can't build about Binary type package." + BuildCommServer.send_end(req) + raise "Can't build about Binary type package." + end # create new job for os in os_list @@ -342,7 +350,7 @@ class SocketJobRequestListener #puts "Received QUERY JOB" # gather all jobs to show - job_list = @parent_server.jobmgr.jobs + @parent_server.jobmgr.internal_jobs + job_list = @parent_server.jobmgr.jobs + @parent_server.jobmgr.internal_jobs + @parent_server.jobmgr.reverse_build_jobs # send the status BuildCommServer.send_begin(req) diff --git a/src/builder/Builder.rb b/src/builder/Builder.rb index c807403..9fa01a5 100644 --- a/src/builder/Builder.rb +++ b/src/builder/Builder.rb @@ -220,7 +220,7 @@ class Builder else for l in local_dep_pkgs @log.info( "Installing local pacakge...#{l}", Log::LV_USER) - cl.install_local_pkg(l,false) + cl.install_local_pkg(l,true,false) end end end @@ -424,7 +424,7 @@ class Builder # write pkginfo.manifest def write_pkginfo_files(pkginfo,os,src_path) - # get category + # get category os_category = Utils.get_os_category( os ) for pkg in pkginfo.packages @@ -433,9 +433,9 @@ class Builder next end - # install script files - copy_post_install_script(pkg,os,src_path); - copy_post_remove_script(pkg,os,src_path); + # install/remove script files + if not copy_post_install_script(pkg,os,src_path) then return false end + if not copy_post_remove_script(pkg,os,src_path) then return false end # write manifest file install_dir = "#{src_path}/package/#{pkg.package_name}.package.#{os}" @@ -443,7 +443,7 @@ class Builder # if there is no intall directory, error if not File.exist? install_dir then install_dir = "#{src_path}/package/#{pkg.package_name}.package.#{os_category}" - + if not File.exist? install_dir then @log.error( "Following directory must be created before writing pkginfo.manifest", Log::LV_USER) @log.error( " * package/#{pkg.package_name}.package.#{os}", Log::LV_USER) @@ -463,19 +463,27 @@ class Builder # copy post-install script def copy_post_install_script(pkg,os,src_path) - + tar = nil + src = nil - # get category - os_category = Utils.get_os_category( os ) + # get category + os_category_list = [] + pkg.os_list.each do |cos| + os_category_list.push Utils.get_os_category(cos) + end - if File.exist? "#{src_path}/package/#{pkg.package_name}.install.#{os}" - src = "#{src_path}/package/#{pkg.package_name}.install.#{os}" - else - if File.exist? "#{src_path}/package/#{pkg.package_name}.install.#{os_category}" - src = "#{src_path}/package/#{pkg.package_name}.install.#{os_category}" - else - src = nil + # check compatable os + (pkg.os_list + os_category_list).uniq.each do |cos| + if File.exist? "#{src_path}/package/#{pkg.package_name}.install.#{cos}" then + if src.nil? then + src = "#{src_path}/package/#{pkg.package_name}.install.#{cos}" + else + @log.error( "compatable package can have only one install script\n but you have another availabe install scripts", Log::LV_USER) + @log.error( " * package/#{File.basename src}", Log::LV_USER) + @log.error( " * package/#{pkg.package_name}.install.#{cos}", Log::LV_USER) + return false + end end end @@ -483,8 +491,9 @@ class Builder # if there is no intall directory, error if not File.exist? install_dir then + os_category = Utils.get_os_category( os ) install_dir = "#{src_path}/package/#{pkg.package_name}.package.#{os_category}" - + if not File.exist? install_dir then @log.error( "Following directory must be created before writing pkginfo.manifest", Log::LV_USER) @log.error( " * package/#{pkg.package_name}.package.#{os}", Log::LV_USER) @@ -499,31 +508,39 @@ class Builder tar = "#{install_dir}/install.BAT" else puts "Unknown OS: #{os} " - return + return true end FileUtils.cp(src, tar) - end - - return + end + + return true end # copy post-remove script def copy_post_remove_script(pkg,os,src_path) - + tar = nil + src = nil - # get category - os_category = Utils.get_os_category( os ) + # get category + os_category_list = [] + pkg.os_list.each do |cos| + os_category_list.push Utils.get_os_category(cos) + end - if File.exist? "#{src_path}/package/#{pkg.package_name}.remove.#{os}" - src = "#{src_path}/package/#{pkg.package_name}.remove.#{os}" - else - if File.exist? "#{src_path}/package/#{pkg.package_name}.remove.#{os_category}" - src = "#{src_path}/package/#{pkg.package_name}.remove.#{os_category}" - else - src = nil + # check compatable os + (pkg.os_list + os_category_list).uniq.each do |cos| + if File.exist? "#{src_path}/package/#{pkg.package_name}.remove.#{cos}" then + if src.nil? then + src = "#{src_path}/package/#{pkg.package_name}.remove.#{cos}" + else + @log.error( "compatable package can have only one remove script but you have another availabe remove scripts", Log::LV_USER) + @log.error( " * package/#{File.basename src}", Log::LV_USER) + @log.error( " * package/#{pkg.package_name}.remove.#{cos}", Log::LV_USER) + return false + end end end @@ -531,8 +548,9 @@ class Builder # if there is no intall directory, error if not File.exist? install_dir then + os_category = Utils.get_os_category( os ) install_dir = "#{src_path}/package/#{pkg.package_name}.package.#{os_category}" - + if not File.exist? install_dir then @log.error( "Following directory must be created before writing pkginfo.manifest", Log::LV_USER) @log.error( " * package/#{pkg.package_name}.package.#{os}", Log::LV_USER) @@ -547,11 +565,12 @@ class Builder tar = "#{install_dir}/remove.BAT" else puts "Unknown OS: #{os} " - return + return true end FileUtils.cp(src, tar) - end + end + return true end diff --git a/src/common/ScheduledActionHandler.rb b/src/common/ScheduledActionHandler.rb index 2a0c18a..9a51cd0 100644 --- a/src/common/ScheduledActionHandler.rb +++ b/src/common/ScheduledActionHandler.rb @@ -85,8 +85,8 @@ class ScheduledActionHandler end end - # sleep 1 min - sleep 5 + # sleep 10 sec + sleep 10 end end diff --git a/src/common/fileTransfer.rb b/src/common/fileTransfer.rb index 4c6c85d..32112c5 100644 --- a/src/common/fileTransfer.rb +++ b/src/common/fileTransfer.rb @@ -3,13 +3,7 @@ require 'socket' class FileTransfer - @@log = nil - - def FileTransfer.set_logger(logger) - @@log = logger - end - - def FileTransfer.putfile(ip, port, username, passwd, bpath) + def FileTransfer.putfile(ip, port, username, passwd, bpath, logger) filename = File.basename(bpath) uniqdir = Utils.create_uniq_name ftp_filepath = File.join(uniqdir, filename) @@ -21,31 +15,31 @@ class FileTransfer else ftp.connect(ip, port) end - @@log.info "[FTP log] Connected FTP server (#{ip}:#{port})" + logger.info "[FTP log] Connected FTP server (#{ip}:#{port})" ftp.login(username, passwd) ftp.binary = true ftp.mkdir(uniqdir) ftp.chdir(uniqdir) ftp.put(bpath) - @@log.info "[FTP log] Put a file" - @@log.info "[FTP log] from \"#{bpath}\" to \"#{ftp_filepath}\"" + logger.info "[FTP log] Put a file" + logger.info "[FTP log] from \"#{bpath}\" to \"#{ftp_filepath}\"" files = ftp.list(filename) if files.empty? then - @@log.error "[FTP log] Failed to upload file (#{filename} does not exist)" + logger.error "[FTP log] Failed to upload file (#{filename} does not exist)" return nil end ftp.quit - @@log.info "[FTP log] Disconnected FTP server" + logger.info "[FTP log] Disconnected FTP server" rescue => e - @@log.error "[FTP log] Exception" - @@log.error e.message - @@log.error e.backtrace.inspect + logger.error "[FTP log] Exception" + logger.error e.message + logger.error e.backtrace.inspect return nil end return ftp_filepath end - def FileTransfer.getfile(ip, port, username, passwd, bpath, target) + def FileTransfer.getfile(ip, port, username, passwd, bpath, target, logger) dirname = File.dirname(bpath) filename = File.basename(bpath) @@ -63,29 +57,29 @@ class FileTransfer else ftp.connect(ip, port) end - @@log.info "[FTP log] Connected FTP server (#{ip}:#{port})" + logger.info "[FTP log] Connected FTP server (#{ip}:#{port})" ftp.login(username, passwd) ftp.binary = true ftp.chdir(dirname) ftp.get(filename, dst_file) - @@log.info "[FTP log] Get a file" - @@log.info "[FTP log] from \"#{bpath}\" to \"#{dst_file}\"" + logger.info "[FTP log] Get a file" + logger.info "[FTP log] from \"#{bpath}\" to \"#{dst_file}\"" ftp.quit - @@log.info "[FTP log] Disconnected FTP server" + logger.info "[FTP log] Disconnected FTP server" rescue => e - @@log.error "[FTP log] Exception" - @@log.error e.message - @@log.error e.backtrace.inspect + logger.error "[FTP log] Exception" + logger.error e.message + logger.error e.backtrace.inspect return nil end if not File.exist? dst_file then - @@log.error "[FTP log] Failed to download file (#{dst_file} does not exist)" + logger.error "[FTP log] Failed to download file (#{dst_file} does not exist)" return nil end return bpath end - def FileTransfer.cleandir(ip, port, username, passwd, path) + def FileTransfer.cleandir(ip, port, username, passwd, path, logger) dirname = File.dirname(path) begin @@ -95,7 +89,7 @@ class FileTransfer else ftp.connect(ip, port) end - @@log.info "[FTP log] Connected FTP server (#{ip}:#{port})" + logger.info "[FTP log] Connected FTP server (#{ip}:#{port})" ftp.login(username, passwd) old_dir = ftp.pwd ftp.chdir(dirname) @@ -107,13 +101,13 @@ class FileTransfer end ftp.chdir(old_dir) ftp.rmdir(dirname) - @@log.info "[FTP log] Clean dir (#{dirname})" + logger.info "[FTP log] Clean dir (#{dirname})" ftp.quit - @@log.info "[FTP log] Disconnected FTP server" + logger.info "[FTP log] Disconnected FTP server" rescue => e - @@log.error "[FTP log] Exception" - @@log.error e.message - @@log.error e.backtrace.inspect + logger.error "[FTP log] Exception" + logger.error e.message + logger.error e.backtrace.inspect return nil end diff --git a/src/pkg_server/client.rb b/src/pkg_server/client.rb index d9bd6ba..d353aee 100644 --- a/src/pkg_server/client.rb +++ b/src/pkg_server/client.rb @@ -42,6 +42,8 @@ require "log" require "Version" require "net/ftp" $update_mutex = Mutex.new +$update_os_mutex = Mutex.new +$install_pkg_mutex = Mutex.new class Client # constant @@ -53,7 +55,7 @@ class Client DEFAULT_SERVER_ADDR = "http://172.21.17.55/dibs/unstable" OS_INFO_FILE = "os_info" - attr_accessor :server_addr, :location, :pkg_hash_os, :is_server_remote, :installed_pkg_hash_loc, :archive_pkg_list, :all_dep_list, :log, :support_os_list, :config_dist_path + attr_accessor :server_addr, :location, :pkg_hash_os, :is_server_remote, :installed_pkg_hash_loc, :archive_pkg_list, :all_dep_list, :log, :support_os_list, :config_dist_path, :download_path public # initialize @@ -77,10 +79,12 @@ class Client @all_dep_list = [] @is_server_remote = Utils.is_url_remote(server_addr) @support_os_list = [] - @config_dist_path = CONFIG_PATH + "/" + get_distribution + @config_dist_path = CONFIG_PATH + "/" + get_flat_serveraddr + @download_path = @config_dist_path + "/downloads" # create directory if not File.exist? @config_dist_path then FileUtils.mkdir_p "#{@config_dist_path}" end + if not File.exist? @download_path then FileUtils.mkdir_p "#{@download_path}" end # chop server address, if end with "/" if server_addr.strip.end_with? "/" then server_addr = server_addr.chop end @@ -92,10 +96,6 @@ class Client @log = logger end - FileInstaller.set_logger(@log) - FileDownLoader.set_logger(@log) - FileTransfer.set_logger(@log) - # read installed pkg list, and create hash if not File.exist? @location then FileUtils.mkdir_p "#{@location}" end create_installed_pkg_hash() @@ -138,34 +138,48 @@ class Client end else dependent_pkg_list = [pkg_name] end - surl = nil - addr_arr = @server_addr.split('/') - if addr_arr[-2].eql? "snapshots" then + surl = @server_addr + if is_snapshot_url then surl = @server_addr + "/../.." - else - surl = @server_addr - end + end # download files file_local_path = [] dependent_pkg_list.each do |p| + pkg_name = get_attr_from_pkg(p, os, "name") pkg_path = get_attr_from_pkg(p, os, "path") pkg_ver = get_attr_from_pkg(p, os, "version") + pkg_checksum = get_attr_from_pkg(p, os, "checksum") + pkg_size = get_attr_from_pkg(p, os, "size") if pkg_path.nil? or pkg_ver.nil? then @log.error "\"#{p}\" package does not exist in package server. If it exist in package server, then try \"pkg-cli update\"" return nil end url = surl + pkg_path filename = pkg_path.split('/')[-1] - if not FileDownLoader.download(url, @location) then + + cached_filepath = nil + if Utils.is_linux_like_os( Utils::HOST_OS ) then + cached_filepath = get_cached_filepath(filename, pkg_checksum, pkg_size, os) + if not cached_filepath.nil? then url = cached_filepath end + end + + if not FileDownLoader.download(url, @location, @log) then @log.error "Failed download #{pkg_name} [#{pkg_ver}]" return nil end file_path = File.join(@location, filename) file_local_path.push(file_path) + + if Utils.is_linux_like_os( Utils::HOST_OS ) then + if cached_filepath.nil? then + remove_downloaded_pkgs(pkg_name) + FileUtils.cp(file_path, @download_path) + end + end + @log.info "Downloaded \"#{p} [#{pkg_ver}]\" package file.. OK" - #@log.info " [path : #{file_path}]" end if trace then @@ -176,40 +190,32 @@ class Client @log.info " [path : #{file_local_path.join(", ")}]" return file_local_path - end - - public - # download source package - def download_source(pkg_name, os) + end - # get source file path - src_path = get_attr_from_pkg(pkg_name, os, "src_path") - if src_path.nil? or src_path.empty? then - @log.error "#{pkg_name} package does not have source" - return nil - end - file_url = nil + private + def remove_downloaded_pkgs(pkg_name) + pkg_file_prefix = "#{@download_path}/#{pkg_name}_*" + pkg_files = Dir.glob(pkg_file_prefix) - addr_arr = @server_addr.split('/') - if addr_arr[-2].eql? "snapshots" then - surl = @server_addr + "/../.." + src_path - else - surl = @server_addr + src_path - end + if not pkg_files.nil? then + Utils.execute_shell("rm -f #{pkg_file_prefix}") + end + end - # download file - filename = src_path.split('/')[-1] - if not FileDownLoader.download(surl, @location) then - @log.error "Failed download #{pkg_name} source" - return nil - end - file_local_path = File.join(@location, filename) - @log.info "Downloaded source of #{pkg_name} package.. OK" - @log.info " [path : #{file_local_path}]" + private + def get_cached_filepath(pkg_filename, pkg_checksum, pkg_size, os) + + cached_filepath = "#{@download_path}/#{pkg_filename}" + if File.exist? cached_filepath then + checksum = `sha256sum #{cached_filepath}`.split(" ")[0] + size = `du -b #{cached_filepath}`.split[0].strip + if checksum.eql? pkg_checksum and size.eql? pkg_size then + return cached_filepath + end + end + return nil + end - return file_local_path - end - public # download dependent source def download_dep_source(file_name) @@ -222,7 +228,7 @@ class Client else file_url = @server_addr + "/source/#{file_name}" end - if not FileDownLoader.download(file_url, @location) then + if not FileDownLoader.download(file_url, @location, @log) then @log.error "Failed download #{file_name}" return nil end @@ -352,26 +358,22 @@ class Client path = Utils::HOME + "/tmp/#{uniq_name}" if not File.exist? path then FileUtils.mkdir_p "#{path}" end begin - if not FileInstaller.extract_a_file(pkg_path, manifest_file, path) then + if not FileInstaller.extract_a_file(pkg_path, manifest_file, path, @log) then @log.error "The \"pkginfo.manifest\" file does not exist in \"#{pkg_path}\"" return false end manifest_path = File.join(path, manifest_file) pkg = Parser.read_single_pkginfo_from manifest_path - FileUtils.rm_f(manifest_path) + if File.exists? manifest_path then FileUtils.rm_f(manifest_path) end FileUtils.remove_dir(path, true) rescue Interrupt @log.error "Client: Interrupted.." - FileUtils.rm_f(manifest_path) FileUtils.remove_dir(path, true) - @log.info "Removed #{manifest_path}" @log.info "Removed #{path}" raise Interrupt rescue RuntimeError => e @log.error( e.message, Log::LV_USER) - FileUtils.rm_f(manifest_path) FileUtils.remove_dir(path, true) - @log.info "Removed #{mainfest_path}" @log.info "Removed #{path}" return false end @@ -431,13 +433,37 @@ class Client else dist = File.basename(server) end return dist - end + end + + private + def get_flat_serveraddr() + server = @server_addr + if server.nil? or server.empty? then + @log.error "Server addr is nil" + return nil + end + + server = server.delete ".:/@" + return server + end public # install package # install all install dependency packages def install(pkg_name, os, trace, force) + ret = true + $install_pkg_mutex.synchronize { + ret = install_internal( pkg_name, os, trace, force ) + } + + return ret + end + + + private + def install_internal(pkg_name, os, trace, force) + if trace.nil? then trace = true end if force.nil? then force = false end @@ -451,43 +477,44 @@ class Client @log.error "#{pkg_name} package does not exist in remote package list" return false end + compare_result = compare_version_with_installed_pkg(pkg_name, pkg_ver) if not force then - case compare_result - when -1 then - @log.warn "\"#{pkg_name}\" package version is bigger then remote package version" - return true - when 0 then - @log.warn "\"#{pkg_name}\" package version is same with remote package version" - return true - when 1, 2 then - end - end - - # if enable trace, create all dependent package list - if trace then - dependent_pkg_list = get_install_dependent_packages(pkg_name, os, true, force) - if dependent_pkg_list.nil? then - @log.error "Failed to get dependency for \"#{pkg_name}\" package" - return false - end - else - dependent_pkg_list = [pkg_name] - end - - # TODO: need to compare dependent package version - # install packages including dependent packages - dependent_pkg_list.each do |pkg| - if not install_pkg(pkg, os, force) then - @log.error "#{pkg} does not exist" - return false - end - add_pkg_info(pkg, os) - end - - # write installed package information to file - write_pkg_hash_to_file(nil) + case compare_result + when -1 then + @log.warn "\"#{pkg_name}\" package version is bigger then remote package version" + return true + when 0 then + @log.warn "\"#{pkg_name}\" package version is same with remote package version" + return true + when 1, 2 then + end + end + + # if enable trace, create all dependent package list + if trace then + dependent_pkg_list = get_install_dependent_packages(pkg_name, os, true, force) + if dependent_pkg_list.nil? then + @log.error "Failed to get dependency for \"#{pkg_name}\" package" + return false + end + else + dependent_pkg_list = [pkg_name] + end + + # TODO: need to compare dependent package version + # install packages including dependent packages + dependent_pkg_list.each do |pkg| + if not install_pkg(pkg, os, force) then + @log.error "#{pkg} does not exist" + return false + end + add_pkg_info(pkg, os) + end + + # write installed package information to file + write_pkg_hash_to_file(nil) if trace then @log.info "Installed \"#{pkg_name} [#{pkg_ver}]\" package with all dependent packages.. OK" @@ -495,12 +522,26 @@ class Client else @log.info "Install only \"#{pkg_name} [#{pkg_ver}]\" package.. OK" end + return true end + public # install local package (ignore dependent packages) - def install_local_pkg(pkg_path, force) + def install_local_pkg(pkg_path, trace, force) + + ret = true + $install_pkg_mutex.synchronize { + ret = install_local_pkg_internal(pkg_path, trace, force) + } + + return ret + end + + + private + def install_local_pkg_internal(pkg_path, trace, force) file_name = File.basename(pkg_path) pkg_name = file_name.split('_')[0] @@ -522,60 +563,66 @@ class Client uniq_name = Utils.create_uniq_name path = Utils::HOME + "/tmp/#{uniq_name}" - if not File.exist? path then FileUtils.mkdir_p "#{path}" end + if not File.exist? path then FileUtils.mkdir_p "#{path}" end begin - if not FileInstaller.extract_a_file(pkg_path, manifest_file, path) then + if not FileInstaller.extract_a_file(pkg_path, manifest_file, path, @log) then @log.error "pkginfo.manifest file does not exist in #{pkg_path}" return false end manifest_path = File.join(path, manifest_file) pkg = Parser.read_single_pkginfo_from manifest_path new_pkg_ver = pkg.version - FileUtils.rm_f(manifest_path) FileUtils.remove_dir(path, true) rescue Interrupt @log.error "Client: Interrupted.." - FileUtils.rm_f(manifest_path) FileUtils.remove_dir(path, true) - @log.info "Removed #{mainfest_path}" @log.info "Removed #{path}" raise Interrupt rescue RuntimeError => e @log.error( e.message, Log::LV_USER) - FileUtils.rm_f(manifest_path) - FileUtils.remove_dir(path, true) - @log.info "Removed #{mainfest_path}" + FileUtils.remove_dir(path, true) @log.info "Removed #{path}" return false end - compare_result = compare_version_with_installed_pkg(pkg_name, new_pkg_ver) - if not force then - case compare_result - when -1 then - @log.warn "\"#{pkg_name}\" package version is bigger then remote package version.." - return true - when 0 then - @log.warn "\"#{pkg_name}\" package version is same with remote package version.." - return true - when 1, 2 then - end - end + compare_result = compare_version_with_installed_pkg(pkg_name, new_pkg_ver) + if not force then + case compare_result + when -1 then + @log.warn "installed \"#{pkg_name}\" package version is bigger.." + return true + when 0 then + @log.warn "\"#{pkg_name}\" package version is same with installed package version.." + return true + when 1, 2 then + end + end + + if check_installed_pkg(pkg_name) then + uninstall(pkg_name, false) + end - if check_installed_pkg(pkg_name) then - uninstall(pkg_name, false) - end + if trace then + install_dep_pkgs = pkg.install_dep_list + new_pkg_os = pkg.os + install_dep_pkgs.each do |pkg| + if not install_internal(pkg.package_name, new_pkg_os, true, false) then + @log.warn "#{pkg} package is not installed" + end + end + end - # install package - ret = FileInstaller.install(pkg_name, pkg_path, "binary", @location) + # install package + ret = FileInstaller.install(pkg_name, pkg_path, "binary", @location, @log) - add_local_pkg_info(pkg_name) - write_pkg_hash_to_file(nil) + add_local_pkg_info(pkg_name) + write_pkg_hash_to_file(nil) @log.info "Installed \"#{pkg_path} [#{new_pkg_ver}]\" file.. OK" return true end + public # upgrade package def upgrade(os, trace) @@ -596,7 +643,7 @@ class Client end end - if not install(p, os, trace, false) then + if not install_internal(p, os, trace, false) then @log.error "Failed to install \"#{p}\" package.." return false end @@ -719,7 +766,7 @@ class Client pkg_list.each do |p| if not check_installed_pkg(p) then next end - if not FileInstaller.uninstall(p, type, @location) then + if not FileInstaller.uninstall(p, type, @location, @log) then @log.error "Failed uninstall \"#{pkg_name}\" package" return false end @@ -753,7 +800,7 @@ class Client if File.exist? @location then FileUtils.rm_rf(@location) end FileUtils.mkdir_p(@location) #@pkg_hash_os.clear - #@installed_pkg_hash_loc.clear + @installed_pkg_hash_loc.clear #@archive_pkg_list.clear @log.info "Cleaned \"#{@location}\" path.. OK" end @@ -967,7 +1014,7 @@ class Client if pkg_hash.nil? then return false end pkg = pkg_hash[pkg_name] if pkg.nil? then - @log.warn "There is no \"#{pkg_name}\" remote package information in list" + #@log.warn "There is no \"#{pkg_name}\" remote package information in list" return false end @@ -1017,6 +1064,7 @@ class Client if pkg.nil? then return nil end case attr + when "name" then return pkg.package_name when "path" then return pkg.path when "source" then return pkg.source when "version" then return pkg.version @@ -1025,6 +1073,9 @@ class Client when "build_dep_list" then return pkg.build_dep_list when "install_dep_list" then return pkg.install_dep_list when "attribute" then return pkg.attribute + when "checksum" then return pkg.checksum + when "size" then return pkg.size + end end @@ -1259,14 +1310,11 @@ class Client end # install package - ret = FileInstaller.install(pkg_name, file_local_path, type, @location) - FileUtils.rm_f(file_local_path) + ret = FileInstaller.install(pkg_name, file_local_path, type, @location, @log) FileUtils.remove_dir(tmppath, true) rescue Interrupt @log.error "Client: Interrupted.." - FileUtils.rm_f(file_local_path) FileUtils.remove_dir(tmppath, true) - @log.info "Removed #{file_local_path}" @log.info "Removed #{tmppath}" raise Interrupt end @@ -1361,40 +1409,97 @@ class Client return nil end - @log.info "Added information for \"#{pkg_name}\" package.. OK" + @log.info "Read information for \"#{pkg_name}\" package.. OK" return pkg end private def get_remote_pkg_os_list() - file_url = @server_addr + "/" + OS_INFO_FILE - if(is_server_remote) then - FileDownLoader.download(file_url, @config_dist_path) - File.open( "#{@config_dist_path}/#{OS_INFO_FILE}", "r" ) do |f| - f.each_line do |l| - @support_os_list.push l.strip - end - end - else - if File.exist?(file_url) then - File.open(file_url) do |f| + file_url = @server_addr + "/" + OS_INFO_FILE + if is_snapshot_url then + file_url = @server_addr + "/../../" + OS_INFO_FILE + end + + $update_os_mutex.synchronize { + if(is_server_remote) then + FileDownLoader.download(file_url, @config_dist_path, @log) + File.open( "#{@config_dist_path}/#{OS_INFO_FILE}", "r" ) do |f| f.each_line do |l| @support_os_list.push l.strip end end + else + if File.exist?(file_url) then + File.open(file_url) do |f| + f.each_line do |l| + @support_os_list.push l.strip + end + end + end end - end + } end + # get the lastest snapshot + # from_server : if true, update from server + def get_lastest_snapshot(from_server) + ssinfo_file = "snapshot.info" + file_url = @server_addr + "/" + ssinfo_file + if from_server then + if not FileDownLoader.download(file_url, @config_dist_path, @log) then + @log.warn "Server does not have \"#{ssinfo_file}\" file. This error can be ignored." + return nil + end + else + FileUtils.cp(file_url, @config_dist_path) + end + + file_path = File.join(@config_dist_path, ssinfo_file) + contents = File.open(file_path, "r").read + + _list = contents.split("\n\n") + if _list.nil? or _list == "" or _list.empty? then return nil end + list = _list[-1].split("\n") + if list.nil? or list == "" or list.empty? then return nil end + _path = list[-1].split(":") + if _path.nil? or _path == "" or _path.length != 2 then return nil end + path = _path[1].strip + if path == nil or path == "" then return nil end + + return path + end + + # if url includes snapshot infomation, retuen true + def is_snapshot_url() + addr_arr = @server_addr.split('/') + if addr_arr[-2].eql? "snapshots" then + return true + else + return false + end + end + # from_server : if true, update from server def create_remote_pkg_hash(from_server) + + snapshot_path = "" + + if not is_snapshot_url then + snapshot_path = get_lastest_snapshot(from_server) + @log.info "The lastest snapshot : #{snapshot_path}" + if snapshot_path.nil? then + @log.warn "Failed to get the lastest package list" + snapshot_path = "" + end + end + for os in @support_os_list filename = PKG_LIST_FILE_PREFIX + os - file_url = @server_addr + "/" + filename + file_url = @server_addr + snapshot_path + "/" + filename local_file_path = File.join(@config_dist_path, filename) if from_server then - if not FileDownLoader.download(file_url, @config_dist_path) then + if not FileDownLoader.download(file_url, @config_dist_path, @log) then return false end else @@ -1415,9 +1520,9 @@ class Client end filename = "archive_pkg_list" - file_url = @server_addr + "/" + filename + file_url = @server_addr + snapshot_path + "/" + filename if from_server then - if not FileDownLoader.download(file_url, @config_dist_path) then + if not FileDownLoader.download(file_url, @config_dist_path, @log) then @log.warn "Server does not have \"#{filename}\" file. This error can be ignored." end else diff --git a/src/pkg_server/clientOptParser.rb b/src/pkg_server/clientOptParser.rb index 1290059..0e985c3 100644 --- a/src/pkg_server/clientOptParser.rb +++ b/src/pkg_server/clientOptParser.rb @@ -60,7 +60,7 @@ def option_error_check( options ) when "install-file" then if options[:pkg].nil? or options[:pkg].empty? then - raise ArgumentError, "Usage: pkg-cli install-lpkg -P [-l ] [--force]" + raise ArgumentError, "Usage: pkg-cli install-lpkg -P [-l ] [-u ] [--trace] [--force]" end when "uninstall" then @@ -121,7 +121,7 @@ def option_parse + "\t" + "pkg-cli clean [-l ] [--force]" + "\n" \ + "\t" + "pkg-cli download -P [-o ] [-l ] [-u ] [--trace]" + "\n" \ + "\t" + "pkg-cli install -P [-o ] [-l ] [-u ] [--trace] [--force]" + "\n" \ - + "\t" + "pkg-cli install-file -P [-l ] [--force]" + "\n" \ + + "\t" + "pkg-cli install-file -P [-l ] [-u ] [--trace] [--force]" + "\n" \ + "\t" + "pkg-cli uninstall -P [-l ] [--trace]" + "\n" \ + "\t" + "pkg-cli upgrade [-l ] [-o ] [-u ] [--trace]" + "\n" \ + "\t" + "pkg-cli check-upgrade [-l ] [-o ] [-u ]" + "\n" \ diff --git a/src/pkg_server/distribution.rb b/src/pkg_server/distribution.rb index eee5b18..5c6b9e2 100644 --- a/src/pkg_server/distribution.rb +++ b/src/pkg_server/distribution.rb @@ -332,13 +332,11 @@ class Distribution def get_package_from_file(file_path) tmp_dir = @location + "/" + Utils.create_uniq_name - FileInstaller.set_logger(@log) - #if file extension is .zip then check pkginfo.manifest if File.extname(file_path).eql? ".zip" then FileUtils.mkdir tmp_dir - ret = FileInstaller.extract_a_file(file_path, "pkginfo.manifest", tmp_dir) + ret = FileInstaller.extract_a_file(file_path, "pkginfo.manifest", tmp_dir, @log) else return nil end diff --git a/src/pkg_server/downloader.rb b/src/pkg_server/downloader.rb index 17288fb..98c8121 100644 --- a/src/pkg_server/downloader.rb +++ b/src/pkg_server/downloader.rb @@ -31,30 +31,24 @@ require "utils" class FileDownLoader - @@log = nil - - def FileDownLoader.set_logger(logger) - @@log = logger - end - - def FileDownLoader.download(url, path) + def FileDownLoader.download(url, path, logger) ret = false if not File.directory? path then - @@log.error "\"#{path}\" does not exist" + logger.error "\"#{path}\" does not exist" return ret - end - + end + is_remote = Utils.is_url_remote(url) filename = url.split('/')[-1] fullpath = File.join(path, filename) if is_remote then - ret = Utils.execute_shell_with_log( "wget #{url} -O #{fullpath} -nv", @@log ) + ret = Utils.execute_shell_with_log( "wget #{url} -O #{fullpath} -nv", logger ) else if not File.exist? url then - @@log.error "\"#{url}\" file does not exist" + logger.error "\"#{url}\" file does not exist" return false else ret = system "cp #{url} #{fullpath}" diff --git a/src/pkg_server/installer.rb b/src/pkg_server/installer.rb index 49c7d28..589d1be 100644 --- a/src/pkg_server/installer.rb +++ b/src/pkg_server/installer.rb @@ -41,16 +41,10 @@ class FileInstaller CONFIG_PATH = "#{PackageServerConfig::CONFIG_ROOT}/client" PACKAGE_INFO_DIR = ".info" - @@log = nil - - def FileInstaller.set_logger(logger) - @@log = logger - end - - def FileInstaller.install(package_name, package_file_path, type, target_path) + def FileInstaller.install(package_name, package_file_path, type, target_path, logger) if not File.exist? package_file_path then - @@log.error "\"#{package_file_path}\" file does not exist." + logger.error "\"#{package_file_path}\" file does not exist." return false end @@ -75,11 +69,11 @@ class FileInstaller begin log = log + "##### extract file : #{package_file_path} #####\n" - log = log + extract_file(package_name, package_file_path, path, target_path) + log = log + extract_file(package_name, package_file_path, path, target_path, logger) move_dir(package_name, path, target_path) log = log + "##### execute install script #####\n" - log = log + execute_install_script(package_name, path, target_path) + log = log + execute_install_script(package_name, path, target_path, logger) log = log + "##### move remove script #####\n" move_remove_script(package_name, path, target_path) @@ -87,9 +81,9 @@ class FileInstaller log = log + "##### remove temporary dir : #{path} #####\n" Utils.execute_shell("rm -rf #{path}") rescue Interrupt - @@log.error "FileInstaller: Interrupted.." + logger.error "FileInstaller: Interrupted.." Utils.execute_shell("rm -rf #{path}") - @@log.info "Removed #{path}" + logger.info "Removed #{path}" raise Interrupt end @@ -121,13 +115,13 @@ class FileInstaller end - def FileInstaller.execute_install_script(package_name, path, target_path) + def FileInstaller.execute_install_script(package_name, path, target_path, logger) script_file_prefix = "#{path}/install.*" script_file = Dir.glob(script_file_prefix)[0] log = "" if not script_file.nil? then - @@log.info "Execute \"#{script_file}\" file" + logger.info "Execute \"#{script_file}\" file" if Utils.is_windows_like_os( Utils::HOST_OS ) then cmd = "set INSTALLED_PATH=\"#{target_path}\"& #{script_file}" else @@ -138,7 +132,7 @@ class FileInstaller return log end - def FileInstaller.execute_remove_script(package_name, target_path) + def FileInstaller.execute_remove_script(package_name, target_path, logger) info_path = target_path + "/#{PACKAGE_INFO_DIR}/#{package_name}" if not File.directory? info_path then return false @@ -149,7 +143,7 @@ class FileInstaller log = "" if not script_file.nil? then - @@log.info "Execute \"#{script_file}\" file" + logger.info "Execute \"#{script_file}\" file" if Utils.is_windows_like_os( Utils::HOST_OS ) then cmd = "set INSTALLED_PATH=\"#{target_path}\"& #{script_file}" else @@ -159,7 +153,7 @@ class FileInstaller end end - def FileInstaller.remove_pkg_files(package_name, target_path) + def FileInstaller.remove_pkg_files(package_name, target_path, logger) list_path = target_path + "/#{PACKAGE_INFO_DIR}/#{package_name}" if not File.directory? list_path then @@ -188,17 +182,18 @@ class FileInstaller begin Dir.rmdir(file_path) rescue SystemCallError - @@log.warn "\"#{file_path}\" directory is not empty" + logger.warn "\"#{file_path}\" directory is not empty" end else directories.push(file_path) end elsif File.file? file_path then FileUtils.rm_f(file_path) elsif File.symlink? file_path then File.unlink file_path # if files are already removed by remove script, - else @@log.warn "\"#{file_path}\" does not exist" end + else logger.warn "\"#{file_path}\" does not exist" end end end directories.reverse.each do |path| + if not File.directory? path then next end entries = Dir.entries(path) if entries.include? "." then entries.delete(".") end if entries.include? ".." then entries.delete("..") end @@ -206,7 +201,7 @@ class FileInstaller begin Dir.rmdir(path) rescue SystemCallError - @@log.warn "\"#{file_path}\" directory is not empty" + logger.warn "\"#{file_path}\" directory is not empty" end else next end end @@ -215,11 +210,11 @@ class FileInstaller return true end - def FileInstaller.uninstall(package_name, type, target_path) + def FileInstaller.uninstall(package_name, type, target_path, logger) case type when "binary" then - execute_remove_script(package_name, target_path) - remove_pkg_files(package_name, target_path) + execute_remove_script(package_name, target_path, logger) + remove_pkg_files(package_name, target_path, logger) when "source" then end @@ -232,7 +227,7 @@ class FileInstaller FileUtils.cp "#{source_path}/pkginfo.manifest", config_path end - def FileInstaller.extract_file(package_name, package_file_path, path, target_path) + def FileInstaller.extract_file(package_name, package_file_path, path, target_path, logger) dirname = File.dirname(package_file_path) filename = File.basename(package_file_path) ext = File.extname(filename) @@ -258,7 +253,7 @@ class FileInstaller show_file_list_command = "tar -tf #{_package_file_path}" extract_file_list_command = "tar xf \"#{_package_file_path}\" -C \"#{_path}\"" else - @@log.error "\"#{filename}\" is not supported." + logger.error "\"#{filename}\" is not supported." return nil end @@ -286,12 +281,12 @@ class FileInstaller log = `#{extract_file_list_command}` end - @@log.info "Extracted \"#{filename}\" file.. OK" + logger.info "Extracted \"#{filename}\" file.. OK" if log.nil? then log = "" end return log end - def FileInstaller.extract_a_file(package_file_path, target_file, path) + def FileInstaller.extract_a_file(package_file_path, target_file, path, logger) dirname = File.dirname(package_file_path) filename = File.basename(package_file_path) ext = File.extname(filename) @@ -323,10 +318,10 @@ class FileInstaller end if File.exist? target_file_path then - @@log.info "Extracted \"#{target_file}\" file.." + logger.info "Extracted \"#{target_file}\" file.." return true else - @@log.warn "Failed to extracted \"#{target_file}\" file.." + logger.warn "Failed to extracted \"#{target_file}\" file.." return false end end -- 2.34.1