From 6352157338e07e29ed85662c86d32ad3f89cb456 Mon Sep 17 00:00:00 2001 From: donghee yang Date: Fri, 17 Aug 2012 16:17:23 +0900 Subject: [PATCH] [Title] Added remote package synchonization --- build-svr | 22 ++- src/build_server/BuildServer.rb | 30 +++- src/build_server/BuildServerController.rb | 44 ++++- src/build_server/BuildServerOptionParser.rb | 16 +- src/build_server/PackageSync.rb | 176 ++++++++++++++++++++ test/buildsvr.init | 3 +- 6 files changed, 272 insertions(+), 19 deletions(-) create mode 100644 src/build_server/PackageSync.rb diff --git a/build-svr b/build-svr index fde8b50..42861b6 100755 --- a/build-svr +++ b/build-svr @@ -85,15 +85,19 @@ begin when "stop" BuildServerController.stop_server( option[:name] ) when "add-svr" - svr_result = Utils.parse_server_addr(option[:domain]) - if svr_result.nil? or svr_result.length != 2 then - puts "Server address is incorrect. Tune as following format." - puts " :" - exit 1 - end - pkgsvr_addr = svr_result[0] - pkgsvr_port = svr_result[1] - BuildServerController.add_friend_server( option[:name], pkgsvr_addr, pkgsvr_port ) + if not option[:domain].nil? then + svr_result = Utils.parse_server_addr(option[:domain]) + if svr_result.nil? or svr_result.length != 2 then + puts "Server address is incorrect. Tune as following format." + puts " :" + exit 1 + end + pkgsvr_addr = svr_result[0] + pkgsvr_port = svr_result[1] + BuildServerController.add_friend_server( option[:name], pkgsvr_addr, pkgsvr_port ) + elsif not option[:url].nil? then + BuildServerController.add_remote_package_server( option[:name], option[:url], option[:proxy] ) + end when "add-prj" if not option[:git].nil? then BuildServerController.add_project( option[:name], option[:pid], diff --git a/src/build_server/BuildServer.rb b/src/build_server/BuildServer.rb index 4e6e142..6a5ee76 100644 --- a/src/build_server/BuildServer.rb +++ b/src/build_server/BuildServer.rb @@ -34,6 +34,7 @@ require "LocalBuildJob.rb" require "JobManager.rb" require "JobClean.rb" require "RemoteBuildServer.rb" +require "PackageSync.rb" require "ProjectManager.rb" class BuildServer @@ -57,6 +58,7 @@ class BuildServer attr_accessor :incoming_path, :outgoing_path attr_accessor :cancel_lock attr_accessor :supported_os_list + attr_accessor :remote_pkg_servers CONFIG_ROOT = "#{Utils::HOME}/.build_tools/build_server" HOST_OS = Utils::HOST_OS @@ -70,6 +72,7 @@ class BuildServer @pkgserver_port = pkgsvr_port @pkgserver_id = pkgsvr_id @friend_servers = [] + @remote_pkg_servers = [] @req_listener = [] @finish = false # port number @@ -125,8 +128,6 @@ class BuildServer @log.info "Setting Project Manager..." @prjmgr.load() - # set local package server for cache - @log.info "Setting local package server..." pkgsvr_id = @id pkgsvr_dist = @pkgserver_url.split("/")[-1] @@ -140,6 +141,13 @@ class BuildServer @log.info "Setting Job Cleaner..." @cleaner = JobCleaner.new(self) @cleaner.start + + # set package server synchrontizer + if not @remote_pkg_servers.empty? then + @log.info "Setting Package Server Synchronizer..." + @pkg_sync = PackageServerSynchronizer.new(self) + @pkg_sync.start + end # main loop @log.info "Entering main loop..." @@ -212,6 +220,24 @@ class BuildServer end + # add new remote pkg server + def add_remote_package_server( url, proxy ) + + # if already exit, return false + for entry in @remote_pkg_servers + u = entry[0]; p = entry[1] + + if u == url then + return false + end + end + + @remote_pkg_servers.push [url, proxy] + + return true + end + + # add new target OS. # If already exist, return false , otherwise true def add_target_os( os_name ) diff --git a/src/build_server/BuildServerController.rb b/src/build_server/BuildServerController.rb index ab367bc..78ba035 100644 --- a/src/build_server/BuildServerController.rb +++ b/src/build_server/BuildServerController.rb @@ -181,6 +181,34 @@ class BuildServerController end + # add remote package server + def self.add_remote_package_server(id, url, proxy ) + server = get_server(id) + + # add + if server.add_remote_package_server( url, proxy ) then + + # write config + server_dir = "#{BuildServer::CONFIG_ROOT}/#{server.id}" + f = File.open( "#{server_dir}/remote_pkg_servers", "a" ) + if not proxy.nil? then + f.puts "#{url}|#{proxy}" + else + f.puts "#{url}|" + end + f.close + + puts "Remote package server is added!" + + return true + else + puts "The server already exists in list!" + + return false + end + end + + # add supported target os def self.add_target_os( id, os_name ) # TODO:check os foramt @@ -299,7 +327,7 @@ class BuildServerController # get supported os for project. # if not specified, all supported os of the server will be used - if os_string == "default" then + if os_string.nil? or os_string == "default" then os_list = server.supported_os_list else os_list = os_string.strip.split(",") @@ -344,7 +372,7 @@ class BuildServerController # get supported os for project. # if not specified, all supported os of the server will be used - if os_string.nil? then + if os_string.nil? or os_string == "default" then os_list = server.supported_os_list else os_list = os_string.strip.split(",") @@ -580,6 +608,18 @@ class BuildServerController end end + # check remote package server + if File.exist? "#{server_dir}/remote_pkg_servers" then + File.open( "#{server_dir}/remote_pkg_servers", "r" ) do |f| + f.each_line do |l| + if l.split("|").count < 2 then next end + url = l.split("|")[0].strip + proxy = l.split("|")[1].strip + obj.add_remote_package_server( url, proxy ) + end + end + end + # check supported os if File.exist? "#{server_dir}/supported_os_list" then File.open( "#{server_dir}/supported_os_list", "r" ) do |f| diff --git a/src/build_server/BuildServerOptionParser.rb b/src/build_server/BuildServerOptionParser.rb index 38c392b..6f496a7 100644 --- a/src/build_server/BuildServerOptionParser.rb +++ b/src/build_server/BuildServerOptionParser.rb @@ -56,9 +56,10 @@ def option_error_check( options ) end when "add-svr" - if options[:name].nil? or options[:name].empty? or - options[:domain].nil? or options[:domain].empty? then - raise ArgumentError, "Usage: build-svr add-svr -n -d " + if options[:name].nil? or options[:name].empty? or + ((options[:domain].nil? or options[:domain].empty?) and + (options[:url].nil? or options[:url].empty?)) then + raise ArgumentError, "Usage: build-svr add-svr -n [-d ] [-u ] [--proxy ]" end when "add-prj" @@ -97,7 +98,7 @@ def option_parse + "\t" + "remove Remove the build-server." + "\n" \ + "\t" + "start Start the build-server." + "\n" \ + "\t" + "stop Stop the build-server." + "\n" \ - + "\t" + "add-svr Add build-server for support multi-OS or distribute build job." + "\n" \ + + "\t" + "add-svr Add remote build/package server for support multi-OS or distribute build job." + "\n" \ + "\t" + "add-prj Register information for project what you want build berfore building a project." + "\n" \ + "\t" + "register Register the package to the build-server." + "\n" \ + "\t" + "fullbuild Build all your projects and upload them to package server." + "\n" \ @@ -106,7 +107,7 @@ def option_parse + "\t" + "build-svr remove -n " + "\n" \ + "\t" + "build-svr start -n -p " + "\n" \ + "\t" + "build-svr stop -n " + "\n" \ - + "\t" + "build-svr add-svr -n -d " + "\n" \ + + "\t" + "build-svr add-svr -n [-d ] [-u ] [--proxy ]" + "\n" \ + "\t" + "build-svr add-prj -n -N [-g ] [-b ] [-P ] [-w ] [-o ]" + "\n" \ + "\t" + "build-svr add-os -n -o " + "\n" \ + "\t" + "build-svr register -n -P " + "\n" \ @@ -127,6 +128,11 @@ def option_parse options[:url] = url end + options[:proxy] = nil + opts.on( '--proxy ', 'proxy url: http://172.21.111.100:2222' ) do|proxy| + options[:proxy] = proxy + end + opts.on( '-d', '--address ', 'server address: 127.0.0.1:2224' ) do|domain| options[:domain] = domain end diff --git a/src/build_server/PackageSync.rb b/src/build_server/PackageSync.rb new file mode 100644 index 0000000..67fbaab --- /dev/null +++ b/src/build_server/PackageSync.rb @@ -0,0 +1,176 @@ +=begin + + PackageSync.rb + +Copyright (c) 2000 - 2011 Samsung Electronics Co., Ltd. All rights reserved. + +Contact: +Taejun Ha +Jiil Hyoun +Donghyuk Yang +DongHee Yang + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Contributors: +- S-Core Co., Ltd +=end + +require "fileutils" +require "thread" +$LOAD_PATH.unshift File.dirname(__FILE__) +$LOAD_PATH.unshift File.dirname(File.dirname(__FILE__))+"/common" +require "BuildServer.rb" +require "Action.rb" +require "ScheduledActionHandler.rb" + + +class PackageSyncAction < Action + @@new_id = 0 + + def initialize( time, url, proxy, server ) + super(time, 600) + my_id = @@new_id + @@new_id += 1 + @pkgsvr_url = url + @proxy = proxy + @server = server + @pkgsvr_client = nil + @main_client = nil + @sync_root = "#{@server.path}/sync/#{my_id}" + @download_path = "#{@sync_root}/remote" + @original_path = "#{@sync_root}/main" + end + + + def init + # create directory + if File.exist? @download_path then + FileUtils.rm_rf @download_path + FileUtils.rm_rf @original_path + else + FileUtils.mkdir_p @download_path + FileUtils.mkdir_p @original_path + end + + # create client + @pkgsvr_client = Client.new( @pkgsvr_url, @download_path, @server.log ) + @main_client = Client.new( @server.pkgserver_url, @original_path, @server.log ) + end + + + def execute + @server.log.info "Executing package-sync action for server \"#{@pkgsvr_url}\"" + + # check update + pkgs = check_package_update + + # if updates are found, download them + downloaded_files = [] + pkgs.each { |pkg| + pkg_name=pkg[0]; os=pkg[1] + + files = @pkgsvr_client.download(pkg_name, os, false) + downloaded_files += files + } + + # request to register + registered_jobs = [] + downloaded_files.each { |file_path| + @server.log.info "Creating new job for registering \"#{file_path}\"" + new_job = @server.jobmgr.create_new_register_job( file_path ) + logger = JobLog.new( new_job, nil ) + new_job.set_logger(logger) + logger.init + + # add + @server.jobmgr.add_job( new_job ) + registered_jobs.push new_job + } + + # wait for finish all jobs + all_jobs_finished = false + while not all_jobs_finished + unfinished_jobs = registered_jobs.select { |j| + (j.status != "ERROR" and j.status != "FINISHED" and j.status != "CANCELED") + } + if unfinished_jobs.empty? then + all_jobs_finished = true + else + sleep 10 + end + end + + # remove files + downloaded_files.each { |file_path| + @server.log.info "Removed downloaded file: \"#{file_path}\"" + FileUtils.rm_rf file_path + } + end + + + protected + def check_package_update + pkgs = [] + + # update + @pkgsvr_client.update() + @main_client.update() + + # for all BINARY project + bin_prjs = @server.prjmgr.projects.select { |p| (p.type == "BINARY") } + bin_prjs.each { |p| + pkg_name = p.pkg_name + p.os_list.each { |os| + # get pkg version in server + main_ver = @main_client.get_attr_from_pkg(pkg_name, os, "version") + if main_ver.nil? then next end + remote_ver = @pkgsvr_client.get_attr_from_pkg(pkg_name, os, "version") + if remote_ver.nil? then next end + + if Version.new(main_ver) < Version.new(remote_ver) then + pkgs.push [pkg_name, os] + end + } + } + + return pkgs + end + +end + + +class PackageServerSynchronizer + attr_accessor :quit + + # init + def initialize( server ) + @server = server + @handler = ScheduledActionHandler.new + end + + + # start thread + def start() + + time = Time.new + 60 + @server.remote_pkg_servers.each { |entry| + url=entry[0]; proxy=entry[1] + @handler.register( PackageSyncAction.new(time, url, proxy, @server) ) + @server.log.info "Registered package-sync action for server \"#{url}\"" + } + + # start handler + @handler.start + end +end diff --git a/test/buildsvr.init b/test/buildsvr.init index 31ece30..0792aa5 100755 --- a/test/buildsvr.init +++ b/test/buildsvr.init @@ -3,7 +3,8 @@ rm -rf buildsvr01 mkdir buildsvr01 cd buildsvr01 ../../build-svr remove -n testserver3 -../../build-svr create -n testserver3 -u `pwd`/../pkgsvr01/unstable -d 127.0.0.1:3333 -t ftp://tmax:tmax@172.21.111.217 +../../build-svr create -n testserver3 -u `pwd`/../pkgsvr01/unstable -d 127.0.0.1:3333 -t ftp://ftpuser:ftpuser@127.0.0.1 +../../build-svr add-svr -n testserver3 -u `pwd`/../pkgsvr02/unstable cd .. cd git01 rm -rf a -- 2.34.1