From 128d318089f83c667e6790c80e1a106212b3dcc1 Mon Sep 17 00:00:00 2001 From: donghee yang Date: Sat, 27 Oct 2012 12:27:13 +0900 Subject: [PATCH] [Title] Modified to update DB when job status changed --- src/build_server/BinaryUploadProject.rb | 17 +++- src/build_server/BuildJob.rb | 7 -- src/build_server/BuildServer.rb | 4 +- src/build_server/CommonJob.rb | 39 +++++++++- src/build_server/CommonProject.rb | 5 ++ src/build_server/JobManager.rb | 98 ++++++++++++++++-------- src/build_server/MultiBuildJob.rb | 11 +-- src/build_server/PackageSync.rb | 46 ++++++----- src/build_server/ProjectManager.rb | 4 +- src/build_server/RegisterPackageJob.rb | 18 ++--- test/build-server.multi-svr2/01.testcase | 2 +- 11 files changed, 167 insertions(+), 84 deletions(-) diff --git a/src/build_server/BinaryUploadProject.rb b/src/build_server/BinaryUploadProject.rb index d31f029..c7c89dd 100644 --- a/src/build_server/BinaryUploadProject.rb +++ b/src/build_server/BinaryUploadProject.rb @@ -47,6 +47,18 @@ class BinaryUploadProject < CommonProject # create new job def create_new_job( filename, dock = "0" ) + file_path = "#{@server.transport_path}/#{dock}/#{filename}" + new_job = create_new_job_from_local_file( file_path ) + if not new_job.nil? then + new_job.set_auto_remove(true) + end + + return new_job + end + + + def create_new_job_from_local_file( file_path ) + filename = File.basename(file_path) new_name = filename.sub(/(.*)_(.*)_(.*)\.zip/,'\1,\2,\3') pkg_name = new_name.split(",")[0] os = new_name.split(",")[2] @@ -64,9 +76,8 @@ class BinaryUploadProject < CommonProject end # check package info - file_path = "#{@server.transport_path}/#{dock}/#{filename}" if not File.exist? file_path then - @server.log.error( "file not exists in #{@server.transport_path}/#{dock}/#{filename}", Log::LV_USER) + @server.log.error( "file not exists in #{file_path}", Log::LV_USER) return nil end @@ -112,8 +123,6 @@ class BinaryUploadProject < CommonProject end new_job = RegisterPackageJob.new( file_path, self, @server, nil, @dist_name ) - - return new_job end diff --git a/src/build_server/BuildJob.rb b/src/build_server/BuildJob.rb index 9a33bea..3f786c5 100644 --- a/src/build_server/BuildJob.rb +++ b/src/build_server/BuildJob.rb @@ -45,7 +45,6 @@ require "CommonJob.rb" class BuildJob < CommonJob - attr_accessor :pre_jobs, :os, :type attr_accessor :pkginfo, :source_path attr_accessor :pkgsvr_client, :thread attr_accessor :rev_fail_projects, :rev_success_jobs @@ -76,7 +75,6 @@ class BuildJob < CommonJob @source_path = @job_root+"/temp" @job_working_dir=@job_root+"/works" @buildroot_dir = "#{@job_root}/buildroot" - @pre_jobs = [] #pre-requisite jobs # this item will be initialized on pre-verify @pkginfo = nil @@ -112,11 +110,6 @@ class BuildJob < CommonJob end - def get_project() - return @project - end - - def get_distribution_name() return @project.dist_name end diff --git a/src/build_server/BuildServer.rb b/src/build_server/BuildServer.rb index df9e656..946fb93 100644 --- a/src/build_server/BuildServer.rb +++ b/src/build_server/BuildServer.rb @@ -556,7 +556,7 @@ class BuildServer # USERS/GROUPS db.do "CREATE TABLE IF NOT EXISTS users(id INTEGER PRIMARY KEY #{inc}, email VARCHAR(256), password_hash VARCHAR(256), password_salt VARCHAR(256), name VARCHAR(256), group_id INTEGER)" db.do "CREATE TABLE IF NOT EXISTS groups(id INTEGER PRIMARY KEY #{inc}, name VARCHAR(256))" - db.do "CREATE TABLE IF NOT EXISTS group_project_acesss(group_id INTEGER PRIMARY KEY, prj_id INTEGER PRIMARY KEY, build VARCHAR(32))" + db.do "CREATE TABLE IF NOT EXISTS group_project_acesss(group_id INTEGER, prj_id INTEGER, build VARCHAR(32))" # PROJECTS db.do "CREATE TABLE IF NOT EXISTS projects(id INTEGER PRIMARY KEY #{inc}, name VARCHAR(32), ptype VARCHAR(32), password VARCHAR(32), dist_name VARCHAR(32))" @@ -567,7 +567,7 @@ class BuildServer db.do "CREATE TABLE IF NOT EXISTS project_packages(prj_id INTEGER, pkg_ver VARCHAR(64), pkg_name VARCHAR(64), pkg_os VARCHAR(32))" # JOBS - db.do "CREATE TABLE IF NOT EXISTS jobs(id INTEGER PRIMARY KEY, prj_id INTEGER, jtype VARCHAR(32), os_name VARCHAR(32), user_id INTEGER, parent_job_id INTEGER, status VARCHAR(32) )" + db.do "CREATE TABLE IF NOT EXISTS jobs(id INTEGER PRIMARY KEY, jtype VARCHAR(32), os_name VARCHAR(32), user_id INTEGER, parent_job_id INTEGER, status VARCHAR(32), prj_id INTEGER, dist_name VARCHAR(32) )" rescue DBI::DatabaseError => e puts "DB Creation failed!" diff --git a/src/build_server/CommonJob.rb b/src/build_server/CommonJob.rb index 5def3f0..3d20cc5 100644 --- a/src/build_server/CommonJob.rb +++ b/src/build_server/CommonJob.rb @@ -33,7 +33,7 @@ require "utils.rb" class CommonJob - attr_accessor :id, :server, :log, :status + attr_accessor :id, :server, :log, :os, :type, :pre_jobs, :status # initialize public @@ -47,6 +47,11 @@ class CommonJob @parent = nil @sub_jobs = [] + @os = "Unknown" + @type = "Unknown" + @pre_jobs = [] #pre-requisite jobs + @project = nil + @user_id = 0 @status = "JUST_CREATED" @log = nil @@ -99,6 +104,18 @@ class CommonJob end + public + def get_project() + return @project + end + + + public + def set_project( project ) + @project = project + end + + # execute public def execute(sync=false) @@ -162,6 +179,26 @@ class CommonJob return pid, status end + + # save to db + def save(db) + if @project.nil? then + prj_id = -1 + else + prj_id = @project.get_project_id() + end + dist_name = get_distribution_name() + parent_id = @parent.nil? ? -1 : @parent.id + + row=db.select_one("SELECT * FROM jobs WHERE id=#{@id}") + if row.nil? then + db.do "INSERT INTO jobs VALUES (#{@id},'#{@type}','#{@os}', #{@user_id}, #{parent_id}, '#{@status}', #{prj_id}, '#{dist_name}')" + else + db.do "UPDATE jobs SET status='#{@status}' WHERE id=#{@id}" + end + end + + # # PROTECTED METHODS # diff --git a/src/build_server/CommonProject.rb b/src/build_server/CommonProject.rb index c5e83f0..c22c1b9 100644 --- a/src/build_server/CommonProject.rb +++ b/src/build_server/CommonProject.rb @@ -228,6 +228,11 @@ class CommonProject end + def get_project_id() + return @prj_id + end + + protected def self.load_row(name, dist_name, db) row = db.select_one("SELECT * FROM projects WHERE name='#{name}' and dist_name='#{dist_name}'") diff --git a/src/build_server/JobManager.rb b/src/build_server/JobManager.rb index 5247d61..385beeb 100644 --- a/src/build_server/JobManager.rb +++ b/src/build_server/JobManager.rb @@ -42,7 +42,7 @@ class JobManager # initialize def initialize( parent ) - @parent = parent + @server = parent @jobs = [] @internal_jobs = [] @reverse_build_jobs = [] @@ -56,7 +56,7 @@ class JobManager # initialize def init() # load latest job idx if exist - file_path = "#{BuildServer::CONFIG_ROOT}/#{@parent.id}/latest_job" + file_path = "#{BuildServer::CONFIG_ROOT}/#{@server.id}/latest_job" if File.exist? file_path then latest_idx = -1 File.open( file_path, "r" ) do |f| @@ -79,7 +79,7 @@ class JobManager @latest_job_touch.synchronize do new_idx = @new_job_index - file_path = "#{BuildServer::CONFIG_ROOT}/#{@parent.id}/latest_job" + file_path = "#{BuildServer::CONFIG_ROOT}/#{@server.id}/latest_job" File.open( file_path, "w" ) do |f| f.puts "#{@new_job_index}" end @@ -92,25 +92,27 @@ class JobManager def create_new_register_job( file_path, dist_name ) - return RegisterPackageJob.new( file_path, nil, @parent, nil, dist_name ) + return RegisterPackageJob.new( file_path, nil, @server, nil, dist_name ) end # add a normal job def add_job ( new_job ) - @parent.log.info "Added new job \"#{new_job.id}\"" - new_job.status = "JUST_CREATED" + @server.log.info "Added new job \"#{new_job.id}\"" + save_job_status(new_job) @jobs.push( new_job ) end # add internal job for multi-build job def add_internal_job( new_job ) - @parent.log.info "Added new job \"#{new_job.id}\"" + @server.log.info "Added new job \"#{new_job.id}\"" + save_job_status(new_job) @internal_jobs.push( new_job ) end # add reverse build chek job def add_reverse_build_job( new_job ) - @parent.log.info "Added new job \"#{new_job.id}\"" + @server.log.info "Added new job \"#{new_job.id}\"" + save_job_status(new_job) @reverse_build_jobs.push( new_job ) end @@ -129,34 +131,38 @@ class JobManager def initialize_job ( job ) job.status = "INITIALIZING" Thread.new do + save_job_status(job) begin # init if not job.init or job.status == "ERROR" then if job.cancel_state == "NONE" then job.status = "ERROR" end - @parent.log.info "Adding the job \"#{job.id}\" is canceled" + @server.log.info "Adding the job \"#{job.id}\" is canceled" job.terminate() + save_job_status(job) Thread.current.exit end if job.status != "FINISHED" then job.status = "WAITING" + save_job_status(job) end - @parent.log.info "Checking the job \"#{job.id}\" was finished!" + @server.log.info "Checking the job \"#{job.id}\" was finished!" rescue => e - @parent.log.error e.message - @parent.log.error e.backtrace.inspect + @server.log.error e.message + @server.log.error e.backtrace.inspect end end - @parent.log.info "Job \"#{job.id}\" entered INITIALIZING status" + @server.log.info "Job \"#{job.id}\" entered INITIALIZING status" end #execute def execute(job) job.status = "WORKING" + save_job_status(job) # start build job.execute - @parent.log.info "Moved the job \"#{job.id}\" to working job list" + @server.log.info "Moved the job \"#{job.id}\" to working job list" end @@ -168,15 +174,16 @@ class JobManager if job.execute() then # status change & job control job.status = "REMOTE_WORKING" - @parent.log.info "Moved the job \"#{job.id}\" to remote job list" + save_job_status(job) + @server.log.info "Moved the job \"#{job.id}\" to remote job list" else - @parent.log.info "Moving the job \"#{job.id}\" to remote failed" + @server.log.info "Moving the job \"#{job.id}\" to remote failed" end end def cancel_job( job) job.cancel_state = "WORKING" - @parent.log.info "Creating thread for canceling the job \"#{job.id}\"" + @server.log.info "Creating thread for canceling the job \"#{job.id}\"" Thread.new do begin #terminate job thread @@ -190,12 +197,13 @@ class JobManager # cancel finished job.status = "CANCELED" + save_job_status(job) # call terminate process for job job.terminate rescue => e - @parent.log.error e.message - @parent.log.error e.backtrace.inspect + @server.log.error e.message + @server.log.error e.backtrace.inspect end end end @@ -212,13 +220,16 @@ class JobManager job_list.each do |job| # if "ERROR", "FINISHED", "CANCELED" remove it from list if job.status == "ERROR" - @parent.log.info "Job \"#{job.id}\" is stopped by ERROR" + save_job_status(job) + @server.log.info "Job \"#{job.id}\" is stopped by ERROR" @reverse_build_jobs.delete job elsif job.status == "FINISHED" - @parent.log.info "Job \"#{job.id}\" is removed by FINISH status" + save_job_status(job) + @server.log.info "Job \"#{job.id}\" is removed by FINISH status" @reverse_build_jobs.delete job elsif job.status == "CANCELED" - @parent.log.info "Job \"#{job.id}\" is removed by CANCELED status" + save_job_status(job) + @server.log.info "Job \"#{job.id}\" is removed by CANCELED status" @reverse_build_jobs.delete job end @@ -233,13 +244,16 @@ class JobManager job_list.each do |job| # if "ERROR", "FINISHED", "CANCELED" remove it from list if job.status == "ERROR" - @parent.log.info "Job \"#{job.id}\" is stopped by ERROR" + save_job_status(job) + @server.log.info "Job \"#{job.id}\" is stopped by ERROR" @internal_jobs.delete job elsif job.status == "FINISHED" - @parent.log.info "Job \"#{job.id}\" is removed by FINISH status" + save_job_status(job) + @server.log.info "Job \"#{job.id}\" is removed by FINISH status" @internal_jobs.delete job elsif job.status == "CANCELED" - @parent.log.info "Job \"#{job.id}\" is removed by CANCELED status" + save_job_status(job) + @server.log.info "Job \"#{job.id}\" is removed by CANCELED status" @internal_jobs.delete job end @@ -254,13 +268,16 @@ class JobManager job_list.each do |job| # if "ERROR", "FINISHED", "CANCELED" remove it from list if job.status == "ERROR" - @parent.log.info "Job \"#{job.id}\" is stopped by ERROR" + save_job_status(job) + @server.log.info "Job \"#{job.id}\" is stopped by ERROR" @jobs.delete job elsif job.status == "FINISHED" - @parent.log.info "Job \"#{job.id}\" is removed by FINISH status" + save_job_status(job) + @server.log.info "Job \"#{job.id}\" is removed by FINISH status" @jobs.delete job elsif job.status == "CANCELED" - @parent.log.info "Job \"#{job.id}\" is removed by CANCELED status" + save_job_status(job) + @server.log.info "Job \"#{job.id}\" is removed by CANCELED status" @jobs.delete job end @@ -275,8 +292,9 @@ class JobManager not job.is_connected? then job.status = "ERROR" + save_job_status(job) @jobs.delete( job ) - @parent.log.info "Job \"#{job.id}\" is disconnected by user. Removed!" + @server.log.info "Job \"#{job.id}\" is disconnected by user. Removed!" end end @@ -286,8 +304,8 @@ class JobManager # available job not exist?, continue if not job.nil? then # oherwise, check remote server - rserver = @parent.get_available_server( job ) - if rserver != nil and rserver == @parent then + rserver = @server.get_available_server( job ) + if rserver != nil and rserver == @server then execute(job) elsif rserver != nil then execute_remote(job, rserver) @@ -425,6 +443,24 @@ class JobManager end + def save_job_status(job) + begin + db = @server.get_db_connection() + db['AutoCommit'] = false + db.transaction do |dbh| + return job.save(dbh) + end + + rescue DBI::DatabaseError => e + @server.log.error "DB failed!" + @server.log.error e.errstr + @server.log.error e.backtrace.inspect + + ensure + db['AutoCommit'] = true if not db.nil? + end + end + protected # select the job whith no build-dependency problem # if "check_dep_wait" is true, it will check the build dependency diff --git a/src/build_server/MultiBuildJob.rb b/src/build_server/MultiBuildJob.rb index 1f4a6e2..5b31daf 100644 --- a/src/build_server/MultiBuildJob.rb +++ b/src/build_server/MultiBuildJob.rb @@ -43,7 +43,6 @@ require "CommonJob.rb" class MultiBuildJob < CommonJob - attr_accessor :pre_jobs, :os, :type attr_accessor :source_path, :cancel_state attr_accessor :pkgsvr_client, :thread @@ -52,7 +51,6 @@ class MultiBuildJob < CommonJob super(server) @log = nil @type = "MULTIBUILD" - @os = "Unknown" @host_os = Utils::HOST_OS @pkgsvr_url = nil @@ -62,7 +60,6 @@ class MultiBuildJob < CommonJob @source_path = @job_root+"/temp" @job_working_dir=@job_root+"/works" @buildroot_dir = "#{@job_root}/buildroot" - @pre_jobs = [] #pre-requisite jobs @cancel_state = "NONE" end @@ -351,10 +348,10 @@ class MultiBuildJob < CommonJob # init finished, add internal_jobs @server.jobmgr.add_internal_job(job) @log.info( "Added new job \"#{job.get_project().name}\" for #{job.os}! (#{job.id})", - Log::LV_USER) - if not @server.job_log_url.empty? then - @log.info( " * Log URL : #{@server.job_log_url}/#{job.id}/log", Log::LV_USER) - end + Log::LV_USER) + if not @server.job_log_url.empty? then + @log.info( " * Log URL : #{@server.job_log_url}/#{job.id}/log", Log::LV_USER) + end end end diff --git a/src/build_server/PackageSync.rb b/src/build_server/PackageSync.rb index c3498da..523a0bb 100644 --- a/src/build_server/PackageSync.rb +++ b/src/build_server/PackageSync.rb @@ -92,24 +92,28 @@ class PackageSyncAction < Action # if updates are found, download them downloaded_files = [] pkgs.each do |pkg| - pkg_name=pkg[0]; os=pkg[1] - - files = @pkgsvr_client.download(pkg_name, os, false) - downloaded_files += files - end - - # request to register - registered_jobs = [] - downloaded_files.each do |file_path| - @server.log.info "Creating new job for registering \"#{file_path}\"" - new_job = @server.jobmgr.create_new_register_job( file_path, @dist_name ) - logger = JobLog.new( new_job, nil ) - new_job.set_logger(logger) - logger.init - - # add - @server.jobmgr.add_job( new_job ) - registered_jobs.push new_job + pkg_name=pkg[0]; os=pkg[1]; prj=pkg[2] + + downloaded_files = @pkgsvr_client.download(pkg_name, os, false) + + # request to register + registered_jobs = [] + downloaded_files.each do |file_path| + @server.log.info "Creating new job for registering \"#{file_path}\"" + new_job = prj.create_new_job_from_local_file( file_path ) + if new_job.nil? then + @server.log.error "Creating job failed: #{prj.name} #{pkg_name} #{@dist_name}" + next + end + + logger = JobLog.new( new_job, nil ) + new_job.set_logger(logger) + logger.init + + # add + @server.jobmgr.add_job( new_job ) + registered_jobs.push new_job + end end # wait for finish all jobs @@ -142,7 +146,9 @@ class PackageSyncAction < Action @main_client.update() # for all BINARY project - bin_prjs = @server.prjmgr.get_all_projects().select { |p| (p.type == "BINARY") } + bin_prjs = @server.prjmgr.get_all_projects().select { |p| + (p.type == "BINARY" and p.dist_name == @dist_name) + } bin_prjs.each do |p| pkg_name = p.pkg_name p.os_list.each do |os| @@ -152,7 +158,7 @@ class PackageSyncAction < Action if remote_ver.nil? then next end if main_ver.nil? or Version.new(main_ver) < Version.new(remote_ver) then - pkgs.push [pkg_name, os] + pkgs.push [pkg_name, os, p] end end end diff --git a/src/build_server/ProjectManager.rb b/src/build_server/ProjectManager.rb index d832649..7cef952 100644 --- a/src/build_server/ProjectManager.rb +++ b/src/build_server/ProjectManager.rb @@ -87,7 +87,7 @@ class ProjectManager rows.each do |row| prj_name = row['name'] prj_dist = row['dist_name'] - prj_type = row['type'] + prj_type = row['ptype'] if prj_type == "GIT" then result.push GitBuildProject.load(prj_name, prj_dist, @server, dbh) else @@ -374,7 +374,7 @@ class ProjectManager def get_project_internal(name, dist_name, db) row = db.select_one("SELECT * FROM projects WHERE name='#{name}' and dist_name='#{dist_name}'") if row.nil? then return nil end - prj_type = row['type'] + prj_type = row['ptype'] if prj_type == "GIT" then return GitBuildProject.load(name, dist_name, @server, db) diff --git a/src/build_server/RegisterPackageJob.rb b/src/build_server/RegisterPackageJob.rb index a95aed5..8d66523 100644 --- a/src/build_server/RegisterPackageJob.rb +++ b/src/build_server/RegisterPackageJob.rb @@ -43,7 +43,6 @@ require "CommonJob.rb" class RegisterPackageJob < CommonJob - attr_accessor :pre_jobs, :os, :type attr_accessor :source_path attr_accessor :pkgsvr_client, :thread, :pkg_type attr_accessor :pkg_name, :pkginfo, :cancel_state @@ -70,7 +69,6 @@ class RegisterPackageJob < CommonJob @job_working_dir=@job_root+"/works" @buildroot_dir = "#{@job_root}/buildroot" @cancel_state = "NONE" - @pre_jobs = [] @local_path=local_path @file_path = nil @@ -87,18 +85,14 @@ class RegisterPackageJob < CommonJob end @pkginfo = nil #This info is valid only for BINARY package @project = project - if not dist_name.nil? then + if not dist_name.nil? then @dist_name = dist_name elsif not @project.nil? then @dist_name = @project.dist_name else @dist_name = "BASE" end - end - - - def get_project() - return @project + @auto_remove = false end @@ -116,6 +110,12 @@ class RegisterPackageJob < CommonJob return false end + + def set_auto_remove(value) + @auto_remove=value + end + + # def init # mkdir @@ -146,7 +146,7 @@ class RegisterPackageJob < CommonJob @status = "ERROR" return false else - if not @project.nil? then + if @auto_remove then # if remote upload remove file and its directory FileUtils.mv(@local_path, @file_path) FileUtils.rm_rf("#{File.dirname(@local_path)}") diff --git a/test/build-server.multi-svr2/01.testcase b/test/build-server.multi-svr2/01.testcase index f813f07..07ec572 100644 --- a/test/build-server.multi-svr2/01.testcase +++ b/test/build-server.multi-svr2/01.testcase @@ -2,7 +2,7 @@ #EXEC ../../pkg-svr register -n pkgsvr02 -d unstable -P bin/bin_0.0.1_ubuntu-32.zip sleep 25 -../../pkg-cli list-rpkg -P bin -u `pwd`/pkgsvr02/unstable +../../pkg-cli list-rpkg -P bin -u `pwd`/pkgsvr01/unstable #POST-EXEC #EXPECT Archive: bin/bin_0.0.1_ubuntu-32.zip -- 2.34.1