From 2ee44301db1f26a7b6dc3095f2b78d6bfe408f14 Mon Sep 17 00:00:00 2001 From: donghee yang Date: Fri, 26 Oct 2012 22:14:33 +0900 Subject: [PATCH] [Title] Moved to all project infomation to DB --- src/build_server/BinaryUploadProject.rb | 59 +++----- src/build_server/BuildServer.rb | 16 +- src/build_server/CommonProject.rb | 190 ++++++++++++++++++++++-- src/build_server/GitBuildJob.rb | 6 +- src/build_server/GitBuildProject.rb | 174 +++------------------- src/build_server/ProjectManager.rb | 5 +- 6 files changed, 236 insertions(+), 214 deletions(-) diff --git a/src/build_server/BinaryUploadProject.rb b/src/build_server/BinaryUploadProject.rb index e0d6ee6..d31f029 100644 --- a/src/build_server/BinaryUploadProject.rb +++ b/src/build_server/BinaryUploadProject.rb @@ -117,68 +117,43 @@ class BinaryUploadProject < CommonProject end - def include_package?(name, version=nil, os=nil) - return (name == @pkg_name) - end - - - def save_db() - config_file = "#{@path}/build" - File.open( config_file, "w" ) do |f| - f.puts "TYPE=BINARY" - f.puts "PACKAGE_NAME=#{@pkg_name}" - end - return true - end - - - def load_db() - # git - config_file = "#{@path}/build" - pkg_name=nil - File.open( config_file, "r" ) do |f| - f.each_line do |l| - idx = l.index("=") + 1 - length = l.length - idx - - if l.start_with?("PACKAGE_NAME=") - pkg_name = l[idx,length].strip - else - next - end - end - end - - @pkg_name = pkg_name - return true - end - - def self.load(name, dist_name, server, db) - row, prj_os_list = load_row(name, dist_name, db) + row, prj_os_list, source_info, package_info = load_row(name, dist_name, db) prj_id = row['id'] prj_name = row['name'] prj_passwd = row['password'] prj_dist = row['dist_name'] new_project = BinaryUploadProject.new(prj_name, server, prj_os_list, prj_dist) - if not prj_passwd.empty? then new_project.passwd = prj_passwd end + new_project.set_project_id( prj_id ) + new_project.set_source_info( source_info ) + new_project.set_package_info( package_info ) - if not new_project.load_db() then raise RuntimeError, "Project DB load failed!" end + row=db.select_one("SELECT * FROM project_bins WHERE prj_id=#{prj_id}") + if row.nil? then return nil end + new_project.pkg_name=row['pkg_name'] return new_project end def save(db) - save_common(db) + is_new = save_common(db) init() - if not save_db() then raise RuntimeError, "Project DB load failed!" end + + if is_new then + db.do "INSERT INTO project_bins VALUES (#{@prj_id},'#{@pkg_name}')" + else + db.do "UPDATE project_bins SET pkg_name='#{@pkg_name}' WHERE prj_id=#{@prj_id})" + end end def unload(db) unload_common(db) + if @prj_id != -1 then + db.do "DELETE FROM project_bins WHERE prj_id=#{@prj_id}" + end end end diff --git a/src/build_server/BuildServer.rb b/src/build_server/BuildServer.rb index ea4fa8d..b148d95 100644 --- a/src/build_server/BuildServer.rb +++ b/src/build_server/BuildServer.rb @@ -182,6 +182,7 @@ class BuildServer end rescue => e @log.error( e.message, Log::LV_USER) + @log.error e.backtrace.inspect end if(@upgrade) @@ -546,7 +547,6 @@ class BuildServer else inc="AUTOINCREMENT" end - # create table db.do "CREATE TABLE IF NOT EXISTS distributions(name VARCHAR(32) PRIMARY KEY, pkgsvr_url VARCHAR(256), pkgsvr_addr VARCHAR(64))" db.do "CREATE TABLE IF NOT EXISTS remote_build_servers(id INTEGER PRIMARY KEY #{inc}, svr_addr VARCHAR(64))" @@ -556,6 +556,13 @@ class BuildServer db.do "CREATE TABLE IF NOT EXISTS project_os(prj_id INTEGER, os_name VARCHAR(32))" db.do "CREATE TABLE IF NOT EXISTS users(user_id INTEGER PRIMARY KEY #{inc}, email VARCHAR(256), password VARCHAR(256), password_salt VARCHAR(256), user_name VARCHAR(256), group_id VARCHAR(256))" db.do "CREATE TABLE IF NOT EXISTS groups(group_id INTEGER PRIMARY KEY #{inc}, group_name VARCHAR(256))" + # Projects + db.do "CREATE TABLE IF NOT EXISTS project_gits(prj_id INTEGER, git_repos VARCHAR(128), git_branch VARCHAR(32))" + db.do "CREATE TABLE IF NOT EXISTS project_sources(prj_id INTEGER, pkg_ver VARCHAR(64), location VARCHAR(256))" + db.do "CREATE TABLE IF NOT EXISTS project_bins(prj_id INTEGER, pkg_name VARCHAR(64))" + + db.do "CREATE TABLE IF NOT EXISTS project_packages(prj_id INTEGER, pkg_ver VARCHAR(64), pkg_name VARCHAR(64), pkg_os VARCHAR(32))" + rescue DBI::DatabaseError => e puts "DB Creation failed!" puts e.errstr @@ -569,6 +576,10 @@ class BuildServer def get_db_connection() if not @db.nil? and @db.connected? then return @db end + if not @db.nil? then + @db.disconnect + @db = nil + end begin db = DBI.connect("DBI:#{@db_dsn}", @db_user, @db_passwd) @@ -582,7 +593,8 @@ class BuildServer return nil end - return db + @db = db + return @db end end diff --git a/src/build_server/CommonProject.rb b/src/build_server/CommonProject.rb index 9e2406c..64f2157 100644 --- a/src/build_server/CommonProject.rb +++ b/src/build_server/CommonProject.rb @@ -35,13 +35,15 @@ class CommonProject # initialize def initialize( name, type, server, os_list, dist_name="BASE" ) - @project_id = -1 + @prj_id = -1 @name = name @type = type @passwd = "" @os_list = os_list @server = server @dist_name = dist_name + @source_info = {} + @package_info = {} if @dist_name == "BASE" then @path = "#{@server.path}/projects/#{@name}" else @@ -119,6 +121,114 @@ class CommonProject end + def set_source_info(info) + @source_info = info + end + + + def save_source_info(ver, info) + @source_info[ver] = info + + # save to db + begin + db = @server.get_db_connection() + db['AutoCommit'] = false + db.transaction do |dbh| + save_source_info_internal(ver, info, dbh) + end + + rescue DBI::DatabaseError => e + @server.log.error "DB update failed!" + @server.log.error e.errstr + @server.log.error e.backtrace.inspect + return false + + ensure + db['AutoCommit'] = true if not db.nil? + end + end + + + def get_source_info(ver) + return @source_info[ver] + end + + + def set_package_info(info) + @package_info = info + end + + + def save_package_info(ver, pkg_name, pkg_os) + if @package_info[ver].nil? then + @package_info[ver] = [] + end + @package_info[ver].push [pkg_name, pkg_os] + + # save to db + begin + db = @server.get_db_connection() + db['AutoCommit'] = false + db.transaction do |dbh| + save_package_info_internal(ver, pkg_name, pkg_os, dbh) + end + + rescue DBI::DatabaseError => e + @server.log.error "DB update failed!" + @server.log.error e.errstr + @server.log.error e.backtrace.inspect + return false + + ensure + db['AutoCommit'] = true if not db.nil? + end + end + + + # get latest package version + def get_latest_version() + versions = @package_info.keys + if not versions.empty? then + versions.sort! {|x,y| Version.new(x).compare(Version.new(y)) } + return versions[-1] + else + return nil + end + end + + + # get all package version + def get_all_versions() + return @package_info.keys + end + + + def include_package?(pkg_name, ver=nil, os=nil) + # check version first + if not ver.nil? then + ver = get_latest_version() + end + + if ver.nil? or @package_info[ver].nil? then return false end + if not os.nil? and not @os_list.include? os then return false end + + # check name and version + @package_info[ver].each do |pkg| + if pkg_name == pkg[0] and os == pkg[1] then + return true + end + end + + return false + end + + + def set_project_id(id) + @prj_id = id + end + + + protected def self.load_row(name, dist_name, db) row = db.select_one("SELECT * FROM projects WHERE name='#{name}' and dist_name='#{dist_name}'") if row.nil? then return nil end @@ -131,39 +241,89 @@ class CommonProject os_list.push r['os_name'] end - return row, os_list + # get source info + source_info = {} + rows=db.select_all("SELECT * FROM project_sources WHERE prj_id=#{prj_id}") + rows.each do |r| + source_info[r['pkg_ver']] = r['location'] + end + + # get package info + package_info = {} + rows=db.select_all("SELECT * FROM project_packages WHERE prj_id=#{prj_id}") + rows.each do |r| + if package_info[r['pkg_ver']].nil? then + package_info[r['pkg_ver']] = [] + end + package_info[r['pkg_ver']].push [r['pkg_name'], r['pkg_os']] + end + + return row, os_list, source_info, package_info end protected def save_common(db) - row = db.select_one("SELECT * FROM projects WHERE name='#{@name}' and dist_name='#{@dist_name}'") - if row.nil? then + if @prj_id == -1 then db.do "INSERT INTO projects (name,type,password,dist_name) VALUES ('#{@name}','#{@type}','#{@passwd}','#{@dist_name}')" case @server.db_dsn - when /^SQLite3:/ then prj_id = db.select_one("select last_insert_rowid()")[0] - when /^Mysql:/ then prj_id = db.func(:insert_id) - else prj_id = db.select_one("select last_insert_rowid()")[0] + when /^SQLite3:/ then @prj_id = db.select_one("select last_insert_rowid()")[0] + when /^Mysql:/ then @prj_id = db.func(:insert_id) + else @prj_id = db.select_one("select last_insert_rowid()")[0] end @os_list.each do |os| - db.do "INSERT INTO project_os VALUES('#{prj_id}','#{os}')" + db.do "INSERT INTO project_os VALUES('#{@prj_id}','#{os}')" end + + return true else db.do "UPDATE projects SET type='#{@type}',password='#{@passwd}' WHERE name='#{@name}' and dist_name='#{@dist_name}')" - db.do "DELETE FROM project_os WHERE prj_id=#{row['prj_id']}" - prj_id = row['prj_id'] + db.do "DELETE FROM project_os WHERE prj_id=#{@prj_id}" @os_list.each do |os| - db.do "INSERT INTO project_os VALUES('#{prj_id}','#{os}')" + db.do "INSERT INTO project_os VALUES('#{@prj_id}','#{os}')" end + + @source_info.each do |src_ver,info| + save_source_info_internal(src_ver, info, db) + end + + @package_info.each do |src_ver,pkg_name_os_pair| + pkg_name_os_pair.each do |pkg_name, pkg_os| + save_package_info_internal(src_ver, pkg_name, pkg_os, db) + end + end + + return false end end - + + protected + def save_source_info_internal(src_ver, info, db) + row1=db.select_one("SELECT * FROM project_sources WHERE prj_id=#{@prj_id} and pkg_ver='#{src_ver}'") + if row1.nil? then + db.do "INSERT INTO project_sources VALUES(#{@prj_id},'#{src_ver}','#{info}')" + end + end + + + protected + def save_package_info_internal(src_ver, pkg_name, pkg_os, db) + row1=db.select_one("SELECT * FROM project_packages WHERE prj_id=#{@prj_id} and pkg_ver='#{src_ver}' and pkg_name='#{pkg_name}' and pkg_os='#{pkg_os}'") + if row1.nil? then + db.do "INSERT INTO project_packages VALUES(#{@prj_id},'#{src_ver}','#{pkg_name}','#{pkg_os}')" + end + end + + + # return its prject id and if not exist?, return -1 protected def unload_common(db) - row = db.select_one("SELECT * FROM projects WHERE name='#{@name}' and dist_name='#{@dist_name}'") - if row.nil? then return end + row = db.select_one("SELECT * FROM projects WHERE id=#{@prj_id}") + if row.nil? then return -1 end db.do("DELETE FROM projects WHERE name='#{@name}' and dist_name='#{@dist_name}'") - db.do "DELETE FROM project_os WHERE prj_id=#{row['id']}" + db.do "DELETE FROM project_os WHERE prj_id=#{@prj_id}" + db.do "DELETE FROM project_sources WHERE prj_id=#{@prj_id}" + db.do "DELETE FROM project_packages WHERE prj_id=#{@prj_id}" end end diff --git a/src/build_server/GitBuildJob.rb b/src/build_server/GitBuildJob.rb index 8d1252c..dfd0da5 100644 --- a/src/build_server/GitBuildJob.rb +++ b/src/build_server/GitBuildJob.rb @@ -61,9 +61,9 @@ class GitBuildJob < BuildJob # if succeeded, register source info and copy pkginfo.manifest @log.info( "Updating the source info for project \"#{@project.name}\"" , Log::LV_USER) - @project.add_source_info( @pkginfo.get_version(), @git_commit) - @project.copy_package_info( @pkginfo.get_version(), - "#{@source_path}/package/pkginfo.manifest") + @project.save_source_info( @pkginfo.get_version(), @git_commit) + @project.save_package_info_from_manifest( @pkginfo.get_version(), + "#{@source_path}/package/pkginfo.manifest", @os) @project.set_log_cnt( @log.cnt ) @project.write_ext_info diff --git a/src/build_server/GitBuildProject.rb b/src/build_server/GitBuildProject.rb index 18db1e5..0f69e58 100644 --- a/src/build_server/GitBuildProject.rb +++ b/src/build_server/GitBuildProject.rb @@ -45,8 +45,6 @@ class GitBuildProject < CommonProject super(name, "GIT", server, os_list, dist_name) @repository = repos @branch = branch - @source_infos = {} - @package_infos = {} end @@ -84,161 +82,24 @@ class GitBuildProject < CommonProject return new_job end - # get latest package version - def get_latest_version() - versions = @package_infos.keys - if not versions.empty? then - versions.sort! {|x,y| Version.new(x).compare(Version.new(y)) } - return versions[-1] - else - return nil - end - end - - - # get all package version - def get_all_versions() - return @package_infos.keys - end - - # add source source info - def add_source_info( version, info ) - @source_infos[version] = info - - # write to file - sources_file = "#{@path}/sources" - File.open( sources_file, "w" ) do |f| - @source_infos.each do |key,value| - f.puts "#{key},#{value}" - end - end - end - - - # get source info - def get_source_info( version ) - return @source_infos[version] - end - - - # add package info - def add_package_info( version, path ) + # save package info + def save_package_info_from_manifest(version, file_path, os) begin - pkginfo =PackageManifest.new(path) + pkginfo =PackageManifest.new(file_path) rescue => e - puts e.message + @server.log.error e.message return end - @package_infos[version] = pkginfo - end - - - # get package info - def get_package_info( version ) - return @package_infos[version] - end - - # copy package info - def copy_package_info(version, file_path) - # check pkginfo directory - pkginfo_dir = "#{@path}/pkginfos" - if not File.exist? pkginfo_dir then - FileUtils.mkdir_p pkginfo_dir + pkginfo.get_target_packages(os).each do |pkg| + save_package_info(pkg.version, pkg.package_name, os) end - - # copy - pkginfo_file = "#{pkginfo_dir}/#{version}.manifest" - FileUtils.cp(file_path, pkginfo_file) - - add_package_info(version, pkginfo_file) - end - - - def include_package?(name, version=nil, os=nil) - # check version first - if not version.nil? then - version = get_latest_version() - end - - if version.nil? or @package_infos[version].nil? then return false end - - # check supported os - if not os.nil? and not @os_list.include? os then return false end - - # check name and version - pkginfo=@package_infos[version] - pkg_list = os.nil? ? pkginfo.packages : pkginfo.get_target_packages(os) - pkg_list.each do |pkg| - if pkg.package_name.eql? name then return true end - end - - return false - end - - def save_db() - config_file = "#{@path}/build" - File.open( config_file, "w" ) do |f| - f.puts "TYPE=GIT" - f.puts "GIT_REPOSITORY=#{@repository}" - f.puts "GIT_BRANCH=#{@branch}" - end - return true - end - - - def load_db() - # git - config_file = "#{@path}/build" - repos="none" - branch="master" - File.open( config_file, "r" ) do |f| - f.each_line do |l| - idx = l.index("=") + 1 - length = l.length - idx - - if l.start_with?("GIT_REPOSITORY=") - repos = l[idx,length].strip - elsif l.start_with?("GIT_BRANCH=") - branch = l[idx,length].strip - else - next - end - end - end - @repository = repos - @branch = branch - - # read source info - sources_file = "#{@path}/sources" - if File.exist? sources_file then - File.open(sources_file, "r") do |f| - f.each_line do |l| - version = l.split(",")[0].strip - info = l.split(",")[1].strip - - add_source_info( version, info ) - end - end - end - - # read pkginfo - pkginfo_dir = "#{@path}/pkginfos" - if not File.exist? pkginfo_dir then FileUtils.mkdir_p pkginfo_dir end - Dir.new(pkginfo_dir).entries.each do |file| - if file.eql? "." or file.eql? ".." then next end - - vlen = file.length - ".manifest".length - version = file[0,vlen] - add_package_info( version, "#{pkginfo_dir}/#{file}" ) - end - return true end def self.load(name, dist_name, server, db) - row, prj_os_list = load_row(name, dist_name, db) + row, prj_os_list, source_info, package_info = load_row(name, dist_name, db) if row.nil? then return nil end prj_id = row['id'] @@ -247,23 +108,36 @@ class GitBuildProject < CommonProject prj_dist = row['dist_name'] new_project = GitBuildProject.new(prj_name, server, prj_os_list, prj_dist) - if not prj_passwd.empty? then new_project.passwd = prj_passwd end + new_project.set_project_id( prj_id ) + new_project.set_source_info( source_info ) + new_project.set_package_info( package_info ) - if not new_project.load_db() then raise RuntimeError, "Project DB load failed!" end + row=db.select_one("SELECT * FROM project_gits WHERE prj_id=#{prj_id}") + if row.nil? then return nil end + new_project.repository=row['git_repos'] + new_project.branch=row['git_branch'] return new_project end def save(db) - save_common(db) + is_new = save_common(db) init() - if not save_db() then raise RuntimeError, "Project DB load failed!" end + + if is_new then + db.do "INSERT INTO project_gits VALUES (#{@prj_id},'#{@repository}','#{@branch}')" + else + db.do "UPDATE project_gits SET git_repos='#{@repository}',git_branch='#{@branch}' WHERE prj_id=#{@prj_id})" + end end def unload(db) unload_common(db) + if @prj_id != -1 then + db.do "DELETE FROM project_gits WHERE prj_id=#{@prj_id}" + end end end diff --git a/src/build_server/ProjectManager.rb b/src/build_server/ProjectManager.rb index a952756..d832649 100644 --- a/src/build_server/ProjectManager.rb +++ b/src/build_server/ProjectManager.rb @@ -300,12 +300,13 @@ class ProjectManager end - def get_project_from_package_name(pkg_name, dist_name="BASE") + def get_project_from_package_name(pkg_name, dist_name) get_all_projects().each do |prj| + if prj.type != "BINARY" then next end # check project's distribution if prj.dist_name != dist_name then next end # check project provide target package - if prj.include_package?(pkg_name) then + if prj.pkg_name == pkg_name then return prj end end -- 2.34.1