[Title] Added exception handler for thread to prevent server to die

diff --git a/src/build_server/BuildJob.rb b/src/build_server/BuildJob.rb
index e9dd114e9a489e519fd09985ea5b75c7a5a344cd..8051381de5ba5c3e096a56e1c986ab0035a30914 100644 (file)
--- a/src/build_server/BuildJob.rb
+++ b/src/build_server/BuildJob.rb
@@ -187,12 +187,12 @@ class BuildJob
                @log.info( "Invoking a thread for building Job #{@id}", Log::LV_USER)
                if @status == "ERROR" then return  end
                @thread = Thread.new {
-                       # main
-                       thread_main()
-                       
-                       # close 
-                       if not is_sub_job? then
-                               terminate()
+                       begin
+                               thread_main()
+                               if not is_sub_job? then terminate() end
+                       rescue => e
+                               @log.error e.message
+                               @log.error e.backtrace.inspect
                        end
                }
 

diff --git a/src/build_server/FullBuildJob.rb b/src/build_server/FullBuildJob.rb
index 31778c0345ee663ff5748a379a2238c3dc6a5e57..5c83feeb55bb12ee9d975873ab032661618e3429 100644 (file)
--- a/src/build_server/FullBuildJob.rb
+++ b/src/build_server/FullBuildJob.rb
@@ -72,11 +72,13 @@ class FullBuildJob
                @log.info( "Invoking a thread for FULL-BUILD Job #{@id}", Log::LV_USER)
                if @status == "ERROR" then return  end
                @thread = Thread.new {
-                       # main
-                       thread_main()
-                       
-                       # close 
-                       terminate()
+                       begin
+                               thread_main()
+                               terminate()
+                       rescue => e
+                               @log.error e.message
+                               @log.error e.backtrace.inspect
+                       end
                }
 
                if sync then

diff --git a/src/build_server/JobClean.rb b/src/build_server/JobClean.rb
index 4a6f5856bd0f719a073af2ada1707a7674e72c95..64265d7140341672338f0a7859a3593f31fcb536 100644 (file)
--- a/src/build_server/JobClean.rb
+++ b/src/build_server/JobClean.rb
@@ -58,10 +58,19 @@ class JobCleanAction < Action
 
 
        def execute
-       
                # Start to clean job
                @server.log.info "Executing clean action for the job #{@job_id}"
+               begin
+                       execute_internal()
+               rescue => e
+                       @server.log.error e.message
+                       @server.log.error e.backtrace.inspect
+               end
+       end
+
 
+       private
+       def execute_internal()
                # remove directories
                if File.exist? "#{@job_path}/buildroot" then
                        FileUtils.rm_rf "#{@job_path}/buildroot"

diff --git a/src/build_server/JobManager.rb b/src/build_server/JobManager.rb
index c1d916d6878098d04ec229adce30791fc1f52b2d..e8e1d6467fd9fa50da8bf55b8ca21b4a190fc7c7 100644 (file)
--- a/src/build_server/JobManager.rb
+++ b/src/build_server/JobManager.rb
@@ -129,17 +129,22 @@ class JobManager
        def initialize_job ( job )
                job.status = "INITIALIZING"
                Thread.new {
-                       # init
-                       if not job.init or job.status == "ERROR" then
-                               if job.cancel_state == "NONE" then job.status = "ERROR" end
-                               @parent.log.info "Adding the job \"#{job.id}\" is canceled"
-                               job.terminate()
-                               Thread.current.exit
-                       end
-                       if job.status != "FINISHED" then
-                               job.status = "WAITING"
+                       begin
+                               # init
+                               if not job.init or job.status == "ERROR" then
+                                       if job.cancel_state == "NONE" then job.status = "ERROR" end
+                                       @parent.log.info "Adding the job \"#{job.id}\" is canceled"
+                                       job.terminate()
+                                       Thread.current.exit
+                               end
+                               if job.status != "FINISHED" then
+                                       job.status = "WAITING"
+                               end
+                               @parent.log.info "Checking the job \"#{job.id}\" was finished!"
+                       rescue => e
+                               @parent.log.error e.message
+                               @parent.log.error e.backtrace.inspect
                        end
-                       @parent.log.info "Checking the job \"#{job.id}\" was finished!"
                }
                @parent.log.info "Job \"#{job.id}\" entered INITIALIZING status" 
        end
@@ -173,21 +178,25 @@ class JobManager
                job.cancel_state = "WORKING"
                @parent.log.info "Creating thread for canceling the job \"#{job.id}\"" 
                Thread.new {
-                       # thread terminate
-                       if not job.thread.nil? then
+                       begin
                                #terminate job thread
-                               job.thread.terminate
-                               job.thread = nil
-                       end
+                               if not job.thread.nil? then
+                                       job.thread.terminate
+                                       job.thread = nil
+                               end
 
-                       # job cacncel
-                       job.cancel
+                               # job cacncel
+                               job.cancel
 
-                       # cancel finished
-                       job.status = "CANCELED"
+                               # cancel finished
+                               job.status = "CANCELED"
 
-                       # call terminate process for job
-                       job.terminate
+                               # call terminate process for job
+                               job.terminate
+                       rescue => e
+                               @parent.log.error e.message
+                               @parent.log.error e.backtrace.inspect
+                       end
                }
        end
 

diff --git a/src/build_server/MultiBuildJob.rb b/src/build_server/MultiBuildJob.rb
index a37c2bac5041789de3eea3d3906e4609351d1176..5c5175d0e4ef0cf6540efdb53bf058124d8da352 100644 (file)
--- a/src/build_server/MultiBuildJob.rb
+++ b/src/build_server/MultiBuildJob.rb
@@ -84,11 +84,16 @@ class MultiBuildJob
                @log.info( "Invoking a thread for MULTI-BUILD Job #{@id}", Log::LV_USER)
                if @status == "ERROR" then return  end
                @thread = Thread.new {
-                       # main
-                       thread_main()
+                       begin
+                               # main
+                               thread_main()
                        
-                       # close 
-                       terminate()
+                               # close 
+                               terminate()
+                       rescue => e
+                               @log.error e.message
+                               @log.error e.backtrace.inspect
+                       end     
                }
 
                if sync then

diff --git a/src/build_server/PackageSync.rb b/src/build_server/PackageSync.rb
index 07655ef7c6762b86a6502d7403069a5f6537f287..f395856d4e6ea3301a83c613bdc4362492678e17 100644 (file)
--- a/src/build_server/PackageSync.rb
+++ b/src/build_server/PackageSync.rb
@@ -71,7 +71,18 @@ class PackageSyncAction < Action
 
        def execute
                @server.log.info "Executing package-sync action for server \"#{@pkgsvr_url}\""
+               
+               begin
+                       execute_internal()
+               rescue => e
+                       @server.log.error e.message
+                       @server.log.error e.backtrace.inspect
+               end
+       end
+
 
+       private
+       def execute_internal()
                # check update
                pkgs = check_package_update
 

diff --git a/src/build_server/RegisterPackageJob.rb b/src/build_server/RegisterPackageJob.rb
index d77fb9fe6ef014d8a45195ed7fa498e8acfced7b..0eeebdb7a1172e7d6c9589624b6e76885effe2f3 100644 (file)
--- a/src/build_server/RegisterPackageJob.rb
+++ b/src/build_server/RegisterPackageJob.rb
@@ -106,11 +106,13 @@ class RegisterPackageJob
                @log.info( "Invoking a thread for REGISTER Job #{@id}", Log::LV_USER)
                if @status == "ERROR" then return  end
                @thread = Thread.new {
-                       # main
-                       thread_main()
-                       
-                       # close 
-                       terminate()
+                       begin
+                               thread_main()
+                               terminate()
+                       rescue => e
+                               @log.error e.message
+                               @log.error e.backtrace.inspect
+                       end
                }
 
                if sync then

diff --git a/src/build_server/SocketJobRequestListener.rb b/src/build_server/SocketJobRequestListener.rb
index f09a0565fc5417b2b23a20ba59d7f7ed0da4c1fc..1278aeecf1e52b1a0e6f1e2821f3da7a8d42315d 100644 (file)
--- a/src/build_server/SocketJobRequestListener.rb
+++ b/src/build_server/SocketJobRequestListener.rb
@@ -45,7 +45,15 @@ class SocketJobRequestListener
        # start listening
        def start()
                @thread = Thread.new {
-                       main()          
+                       # make loop recover when unhandled exception occurred
+                       while not @finish_loop
+                               begin
+                                       main()          
+                               rescue => e
+                                       @log.error e.message
+                                       @log.error e.backtrace.inspect
+                               end
+                       end
                }
        end
 
@@ -123,11 +131,23 @@ class SocketJobRequestListener
                        handle_cmd_register( req_line, req )
                when "DOWNLOAD"
             Thread.new {
-                       handle_cmd_download( req_line, req )
+                               begin
+                               handle_cmd_download( req_line, req )
+                               rescue => e
+                                       @log.error "Transfering file failed!"
+                                       @log.error e.message
+                                       @log.error e.backtrace.inspect
+                               end
             }             
                when "UPLOAD"
             Thread.new {
-                       handle_cmd_upload( req_line, req )
+                               begin
+                               handle_cmd_upload( req_line, req )
+                               rescue => e
+                                       @log.error "Transfering file failed!"
+                                       @log.error e.message
+                                       @log.error e.backtrace.inspect
+                               end
             }                
                else
                        @log.info "Received Unknown REQ: #{req_line}" 
@@ -621,15 +641,9 @@ class SocketJobRequestListener
                dock_num = tok[1].strip
 
                BuildCommServer.send_begin(req)
-               begin
-                       incoming_dir = "#{@parent_server.transport_path}/#{dock_num}"
-                       if not File.exist? incoming_dir then FileUtils.mkdir_p incoming_dir end
-                       @comm_server.receive_file( req, incoming_dir )
-               rescue => e
-                       @log.error "Failed to receive file"
-                       @log.error e.message
-                       @log.error e.backtrace.inspect
-               end
+               incoming_dir = "#{@parent_server.transport_path}/#{dock_num}"
+               if not File.exist? incoming_dir then FileUtils.mkdir_p incoming_dir end
+               @comm_server.receive_file( req, incoming_dir )
                BuildCommServer.send_end(req)
        end
 
@@ -650,21 +664,15 @@ class SocketJobRequestListener
                @log.info "Received a request for download file : #{file_name}"
                outgoing_dir = "#{@parent_server.transport_path}/#{dock_num}"
                BuildCommServer.send_begin(req)
-               begin
-                       @log.info "Sending requested file...: #{file_name}"
-                       @comm_server.send_file(req, "#{outgoing_dir}/#{file_name}")
-                       # remove file if "dock" defined
-                       if dock_num != "0" then
-                               @log.info "Removing requested file...: #{file_name}"
-                               FileUtils.rm_rf "#{outgoing_dir}/#{file_name}"
-                               if Utils.directory_emtpy?(outgoing_dir) then 
-                                       FileUtils.rm_rf "#{outgoing_dir}" 
-                               end
+               @log.info "Sending requested file...: #{file_name}"
+               @comm_server.send_file(req, "#{outgoing_dir}/#{file_name}")
+               # remove file if "dock" defined
+               if dock_num != "0" then
+                       @log.info "Removing requested file...: #{file_name}"
+                       FileUtils.rm_rf "#{outgoing_dir}/#{file_name}"
+                       if Utils.directory_emtpy?(outgoing_dir) then 
+                               FileUtils.rm_rf "#{outgoing_dir}" 
                        end
-               rescue => e
-                       @log.error "transfer failed"
-                       @log.error e.message
-                       @log.error e.backtrace.inspect
                end
 
                BuildCommServer.send_end(req)

diff --git a/src/pkg_server/DistSync.rb b/src/pkg_server/DistSync.rb
index d1605650df95274fd6b8e8d7a1e8cf5f429f2ba1..5fb6df9ec1bb8008d506a32a05a176e000497d12 100644 (file)
--- a/src/pkg_server/DistSync.rb
+++ b/src/pkg_server/DistSync.rb
@@ -49,10 +49,19 @@ class DistSyncAction < Action
 
 
        def execute
-       
-               # Start to clean job
+               # Start to sync job
                @pkgserver.log.info "Executing sync action for the #{@dist_name}"
+               begin
+                       execute_internal()
+               rescue => e
+                       @pkgserver.log.error e.message
+                       @pkgserver.log.error e.backtrace.inspect
+               end
+       end
+
 
+       private 
+       def execute_internal()
                # update pkg info
                @pkgserver.reload_dist_package