From 139146d39f2c8aa30ea441f79ed72b9996f9fa5f Mon Sep 17 00:00:00 2001 From: hyokeun Date: Thu, 21 Dec 2017 16:43:52 +0900 Subject: [PATCH] Terminate VMs if healthy check failed Change-Id: I1f38d7fbd8a312e13f0cab756f2da932e589d20d --- job_control_ondemand_slaves.groovy | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/job_control_ondemand_slaves.groovy b/job_control_ondemand_slaves.groovy index 81bb469..038c415 100644 --- a/job_control_ondemand_slaves.groovy +++ b/job_control_ondemand_slaves.groovy @@ -341,10 +341,12 @@ def check_healthy_status(worker_conf, vm_list) { def ret_val = execute_command("ssh", ssh_options + " -X " + ssh_username + "@" + ssh_hostname + " " + docker_command, verbose=false, return_stdout=true) if (!ret_val.contains(" Active: active (running)") || !ret_val.contains("Checking for obsworker: ..running")) { - println 'HEALTHY:FAIL for ' + ssh_hostname + '\nRestarting docker...' - docker_command = "sudo docker-compose -f /root/docker-compose.yml up -d" - execute_command("ssh", ssh_options + " -X " + ssh_username + "@" + ssh_hostname + " " + docker_command, - verbose=false, return_stdout=true) + println 'HEALTHY:FAIL for ' + ssh_hostname + '\nShutdown the VM right now!' + terminate_aws_ec2_instances([inst["instance_id"]]) + //println 'HEALTHY:FAIL for ' + ssh_hostname + '\nRestarting docker...' + //docker_command = "sudo docker-compose -f /root/docker-compose.yml up -d" + //execute_command("ssh", ssh_options + " -X " + ssh_username + "@" + ssh_hostname + " " + docker_command, + // verbose=false, return_stdout=true) } } -- 2.7.4