From b6b5f3344572bd84afed620505182f05ac5c7a2a Mon Sep 17 00:00:00 2001 From: Adam Michalski Date: Fri, 22 Nov 2024 16:38:17 +0100 Subject: [PATCH] Add timeout handling for RW-Upgrade scripts Introduced timeout handling for RW-Upgrade scripts execution. - Added `run-with-timeout.sh` for executing scripts with a specified default timeout. - Modified `update.sh.in` to: - Use `run-with-timeout.sh` with the default timeout value. - Parse `###Timeout = ` in each script for custom timeouts. - Default to `INDIVIDUAL_SCRIPT_TIMEOUT` if no custom timeout is set. - Reads /etc/upgrade/conf.d/rw-upgrade-scripts.conf and sets the timeout value for all scripts based on it, otherwise sets it to 25 seconds. - Ensure backward compatibility for scripts without custom timeouts. - Improve error handling for individual and total timeout violations. This ensures stricter control over script execution timing and improves the reliability of the RW-Upgrade process. Change-Id: Id80f8912e7e4a96251e76cd87ea0131ed2700215 --- packaging/upgrade.spec | 4 +- scripts/rw-upgrade/CMakeLists.txt | 7 +++ scripts/rw-upgrade/run-with-timeout.sh | 60 ++++++++++++++++++++++ scripts/rw-upgrade/rw-upgrade-scripts.conf | 1 + scripts/rw-upgrade/update.sh.in | 43 +++++++++++++++- 5 files changed, 113 insertions(+), 2 deletions(-) create mode 100644 scripts/rw-upgrade/run-with-timeout.sh create mode 100644 scripts/rw-upgrade/rw-upgrade-scripts.conf diff --git a/packaging/upgrade.spec b/packaging/upgrade.spec index 63bb69c..a4d9259 100644 --- a/packaging/upgrade.spec +++ b/packaging/upgrade.spec @@ -3,7 +3,7 @@ Name: upgrade Summary: Upgrade support for Tizen -Version: 8.1.1 +Version: 9.0.0 Release: 0 Group: System License: MIT, Apache-2.0 @@ -228,6 +228,7 @@ fi %{upgrade_scripts_dir}/offline-update-finalize.sh %{upgrade_scripts_dir}/update-init.sh %{upgrade_scripts_dir}/offline-update-post.sh +%{upgrade_scripts_dir}/run-with-timeout.sh %{upgrade_scripts_dir}/update.sh %{upgrade_scripts_dir}/update-verify.sh %{upgrade_scripts_dir}/online-update.sh @@ -236,6 +237,7 @@ fi %{upgrade_scripts_dir}/online-update-verify/ %{upgrade_scripts_dir}/online-update-verify.sh %{upgrade_scripts_dir}/online-update-success.sh +%{_sysconfdir}/upgrade/conf.d/rw-upgrade-scripts.conf %files -n parse-dynparts %manifest upgrade.manifest diff --git a/scripts/rw-upgrade/CMakeLists.txt b/scripts/rw-upgrade/CMakeLists.txt index 4bae122..51fcb7e 100644 --- a/scripts/rw-upgrade/CMakeLists.txt +++ b/scripts/rw-upgrade/CMakeLists.txt @@ -45,5 +45,12 @@ INSTALL(FILES online-update.sh online-update-verify.sh online-update-success.sh + run-with-timeout.sh DESTINATION ${UPGRADE_SCRIPTS_DIR} PERMISSIONS OWNER_READ OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE) + +INSTALL(FILES + rw-upgrade-scripts.conf + DESTINATION + /etc/upgrade/conf.d/ + PERMISSIONS OWNER_READ GROUP_READ) diff --git a/scripts/rw-upgrade/run-with-timeout.sh b/scripts/rw-upgrade/run-with-timeout.sh new file mode 100644 index 0000000..4f107ce --- /dev/null +++ b/scripts/rw-upgrade/run-with-timeout.sh @@ -0,0 +1,60 @@ +#!/bin/bash +PATH=/bin:/usr/bin:/sbin:/usr/sbin + +# This script executes a given command with a specified timeout. +# If the command does not complete within the timeout period, the +# script will first attempt to terminate the process gracefully +# using SIGTERM. If the process is still running after 1 second, +# it will forcefully terminate it using SIGKILL. The script reports +# whether the command finished successfully, was terminated due to +# timeout, or exited with an error code. +# +# Usage: ./run_with_timeout.sh [args...] + +# Common exit code for timeout +TIMEOUT_EXIT_CODE=124 + +# Check if at least two arguments are passed +if [ "$#" -lt 2 ]; then + echo "Usage: $0 [args...]" + exit 1 +fi + +# Parse timeout and command +timeout_seconds=$1 +shift + +# Start the command in the background +"$@" & +child_pid=$! + +# Monitor the child process +elapsed_seconds=0 +interval_seconds=1 # Check every 1 second +while kill -0 $child_pid 2>/dev/null; do + if [ "$elapsed_seconds" -ge "$timeout_seconds" ]; then + # Timeout reached, send SIGTERM + kill -SIGTERM $child_pid + sleep 1 + # Check if the process is still running and send SIGKILL if necessary + if kill -0 $child_pid 2>/dev/null; then + kill -SIGKILL $child_pid + fi + echo "Process terminated due to timeout" + exit $TIMEOUT_EXIT_CODE + fi + sleep $interval_seconds + elapsed_seconds=$((elapsed_seconds + interval_seconds)) +done + +# Wait for the child process to complete and capture its exit status +wait $child_pid +exit_status=$? + +if [ "$exit_status" -eq 0 ]; then + echo "Process finished successfully" +else + echo "Process finished with exit status $exit_status" +fi + +exit $exit_status diff --git a/scripts/rw-upgrade/rw-upgrade-scripts.conf b/scripts/rw-upgrade/rw-upgrade-scripts.conf new file mode 100644 index 0000000..e636e72 --- /dev/null +++ b/scripts/rw-upgrade/rw-upgrade-scripts.conf @@ -0,0 +1 @@ +rw_all_scripts_timeout=180 diff --git a/scripts/rw-upgrade/update.sh.in b/scripts/rw-upgrade/update.sh.in index b19c7b1..b73dc16 100644 --- a/scripts/rw-upgrade/update.sh.in +++ b/scripts/rw-upgrade/update.sh.in @@ -21,6 +21,12 @@ RW_MACRO=@UPGRADE_SCRIPTS_DIR@/rw-update-macro.inc RW_UPDATE_FLAG=/opt/.do_rw_update SCRIPT_NAME=$(basename $0) +TIMEOUT_CONFIG_FILE=/etc/upgrade/conf.d/rw-upgrade-scripts.conf +INVOKE_SCRIPT_WITH_TIMEOUT=@UPGRADE_SCRIPTS_DIR@/run-with-timeout.sh + +# Defaults in seconds. Values determined based on measurement of script execution time. +INDIVIDUAL_SCRIPT_TIMEOUT=70 # override via `###Timeout=xxx` in given script +ALL_SCRIPTS_TIMEOUT=180 # override via /etc/upgrade/conf.d/rw-upgrade-scripts.conf DEBUG() { @@ -120,6 +126,18 @@ then UPDATE_PREPARE_ERR=1 fi +# Load timeout config file +if [ -f "$TIMEOUT_CONFIG_FILE" ]; then + while IFS='=' read -r name value; do + name=$(echo "$name" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') + value=$(echo "$value" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') + + if [ "$name" = "rw_all_scripts_timeout" ]; then + ALL_SCRIPTS_TIMEOUT="$value" + fi + done < "$CFG_DIR/$MAIN_CFG_FILE" +fi + # Execute update scripts if [ ${UPDATE_PREPARE_ERR} = "1" ] then @@ -135,9 +153,20 @@ else UPDATE_SCRIPTS=`/bin/ls ${UPDATE_SCRIPT_DIR}` NOTIFY "TOTAL TASKS: ${TOTAL_TASKS}" + ALL_SCRIPTS_START_TIME=$(date +%s) for UPSCRIPT in ${UPDATE_SCRIPTS}; do NOTIFY "CURRENT TASK: ${CURRENT_TASK} ${UPSCRIPT}" - ${SHELL} ${UPDATE_SCRIPT_DIR}/${UPSCRIPT} + # Default timeout + TIMEOUT=${INDIVIDUAL_SCRIPT_TIMEOUT} + # Check if the script has a custom timeout + if [ -f "${UPDATE_SCRIPT_DIR}/${UPSCRIPT}" ]; then + # Look for ###Timeout in the script + SCRIPT_TIMEOUT=$(grep -E '^###Timeout[[:space:]]*=[[:space:]]*[0-9]+' "${UPDATE_SCRIPT_DIR}/${UPSCRIPT}" | awk -F= '{print $2}' | tr -d '[:space:]') + if [[ ${SCRIPT_TIMEOUT} =~ ^[0-9]+$ ]]; then + TIMEOUT=${SCRIPT_TIMEOUT} + fi + fi + ${SHELL} ${INVOKE_SCRIPT_WITH_TIMEOUT} ${TIMEOUT} ${SHELL} ${UPDATE_SCRIPT_DIR}/${UPSCRIPT} if [ $? -ne 0 ]; then ERROR "[FAIL] ${UPSCRIPT}" UPDATE_PROGRESS_ERR=1 @@ -150,6 +179,18 @@ else CURRENT_TASK=$(( ${CURRENT_TASK} + 1 )) done + # Calculate the elapsed time + CURRENT_TIME=$(date +%s) + TIME_PASSED=$((CURRENT_TIME - ALL_SCRIPTS_START_TIME)) + + # Check if the total time exceeded the allowed timeout + if [ "$TIME_PASSED" -ge "$ALL_SCRIPTS_TIMEOUT" ]; then + echo "The entire loop took too long (time_passed=$TIME_PASSED seconds, timeout=$ALL_SCRIPTS_TIMEOUT seconds)." + UPDATE_PROGRESS_ERR=1 + else + echo "The entire loop completed within the allowed time (time_passed=$TIME_PASSED seconds)." + fi + write_version_info fi -- 2.34.1