Add timeout handling for RW-Upgrade scripts 45/318245/1 accepted/tizen_9.0_unified tizen_9.0 accepted/tizen/9.0/unified/20250117.022453
authorAdam Michalski <a.michalski2@partner.samsung.com>
Fri, 22 Nov 2024 15:38:17 +0000 (16:38 +0100)
committerMateusz Moscicki <m.moscicki2@partner.samsung.com>
Tue, 14 Jan 2025 16:30:37 +0000 (17:30 +0100)
Introduced timeout handling for RW-Upgrade scripts execution.

- Added `run-with-timeout.sh` for executing scripts with a specified
  default timeout.
- Modified `update.sh.in` to:
  - Use `run-with-timeout.sh` with the default timeout value.
  - Parse `###Timeout = <value>` in each script for custom timeouts.
  - Default to `INDIVIDUAL_SCRIPT_TIMEOUT` if no custom timeout is set.
  - Reads /etc/upgrade/conf.d/rw-upgrade-scripts.conf and sets the
    timeout value for all scripts based on it, otherwise sets it to
    25 seconds.
  - Ensure backward compatibility for scripts without custom timeouts.
  - Improve error handling for individual and total timeout violations.

This ensures stricter control over script execution timing and
improves the reliability of the RW-Upgrade process.

Change-Id: Id80f8912e7e4a96251e76cd87ea0131ed2700215

packaging/upgrade.spec
scripts/rw-upgrade/CMakeLists.txt
scripts/rw-upgrade/run-with-timeout.sh [new file with mode: 0644]
scripts/rw-upgrade/rw-upgrade-scripts.conf [new file with mode: 0644]
scripts/rw-upgrade/update.sh.in

index 63bb69cfa51676d5e44115fadcaff34be37cd63f..a4d9259f7b3d463145fcea3555979e5b4796c9a8 100644 (file)
@@ -3,7 +3,7 @@
 
 Name:          upgrade
 Summary:       Upgrade support for Tizen
-Version:       8.1.1
+Version:       9.0.0
 Release:       0
 Group:         System
 License:       MIT, Apache-2.0
@@ -228,6 +228,7 @@ fi
 %{upgrade_scripts_dir}/offline-update-finalize.sh
 %{upgrade_scripts_dir}/update-init.sh
 %{upgrade_scripts_dir}/offline-update-post.sh
+%{upgrade_scripts_dir}/run-with-timeout.sh
 %{upgrade_scripts_dir}/update.sh
 %{upgrade_scripts_dir}/update-verify.sh
 %{upgrade_scripts_dir}/online-update.sh
@@ -236,6 +237,7 @@ fi
 %{upgrade_scripts_dir}/online-update-verify/
 %{upgrade_scripts_dir}/online-update-verify.sh
 %{upgrade_scripts_dir}/online-update-success.sh
+%{_sysconfdir}/upgrade/conf.d/rw-upgrade-scripts.conf
 
 %files -n parse-dynparts
 %manifest upgrade.manifest
index 4bae122cc59db3786caf525379c61b1498dfec19..51fcb7e9e218ef32909f7debe53da22b845d2678 100644 (file)
@@ -45,5 +45,12 @@ INSTALL(FILES
        online-update.sh
        online-update-verify.sh
        online-update-success.sh
+       run-with-timeout.sh
        DESTINATION ${UPGRADE_SCRIPTS_DIR}
        PERMISSIONS OWNER_READ OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE)
+
+INSTALL(FILES
+       rw-upgrade-scripts.conf
+       DESTINATION
+       /etc/upgrade/conf.d/
+       PERMISSIONS OWNER_READ GROUP_READ)
diff --git a/scripts/rw-upgrade/run-with-timeout.sh b/scripts/rw-upgrade/run-with-timeout.sh
new file mode 100644 (file)
index 0000000..4f107ce
--- /dev/null
@@ -0,0 +1,60 @@
+#!/bin/bash
+PATH=/bin:/usr/bin:/sbin:/usr/sbin
+
+# This script executes a given command with a specified timeout.
+# If the command does not complete within the timeout period, the
+# script will first attempt to terminate the process gracefully
+# using SIGTERM. If the process is still running after 1 second,
+# it will forcefully terminate it using SIGKILL. The script reports
+# whether the command finished successfully, was terminated due to
+# timeout, or exited with an error code.
+#
+# Usage: ./run_with_timeout.sh <timeout_in_seconds> <command> [args...]
+
+# Common exit code for timeout
+TIMEOUT_EXIT_CODE=124
+
+# Check if at least two arguments are passed
+if [ "$#" -lt 2 ]; then
+    echo "Usage: $0 <timeout_in_seconds> <command> [args...]"
+    exit 1
+fi
+
+# Parse timeout and command
+timeout_seconds=$1
+shift
+
+# Start the command in the background
+"$@" &
+child_pid=$!
+
+# Monitor the child process
+elapsed_seconds=0
+interval_seconds=1  # Check every 1 second
+while kill -0 $child_pid 2>/dev/null; do
+    if [ "$elapsed_seconds" -ge "$timeout_seconds" ]; then
+        # Timeout reached, send SIGTERM
+        kill -SIGTERM $child_pid
+        sleep 1
+        # Check if the process is still running and send SIGKILL if necessary
+        if kill -0 $child_pid 2>/dev/null; then
+            kill -SIGKILL $child_pid
+        fi
+        echo "Process terminated due to timeout"
+        exit $TIMEOUT_EXIT_CODE
+    fi
+    sleep $interval_seconds
+    elapsed_seconds=$((elapsed_seconds + interval_seconds))
+done
+
+# Wait for the child process to complete and capture its exit status
+wait $child_pid
+exit_status=$?
+
+if [ "$exit_status" -eq 0 ]; then
+    echo "Process finished successfully"
+else
+    echo "Process finished with exit status $exit_status"
+fi
+
+exit $exit_status
diff --git a/scripts/rw-upgrade/rw-upgrade-scripts.conf b/scripts/rw-upgrade/rw-upgrade-scripts.conf
new file mode 100644 (file)
index 0000000..e636e72
--- /dev/null
@@ -0,0 +1 @@
+rw_all_scripts_timeout=180
index b19c7b1b0f037a26fb1602545eabfa189f5906ef..b73dc16bab48a1c1570993439ba156d757b20866 100644 (file)
@@ -21,6 +21,12 @@ RW_MACRO=@UPGRADE_SCRIPTS_DIR@/rw-update-macro.inc
 RW_UPDATE_FLAG=/opt/.do_rw_update
 
 SCRIPT_NAME=$(basename $0)
+TIMEOUT_CONFIG_FILE=/etc/upgrade/conf.d/rw-upgrade-scripts.conf
+INVOKE_SCRIPT_WITH_TIMEOUT=@UPGRADE_SCRIPTS_DIR@/run-with-timeout.sh
+
+# Defaults in seconds. Values determined based on measurement of script execution time.
+INDIVIDUAL_SCRIPT_TIMEOUT=70 # override via `###Timeout=xxx` in given script
+ALL_SCRIPTS_TIMEOUT=180 # override via /etc/upgrade/conf.d/rw-upgrade-scripts.conf
 
 DEBUG()
 {
@@ -120,6 +126,18 @@ then
        UPDATE_PREPARE_ERR=1
 fi
 
+# Load timeout config file
+if [ -f "$TIMEOUT_CONFIG_FILE" ]; then
+       while IFS='=' read -r name value; do
+               name=$(echo "$name" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')
+               value=$(echo "$value" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')
+
+               if [ "$name" = "rw_all_scripts_timeout" ]; then
+                       ALL_SCRIPTS_TIMEOUT="$value"
+               fi
+       done < "$CFG_DIR/$MAIN_CFG_FILE"
+fi
+
 # Execute update scripts
 if [ ${UPDATE_PREPARE_ERR} = "1" ]
 then
@@ -135,9 +153,20 @@ else
 
        UPDATE_SCRIPTS=`/bin/ls ${UPDATE_SCRIPT_DIR}`
        NOTIFY "TOTAL TASKS: ${TOTAL_TASKS}"
+       ALL_SCRIPTS_START_TIME=$(date +%s)
        for UPSCRIPT in ${UPDATE_SCRIPTS}; do
                NOTIFY "CURRENT TASK: ${CURRENT_TASK} ${UPSCRIPT}"
-               ${SHELL} ${UPDATE_SCRIPT_DIR}/${UPSCRIPT}
+               # Default timeout
+               TIMEOUT=${INDIVIDUAL_SCRIPT_TIMEOUT}
+               # Check if the script has a custom timeout
+               if [ -f "${UPDATE_SCRIPT_DIR}/${UPSCRIPT}" ]; then
+                       # Look for ###Timeout in the script
+                       SCRIPT_TIMEOUT=$(grep -E '^###Timeout[[:space:]]*=[[:space:]]*[0-9]+' "${UPDATE_SCRIPT_DIR}/${UPSCRIPT}" | awk -F= '{print $2}' | tr -d '[:space:]')
+                       if [[ ${SCRIPT_TIMEOUT} =~ ^[0-9]+$ ]]; then
+                               TIMEOUT=${SCRIPT_TIMEOUT}
+                       fi
+               fi
+               ${SHELL} ${INVOKE_SCRIPT_WITH_TIMEOUT} ${TIMEOUT} ${SHELL} ${UPDATE_SCRIPT_DIR}/${UPSCRIPT}
                if [ $? -ne 0 ]; then
                        ERROR "[FAIL] ${UPSCRIPT}"
                        UPDATE_PROGRESS_ERR=1
@@ -150,6 +179,18 @@ else
                CURRENT_TASK=$(( ${CURRENT_TASK} + 1 ))
        done
 
+       # Calculate the elapsed time
+       CURRENT_TIME=$(date +%s)
+       TIME_PASSED=$((CURRENT_TIME - ALL_SCRIPTS_START_TIME))
+
+       # Check if the total time exceeded the allowed timeout
+       if [ "$TIME_PASSED" -ge "$ALL_SCRIPTS_TIMEOUT" ]; then
+               echo "The entire loop took too long (time_passed=$TIME_PASSED seconds, timeout=$ALL_SCRIPTS_TIMEOUT seconds)."
+               UPDATE_PROGRESS_ERR=1
+       else
+               echo "The entire loop completed within the allowed time (time_passed=$TIME_PASSED seconds)."
+       fi
+
        write_version_info
 fi