From 359f78aa48584394cb9283c49e11bee62871db60 Mon Sep 17 00:00:00 2001 From: Aleksander Mistewicz Date: Mon, 7 Nov 2016 11:07:16 +0100 Subject: [PATCH] Make TITS profile independent Overall TITS adjustments Change-Id: I8a9b1255a122efa50a8275a558c0bc01174ad4fa Signed-off-by: Aleksander Mistewicz --- tct/common.sh | 2 +- tct/flash/flash.sh | 6 ++-- tct/minnow_prepare_flash_conf.sh | 2 +- tct/odroid_prepare_flash_conf.sh | 2 +- tsp/common.sh | 60 +++++++++++++++++++++++++++++++ tsp/cronfile.example | 4 +-- tsp/jobs/common_prep_flash_conf.sh | 29 ++++++++++----- tsp/jobs/img_test_common.sh | 67 ++++++++++++++++++++++++++--------- tsp/jobs/publish.sh | 4 +-- tsp/jobs/trigger_downloads.sh | 39 ++++++++++++-------- tsp/jobs/watch_diskspace.sh | 2 +- tsp/scripts/common_prep_flash_conf.sh | 19 +++++----- tsp/scripts/free_diskspace.sh | 13 +++---- 13 files changed, 184 insertions(+), 65 deletions(-) diff --git a/tct/common.sh b/tct/common.sh index ac8cb9a..ba5a1ff 100755 --- a/tct/common.sh +++ b/tct/common.sh @@ -31,7 +31,7 @@ die() { INC_SLEEP=10 INIT_SLEEP=60 RESTART_SLEEP=80 -MAX_SLEEP=120 +MAX_SLEEP=140 TL_IP_CONF="/etc/opt/testlab-major/ip_addr" SCREENSHOOTER_RPM="/opt/screenshooter.rpm" diff --git a/tct/flash/flash.sh b/tct/flash/flash.sh index 547ccf8..e81493c 100755 --- a/tct/flash/flash.sh +++ b/tct/flash/flash.sh @@ -36,20 +36,20 @@ uuid2dev() { UUID="$(cat "${FILE}")" DEV="$(readlink "/dev/disk/by-uuid/${UUID}")" - SLEEP=2 + SLEEP=10 echo "Searching: ${UUID}" >&2 while [ -z "${DEV}" ] && [ "${SLEEP}" -le 30 ] do - if [ -n "${SDMUX}" ] && [ "${SLEEP}" -ge 10 ] + if [ -n "${SDMUX}" ] && [ "${SLEEP}" -ge 20 ] then echo "Restarting sdmux: ${SDMUX}" >&2 switch2device "${SDMUX}" sdmux_init "${SDMUX}" fi sleep "${SLEEP}" - SLEEP=$((SLEEP+2)) + SLEEP=$((SLEEP+10)) DEV="$(readlink "/dev/disk/by-uuid/${UUID}")" done diff --git a/tct/minnow_prepare_flash_conf.sh b/tct/minnow_prepare_flash_conf.sh index 09a4a42..cf271f4 100755 --- a/tct/minnow_prepare_flash_conf.sh +++ b/tct/minnow_prepare_flash_conf.sh @@ -46,7 +46,7 @@ switch2testserver "${SDMUX}" echo "### flash ###" UUID_FILE="/var/tmp/uuid-${SDMUX}" -DEV_SDCARD=$(uuid2dev "${UUID_FILE}" "${SDMUX}") +DEV_SDCARD=$(uuid2dev "${UUID_FILE}" "${SDMUX}") || die "sdcard not found!" echo "Bmaptool: start" test -b "${DEV_SDCARD}" || die "File does not exist or is not block special" diff --git a/tct/odroid_prepare_flash_conf.sh b/tct/odroid_prepare_flash_conf.sh index cefd345..c9e774d 100755 --- a/tct/odroid_prepare_flash_conf.sh +++ b/tct/odroid_prepare_flash_conf.sh @@ -57,7 +57,7 @@ unmount_image "${IMAGE_MOUNTPOINT}" echo "### flash ###" UUID_FILE="/var/tmp/uuid-${SDMUX}" -DEV_SDCARD=$(uuid2dev "${UUID_FILE}" "${SDMUX}") +DEV_SDCARD=$(uuid2dev "${UUID_FILE}" "${SDMUX}") || die "sdcard not found!" cd "${IMAGE_UNPACKED}" "${TESTLAB_SCRIPTS}/flash/flash_dd.sh" "${DEV_SDCARD}" diff --git a/tsp/common.sh b/tsp/common.sh index e6ba407..50146ea 100755 --- a/tsp/common.sh +++ b/tsp/common.sh @@ -35,6 +35,8 @@ WS_DOWNLOAD="${WS}/dwn" WS_TEST="${WS}/img_test" WS_TEST_REMOTE="${WS}/img_test_remote" WS_PUBLISH="${WS}/publish" +SNAPSHOT_WS_DOWNLOAD="${WS_DOWNLOAD}/snapshot" +SNAPSHOT_WS_DOWNLOAD_NEW="${WS_DOWNLOAD}/new_snapshot" DBNAME="results.db3" DBPATH="${WS_PUBLISH}/${DBNAME}" @@ -105,3 +107,61 @@ update_db() { (select d.did from device d where d.dname='$TARGET'));" lockfile-remove "${DBPATH}" } + +get_userspace_dir() { + TARGET="$1" + DWN_WORKDIR="$2" + test -n "$TARGET" || die "Missing argument: target" + test -n "$DWN_WORKDIR" || die "Missing argument: dwn_workdir" + case "$TARGET" in + *odroid*) + USR_DIR="$(find "${DWN_WORKDIR}/" -type d -name "*-wayland*-armv7l-odroidu3")" + ;; + *headless-artik*) + USR_DIR="$(find "${DWN_WORKDIR}/" -type d -name "*-headless-*-armv7l-artik")" + ;; + *artik*) + USR_DIR="$(find "${DWN_WORKDIR}/" -type d -name "*-wayland*-armv7l-artik")" + ;; + *minnow32*) + USR_DIR="$(find "${DWN_WORKDIR}/" -type d -name "*-wayland-efi-i586")" + ;; + *minnow64*) + USR_DIR="$(find "${DWN_WORKDIR}/" -type d -name "*-wayland-efi-x86_64")" + ;; + *mbr64*) + USR_DIR="$(find "${DWN_WORKDIR}/" -type d -name "*-minimal-mbr-x86_64")" + ;; + *emulator32*) + USR_DIR="$(find "${DWN_WORKDIR}/" -type d -name "*-emulator-wayland-mbr-i586")" + ;; + *) + die "Unknown target: ${TARGET}" + ;; + esac + test -n "${USR_DIR}" || die "Userspace directory not found" + echo "${USR_DIR}" +} + +get_boot_dir() { + TARGET="$1" + DWN_WORKDIR="$2" + test -n "$TARGET" || die "Missing argument: target" + test -n "$DWN_WORKDIR" || die "Missing argument: dwn_workdir" + case "$TARGET" in + *odroid*) + BOOT_DIR="$(find "${DWN_WORKDIR}/" -type d -name "*-boot-armv7l-odroidu3")" + ;; + *artik5*) + BOOT_DIR="$(find "${DWN_WORKDIR}/" -type d -name "*-boot-armv7l-artik5")" + ;; + *artik10*) + BOOT_DIR="$(find "${DWN_WORKDIR}/" -type d -name "*-boot-armv7l-artik10")" + ;; + *) + die "Unknown target: ${TARGET}" + ;; + esac + test -n "${BOOT_DIR}" || die "Userspace directory not found" + echo "${BOOT_DIR}" +} diff --git a/tsp/cronfile.example b/tsp/cronfile.example index 4ab7e99..42c8794 100644 --- a/tsp/cronfile.example +++ b/tsp/cronfile.example @@ -1,2 +1,2 @@ -*/5 * * * * /opt/testlab-major/tsp/jobs/watcher.sh -0 2 * * * /opt/testlab-major/tsp/jobs/watch_diskspace.sh +*/10 * * * * /opt/testlab-major/tsp/jobs/watcher.sh +0 */4 * * * /opt/testlab-major/tsp/jobs/watch_diskspace.sh diff --git a/tsp/jobs/common_prep_flash_conf.sh b/tsp/jobs/common_prep_flash_conf.sh index 9499195..fabec78 100755 --- a/tsp/jobs/common_prep_flash_conf.sh +++ b/tsp/jobs/common_prep_flash_conf.sh @@ -33,24 +33,37 @@ cd "${WORKDIR}" WORKDIR_REMOTE="${WS_TEST_REMOTE}/${BUILD_NR}" DWN_WORKDIR="${WS_DOWNLOAD}/${BUILD_NR}" +SR="$(project2sr "${DWN_WORKDIR}/project.conf")" +PROFILE="$(sr2profile "${SR}")" + # Update status in database -update_db "$(project2sr "${DWN_WORKDIR}/project-${TARGET}.conf")" "Testing" "$BUILD_NR" "$TARGET" +update_db "${SR}" "Testing" "$BUILD_NR" "$TARGET" +USR_IMG_DIR="$(get_userspace_dir "${TARGET}" "${DWN_WORKDIR}")" case "$TARGET" in *odroid*|*artik*) - BOOT_IMG="$(ls "${DWN_WORKDIR}"/tizen-common_*-boot-*.tar.gz)" - USERSPACE_IMG="$(ls "${DWN_WORKDIR}"/tizen-common_*-3parts-*.tar.gz)" + BOOT_IMG="$(ls "$(get_boot_dir "${TARGET}" "${DWN_WORKDIR}")"/*.tar.gz)" + test -f "${BOOT_IMG}" || BOOT_IMG="$(ls "$(get_boot_dir "${TARGET}" "${SNAPSHOT_WS_DOWNLOAD}-${PROFILE}")"/*.tar.gz)" + USERSPACE_IMG="$(ls "${USR_IMG_DIR}"/*.tar.gz)" + test -f "${USERSPACE_IMG}" || USERSPACE_IMG="$(ls "$(get_userspace_dir "${TARGET}" "${SNAPSHOT_WS_DOWNLOAD}-${PROFILE}")"/*.tar.gz)" + # TODO: Both images shouldn't be replaced by snapshot's at the same time ;; *minnow*) - USERSPACE_IMG="$(ls "${DWN_WORKDIR}"/tizen-common_*-wayland-*-sda.raw.bz2)" - USERSPACE_IMG_BMAP="$(ls "${DWN_WORKDIR}"/tizen-common_*-wayland-*-sda.bmap)" + USERSPACE_IMG="$(ls "${USR_IMG_DIR}"/*.raw.bz2)" + USERSPACE_IMG_BMAP="$(ls "${USR_IMG_DIR}"/*.bmap)" ;; *mbr64*) - USERSPACE_IMG="$(ls "${DWN_WORKDIR}"/tizen-common_*-minimal-mbr-*.tar.gz)" - KERNEL="$(ls "${DWN_WORKDIR}"/tizen-common_*-minimal-mbr-*-vmlinuz-*)" + USERSPACE_IMG="$(ls "${USR_IMG_DIR}"/*.tar.gz)" + KERNEL="$(ls "${USR_IMG_DIR}"/*-vmlinuz-*)" + ;; + *) + die "Unknown target: ${TARGET}" ;; esac +test -z "${BOOT_IMG}" || test -f "${BOOT_IMG}" || die "No boot image found: ${BOOT_IMG}!" +test -f "${USERSPACE_IMG}" || die "No userspace image found: ${USERSPACE_IMG}!" + FAIL_CNT="$3" test -n "${FAIL_CNT}" || FAIL_CNT=0 @@ -66,7 +79,7 @@ retrigger() { tsrun_target "${TARGET#headless-}" -L "RETRIGGERED_$(echo "$TARGET" | tr '[:lower:]' '[:upper:]')_PREPARE_FLASH_CONF" \ sh "${TSP_DIR}/jobs/common_prep_flash_conf.sh" "$BUILD_NR" "${TARGET}" "$((FAIL_CNT + 1))" # Update status in database - update_db "$(project2sr "${DWN_WORKDIR}/project-${TARGET}.conf")" "In test queue" "$BUILD_NR" "$TARGET" + update_db "${SR}" "In test queue" "$BUILD_NR" "$TARGET" exit 0 } diff --git a/tsp/jobs/img_test_common.sh b/tsp/jobs/img_test_common.sh index 40b9ce0..304742f 100755 --- a/tsp/jobs/img_test_common.sh +++ b/tsp/jobs/img_test_common.sh @@ -20,21 +20,32 @@ export TSP_DIR="$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd -P)/.." . "${TSP_DIR}/common.sh" -test -n "$1" || die "Missing argument: build nr" - -WORKDIR="${WS_TEST}/$1" -DWN_WORKDIR="${WS_DOWNLOAD}/$1" +BUILD_NR="$1" +WORKDIR="${WS_TEST}/${BUILD_NR}" +DWN_WORKDIR="${WS_DOWNLOAD}/${BUILD_NR}" TARGET="$2" +test -n "$BUILD_NR" || die "Missing argument: build_nr" +test -n "$TARGET" || die "Missing argument: target" + +# update symlink (all images are downloaded) +SR="$(project2sr "${DWN_WORKDIR}/project.conf")" +PROFILE="$(sr2profile "${SR}")" +test -L "${SNAPSHOT_WS_DOWNLOAD_NEW}" && mv -T "${SNAPSHOT_WS_DOWNLOAD_NEW}-${PROFILE}" "${SNAPSHOT_WS_DOWNLOAD}-${PROFILE}" + +rm -rf "${WORKDIR}" +mkdir -p "${WORKDIR}" + check_diff() { - DIFF_FILE="$1" - JOB_NR="$2" - TARGET="$3" - test -n "$1" || die "Missing argument: diff file" - test -n "$2" || die "Missing argument: build nr" - test -n "$3" || die "Missing argument: target" + DIFF_FILE1="$1" + DIFF_FILE2="$2" + JOB_NR="$3" + TARGET="$4" + test -n "${DIFF_FILE1}" || die "Missing argument: diff_file1" + test -n "${JOB_NR}" || die "Missing argument: build_nr" + test -n "${TARGET}" || die "Missing argument: target" test -n "${TSP_DIR}" || die "Missing environment variable: TSP_DIR" - if grep -q "identical" "${DIFF_FILE}" + if ( grep -q "identical" "${DIFF_FILE1}" ) && ( [ -z "${DIFF_FILE2}" ] || grep -q "identical" "${DIFF_FILE2}" ) then echo "Image is unchanged in respect to snapshot" tspwb -L PUBLISH sh "${TSP_DIR}/jobs/publish.sh" "${JOB_NR}" "${TARGET}" @@ -42,16 +53,40 @@ check_diff() { fi } +report_no_image() { + DWN_WORKDIR="$1" + JOB_NR="$2" + TARGET="$3" + test -n "${DWN_WORKDIR}" || die "Missing argument: dwn_workdir" + test -n "${JOB_NR}" || die "Missing argument: build_nr" + test -n "${TARGET}" || die "Missing argument: target" + test -n "${SR}" || die "Missing argument: sr" + echo "Report missing image" + update_db "$SR" "No image" "$JOB_NR" "$TARGET" + return 1 +} + test -n "$TARGET" || die "Missing argument: target" -DIFF_FILE="${DWN_WORKDIR}/diff-${TARGET}.report" +USR_IMG_DIR="$(get_userspace_dir "${TARGET}" "${DWN_WORKDIR}")" \ + || report_no_image "${DWN_WORKDIR}" "${BUILD_NR}" "${TARGET}" || die "No image" -rm -rf "${WORKDIR}" -if check_diff "${DIFF_FILE}" "$1" "${TARGET}"; +DIFF_FILE1="${USR_IMG_DIR}/diff.report" +case "${TARGET}" in + *artik*|*odroid*) + DIFF_FILE2="$(get_boot_dir "${TARGET}" "${DWN_WORKDIR}")/diff.report" \ + || report_no_image "${DWN_WORKDIR}" "${BUILD_NR}" "${TARGET}" || die "No image" + ;; + *) + DIFF_FILE2="" + ;; +esac + +if check_diff "${DIFF_FILE1}" "${DIFF_FILE2}" "${BUILD_NR}" "${TARGET}" then echo "Add common_prep_flash_conf job to the $TARGET queue" tsrun_target "${TARGET#headless-}" -L "$(echo "$TARGET" | tr '[:lower:]' '[:upper:]')_PREPARE_FLASH_CONF" \ - sh "${TSP_DIR}/jobs/common_prep_flash_conf.sh" "$1" "${TARGET}" + sh "${TSP_DIR}/jobs/common_prep_flash_conf.sh" "${BUILD_NR}" "${TARGET}" # Update status in database - update_db "$(project2sr "${DWN_WORKDIR}/project-${TARGET}.conf")" "In test queue" "$1" "$TARGET" + update_db "$SR" "In test queue" "${BUILD_NR}" "$TARGET" fi diff --git a/tsp/jobs/publish.sh b/tsp/jobs/publish.sh index d684abd..bc90920 100755 --- a/tsp/jobs/publish.sh +++ b/tsp/jobs/publish.sh @@ -97,8 +97,8 @@ compare_with_snapshot() { echo "OK" } -PROJECT_FILE="${DWN_WORKDIR}/project-${TARGET}.conf" -DIFF_FILE="${DWN_WORKDIR}/diff-${TARGET}.report" +PROJECT_FILE="${DWN_WORKDIR}/project.conf" +DIFF_FILE="$(get_userspace_dir "${TARGET}" "${DWN_WORKDIR}")/diff.report" SYSCTL_FILE="${IMG_WORKDIR}/sysctl.result" SCREENSHOT_FILE="${IMG_WORKDIR}/screenshot.png" SNAPSHOT_SCREENSHOT_FILE="${WS_TEST}/snapshot_screenshot_${TARGET}.png" diff --git a/tsp/jobs/trigger_downloads.sh b/tsp/jobs/trigger_downloads.sh index 6db9001..53c12a2 100755 --- a/tsp/jobs/trigger_downloads.sh +++ b/tsp/jobs/trigger_downloads.sh @@ -33,7 +33,7 @@ touch timestamp touch timestamp_snapshot # Crawl given url -timeout 120 "${TSP_DIR}/scripts/crawler.py" --log INFO "${URL}" || die "Crawl failed" "$?" +timeout 240 "${TSP_DIR}/scripts/crawler.py" --log DEBUG "${URL}" || die "Crawl failed" "$?" # Read next buid_nr touch next_dwn @@ -48,29 +48,40 @@ then else cat modified_urls >> unique_dispatch fi + +# Choose image_map +IMAGE_MAP="${TSP_DIR}/image_map" +test -f "${IMAGE_MAP}" || IMAGE_MAP="${TSP_DIR}/image_map.example" + while read -r url do test -n "${url}" || continue echo "Processing: ${url}" - if [ "$url" = "*latest*" ] - then - echo "Skipping symlink url" - continue - fi # kill currently running jobs for this url for i in $(tsmaster | awk -v URL="$url" '$2 ~ "running" && $0 ~ URL {print $1}'); do kill "$(tsmaster -p "$i")"; done + next=$((next+1)) + dwn_next="${next}" + # prepare download directory + rm -rf "${WS_DOWNLOAD}/$dwn_next" + mkdir -p "${WS_DOWNLOAD}/$dwn_next" + cd "${WS_DOWNLOAD}/$dwn_next" + # download images + dwn_nr=$(tsmaster -L "$dwn_next" python "${TSP_DIR}/scripts/download_all.py" --log=INFO -m "${IMAGE_MAP}" "$url") + case "${url}" in + *snapshots*) + echo "Url is a snapshot, create/update symlink" + ln -Tsf "${WS_DOWNLOAD}/${dwn_next}" "${SNAPSHOT_WS_DOWNLOAD_NEW}-$(url2profile "${url}")" + ;; + esac for target in "minnow32" "minnow64" "odroid" "mbr64" "headless-artik5" "headless-artik10" "artik5" "artik10" do - rm -rf "${WS_DOWNLOAD}/$next" - mkdir -p "${WS_DOWNLOAD}/$next" - cd "${WS_DOWNLOAD}/$next" - # Download image for $target - nr=$(tsmaster -L "$next" python "${TSP_DIR}/scripts/download_image.py" --log=INFO -t "${target}" "$url") - nr=$(tsmaster -D "$nr" -L "IMAGE_TEST" \ - sh "${TSP_DIR}/jobs/img_test_common.sh" "$next" "$target") + next=$((next+1)) + # prepare download directory + rm -rf "${WS_DOWNLOAD}/${next}" + ln -sf "${WS_DOWNLOAD}/${dwn_next}" "${WS_DOWNLOAD}/${next}" + tsmaster -D "$dwn_nr" -L "IMAGE_TEST" sh "${TSP_DIR}/jobs/img_test_common.sh" "$next" "$target" # Update status in database update_db "$(url2sr "$url")" "Downloading" "$next" "$target" - next=$((next+1)) done done < unique_dispatch cd "${WS_WATCHER}" diff --git a/tsp/jobs/watch_diskspace.sh b/tsp/jobs/watch_diskspace.sh index 0ea233c..aac0baf 100755 --- a/tsp/jobs/watch_diskspace.sh +++ b/tsp/jobs/watch_diskspace.sh @@ -24,7 +24,7 @@ export TSP_DIR="$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd -P)/.." PER="$(df -P "${WS}" | grep -o "[0-9][0-9]%.*$")" # Leave only numerical value. PER="${PER%\%*}" -if [ "$PER" -ge 70 ] +if [ "$PER" -ge 30 ] then nr="$(tsmaster -L DISK_CLEANUP sh "${TSP_DIR}/scripts/free_diskspace.sh")" # make the above job urgent diff --git a/tsp/scripts/common_prep_flash_conf.sh b/tsp/scripts/common_prep_flash_conf.sh index 48ddbed..1eca9b4 100755 --- a/tsp/scripts/common_prep_flash_conf.sh +++ b/tsp/scripts/common_prep_flash_conf.sh @@ -26,20 +26,23 @@ cd "${WORKDIR_REMOTE}" || exit ACTUAL_TARGET="${TARGET#headless-}" case "$TARGET" in *odroid*) - BOOT_IMG="$(pwd)/$(ls tizen-common_*-boot-*odroid*.tar.gz)" - USR_IMG="$(pwd)/$(ls tizen-common_*-wayland-*odroid*.tar.gz)" + BOOT_IMG="$(pwd)/$(ls tizen-*-boot-*odroid*.tar.gz)" + USR_IMG="$(pwd)/$(ls tizen-*-wayland-*odroid*.tar.gz)" ;; *artik*) - BOOT_IMG="$(pwd)/$(ls tizen-common_*-boot-*artik*.tar.gz)" - USR_IMG="$(pwd)/$(ls tizen-common_*-3parts-*artik*.tar.gz)" + BOOT_IMG="$(pwd)/$(ls tizen-*-boot-*artik*.tar.gz)" + USR_IMG="$(pwd)/$(ls tizen-*-3parts-*artik*.tar.gz)" ;; *minnow*) - USR_IMG="$(pwd)/$(ls tizen-common_*_common-*-sda.raw.bz2)" - USR_IMG_BMAP="$(pwd)/$(ls tizen-common_*_common-*-sda.bmap)" + USR_IMG="$(pwd)/$(ls tizen-*.raw.bz2)" + USR_IMG_BMAP="$(pwd)/$(ls tizen-*.bmap)" ;; *mbr64*) - USR_IMG="$(pwd)/$(ls tizen-common_*-minimal-mbr-*.tar.gz)" - KERNEL="$(pwd)/$(ls tizen-common_*-minimal-mbr-*-vmlinuz-*)" + USR_IMG="$(pwd)/$(ls tizen-*-minimal-mbr-x86_64.tar.gz)" + KERNEL="$(pwd)/$(ls *-vmlinuz-*)" + ;; + *) + die "Unrecognized target" ;; esac diff --git a/tsp/scripts/free_diskspace.sh b/tsp/scripts/free_diskspace.sh index 9adf7d6..d497092 100755 --- a/tsp/scripts/free_diskspace.sh +++ b/tsp/scripts/free_diskspace.sh @@ -25,16 +25,13 @@ export TSP_DIR="$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd -P)/.." for i in $(find "${WS_DOWNLOAD}" -maxdepth 1 -mindepth 1 -type d -mtime +1) do - DIFF_FILE="$(ls "$i"/diff-*.report)" - if [ -z "${DIFF_FILE}" ] - then - echo "Missing diff-*.report in: " "$i" - else - if grep -q "Snapshot" "${DIFF_FILE}" + for diff_file in $(find "$i" -type f -name diff.report) + do + if grep -q "Snapshot" "${diff_file}" then echo "Snapshot, skipping:" "$i" continue fi - fi - rm -vr "$i" + rm -vr "$(dirname "${diff_file}")" + done done -- 2.7.4