Make TITS profile independent 27/108727/3
authorAleksander Mistewicz <a.mistewicz@samsung.com>
Mon, 7 Nov 2016 10:07:16 +0000 (11:07 +0100)
committerAleksander Mistewicz <a.mistewicz@samsung.com>
Mon, 16 Jan 2017 10:07:49 +0000 (11:07 +0100)
Overall TITS adjustments

Change-Id: I8a9b1255a122efa50a8275a558c0bc01174ad4fa
Signed-off-by: Aleksander Mistewicz <a.mistewicz@samsung.com>
13 files changed:
tct/common.sh
tct/flash/flash.sh
tct/minnow_prepare_flash_conf.sh
tct/odroid_prepare_flash_conf.sh
tsp/common.sh
tsp/cronfile.example
tsp/jobs/common_prep_flash_conf.sh
tsp/jobs/img_test_common.sh
tsp/jobs/publish.sh
tsp/jobs/trigger_downloads.sh
tsp/jobs/watch_diskspace.sh
tsp/scripts/common_prep_flash_conf.sh
tsp/scripts/free_diskspace.sh

index ac8cb9a..ba5a1ff 100755 (executable)
@@ -31,7 +31,7 @@ die() {
 INC_SLEEP=10
 INIT_SLEEP=60
 RESTART_SLEEP=80
-MAX_SLEEP=120
+MAX_SLEEP=140
 
 TL_IP_CONF="/etc/opt/testlab-major/ip_addr"
 SCREENSHOOTER_RPM="/opt/screenshooter.rpm"
index 547ccf8..e81493c 100755 (executable)
@@ -36,20 +36,20 @@ uuid2dev() {
 
     UUID="$(cat "${FILE}")"
     DEV="$(readlink "/dev/disk/by-uuid/${UUID}")"
-    SLEEP=2
+    SLEEP=10
 
     echo "Searching: ${UUID}" >&2
 
     while [ -z "${DEV}" ] && [ "${SLEEP}" -le 30 ]
     do
-        if [ -n "${SDMUX}" ] && [ "${SLEEP}" -ge 10 ]
+        if [ -n "${SDMUX}" ] && [ "${SLEEP}" -ge 20 ]
         then
             echo "Restarting sdmux: ${SDMUX}" >&2
             switch2device "${SDMUX}"
             sdmux_init "${SDMUX}"
         fi
         sleep "${SLEEP}"
-        SLEEP=$((SLEEP+2))
+        SLEEP=$((SLEEP+10))
         DEV="$(readlink "/dev/disk/by-uuid/${UUID}")"
     done
 
index 09a4a42..cf271f4 100755 (executable)
@@ -46,7 +46,7 @@ switch2testserver "${SDMUX}"
 echo "### flash ###"
 
 UUID_FILE="/var/tmp/uuid-${SDMUX}"
-DEV_SDCARD=$(uuid2dev "${UUID_FILE}" "${SDMUX}")
+DEV_SDCARD=$(uuid2dev "${UUID_FILE}" "${SDMUX}") || die "sdcard not found!"
 
 echo "Bmaptool: start"
 test -b "${DEV_SDCARD}" || die "File does not exist or is not block special"
index cefd345..c9e774d 100755 (executable)
@@ -57,7 +57,7 @@ unmount_image "${IMAGE_MOUNTPOINT}"
 echo "### flash ###"
 
 UUID_FILE="/var/tmp/uuid-${SDMUX}"
-DEV_SDCARD=$(uuid2dev "${UUID_FILE}" "${SDMUX}")
+DEV_SDCARD=$(uuid2dev "${UUID_FILE}" "${SDMUX}") || die "sdcard not found!"
 
 cd "${IMAGE_UNPACKED}"
 "${TESTLAB_SCRIPTS}/flash/flash_dd.sh" "${DEV_SDCARD}"
index e6ba407..50146ea 100755 (executable)
@@ -35,6 +35,8 @@ WS_DOWNLOAD="${WS}/dwn"
 WS_TEST="${WS}/img_test"
 WS_TEST_REMOTE="${WS}/img_test_remote"
 WS_PUBLISH="${WS}/publish"
+SNAPSHOT_WS_DOWNLOAD="${WS_DOWNLOAD}/snapshot"
+SNAPSHOT_WS_DOWNLOAD_NEW="${WS_DOWNLOAD}/new_snapshot"
 
 DBNAME="results.db3"
 DBPATH="${WS_PUBLISH}/${DBNAME}"
@@ -105,3 +107,61 @@ update_db() {
         (select d.did from device d where d.dname='$TARGET'));"
     lockfile-remove "${DBPATH}"
 }
+
+get_userspace_dir() {
+    TARGET="$1"
+    DWN_WORKDIR="$2"
+    test -n "$TARGET" || die "Missing argument: target"
+    test -n "$DWN_WORKDIR" || die "Missing argument: dwn_workdir"
+    case "$TARGET" in
+        *odroid*)
+            USR_DIR="$(find "${DWN_WORKDIR}/" -type d -name "*-wayland*-armv7l-odroidu3")"
+            ;;
+        *headless-artik*)
+            USR_DIR="$(find "${DWN_WORKDIR}/" -type d -name "*-headless-*-armv7l-artik")"
+            ;;
+        *artik*)
+            USR_DIR="$(find "${DWN_WORKDIR}/" -type d -name "*-wayland*-armv7l-artik")"
+            ;;
+        *minnow32*)
+            USR_DIR="$(find "${DWN_WORKDIR}/" -type d -name "*-wayland-efi-i586")"
+            ;;
+        *minnow64*)
+            USR_DIR="$(find "${DWN_WORKDIR}/" -type d -name "*-wayland-efi-x86_64")"
+            ;;
+        *mbr64*)
+            USR_DIR="$(find "${DWN_WORKDIR}/" -type d -name "*-minimal-mbr-x86_64")"
+            ;;
+        *emulator32*)
+            USR_DIR="$(find "${DWN_WORKDIR}/" -type d -name "*-emulator-wayland-mbr-i586")"
+            ;;
+        *)
+            die "Unknown target: ${TARGET}"
+            ;;
+    esac
+    test -n "${USR_DIR}" || die "Userspace directory not found"
+    echo "${USR_DIR}"
+}
+
+get_boot_dir() {
+    TARGET="$1"
+    DWN_WORKDIR="$2"
+    test -n "$TARGET" || die "Missing argument: target"
+    test -n "$DWN_WORKDIR" || die "Missing argument: dwn_workdir"
+    case "$TARGET" in
+        *odroid*)
+            BOOT_DIR="$(find "${DWN_WORKDIR}/" -type d -name "*-boot-armv7l-odroidu3")"
+            ;;
+        *artik5*)
+            BOOT_DIR="$(find "${DWN_WORKDIR}/" -type d -name "*-boot-armv7l-artik5")"
+            ;;
+        *artik10*)
+            BOOT_DIR="$(find "${DWN_WORKDIR}/" -type d -name "*-boot-armv7l-artik10")"
+            ;;
+        *)
+            die "Unknown target: ${TARGET}"
+            ;;
+    esac
+    test -n "${BOOT_DIR}" || die "Userspace directory not found"
+    echo "${BOOT_DIR}"
+}
index 4ab7e99..42c8794 100644 (file)
@@ -1,2 +1,2 @@
-*/5 * * * * /opt/testlab-major/tsp/jobs/watcher.sh
-0 2 * * * /opt/testlab-major/tsp/jobs/watch_diskspace.sh
+*/10 * * * * /opt/testlab-major/tsp/jobs/watcher.sh
+0 */4 * * * /opt/testlab-major/tsp/jobs/watch_diskspace.sh
index 9499195..fabec78 100755 (executable)
@@ -33,24 +33,37 @@ cd "${WORKDIR}"
 WORKDIR_REMOTE="${WS_TEST_REMOTE}/${BUILD_NR}"
 DWN_WORKDIR="${WS_DOWNLOAD}/${BUILD_NR}"
 
+SR="$(project2sr "${DWN_WORKDIR}/project.conf")"
+PROFILE="$(sr2profile "${SR}")"
+
 # Update status in database
-update_db "$(project2sr "${DWN_WORKDIR}/project-${TARGET}.conf")" "Testing" "$BUILD_NR" "$TARGET"
+update_db "${SR}" "Testing" "$BUILD_NR" "$TARGET"
 
+USR_IMG_DIR="$(get_userspace_dir "${TARGET}" "${DWN_WORKDIR}")"
 case "$TARGET" in
     *odroid*|*artik*)
-        BOOT_IMG="$(ls "${DWN_WORKDIR}"/tizen-common_*-boot-*.tar.gz)"
-        USERSPACE_IMG="$(ls "${DWN_WORKDIR}"/tizen-common_*-3parts-*.tar.gz)"
+        BOOT_IMG="$(ls "$(get_boot_dir "${TARGET}" "${DWN_WORKDIR}")"/*.tar.gz)"
+        test -f "${BOOT_IMG}" || BOOT_IMG="$(ls "$(get_boot_dir "${TARGET}" "${SNAPSHOT_WS_DOWNLOAD}-${PROFILE}")"/*.tar.gz)"
+        USERSPACE_IMG="$(ls "${USR_IMG_DIR}"/*.tar.gz)"
+        test -f "${USERSPACE_IMG}" || USERSPACE_IMG="$(ls "$(get_userspace_dir "${TARGET}" "${SNAPSHOT_WS_DOWNLOAD}-${PROFILE}")"/*.tar.gz)"
+        # TODO: Both images shouldn't be replaced by snapshot's at the same time
         ;;
     *minnow*)
-        USERSPACE_IMG="$(ls "${DWN_WORKDIR}"/tizen-common_*-wayland-*-sda.raw.bz2)"
-        USERSPACE_IMG_BMAP="$(ls "${DWN_WORKDIR}"/tizen-common_*-wayland-*-sda.bmap)"
+        USERSPACE_IMG="$(ls "${USR_IMG_DIR}"/*.raw.bz2)"
+        USERSPACE_IMG_BMAP="$(ls "${USR_IMG_DIR}"/*.bmap)"
         ;;
     *mbr64*)
-        USERSPACE_IMG="$(ls "${DWN_WORKDIR}"/tizen-common_*-minimal-mbr-*.tar.gz)"
-        KERNEL="$(ls "${DWN_WORKDIR}"/tizen-common_*-minimal-mbr-*-vmlinuz-*)"
+        USERSPACE_IMG="$(ls "${USR_IMG_DIR}"/*.tar.gz)"
+        KERNEL="$(ls "${USR_IMG_DIR}"/*-vmlinuz-*)"
+        ;;
+    *)
+        die "Unknown target: ${TARGET}"
         ;;
 esac
 
+test -z "${BOOT_IMG}" || test -f "${BOOT_IMG}" || die "No boot image found: ${BOOT_IMG}!"
+test -f "${USERSPACE_IMG}" || die "No userspace image found: ${USERSPACE_IMG}!"
+
 FAIL_CNT="$3"
 test -n "${FAIL_CNT}" || FAIL_CNT=0
 
@@ -66,7 +79,7 @@ retrigger() {
     tsrun_target "${TARGET#headless-}" -L "RETRIGGERED_$(echo "$TARGET" | tr '[:lower:]' '[:upper:]')_PREPARE_FLASH_CONF" \
         sh "${TSP_DIR}/jobs/common_prep_flash_conf.sh" "$BUILD_NR" "${TARGET}" "$((FAIL_CNT + 1))"
     # Update status in database
-    update_db "$(project2sr "${DWN_WORKDIR}/project-${TARGET}.conf")" "In test queue" "$BUILD_NR" "$TARGET"
+    update_db "${SR}" "In test queue" "$BUILD_NR" "$TARGET"
     exit 0
 }
 
index 40b9ce0..304742f 100755 (executable)
@@ -20,21 +20,32 @@ export TSP_DIR="$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd -P)/.."
 
 . "${TSP_DIR}/common.sh"
 
-test -n "$1" || die "Missing argument: build nr"
-
-WORKDIR="${WS_TEST}/$1"
-DWN_WORKDIR="${WS_DOWNLOAD}/$1"
+BUILD_NR="$1"
+WORKDIR="${WS_TEST}/${BUILD_NR}"
+DWN_WORKDIR="${WS_DOWNLOAD}/${BUILD_NR}"
 TARGET="$2"
 
+test -n "$BUILD_NR" || die "Missing argument: build_nr"
+test -n "$TARGET" || die "Missing argument: target"
+
+# update symlink (all images are downloaded)
+SR="$(project2sr "${DWN_WORKDIR}/project.conf")"
+PROFILE="$(sr2profile "${SR}")"
+test -L "${SNAPSHOT_WS_DOWNLOAD_NEW}" && mv -T "${SNAPSHOT_WS_DOWNLOAD_NEW}-${PROFILE}" "${SNAPSHOT_WS_DOWNLOAD}-${PROFILE}"
+
+rm -rf "${WORKDIR}"
+mkdir -p "${WORKDIR}"
+
 check_diff() {
-    DIFF_FILE="$1"
-    JOB_NR="$2"
-    TARGET="$3"
-    test -n "$1" || die "Missing argument: diff file"
-    test -n "$2" || die "Missing argument: build nr"
-    test -n "$3" || die "Missing argument: target"
+    DIFF_FILE1="$1"
+    DIFF_FILE2="$2"
+    JOB_NR="$3"
+    TARGET="$4"
+    test -n "${DIFF_FILE1}" || die "Missing argument: diff_file1"
+    test -n "${JOB_NR}" || die "Missing argument: build_nr"
+    test -n "${TARGET}" || die "Missing argument: target"
     test -n "${TSP_DIR}" || die "Missing environment variable: TSP_DIR"
-    if grep -q "identical" "${DIFF_FILE}"
+    if ( grep -q "identical" "${DIFF_FILE1}" ) && ( [ -z "${DIFF_FILE2}" ] || grep -q "identical" "${DIFF_FILE2}" )
     then
         echo "Image is unchanged in respect to snapshot"
         tspwb -L PUBLISH sh "${TSP_DIR}/jobs/publish.sh" "${JOB_NR}" "${TARGET}"
@@ -42,16 +53,40 @@ check_diff() {
     fi
 }
 
+report_no_image() {
+    DWN_WORKDIR="$1"
+    JOB_NR="$2"
+    TARGET="$3"
+    test -n "${DWN_WORKDIR}" || die "Missing argument: dwn_workdir"
+    test -n "${JOB_NR}" || die "Missing argument: build_nr"
+    test -n "${TARGET}" || die "Missing argument: target"
+    test -n "${SR}" || die "Missing argument: sr"
+    echo "Report missing image"
+    update_db "$SR" "No image" "$JOB_NR" "$TARGET"
+    return 1
+}
+
 test -n "$TARGET" || die "Missing argument: target"
 
-DIFF_FILE="${DWN_WORKDIR}/diff-${TARGET}.report"
+USR_IMG_DIR="$(get_userspace_dir "${TARGET}" "${DWN_WORKDIR}")" \
+    || report_no_image "${DWN_WORKDIR}" "${BUILD_NR}" "${TARGET}" || die "No image"
 
-rm -rf "${WORKDIR}"
-if check_diff "${DIFF_FILE}" "$1" "${TARGET}";
+DIFF_FILE1="${USR_IMG_DIR}/diff.report"
+case "${TARGET}" in
+    *artik*|*odroid*)
+        DIFF_FILE2="$(get_boot_dir "${TARGET}" "${DWN_WORKDIR}")/diff.report" \
+            || report_no_image "${DWN_WORKDIR}" "${BUILD_NR}" "${TARGET}" || die "No image"
+        ;;
+    *)
+        DIFF_FILE2=""
+        ;;
+esac
+
+if check_diff "${DIFF_FILE1}" "${DIFF_FILE2}" "${BUILD_NR}" "${TARGET}"
 then
     echo "Add common_prep_flash_conf job to the $TARGET queue"
     tsrun_target "${TARGET#headless-}" -L "$(echo "$TARGET" | tr '[:lower:]' '[:upper:]')_PREPARE_FLASH_CONF" \
-        sh "${TSP_DIR}/jobs/common_prep_flash_conf.sh" "$1" "${TARGET}"
+        sh "${TSP_DIR}/jobs/common_prep_flash_conf.sh" "${BUILD_NR}" "${TARGET}"
     # Update status in database
-    update_db "$(project2sr "${DWN_WORKDIR}/project-${TARGET}.conf")" "In test queue" "$1" "$TARGET"
+    update_db "$SR" "In test queue" "${BUILD_NR}" "$TARGET"
 fi
index d684abd..bc90920 100755 (executable)
@@ -97,8 +97,8 @@ compare_with_snapshot() {
     echo "OK"
 }
 
-PROJECT_FILE="${DWN_WORKDIR}/project-${TARGET}.conf"
-DIFF_FILE="${DWN_WORKDIR}/diff-${TARGET}.report"
+PROJECT_FILE="${DWN_WORKDIR}/project.conf"
+DIFF_FILE="$(get_userspace_dir "${TARGET}" "${DWN_WORKDIR}")/diff.report"
 SYSCTL_FILE="${IMG_WORKDIR}/sysctl.result"
 SCREENSHOT_FILE="${IMG_WORKDIR}/screenshot.png"
 SNAPSHOT_SCREENSHOT_FILE="${WS_TEST}/snapshot_screenshot_${TARGET}.png"
index 6db9001..53c12a2 100755 (executable)
@@ -33,7 +33,7 @@ touch timestamp
 touch timestamp_snapshot
 
 # Crawl given url
-timeout 120 "${TSP_DIR}/scripts/crawler.py" --log INFO "${URL}" || die "Crawl failed" "$?"
+timeout 240 "${TSP_DIR}/scripts/crawler.py" --log DEBUG "${URL}" || die "Crawl failed" "$?"
 
 # Read next buid_nr
 touch next_dwn
@@ -48,29 +48,40 @@ then
 else
     cat modified_urls >> unique_dispatch
 fi
+
+# Choose image_map
+IMAGE_MAP="${TSP_DIR}/image_map"
+test -f "${IMAGE_MAP}" || IMAGE_MAP="${TSP_DIR}/image_map.example"
+
 while read -r url
 do
     test -n "${url}" || continue
     echo "Processing: ${url}"
-    if [ "$url" = "*latest*" ]
-    then
-        echo "Skipping symlink url"
-        continue
-    fi
     # kill currently running jobs for this url
     for i in $(tsmaster | awk -v URL="$url" '$2 ~ "running" && $0 ~ URL {print $1}'); do kill "$(tsmaster -p "$i")"; done
+    next=$((next+1))
+    dwn_next="${next}"
+    # prepare download directory
+    rm -rf "${WS_DOWNLOAD}/$dwn_next"
+    mkdir -p "${WS_DOWNLOAD}/$dwn_next"
+    cd "${WS_DOWNLOAD}/$dwn_next"
+    # download images
+    dwn_nr=$(tsmaster -L "$dwn_next" python "${TSP_DIR}/scripts/download_all.py" --log=INFO -m "${IMAGE_MAP}" "$url")
+    case "${url}" in
+        *snapshots*)
+            echo "Url is a snapshot, create/update symlink"
+            ln -Tsf "${WS_DOWNLOAD}/${dwn_next}" "${SNAPSHOT_WS_DOWNLOAD_NEW}-$(url2profile "${url}")"
+            ;;
+    esac
     for target in "minnow32" "minnow64" "odroid" "mbr64" "headless-artik5" "headless-artik10" "artik5" "artik10"
     do
-        rm -rf "${WS_DOWNLOAD}/$next"
-        mkdir -p "${WS_DOWNLOAD}/$next"
-        cd "${WS_DOWNLOAD}/$next"
-        # Download image for $target
-        nr=$(tsmaster -L "$next" python "${TSP_DIR}/scripts/download_image.py" --log=INFO -t "${target}" "$url")
-        nr=$(tsmaster -D "$nr" -L "IMAGE_TEST" \
-            sh "${TSP_DIR}/jobs/img_test_common.sh" "$next" "$target")
+        next=$((next+1))
+        # prepare download directory
+        rm -rf "${WS_DOWNLOAD}/${next}"
+        ln -sf "${WS_DOWNLOAD}/${dwn_next}" "${WS_DOWNLOAD}/${next}"
+        tsmaster -D "$dwn_nr" -L "IMAGE_TEST" sh "${TSP_DIR}/jobs/img_test_common.sh" "$next" "$target"
         # Update status in database
         update_db "$(url2sr "$url")" "Downloading" "$next" "$target"
-        next=$((next+1))
     done
 done < unique_dispatch
 cd "${WS_WATCHER}"
index 0ea233c..aac0baf 100755 (executable)
@@ -24,7 +24,7 @@ export TSP_DIR="$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd -P)/.."
 PER="$(df -P "${WS}" | grep -o "[0-9][0-9]%.*$")"
 # Leave only numerical value.
 PER="${PER%\%*}"
-if [ "$PER" -ge 70 ]
+if [ "$PER" -ge 30 ]
 then
     nr="$(tsmaster -L DISK_CLEANUP sh "${TSP_DIR}/scripts/free_diskspace.sh")"
     # make the above job urgent
index 48ddbed..1eca9b4 100755 (executable)
@@ -26,20 +26,23 @@ cd "${WORKDIR_REMOTE}" || exit
 ACTUAL_TARGET="${TARGET#headless-}"
 case "$TARGET" in
     *odroid*)
-        BOOT_IMG="$(pwd)/$(ls tizen-common_*-boot-*odroid*.tar.gz)"
-        USR_IMG="$(pwd)/$(ls tizen-common_*-wayland-*odroid*.tar.gz)"
+        BOOT_IMG="$(pwd)/$(ls tizen-*-boot-*odroid*.tar.gz)"
+        USR_IMG="$(pwd)/$(ls tizen-*-wayland-*odroid*.tar.gz)"
         ;;
     *artik*)
-        BOOT_IMG="$(pwd)/$(ls tizen-common_*-boot-*artik*.tar.gz)"
-        USR_IMG="$(pwd)/$(ls tizen-common_*-3parts-*artik*.tar.gz)"
+        BOOT_IMG="$(pwd)/$(ls tizen-*-boot-*artik*.tar.gz)"
+        USR_IMG="$(pwd)/$(ls tizen-*-3parts-*artik*.tar.gz)"
         ;;
     *minnow*)
-        USR_IMG="$(pwd)/$(ls tizen-common_*_common-*-sda.raw.bz2)"
-        USR_IMG_BMAP="$(pwd)/$(ls tizen-common_*_common-*-sda.bmap)"
+        USR_IMG="$(pwd)/$(ls tizen-*.raw.bz2)"
+        USR_IMG_BMAP="$(pwd)/$(ls tizen-*.bmap)"
         ;;
     *mbr64*)
-        USR_IMG="$(pwd)/$(ls tizen-common_*-minimal-mbr-*.tar.gz)"
-        KERNEL="$(pwd)/$(ls tizen-common_*-minimal-mbr-*-vmlinuz-*)"
+        USR_IMG="$(pwd)/$(ls tizen-*-minimal-mbr-x86_64.tar.gz)"
+        KERNEL="$(pwd)/$(ls *-vmlinuz-*)"
+        ;;
+    *)
+        die "Unrecognized target"
         ;;
 esac
 
index 9adf7d6..d497092 100755 (executable)
@@ -25,16 +25,13 @@ export TSP_DIR="$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd -P)/.."
 
 for i in $(find "${WS_DOWNLOAD}" -maxdepth 1 -mindepth 1 -type d -mtime +1)
 do
-    DIFF_FILE="$(ls "$i"/diff-*.report)"
-    if [ -z "${DIFF_FILE}" ]
-    then
-        echo "Missing diff-*.report in: " "$i"
-    else
-        if grep -q "Snapshot" "${DIFF_FILE}"
+    for diff_file in $(find "$i" -type f -name diff.report)
+    do
+        if grep -q "Snapshot" "${diff_file}"
         then
             echo "Snapshot, skipping:" "$i"
             continue
         fi
-    fi
-    rm -vr "$i"
+        rm -vr "$(dirname "${diff_file}")"
+    done
 done