Enable test_driver to catch failed test (#1409)

* Enable test_driver to catch failed test

Enable tools/test_driver/test_driver.sh to catch failed test, so that
now do not pass any failed test cases. To do this, run_benchmark*.sh and
run_unittest.sh will stop when the tests catch fail.
run_frameworktest.sh will stop after the tests run all models.

Signed-off-by: Yongseop Kim <yons.kim@samsung.com>

diff --git a/tests/framework/run_test.sh b/tests/framework/run_test.sh
index 287d579..0db45ff 100755 (executable)
--- a/tests/framework/run_test.sh
+++ b/tests/framework/run_test.sh
@@ -112,6 +112,7 @@ run_tests()
     done
     echo "======================"
 
+    TOTAL_RESULT=0  # 0(normal) or 1(abnormal)
     i=0
     for TEST_NAME in $SELECTED_TESTS; do
         ((i++))
@@ -152,13 +153,14 @@ run_tests()
         echo "======================"
         # Run driver to test framework
         LD_LIBRARY_PATH="${NNFW_LD_LIBRARY_PATH:=$NNFW_HOME/Product/out/lib}:${LD_LIBRARY_PATH}" $DRIVER_BIN $MODELFILE
-        RESULT=$?
-        if [[ $RESULT -eq 0 ]]; then
+        if [[ $? -eq 0 ]]; then
             echo "ok $i - $TEST_NAME" >> $REPORT_DIR/$TAP_NAME
         else
             echo "not ok $i - $TEST_NAME" >> $REPORT_DIR/$TAP_NAME
+            TOTAL_RESULT=1
         fi
     done
+    return $TOTAL_RESULT
 }
 
 download_tests()

diff --git a/tools/test_driver/run_benchmark.sh b/tools/test_driver/run_benchmark.sh
index 52ec958..a1a0c2f 100755 (executable)
--- a/tools/test_driver/run_benchmark.sh
+++ b/tools/test_driver/run_benchmark.sh
@@ -58,7 +58,13 @@ function get_result_of_benchmark_test()
     local MODEL=$3
     local LOG_FILE=$4
 
+    local RET=0
     $RUN_TEST_SH --driverbin=$DRIVER_BIN $MODEL > $LOG_FILE 2>&1
+    RET=$?
+    if [[ $RET -ne 0 ]]; then
+        echo "Testing $MODEL aborted... exit code: $RET"
+        exit $RET
+    fi
 
     local RESULT=`grep -E '^Mean:' $LOG_FILE | sed -e 's/ms//g' | awk '{print $2}'`
     echo "$RESULT"

diff --git a/tools/test_driver/run_benchmark_acl.sh b/tools/test_driver/run_benchmark_acl.sh
index 2712f55..c6a643b 100755 (executable)
--- a/tools/test_driver/run_benchmark_acl.sh
+++ b/tools/test_driver/run_benchmark_acl.sh
@@ -54,6 +54,7 @@ function run_benchmark_acl()
     local LOG_FILE=""
     local RESULT_FILE=""
     local RESULT=""
+    local RET=0
 
     export COUNT=5
     echo "============================================"
@@ -69,6 +70,11 @@ function run_benchmark_acl()
         RESULT_FILE=$REPORT_DIR/$BENCHMARK_ACL_BIN_BASENAME/acl_neon.result
         echo -n "ACL(NEON)...... "
         $BENCHMARK_ACL_BIN 0 > $LOG_FILE 2>&1
+        RET=$?
+        if [[ $RET -ne 0 ]]; then
+            echo "aborted... exit code: $RET"
+            exit $RET
+        fi
         RESULT=`grep -E '^Mean:' $LOG_FILE | sed -e 's/ms//g' | awk '{print $2}'`
         echo "$RESULT ms"
         echo "ACL(NEON)" $RESULT > $RESULT_FILE
@@ -78,6 +84,11 @@ function run_benchmark_acl()
         RESULT_FILE=$REPORT_DIR/$BENCHMARK_ACL_BIN_BASENAME/acl_opencl.result
         echo -n "ACL(OpenCL).... "
         $BENCHMARK_ACL_BIN 1 > $LOG_FILE 2>&1
+        RET=$?
+        if [[ $RET -ne 0 ]]; then
+            echo "aborted... exit code: $RET"
+            exit $RET
+        fi
         RESULT=`grep -E '^Mean:' $LOG_FILE | sed -e 's/ms//g' | awk '{print $2}'`
         echo "$RESULT ms"
         echo "ACL(OpenCL)" $RESULT > $RESULT_FILE

diff --git a/tools/test_driver/run_frameworktest.sh b/tools/test_driver/run_frameworktest.sh
index 9e7b8c8..8cf0cda 100755 (executable)
--- a/tools/test_driver/run_frameworktest.sh
+++ b/tools/test_driver/run_frameworktest.sh
@@ -70,6 +70,16 @@ $FWTEST_RUN_TEST_SH --driverbin=$FWTEST_DRIVER_BIN \
     --reportdir=$FWTEST_REPORT_DIR \
     --tapname=$FWTEST_TAP_NAME \
     > $FWTEST_REPORT_DIR/$FWTEST_LOG_NAME 2>&1
+FWTEST_RESULT=$?
+if [[ $FWTEST_RESULT -ne 0 ]]; then
+    echo ""
+    cat $FWTEST_REPORT_DIR/$FWTEST_TAP_NAME
+    echo ""
+    echo "$FWTEST_TEST_NAME failed... exit code: $FWTEST_RESULT"
+    echo "============================================"
+    echo ""
+    exit $FWTEST_RESULT
+fi
 
 echo ""
 cat $FWTEST_REPORT_DIR/$FWTEST_TAP_NAME

diff --git a/tools/test_driver/run_unittest.sh b/tools/test_driver/run_unittest.sh
index e2cfc70..5b467c6 100755 (executable)
--- a/tools/test_driver/run_unittest.sh
+++ b/tools/test_driver/run_unittest.sh
@@ -16,6 +16,7 @@
 
 UNITTEST_REPORT_DIR=
 UNITTEST_TEST_DIR=
+UNITTEST_RESULT=0
 
 function Usage()
 {
@@ -68,11 +69,22 @@ for TEST_BIN in `ls $UNITTEST_TEST_DIR --hide=*.skip`; do
     echo "Starting set $num_unittest: $TEST_BIN..."
     echo "============================================"
     $UNITTEST_TEST_DIR/$TEST_BIN $(get_gtest_option)
+    if [[ $? -ne 0 ]]; then
+        UNITTEST_RESULT=$?
+        echo "$TEST_BIN failed... return code: $UNITTEST_RESULT"
+    fi
     echo "============================================"
     echo "Finishing set $num_unittest: $TEST_BIN..."
     echo "============================================"
 done
 
+if [[ $UNITTEST_RESULT -ne 0 ]]; then
+    echo "============================================"
+    echo "Failed unit test... exit code: $UNITTEST_RESULT"
+    echo "============================================"
+    exit $UNITTEST_RESULT
+fi
+
 echo "============================================"
 echo "Completed total $num_unittest set of unittest"
 echo "Unittest end"