From c3b68296401e04814ea991bea2201b380db33193 Mon Sep 17 00:00:00 2001
From: =?utf8?q?=EC=9D=B4=EC=83=81=EA=B7=9C/=EB=8F=99=EC=9E=91=EC=A0=9C?=
 =?utf8?q?=EC=96=B4Lab=28SR=29/Principal=20Engineer/=EC=82=BC=EC=84=B1?=
 =?utf8?q?=EC=A0=84=EC=9E=90?= <sg5.lee@samsung.com>
Date: Wed, 31 Oct 2018 14:50:48 +0900
Subject: [PATCH] Run benchmark for all tflite model under tests/framework
 (#3402)

Related: #3397 #3264

Previously, run_benchmark_op.sh runs the tests in `benchmark_op_models.txt`.
- No one maintains the list.
- It contains only a few tests.
- If the location or folder name is changed, the list should be updated.

Now, run_benchmark_op.sh runs all tests under `tests/frameworks`.
- No one need to update whenever a new model is added/removed, location is changed....

This PR also removes

- option  `--frameworktest_list_file` option from `run_benchmark_op.sh`.
- benchmark_op_list.txt

Signed-off-by: Sanggyu Lee <sg5.lee@samsung.com>
---
 tools/test_driver/benchmark_op_list.txt | 11 -----------
 tools/test_driver/run_benchmark_op.sh   | 33 ++++++++++++++++++---------------
 tools/test_driver/test_driver.sh        |  3 +--
 3 files changed, 19 insertions(+), 28 deletions(-)
 delete mode 100644 tools/test_driver/benchmark_op_list.txt

diff --git a/tools/test_driver/benchmark_op_list.txt b/tools/test_driver/benchmark_op_list.txt
deleted file mode 100644
index 166ddb7..0000000
--- a/tools/test_driver/benchmark_op_list.txt
+++ /dev/null
@@ -1,11 +0,0 @@
-add/4D
-average_pool_2d/avgpool1
-average_pool_2d/avgpool1
-concat/concat1
-conv_2d/convolution1
-conv_2d/convolution2
-div/broadcast
-max_pool_2d/maxpool1
-max_pool_2d/maxpool2
-resize_bilinear
-softmax
diff --git a/tools/test_driver/run_benchmark_op.sh b/tools/test_driver/run_benchmark_op.sh
index c08e93b..54c4a9c 100755
--- a/tools/test_driver/run_benchmark_op.sh
+++ b/tools/test_driver/run_benchmark_op.sh
@@ -58,9 +58,6 @@ do
             MODEL_CACHE_ROOT_PATH=$TEST_LIST_PATH/cache
             MODEL_TEST_ROOT_PATH=$TEST_LIST_PATH/tests
             ;;
-        --frameworktest_list_file=*)
-            FRAMEWORKTEST_LIST_FILE=${i#*=}
-            ;;
     esac
     shift
 done
@@ -95,12 +92,25 @@ function print_result_of_benchmark_test()
 
 function get_benchmark_op_list()
 {
-    if [ ! -z "$FRAMEWORKTEST_LIST_FILE" ]; then
-        BENCHMARK_MODEL_LIST=$(cat "${FRAMEWORKTEST_LIST_FILE}")
-    else
-        BENCHMARK_MODEL_LIST=$(cat "${PURE_ACL_RT_ENV_FILE}")
+    local TEST_DIRS="$@"
+    local TESTS_TO_RUN=""
+
+    if [[ $# -eq 0 ]]; then
+        TEST_DIRS="."
     fi
-    echo "BENCHMARK_MODEL_LIST=> $BENCHMARK_MODEL_LIST"  
+
+    shift $#
+
+    pushd $MODEL_TEST_ROOT_PATH > /dev/null
+    for DIR in $TEST_DIRS; do
+        if [ -d "$DIR" ]; then
+            TESTS_FOUND=$(find "$DIR" -type f -name 'config.sh' -exec dirname {} \;| sed 's|^./||' | sort)
+            TESTS_TO_RUN="$TESTS_TO_RUN $TESTS_FOUND"
+        fi
+    done
+    popd > /dev/null
+
+    BENCHMARK_MODEL_LIST=$(echo "${TESTS_TO_RUN}")
 }
 
 function run_benchmark_test()
@@ -118,13 +128,6 @@ function run_benchmark_test()
         STATUS="enabled"
         source $MODEL_TEST_ROOT_PATH/$MODEL/config.sh
 
-        LOWER_STATUS="$(echo $STATUS | awk '{print tolower($0)}')"
-        if [ "$LOWER_STATUS" == "disabled" ]; then
-            echo ""
-            echo "Skip $MODEL"
-            continue
-        fi
-
         echo "Benchmark test with `basename $DRIVER_BIN` & `echo $MODEL`"
         echo $MODEL >> $BENCHMARK_MODELS_FILE
 
diff --git a/tools/test_driver/test_driver.sh b/tools/test_driver/test_driver.sh
index 72be65d..077d126 100755
--- a/tools/test_driver/test_driver.sh
+++ b/tools/test_driver/test_driver.sh
@@ -282,8 +282,7 @@ if [ "$BENCHMARK_OP_ON" == "true" ]; then
         --runtestsh=$RUN_TEST_SH \
         --driverbin=$DRIVER_BIN \
         --reportdir=$REPORT_DIR/benchmark_op \
-        --modelfilepath=$ARTIFACT_PATH/tests/framework \
-        --frameworktest_list_file=${FRAMEWORKTEST_LIST_FILE:-}
+        --modelfilepath=$ARTIFACT_PATH/tests/framework
 fi
 
 # Run benchmark/acl/benchmark_googlenet, mobilenet and inception_v3
-- 
2.7.4