Refactory benchmark scripts to unify (#8838)

1. Most of the script weren't up-to-date
2. Most of the steps are the same, so move them as a separate functions to
common.sh

Signed-off-by: Poshshoev Dilshodzhon <d.poshshoev@samsung.com>

diff --git a/tests/scripts/common.sh b/tests/scripts/common.sh
index 1e7ab39..60aaf76 100755 (executable)
--- a/tests/scripts/common.sh
+++ b/tests/scripts/common.sh
@@ -16,7 +16,7 @@
 
 MY_PATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 
-function switch_nnfw_kernel_env()
+function switch_nnfw_pacl_kernel_env()
 {
     local switch=$1  # "ON" or "OFF"
     local mode=$2    # "acl" or "neon" or ""
@@ -60,3 +60,33 @@ function print_result_of_benchmark_test()
 
     echo "$NAME $RESULT" > $RESULT_FILE
 }
+
+function print_with_dots()
+{
+    PRINT_WIDTH=45
+    local MSG="$@"
+    pad=$(printf '%0.1s' "."{1..45})
+    padlength=$((PRINT_WIDTH- ${#MSG}))
+    printf '%s' "$MSG"
+    printf '%*.*s ' 0 $padlength "$pad"
+}
+
+
+function run_benchmark_and_print()
+{
+    local WRITE_FILE_NAME=$1
+    local MSG=$2
+    local MODEL=$3
+    local REPORT_MODEL_DIR=$4
+    local PAUSE_TIME_IN_SEC=$5
+    local DRIVER_BIN=$6
+    local BENCHMARK_RUN_TEST_SH=$7
+
+    LOG_FILE=$REPORT_MODEL_DIR/$WRITE_FILE_NAME.txt
+    RESULT_FILE=$REPORT_MODEL_DIR/$WRITE_FILE_NAME.result
+    print_with_dots $MSG
+    RESULT=$(get_result_of_benchmark_test $BENCHMARK_RUN_TEST_SH $DRIVER_BIN $MODEL $LOG_FILE)
+    echo "$RESULT ms"
+    print_result_of_benchmark_test "$MSG" "$RESULT" $RESULT_FILE
+    sleep $PAUSE_TIME_IN_SEC
+}

diff --git a/tests/scripts/py/common.py b/tests/scripts/py/common.py
index 37b40e1..34358b5 100755 (executable)
--- a/tests/scripts/py/common.py
+++ b/tests/scripts/py/common.py
@@ -18,7 +18,7 @@ import os.path
 mypath = os.path.abspath(os.path.dirname(__file__))
 
 
-def switch_nnfw_kernel_env(mode):
+def switch_nnfw_pacl_kernel_env(mode):
     # mode  : "acl" or "neon" or ""
 
     # TODO: Handle whether there is nnfw_kernel_env_list.txt or not
@@ -34,6 +34,6 @@ def switch_nnfw_kernel_env(mode):
 
 if __name__ == "__main__":
     # for test
-    switch_nnfw_kernel_env("acl")
-    switch_nnfw_kernel_env("neon")
-    switch_nnfw_kernel_env("")
+    switch_nnfw_pacl_kernel_env("acl")
+    switch_nnfw_pacl_kernel_env("neon")
+    switch_nnfw_pacl_kernel_env("")

diff --git a/tests/scripts/py/test_driver.py b/tests/scripts/py/test_driver.py
index 39e5315..1e77494 100755 (executable)
--- a/tests/scripts/py/test_driver.py
+++ b/tests/scripts/py/test_driver.py
@@ -352,7 +352,7 @@ def main():
 
     # set acl
     if options.aclenv_on:
-        common.switch_nnfw_kernel_env("acl")
+        common.switch_nnfw_pacl_kernel_env("acl")
 
     # unittest
     if alltest_on or options.unittest_on:

diff --git a/tests/scripts/run_benchmark.sh b/tests/scripts/run_benchmark.sh
index a6f96c9..7cb88f8 100755 (executable)
--- a/tests/scripts/run_benchmark.sh
+++ b/tests/scripts/run_benchmark.sh
@@ -70,33 +70,18 @@ function run_benchmark_test()
         mkdir -p $REPORT_MODEL_DIR
 
         # TFLite+CPU
-        LOG_FILE=$REPORT_MODEL_DIR/tflite_cpu.txt
-        RESULT_FILE=$REPORT_MODEL_DIR/tflite_cpu.result
-        echo -n "TFLite + CPU................... "
         unset USE_NNAPI
-        RESULT=$(get_result_of_benchmark_test $BENCHMARK_RUN_TEST_SH $DRIVER_BIN $MODEL $LOG_FILE)
-        echo "$RESULT ms"
-        print_result_of_benchmark_test "TFLite_CPU" $RESULT $RESULT_FILE
+        run_benchmark_and_print "tflite_cpu" "TFLite+CPU" $MODEL $REPORT_MODEL_DIR 0 $DRIVER_BIN $BENCHMARK_RUN_TEST_SH
 
         # TFLite+NNAPI(CPU fallback)
-        LOG_FILE=$REPORT_MODEL_DIR/tflite_nnapi_cpu.txt
-        RESULT_FILE=$REPORT_MODEL_DIR/tflite_nnapi_cpu.result
-        echo -n "TFLite + NNAPI(CPU)............ "
         export USE_NNAPI=1
-        RESULT=$(get_result_of_benchmark_test $BENCHMARK_RUN_TEST_SH $DRIVER_BIN $MODEL $LOG_FILE)
-        echo "$RESULT ms"
-        print_result_of_benchmark_test "TFLite_NNAPI_CPU" $RESULT $RESULT_FILE
+        run_benchmark_and_print "tflite_nnapi_cpu" "TFLite+NNAPI(CPU fallback)" $MODEL $REPORT_MODEL_DIR 0 $DRIVER_BIN $BENCHMARK_RUN_TEST_SH
 
         # TFLite+NNAPI(ACL)
-        LOG_FILE=$REPORT_MODEL_DIR/tflite_nnapi_acl.txt
-        RESULT_FILE=$REPORT_MODEL_DIR/tflite_nnapi_acl.result
-        echo -n "TFLite + NNAPI(ACL)............ "
-        switch_nnfw_kernel_env "ON" "acl"
-        RESULT=$(get_result_of_benchmark_test $BENCHMARK_RUN_TEST_SH $DRIVER_BIN $MODEL $LOG_FILE)
-        echo "$RESULT ms"
-        print_result_of_benchmark_test "TFLite_NNAPI_ACL" $RESULT $RESULT_FILE
         unset USE_NNAPI
-        switch_nnfw_kernel_env "OFF"
+        switch_nnfw_pacl_kernel_env "ON" "acl"
+        run_benchmark_and_print "tflite_nnapi_acl" "TFLite+NNAPI(ACL)" $MODEL $REPORT_MODEL_DIR 0 $DRIVER_BIN $BENCHMARK_RUN_TEST_SH
+        switch_nnfw_pacl_kernel_env "OFF"
 
         if [[ $i -ne $(echo $BENCHMARK_MODEL_LIST | wc -w)-1 ]]; then
             echo ""

diff --git a/tests/scripts/run_benchmark_op.sh b/tests/scripts/run_benchmark_op.sh
index 2e68049..e2013f0 100755 (executable)
--- a/tests/scripts/run_benchmark_op.sh
+++ b/tests/scripts/run_benchmark_op.sh
@@ -59,33 +59,6 @@ do
     shift
 done
 
-function get_result_of_benchmark_test()
-{
-    local RUN_TEST_SH=$1
-    local DRIVER_BIN=$2
-    local MODEL=$3
-    local LOG_FILE=$4
-
-    local RET=0
-    $RUN_TEST_SH --driverbin=$DRIVER_BIN $MODEL > $LOG_FILE 2>&1
-    RET=$?
-    if [[ $RET -ne 0 ]]; then
-        echo "Testing $MODEL aborted... exit code: $RET"
-        exit $RET
-    fi
-
-    local RESULT=`grep -E '^Mean:' $LOG_FILE | sed -e 's/ms//g' | awk '{print $2}'`
-    echo "$RESULT"
-}
-
-function print_result_of_benchmark_test()
-{
-    local NAME=$1
-    local RESULT=$2
-    local RESULT_FILE=$3
-    echo "$NAME $RESULT" > $RESULT_FILE
-}
-
 function get_benchmark_op_list()
 {
     local TEST_DIRS="$@"
@@ -137,36 +110,18 @@ function run_benchmark_test()
         mkdir -p $REPORT_MODEL_DIR
 
         # TFLite Interpreter
-        LOG_FILE=$REPORT_MODEL_DIR/tflite_cpu_op.txt
-        RESULT_FILE=$REPORT_MODEL_DIR/tflite_cpu_op.result
-        echo -n "TFLite Interpreter............. "
         unset USE_NNAPI
-        RESULT=$(get_result_of_benchmark_test $BENCHMARK_RUN_TEST_SH $DRIVER_BIN $MODEL $LOG_FILE)
-        echo "$RESULT ms"
-        print_result_of_benchmark_test "TFLite_Interpreter" $RESULT $RESULT_FILE
-        sleep $PAUSE_TIME_IN_SEC
+        run_benchmark_and_print "tflite_cpu_op" "TFLite Interpreter" $MODEL $REPORT_MODEL_DIR $PAUSE_TIME_IN_SEC $DRIVER_BIN $BENCHMARK_RUN_TEST_SH
 
         # PureACL-Neon
-        LOG_FILE=$REPORT_MODEL_DIR/tflite_nnrt_acl_neon_op.txt
-        RESULT_FILE=$REPORT_MODEL_DIR/tflite_nnrt_acl_neon_op.result
-        echo -n "PureACL-Neon................... "
         export USE_NNAPI=1
         export NEON=1
-        RESULT=$(get_result_of_benchmark_test $BENCHMARK_RUN_TEST_SH $DRIVER_BIN $MODEL $LOG_FILE)
-        echo "$RESULT ms"
-        print_result_of_benchmark_test "PureACL_Neon" $RESULT $RESULT_FILE
-        sleep $PAUSE_TIME_IN_SEC
+        run_benchmark_and_print "tflite_nnrt_acl_neon_op" "PureACL-Neon" $MODEL $REPORT_MODEL_DIR $PAUSE_TIME_IN_SEC $DRIVER_BIN $BENCHMARK_RUN_TEST_SH
 
         # PureACL-CL
-        LOG_FILE=$REPORT_MODEL_DIR/tflite_nnrt_acl_opencl_op.txt
-        RESULT_FILE=$REPORT_MODEL_DIR/tflite_nnrt_acl_opencl_op.result
         export USE_NNAPI=1
         unset NEON
-        echo -n "PureACL-CL..................... "
-        RESULT=$(get_result_of_benchmark_test $BENCHMARK_RUN_TEST_SH $DRIVER_BIN $MODEL $LOG_FILE)
-        echo "$RESULT ms"
-        print_result_of_benchmark_test "PureACL-CL" $RESULT $RESULT_FILE
-        sleep $PAUSE_TIME_IN_SEC
+        run_benchmark_and_print "tflite_nnrt_acl_opencl_op" "PureACL-CL" $MODEL $REPORT_MODEL_DIR $PAUSE_TIME_IN_SEC $DRIVER_BIN $BENCHMARK_RUN_TEST_SH
 
         # Clean Up
         unset USE_NNAPI NEON

diff --git a/tests/scripts/test_driver.sh b/tests/scripts/test_driver.sh
index ecba478..a637937 100755 (executable)
--- a/tests/scripts/test_driver.sh
+++ b/tests/scripts/test_driver.sh
@@ -192,7 +192,7 @@ fi
 source $TEST_DRIVER_DIR/common.sh
 
 if [ "$ACL_ENV_ON" == "true" ]; then
-    switch_nnfw_kernel_env "ON" "acl"
+    switch_nnfw_pacl_kernel_env "ON" "acl"
 fi
 
 # Run unittest in each part such as Runtime

diff --git a/tests/scripts/test_performance.sh b/tests/scripts/test_performance.sh
index 29f90b9..499e039 100644 (file)
--- a/tests/scripts/test_performance.sh
+++ b/tests/scripts/test_performance.sh
@@ -3,7 +3,6 @@
 MY_PATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 source $MY_PATH/common.sh
 
-PRINT_WIDTH=45
 BACKEND_CNT=3
 # Run profiler BACKEND_CNT+1 times: on each run of the first BACKEND_CNT runs it will
 #     collect metrics for one unmeasured backend. On the last run metrics for data transfer
@@ -24,15 +23,6 @@ fi
 BENCHMARK_REPORT_DIR=$REPORT_DIR/benchmark
 BENCHMARK_MODELS_FILE=$BENCHMARK_REPORT_DIR/benchmark_models.txt
 
-function print_with_dots()
-{
-    local MSG=$1
-    pad=$(printf '%0.1s' "."{1..45})
-    padlength=$((PRINT_WIDTH- ${#MSG}))
-    printf '%s' "$MSG"
-    printf '%*.*s ' 0 $padlength "$pad"
-}
-
 function run_without_sched()
 {
     local RESULT_SCH_INT=$1

diff --git a/tests/tools/tflite_benchmark_model/benchmark_tflite_model.cc b/tests/tools/tflite_benchmark_model/benchmark_tflite_model.cc
index 9bc9c39..16e85fc 100644 (file)
--- a/tests/tools/tflite_benchmark_model/benchmark_tflite_model.cc
+++ b/tests/tools/tflite_benchmark_model/benchmark_tflite_model.cc
@@ -405,7 +405,7 @@ void BenchmarkTfLiteModel::Init() {
 void BenchmarkTfLiteModel::RunImpl() {
   bool use_nnapi = params_.Get<bool>("use_nnapi");
   if (use_nnapi) {
-    if (nnfw_delegate_.Invoke(interpreter.get()) != kTfLiteOk) {
+    if (nnfw_delegate_.Invoke(&interpreter->primary_subgraph()) != kTfLiteOk) {
       TFLITE_LOG(FATAL) << "Failed to invoke!";
     }
   } else {