From: Dilshodzhon Poshshoev/AI Tools Lab /SRR/Engineer/Samsung Electronics Date: Thu, 14 Nov 2019 07:08:43 +0000 (+0300) Subject: Refactory benchmark scripts to unify (#8838) X-Git-Tag: submit/tizen/20191205.083104~289 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=9d412afa96bcfe496234fd57a903880a556468be;p=platform%2Fcore%2Fml%2Fnnfw.git Refactory benchmark scripts to unify (#8838) 1. Most of the script weren't up-to-date 2. Most of the steps are the same, so move them as a separate functions to common.sh Signed-off-by: Poshshoev Dilshodzhon --- diff --git a/tests/scripts/common.sh b/tests/scripts/common.sh index 1e7ab39..60aaf76 100755 --- a/tests/scripts/common.sh +++ b/tests/scripts/common.sh @@ -16,7 +16,7 @@ MY_PATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -function switch_nnfw_kernel_env() +function switch_nnfw_pacl_kernel_env() { local switch=$1 # "ON" or "OFF" local mode=$2 # "acl" or "neon" or "" @@ -60,3 +60,33 @@ function print_result_of_benchmark_test() echo "$NAME $RESULT" > $RESULT_FILE } + +function print_with_dots() +{ + PRINT_WIDTH=45 + local MSG="$@" + pad=$(printf '%0.1s' "."{1..45}) + padlength=$((PRINT_WIDTH- ${#MSG})) + printf '%s' "$MSG" + printf '%*.*s ' 0 $padlength "$pad" +} + + +function run_benchmark_and_print() +{ + local WRITE_FILE_NAME=$1 + local MSG=$2 + local MODEL=$3 + local REPORT_MODEL_DIR=$4 + local PAUSE_TIME_IN_SEC=$5 + local DRIVER_BIN=$6 + local BENCHMARK_RUN_TEST_SH=$7 + + LOG_FILE=$REPORT_MODEL_DIR/$WRITE_FILE_NAME.txt + RESULT_FILE=$REPORT_MODEL_DIR/$WRITE_FILE_NAME.result + print_with_dots $MSG + RESULT=$(get_result_of_benchmark_test $BENCHMARK_RUN_TEST_SH $DRIVER_BIN $MODEL $LOG_FILE) + echo "$RESULT ms" + print_result_of_benchmark_test "$MSG" "$RESULT" $RESULT_FILE + sleep $PAUSE_TIME_IN_SEC +} diff --git a/tests/scripts/py/common.py b/tests/scripts/py/common.py index 37b40e1..34358b5 100755 --- a/tests/scripts/py/common.py +++ b/tests/scripts/py/common.py @@ -18,7 +18,7 @@ import os.path mypath = os.path.abspath(os.path.dirname(__file__)) -def switch_nnfw_kernel_env(mode): +def switch_nnfw_pacl_kernel_env(mode): # mode : "acl" or "neon" or "" # TODO: Handle whether there is nnfw_kernel_env_list.txt or not @@ -34,6 +34,6 @@ def switch_nnfw_kernel_env(mode): if __name__ == "__main__": # for test - switch_nnfw_kernel_env("acl") - switch_nnfw_kernel_env("neon") - switch_nnfw_kernel_env("") + switch_nnfw_pacl_kernel_env("acl") + switch_nnfw_pacl_kernel_env("neon") + switch_nnfw_pacl_kernel_env("") diff --git a/tests/scripts/py/test_driver.py b/tests/scripts/py/test_driver.py index 39e5315..1e77494 100755 --- a/tests/scripts/py/test_driver.py +++ b/tests/scripts/py/test_driver.py @@ -352,7 +352,7 @@ def main(): # set acl if options.aclenv_on: - common.switch_nnfw_kernel_env("acl") + common.switch_nnfw_pacl_kernel_env("acl") # unittest if alltest_on or options.unittest_on: diff --git a/tests/scripts/run_benchmark.sh b/tests/scripts/run_benchmark.sh index a6f96c9..7cb88f8 100755 --- a/tests/scripts/run_benchmark.sh +++ b/tests/scripts/run_benchmark.sh @@ -70,33 +70,18 @@ function run_benchmark_test() mkdir -p $REPORT_MODEL_DIR # TFLite+CPU - LOG_FILE=$REPORT_MODEL_DIR/tflite_cpu.txt - RESULT_FILE=$REPORT_MODEL_DIR/tflite_cpu.result - echo -n "TFLite + CPU................... " unset USE_NNAPI - RESULT=$(get_result_of_benchmark_test $BENCHMARK_RUN_TEST_SH $DRIVER_BIN $MODEL $LOG_FILE) - echo "$RESULT ms" - print_result_of_benchmark_test "TFLite_CPU" $RESULT $RESULT_FILE + run_benchmark_and_print "tflite_cpu" "TFLite+CPU" $MODEL $REPORT_MODEL_DIR 0 $DRIVER_BIN $BENCHMARK_RUN_TEST_SH # TFLite+NNAPI(CPU fallback) - LOG_FILE=$REPORT_MODEL_DIR/tflite_nnapi_cpu.txt - RESULT_FILE=$REPORT_MODEL_DIR/tflite_nnapi_cpu.result - echo -n "TFLite + NNAPI(CPU)............ " export USE_NNAPI=1 - RESULT=$(get_result_of_benchmark_test $BENCHMARK_RUN_TEST_SH $DRIVER_BIN $MODEL $LOG_FILE) - echo "$RESULT ms" - print_result_of_benchmark_test "TFLite_NNAPI_CPU" $RESULT $RESULT_FILE + run_benchmark_and_print "tflite_nnapi_cpu" "TFLite+NNAPI(CPU fallback)" $MODEL $REPORT_MODEL_DIR 0 $DRIVER_BIN $BENCHMARK_RUN_TEST_SH # TFLite+NNAPI(ACL) - LOG_FILE=$REPORT_MODEL_DIR/tflite_nnapi_acl.txt - RESULT_FILE=$REPORT_MODEL_DIR/tflite_nnapi_acl.result - echo -n "TFLite + NNAPI(ACL)............ " - switch_nnfw_kernel_env "ON" "acl" - RESULT=$(get_result_of_benchmark_test $BENCHMARK_RUN_TEST_SH $DRIVER_BIN $MODEL $LOG_FILE) - echo "$RESULT ms" - print_result_of_benchmark_test "TFLite_NNAPI_ACL" $RESULT $RESULT_FILE unset USE_NNAPI - switch_nnfw_kernel_env "OFF" + switch_nnfw_pacl_kernel_env "ON" "acl" + run_benchmark_and_print "tflite_nnapi_acl" "TFLite+NNAPI(ACL)" $MODEL $REPORT_MODEL_DIR 0 $DRIVER_BIN $BENCHMARK_RUN_TEST_SH + switch_nnfw_pacl_kernel_env "OFF" if [[ $i -ne $(echo $BENCHMARK_MODEL_LIST | wc -w)-1 ]]; then echo "" diff --git a/tests/scripts/run_benchmark_op.sh b/tests/scripts/run_benchmark_op.sh index 2e68049..e2013f0 100755 --- a/tests/scripts/run_benchmark_op.sh +++ b/tests/scripts/run_benchmark_op.sh @@ -59,33 +59,6 @@ do shift done -function get_result_of_benchmark_test() -{ - local RUN_TEST_SH=$1 - local DRIVER_BIN=$2 - local MODEL=$3 - local LOG_FILE=$4 - - local RET=0 - $RUN_TEST_SH --driverbin=$DRIVER_BIN $MODEL > $LOG_FILE 2>&1 - RET=$? - if [[ $RET -ne 0 ]]; then - echo "Testing $MODEL aborted... exit code: $RET" - exit $RET - fi - - local RESULT=`grep -E '^Mean:' $LOG_FILE | sed -e 's/ms//g' | awk '{print $2}'` - echo "$RESULT" -} - -function print_result_of_benchmark_test() -{ - local NAME=$1 - local RESULT=$2 - local RESULT_FILE=$3 - echo "$NAME $RESULT" > $RESULT_FILE -} - function get_benchmark_op_list() { local TEST_DIRS="$@" @@ -137,36 +110,18 @@ function run_benchmark_test() mkdir -p $REPORT_MODEL_DIR # TFLite Interpreter - LOG_FILE=$REPORT_MODEL_DIR/tflite_cpu_op.txt - RESULT_FILE=$REPORT_MODEL_DIR/tflite_cpu_op.result - echo -n "TFLite Interpreter............. " unset USE_NNAPI - RESULT=$(get_result_of_benchmark_test $BENCHMARK_RUN_TEST_SH $DRIVER_BIN $MODEL $LOG_FILE) - echo "$RESULT ms" - print_result_of_benchmark_test "TFLite_Interpreter" $RESULT $RESULT_FILE - sleep $PAUSE_TIME_IN_SEC + run_benchmark_and_print "tflite_cpu_op" "TFLite Interpreter" $MODEL $REPORT_MODEL_DIR $PAUSE_TIME_IN_SEC $DRIVER_BIN $BENCHMARK_RUN_TEST_SH # PureACL-Neon - LOG_FILE=$REPORT_MODEL_DIR/tflite_nnrt_acl_neon_op.txt - RESULT_FILE=$REPORT_MODEL_DIR/tflite_nnrt_acl_neon_op.result - echo -n "PureACL-Neon................... " export USE_NNAPI=1 export NEON=1 - RESULT=$(get_result_of_benchmark_test $BENCHMARK_RUN_TEST_SH $DRIVER_BIN $MODEL $LOG_FILE) - echo "$RESULT ms" - print_result_of_benchmark_test "PureACL_Neon" $RESULT $RESULT_FILE - sleep $PAUSE_TIME_IN_SEC + run_benchmark_and_print "tflite_nnrt_acl_neon_op" "PureACL-Neon" $MODEL $REPORT_MODEL_DIR $PAUSE_TIME_IN_SEC $DRIVER_BIN $BENCHMARK_RUN_TEST_SH # PureACL-CL - LOG_FILE=$REPORT_MODEL_DIR/tflite_nnrt_acl_opencl_op.txt - RESULT_FILE=$REPORT_MODEL_DIR/tflite_nnrt_acl_opencl_op.result export USE_NNAPI=1 unset NEON - echo -n "PureACL-CL..................... " - RESULT=$(get_result_of_benchmark_test $BENCHMARK_RUN_TEST_SH $DRIVER_BIN $MODEL $LOG_FILE) - echo "$RESULT ms" - print_result_of_benchmark_test "PureACL-CL" $RESULT $RESULT_FILE - sleep $PAUSE_TIME_IN_SEC + run_benchmark_and_print "tflite_nnrt_acl_opencl_op" "PureACL-CL" $MODEL $REPORT_MODEL_DIR $PAUSE_TIME_IN_SEC $DRIVER_BIN $BENCHMARK_RUN_TEST_SH # Clean Up unset USE_NNAPI NEON diff --git a/tests/scripts/test_driver.sh b/tests/scripts/test_driver.sh index ecba478..a637937 100755 --- a/tests/scripts/test_driver.sh +++ b/tests/scripts/test_driver.sh @@ -192,7 +192,7 @@ fi source $TEST_DRIVER_DIR/common.sh if [ "$ACL_ENV_ON" == "true" ]; then - switch_nnfw_kernel_env "ON" "acl" + switch_nnfw_pacl_kernel_env "ON" "acl" fi # Run unittest in each part such as Runtime diff --git a/tests/scripts/test_performance.sh b/tests/scripts/test_performance.sh index 29f90b9..499e039 100644 --- a/tests/scripts/test_performance.sh +++ b/tests/scripts/test_performance.sh @@ -3,7 +3,6 @@ MY_PATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" source $MY_PATH/common.sh -PRINT_WIDTH=45 BACKEND_CNT=3 # Run profiler BACKEND_CNT+1 times: on each run of the first BACKEND_CNT runs it will # collect metrics for one unmeasured backend. On the last run metrics for data transfer @@ -24,15 +23,6 @@ fi BENCHMARK_REPORT_DIR=$REPORT_DIR/benchmark BENCHMARK_MODELS_FILE=$BENCHMARK_REPORT_DIR/benchmark_models.txt -function print_with_dots() -{ - local MSG=$1 - pad=$(printf '%0.1s' "."{1..45}) - padlength=$((PRINT_WIDTH- ${#MSG})) - printf '%s' "$MSG" - printf '%*.*s ' 0 $padlength "$pad" -} - function run_without_sched() { local RESULT_SCH_INT=$1 diff --git a/tests/tools/tflite_benchmark_model/benchmark_tflite_model.cc b/tests/tools/tflite_benchmark_model/benchmark_tflite_model.cc index 9bc9c39..16e85fc 100644 --- a/tests/tools/tflite_benchmark_model/benchmark_tflite_model.cc +++ b/tests/tools/tflite_benchmark_model/benchmark_tflite_model.cc @@ -405,7 +405,7 @@ void BenchmarkTfLiteModel::Init() { void BenchmarkTfLiteModel::RunImpl() { bool use_nnapi = params_.Get("use_nnapi"); if (use_nnapi) { - if (nnfw_delegate_.Invoke(interpreter.get()) != kTfLiteOk) { + if (nnfw_delegate_.Invoke(&interpreter->primary_subgraph()) != kTfLiteOk) { TFLITE_LOG(FATAL) << "Failed to invoke!"; } } else {