From 847ad18a94ea904252dd41f48a273752858d56db Mon Sep 17 00:00:00 2001 From: Dilshodzhon Poshshoev/AI Tools Lab /SRR/Engineer/Samsung Electronics Date: Thu, 21 Nov 2019 07:09:34 +0300 Subject: [PATCH] Separate benchmark scripts for neurun and pacl (#8945) 1. Rename run_benchmark_op.sh to run_benchmark_pacl.sh 2. Change run_benchmark.sh to use neurun 3. run_benchmark.sh can handle --benchmark_op option using neurun (porting from run_benchmark_pacl.sh) 4. Remove LD_LIBRARY_PATH from usage msg 5. Append _neurun to script name Signed-off-by: Poshshoev Dilshodzhon --- tests/scripts/py/test_driver.py | 1 + tests/scripts/run_benchmark.sh | 103 --------- tests/scripts/run_benchmark_neurun.sh | 245 +++++++++++++++++++++ .../{run_benchmark_op.sh => run_benchmark_pacl.sh} | 0 tests/scripts/test_driver.sh | 41 +++- 5 files changed, 277 insertions(+), 113 deletions(-) delete mode 100755 tests/scripts/run_benchmark.sh create mode 100755 tests/scripts/run_benchmark_neurun.sh rename tests/scripts/{run_benchmark_op.sh => run_benchmark_pacl.sh} (100%) diff --git a/tests/scripts/py/test_driver.py b/tests/scripts/py/test_driver.py index 1e77494..c512342 100755 --- a/tests/scripts/py/test_driver.py +++ b/tests/scripts/py/test_driver.py @@ -14,6 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +# TODO: remove since no needed and old-dated import os import shutil import argparse diff --git a/tests/scripts/run_benchmark.sh b/tests/scripts/run_benchmark.sh deleted file mode 100755 index 7cb88f8..0000000 --- a/tests/scripts/run_benchmark.sh +++ /dev/null @@ -1,103 +0,0 @@ -#!/bin/bash -# -# Copyright (c) 2018 Samsung Electronics Co., Ltd. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -MY_PATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - -source $MY_PATH/common.sh - -BENCHMARK_RUN_TEST_SH= -BENCHMARK_DRIVER_BIN= -BENCHMARK_REPORT_DIR= -BENCHMARK_MODELS_FILE= -BENCHMARK_MODEL_LIST="MODELS/inception_nonslim MODELS/inception_slim MODELS/mobilenet" - -function Usage() -{ - # TODO: Fill this - echo "Usage: ./$0 --reportdir=report" -} - -for i in "$@" -do - case $i in - -h|--help|help) - Usage - exit 1 - ;; - --runtestsh=*) - BENCHMARK_RUN_TEST_SH=${i#*=} - ;; - --driverbin=*) - BENCHMARK_DRIVER_BIN=${i#*=} - ;; - --reportdir=*) - BENCHMARK_REPORT_DIR=${i#*=} - BENCHMARK_MODELS_FILE=$BENCHMARK_REPORT_DIR/benchmark_models.txt - ;; - esac - shift -done - -function run_benchmark_test() -{ - local DRIVER_BIN=$BENCHMARK_DRIVER_BIN - local LOG_FILE= - local RESULT_FILE= - local RESULT= - local REPORT_MODEL_DIR= - - export COUNT=5 - echo "============================================" - local i=0 - for MODEL in $BENCHMARK_MODEL_LIST; do - echo "Benchmark test with `basename $DRIVER_BIN` & `echo $MODEL`" - echo $MODEL >> $BENCHMARK_MODELS_FILE - - REPORT_MODEL_DIR=$BENCHMARK_REPORT_DIR/$MODEL - mkdir -p $REPORT_MODEL_DIR - - # TFLite+CPU - unset USE_NNAPI - run_benchmark_and_print "tflite_cpu" "TFLite+CPU" $MODEL $REPORT_MODEL_DIR 0 $DRIVER_BIN $BENCHMARK_RUN_TEST_SH - - # TFLite+NNAPI(CPU fallback) - export USE_NNAPI=1 - run_benchmark_and_print "tflite_nnapi_cpu" "TFLite+NNAPI(CPU fallback)" $MODEL $REPORT_MODEL_DIR 0 $DRIVER_BIN $BENCHMARK_RUN_TEST_SH - - # TFLite+NNAPI(ACL) - unset USE_NNAPI - switch_nnfw_pacl_kernel_env "ON" "acl" - run_benchmark_and_print "tflite_nnapi_acl" "TFLite+NNAPI(ACL)" $MODEL $REPORT_MODEL_DIR 0 $DRIVER_BIN $BENCHMARK_RUN_TEST_SH - switch_nnfw_pacl_kernel_env "OFF" - - if [[ $i -ne $(echo $BENCHMARK_MODEL_LIST | wc -w)-1 ]]; then - echo "" - fi - i=$((i+1)) - done - echo "============================================" - unset COUNT -} - -if [ ! -e "$BENCHMARK_REPORT_DIR" ]; then - mkdir -p $BENCHMARK_REPORT_DIR -fi - -rm -rf $BENCHMARK_MODELS_FILE - -echo "" -run_benchmark_test -echo "" diff --git a/tests/scripts/run_benchmark_neurun.sh b/tests/scripts/run_benchmark_neurun.sh new file mode 100755 index 0000000..03fa8ae --- /dev/null +++ b/tests/scripts/run_benchmark_neurun.sh @@ -0,0 +1,245 @@ +#!/bin/bash +# +# Copyright (c) 2018 Samsung Electronics Co., Ltd. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +MY_PATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +source $MY_PATH/common.sh + +BENCHMARK_RUN_TEST_SH= +BENCHMARK_DRIVER_BIN= +BENCHMARK_REPORT_DIR= +BENCHMARK_MODELS_FILE= +MODEL_TEST_ROOT_PATH= +TEST_OP="false" +BENCHMARK_MODEL_LIST="MODELS/inception_nonslim MODELS/inception_slim MODELS/mobilenet" +BACKEND_LIST="acl_cl acl_neon cpu" #TODO: accept this list as argument +EXECUTORS="Linear Parallel" #TODO: accept this list as argument + +function Usage() +{ + echo "Usage: ./$0 --reportdir=. --runtestsh=tests/framework/run_test.sh --driverbin=Product/out/bin/tflite_benchmark" +} + +for i in "$@" +do + case $i in + -h|--help|help) + Usage + exit 1 + ;; + --test_op) + TEST_OP="true" + ;; + --runtestsh=*) + BENCHMARK_RUN_TEST_SH=${i#*=} + ;; + --driverbin=*) + BENCHMARK_DRIVER_BIN=${i#*=} + ;; + --reportdir=*) + BENCHMARK_REPORT_DIR=${i#*=} + BENCHMARK_MODELS_FILE=$BENCHMARK_REPORT_DIR/benchmark_models.txt + ;; + --modelfilepath=*) + TEST_LIST_PATH=${i#*=} + MODEL_TEST_ROOT_PATH=$TEST_LIST_PATH/tests + ;; + esac + shift +done + +function get_benchmark_op_list() +{ + local TEST_DIRS="$@" + local TESTS_TO_RUN="" + + if [[ $# -eq 0 ]]; then + TEST_DIRS="." + fi + + shift $# + + pushd $MODEL_TEST_ROOT_PATH > /dev/null + for DIR in $TEST_DIRS; do + if [ -d "$DIR" ]; then + TESTS_FOUND=$(find "$DIR" -type f -name 'config.sh' -exec dirname {} \;| sed 's|^./||' | grep -v '^MODELS/' | sort) + TESTS_TO_RUN="$TESTS_TO_RUN $TESTS_FOUND" + fi + done + popd > /dev/null + + BENCHMARK_MODEL_LIST=$(echo "${TESTS_TO_RUN}") +} + +function profile_for_he_shed() +{ + + local REPORT_MODEL_DIR=$1 + local RUN_TEST_SH=$2 + local BENCHMARK_DRIVER_BIN=$3 + local MODEL=$4 + local PROFILING_RUN_CNT=$5 + + export USE_SCHEDULER=1 + export PROFILING_MODE=1 + export EXECUTOR="Dataflow" + export NEURUN_LOG_ENABLE=1 + + rm "exec_time.json" 2>/dev/null + for ((j = 1 ; j <= $PROFILING_RUN_CNT ; j++)); do + # Save the verbose log of each run + LOG_FILE=$REPORT_MODEL_DIR/tflite_profiling_$j.txt + + print_with_dots "Profiling run #$j out of $PROFILING_RUN_CNT" + + $RUN_TEST_SH --driverbin=$BENCHMARK_DRIVER_BIN $MODEL > $LOG_FILE 2>&1 + RET=$? + if [[ $RET -ne 0 ]]; then + echo "Profiling $MODEL aborted in run#$j... exit code: $RET"xX + exit $RET + fi + echo "finished" + # Save the exec_time.json of each run + cp "exec_time.json" $REPORT_MODEL_DIR/"exec_time_$j.json" + done + unset USE_SCHEDULER PROFILING_MODE EXECUTOR NEURUN_LOG_ENABLE +} + +function run_with_he_scheduler() +{ + local REPORT_MODEL_DIR=$1 + local RUN_TEST_SH=$2 + local BENCHMARK_DRIVER_BIN=$3 + local MODEL=$4 + local EXECUTOR=$5 + + LOG_FILE=$REPORT_MODEL_DIR/tflite_neurun_with_he_scheduler_in_$EXECUTOR.txt + export EXECUTOR=$EXECUTOR + export GRAPH_DOT_DUMP=1 + export USE_SCHEDULER=1 + export NEURUN_LOG_ENABLE=1 + + print_with_dots "TFLite neurun $EXECUTOR with HEScheduler" + + RESULT=$(get_result_of_benchmark_test $RUN_TEST_SH $BENCHMARK_DRIVER_BIN $MODEL $LOG_FILE) + echo "$RESULT ms" + + mv "after_lower.dot" $REPORT_MODEL_DIR/"after_lower_$EXECUTOR.dot" + unset EXECUTOR GRAPH_DOT_DUMP USE_SCHEDULER NEURUN_LOG_ENABLE +} + +function run_neurun_with_all_config() +{ + local MODEL=$1 + local REPORT_MODEL_DIR=$2 + local PAUSE_TIME_IN_SEC=$3 + local BENCHMARK_DRIVER_BIN=$4 + local BENCHMARK_RUN_TEST_SH=$5 + local EXECUTORS=$6 + local BACKEND_LIST=$7 + + export USE_NNAPI=1 + + # Run profiler BACKEND_CNT+1 times: on each run of the first BACKEND_CNT runs it will + # collect metrics for one unmeasured backend. On the last run metrics for data transfer + PROFILING_RUN_CNT=1 + BACKENDS_TO_USE= + for backend in $BACKEND_LIST; do + BACKENDS_TO_USE+=$backend';' + ((++PROFILING_RUN_CNT)) + done + export BACKENDS=$BACKENDS_TO_USE + if [ "$TEST_OP" == "false" ]; then + profile_for_he_shed $REPORT_MODEL_DIR $BENCHMARK_RUN_TEST_SH $BENCHMARK_DRIVER_BIN $MODEL $PROFILING_RUN_CNT + fi + + for executor in $EXECUTORS; do + export EXECUTOR=$executor + if [ "$TEST_OP" == "false" ]; then + run_with_he_scheduler $REPORT_MODEL_DIR $BENCHMARK_RUN_TEST_SH $BENCHMARK_DRIVER_BIN $MODEL $executor + fi + for backend in $BACKEND_LIST; do + export OP_BACKEND_ALLOPS=$backend + run_benchmark_and_print "tflite_neurun_"$executor"_executor_$backend" "TFLite neurun $executor Executor $backend"\ + $MODEL $REPORT_MODEL_DIR 0 $BENCHMARK_DRIVER_BIN $BENCHMARK_RUN_TEST_SH + done + done + unset USE_NNAPI EXECUTOR OP_BACKEND_ALLOPS BACKENDS +} + +function run_benchmark_test() +{ + local LOG_FILE= + local RESULT_FILE= + local RESULT= + local REPORT_MODEL_DIR= + + export COUNT=5 + export NEURUN_LOG_ENABLE=1 + echo + echo "============================================" + echo + date +'%Y-%m-%d %H:%M:%S %s' + echo + local i=0 + for MODEL in $BENCHMARK_MODEL_LIST; do + + STATUS="enabled" + if [ "$TEST_OP" == "true" ]; then + source $MODEL_TEST_ROOT_PATH/$MODEL/config.sh + fi + + # Skip 'disabled' tests + if [ $(tr '[:upper:]' '[:lower:]' <<< "$STATUS") == "disabled" ]; then + continue + fi + + echo "Benchmark test with `basename $BENCHMARK_DRIVER_BIN` & `echo $MODEL`" + echo $MODEL >> $BENCHMARK_MODELS_FILE + + REPORT_MODEL_DIR=$BENCHMARK_REPORT_DIR/$MODEL + mkdir -p $REPORT_MODEL_DIR + + # TFLite+CPU + unset USE_NNAPI + run_benchmark_and_print "tflite_cpu" "TFLite CPU" $MODEL $REPORT_MODEL_DIR 0 $BENCHMARK_DRIVER_BIN $BENCHMARK_RUN_TEST_SH + + # run neurun + run_neurun_with_all_config $MODEL $REPORT_MODEL_DIR 0 $BENCHMARK_DRIVER_BIN $BENCHMARK_RUN_TEST_SH "$EXECUTORS" "$BACKEND_LIST" + + if [[ $i -ne $(echo $BENCHMARK_MODEL_LIST | wc -w)-1 ]]; then + echo "" + fi + i=$((i+1)) + done + echo "============================================" + unset COUNT +} + +if [ ! -e "$BENCHMARK_REPORT_DIR" ]; then + mkdir -p $BENCHMARK_REPORT_DIR +fi + +if [ "$TEST_OP" == "true" ]; then + get_benchmark_op_list +fi + +rm -rf $BENCHMARK_MODELS_FILE + +echo "" +# print the result AND append to log file +run_benchmark_test 2>&1 | tee -a neurun_benchmarks.txt +echo "" diff --git a/tests/scripts/run_benchmark_op.sh b/tests/scripts/run_benchmark_pacl.sh similarity index 100% rename from tests/scripts/run_benchmark_op.sh rename to tests/scripts/run_benchmark_pacl.sh diff --git a/tests/scripts/test_driver.sh b/tests/scripts/test_driver.sh index a637937..d856ed5 100755 --- a/tests/scripts/test_driver.sh +++ b/tests/scripts/test_driver.sh @@ -35,8 +35,9 @@ function Usage() echo "" echo "Following option is only needed when you want to test benchmark." echo "--benchmark_acl - (default=off) run benchmark-acl" - echo "--benchmark - (default=off) run benchmark" - echo "--benchmark_op - (default=off) run benchmark per operation" + echo "--benchmark - (default=off) run benchmark on neurun" + echo "--benchmark_pacl_op - (default=off) run benchmark per operation on pacl" + echo "--benchmark_neurun_op - (default=off) run benchmark per operation on neurun" echo "--benchmark_tflite_model - (default=off) run tflite_benchmark_model" echo "" echo "Following option is used for profiling." @@ -70,7 +71,8 @@ UNITTESTALL_ON="false" FRAMEWORKTEST_ON="false" VERIFICATION_ON="false" BENCHMARK_ON="false" -BENCHMARK_OP_ON="false" +BENCHMARK_PACL_OP_ON="false" +BENCHMARK_NEURUN_OP_ON="false" BENCHMARK_TFLITE_MODEL_ON="false" ACL_ENV_ON="false" PROFILE_ON="false" @@ -135,9 +137,13 @@ do ALLTEST_ON="false" BENCHMARK_ON="true" ;; - --benchmark_op) + --benchmark_pacl_op) ALLTEST_ON="false" - BENCHMARK_OP_ON="true" + BENCHMARK_PACL_OP_ON="true" + ;; + --benchmark_neurun_op) + ALLTEST_ON="false" + BENCHMARK_NEURUN_OP_ON="true" ;; --benchmark_tflite_model) ALLTEST_ON="false" @@ -250,7 +256,7 @@ if [ "$BENCHMARK_ON" == "true" ]; then DRIVER_BIN=$BENCHMARK_DRIVER_BIN fi - $TEST_DRIVER_DIR/run_benchmark.sh \ + $TEST_DRIVER_DIR/run_benchmark_neurun.sh \ --runtestsh=$RUN_TEST_SH \ --driverbin=$DRIVER_BIN \ --reportdir=$REPORT_DIR/benchmark @@ -258,14 +264,29 @@ fi # Run tflite_benchmark from a list of tflite models. # Each model has only one operator. -if [ "$BENCHMARK_OP_ON" == "true" ]; then +if [ "$BENCHMARK_PACL_OP_ON" == "true" ]; then + if [ -z "$BENCHMARK_DRIVER_BIN" ]; then + DRIVER_BIN=$ARTIFACT_PATH/Product/out/bin/tflite_benchmark + else + DRIVER_BIN=$BENCHMARK_DRIVER_BIN + fi + + $TEST_DRIVER_DIR/run_benchmark_pacl.sh \ + --runtestsh=$RUN_TEST_SH \ + --driverbin=$DRIVER_BIN \ + --reportdir=$REPORT_DIR/benchmark_op \ + --modelfilepath=$ARTIFACT_PATH/tests/framework +fi + +if [ "$BENCHMARK_NEURUN_OP_ON" == "true" ]; then if [ -z "$BENCHMARK_DRIVER_BIN" ]; then DRIVER_BIN=$ARTIFACT_PATH/Product/out/bin/tflite_benchmark else DRIVER_BIN=$BENCHMARK_DRIVER_BIN fi - $TEST_DRIVER_DIR/run_benchmark_op.sh \ + $TEST_DRIVER_DIR/run_benchmark_neurun.sh \ + --test_op \ --runtestsh=$RUN_TEST_SH \ --driverbin=$DRIVER_BIN \ --reportdir=$REPORT_DIR/benchmark_op \ @@ -273,10 +294,10 @@ if [ "$BENCHMARK_OP_ON" == "true" ]; then fi # Make json file. Actually, this process is only needed on CI. That's why it is in test_driver.sh. -if [ "$BENCHMARK_ON" == "true" ] || [ "$BENCHMARK_OP_ON" == "true" ]; then +if [ "$BENCHMARK_ON" == "true" ] || [ "$BENCHMARK_PACL_OP_ON" == "true" ] || [ "$BENCHMARK_NEURUN_OP_ON" == "true" ]; then # functions to fill json with benchmark results source $ARTIFACT_PATH/tests/scripts/print_to_json.sh - if [ "$BENCHMARK_OP_ON" == "true" ]; then + if [ "$BENCHMARK_PACL_OP_ON" == "true" ] || [ "$BENCHMARK_NEURUN_OP_ON" == "true" ]; then print_to_json $REPORT_DIR/benchmark_op $REPORT_DIR "benchmark_op_result.json" else print_to_json $REPORT_DIR/benchmark $REPORT_DIR "benchmark_result.json" -- 2.7.4