From f290410657399496e13eb675b1e09956d34f8f9b Mon Sep 17 00:00:00 2001 From: =?utf8?q?=EC=98=A4=ED=98=95=EC=84=9D/On-Device=20Lab=28SR=29/Staff?= =?utf8?q?=20Engineer/=EC=82=BC=EC=84=B1=EC=A0=84=EC=9E=90?= Date: Thu, 5 Dec 2019 09:53:22 +0900 Subject: [PATCH] [runtime/test] Remove unused benchmark script (#9382) Remove unused benchmark script - pacl benchmark - tflite model benchmark Signed-off-by: Hyeongseok Oh --- tests/scripts/run_benchmark_pacl.sh | 149 ---------------------------- tests/scripts/run_benchmark_tflite_model.in | 2 - tests/scripts/run_benchmark_tflite_model.sh | 125 ----------------------- tests/scripts/test_driver.sh | 65 +----------- 4 files changed, 2 insertions(+), 339 deletions(-) delete mode 100755 tests/scripts/run_benchmark_pacl.sh delete mode 100644 tests/scripts/run_benchmark_tflite_model.in delete mode 100755 tests/scripts/run_benchmark_tflite_model.sh diff --git a/tests/scripts/run_benchmark_pacl.sh b/tests/scripts/run_benchmark_pacl.sh deleted file mode 100755 index e2013f0..0000000 --- a/tests/scripts/run_benchmark_pacl.sh +++ /dev/null @@ -1,149 +0,0 @@ -#!/bin/bash -# -# Copyright (c) 2018 Samsung Electronics Co., Ltd. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -MY_PATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -NNFW_HOME="$(dirname $(dirname ${MY_PATH}))" -source $MY_PATH/common.sh - -BENCHMARK_RUN_TEST_SH= -BENCHMARK_DRIVER_BIN= -BENCHMARK_REPORT_DIR= -BENCHMARK_MODELS_FILE= -BENCHMARK_MODEL_LIST= -MODEL_CACHE_ROOT_PATH= -MODEL_TEST_ROOT_PATH= -PURE_ACL_RT_ENV_FILE=$MY_PATH/benchmark_op_list.txt - -function Usage() -{ - # TODO: Fill this - echo "Usage: LD_LIBRARY_PATH=Product/out/lib/pureacl ./$0 --reportdir=report" -} - -for i in "$@" -do - case $i in - -h|--help|help) - Usage - exit 1 - ;; - --runtestsh=*) - BENCHMARK_RUN_TEST_SH=${i#*=} - ;; - --driverbin=*) - BENCHMARK_DRIVER_BIN=${i#*=} - ;; - --reportdir=*) - BENCHMARK_REPORT_DIR=${i#*=} - BENCHMARK_MODELS_FILE=$BENCHMARK_REPORT_DIR/benchmark_op_models.txt - ;; - --modelfilepath=*) - TEST_LIST_PATH=${i#*=} - MODEL_CACHE_ROOT_PATH=$TEST_LIST_PATH/cache - MODEL_TEST_ROOT_PATH=$TEST_LIST_PATH/tests - ;; - esac - shift -done - -function get_benchmark_op_list() -{ - local TEST_DIRS="$@" - local TESTS_TO_RUN="" - - if [[ $# -eq 0 ]]; then - TEST_DIRS="." - fi - - shift $# - - pushd $MODEL_TEST_ROOT_PATH > /dev/null - for DIR in $TEST_DIRS; do - if [ -d "$DIR" ]; then - TESTS_FOUND=$(find "$DIR" -type f -name 'config.sh' -exec dirname {} \;| sed 's|^./||' | grep -v '^MODELS/' | sort) - TESTS_TO_RUN="$TESTS_TO_RUN $TESTS_FOUND" - fi - done - popd > /dev/null - - BENCHMARK_MODEL_LIST=$(echo "${TESTS_TO_RUN}") -} - -function run_benchmark_test() -{ - local DRIVER_BIN=$BENCHMARK_DRIVER_BIN - local LOG_FILE= - local RESULT_FILE= - local RESULT= - local REPORT_MODEL_DIR= - local PAUSE_TIME_IN_SEC=15 - - export COUNT=5 - echo "============================================" - local i=0 - for MODEL in $BENCHMARK_MODEL_LIST; do - STATUS="enabled" - source $MODEL_TEST_ROOT_PATH/$MODEL/config.sh - - # Skip 'disabled' tests - if [ $(tr '[:upper:]' '[:lower:]' <<< "$STATUS") == "disabled" ]; then - continue - fi - - echo "Benchmark test with `basename $DRIVER_BIN` & `echo $MODEL`" - echo $MODEL >> $BENCHMARK_MODELS_FILE - - REPORT_MODEL_DIR=$BENCHMARK_REPORT_DIR/$MODEL - mkdir -p $REPORT_MODEL_DIR - - # TFLite Interpreter - unset USE_NNAPI - run_benchmark_and_print "tflite_cpu_op" "TFLite Interpreter" $MODEL $REPORT_MODEL_DIR $PAUSE_TIME_IN_SEC $DRIVER_BIN $BENCHMARK_RUN_TEST_SH - - # PureACL-Neon - export USE_NNAPI=1 - export NEON=1 - run_benchmark_and_print "tflite_nnrt_acl_neon_op" "PureACL-Neon" $MODEL $REPORT_MODEL_DIR $PAUSE_TIME_IN_SEC $DRIVER_BIN $BENCHMARK_RUN_TEST_SH - - # PureACL-CL - export USE_NNAPI=1 - unset NEON - run_benchmark_and_print "tflite_nnrt_acl_opencl_op" "PureACL-CL" $MODEL $REPORT_MODEL_DIR $PAUSE_TIME_IN_SEC $DRIVER_BIN $BENCHMARK_RUN_TEST_SH - - # Clean Up - unset USE_NNAPI NEON - - if [[ $i -ne $(echo $BENCHMARK_MODEL_LIST | wc -w)-1 ]]; then - echo "" - fi - i=$((i+1)) - done - unset USE_NNAPI - unset COUNT - echo "============================================" -} - -if [ ! -e "$BENCHMARK_REPORT_DIR" ]; then - mkdir -p $BENCHMARK_REPORT_DIR -fi - -get_benchmark_op_list - -rm -rf $BENCHMARK_MODELS_FILE - -echo "" -run_benchmark_test -echo "" diff --git a/tests/scripts/run_benchmark_tflite_model.in b/tests/scripts/run_benchmark_tflite_model.in deleted file mode 100644 index 4a03986..0000000 --- a/tests/scripts/run_benchmark_tflite_model.in +++ /dev/null @@ -1,2 +0,0 @@ -MODELS/inception_nonslim --input_layer=Mul --input_layer_shape=1,299,299,3 --num_threads=1 --num_runs=1 -MODELS/mobilenet --input_layer=input --input_layer_shape=1,128,128,3 --num_threads=1 --num_runs=1 diff --git a/tests/scripts/run_benchmark_tflite_model.sh b/tests/scripts/run_benchmark_tflite_model.sh deleted file mode 100755 index c604b18..0000000 --- a/tests/scripts/run_benchmark_tflite_model.sh +++ /dev/null @@ -1,125 +0,0 @@ -#!/bin/bash -# -# Copyright (c) 2018 Samsung Electronics Co., Ltd. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -NNFW_DIR="$(dirname "$(dirname "${SCRIPT_DIR}")")" -REPORT_DIR="$NNFW_DIR/report/tflite_benchmark_model" -MODEL_ROOT="$NNFW_DIR/tests/framework/tests" -LD_LIBRARY_PATH="$NNFW_DIR/Product/out/lib/pureacl" - -RUN_TEST=$NNFW_DIR/tests/framework/run_test.sh -MODEL_IN=${BASH_SOURCE[0]%.sh}.in -BENCHMARK_BIN=$NNFW_DIR/Product/out/bin/tflite_benchmark_model -MODEL_NAMES= -MODEL_PARAMS= - -source $SCRIPT_DIR/common.sh - -usage() -{ - echo - echo "Usage: LD_LIBRARY_PATH=Product/out/lib/pureacl $(basename ${BASH_SOURCE[0]}) --reportdir=report --modelroot=modelroot" - echo -} - -parse_args() -{ - for i in "$@"; do - case $i in - -h|--help|help) - usage - exit 1 - ;; - --reportdir=*) - REPORT_DIR=${i#*=} - ;; - --modelroot=*) - MODEL_ROOT=${i#*=} - ;; - esac - shift - done -} - -load_input() -{ - mapfile -t MODEL_NAMES < <(cut -f1 "${MODEL_IN}") - mapfile -t MODEL_PARAMS < <(cut -f2- "${MODEL_IN}") - if [ "${#MODEL_NAMES[@]}" -eq 0 ]; then - echo "No model is found. Please check ${MODEL_IN} is correct." - exit 1 - fi -} - -download_models() -{ - $RUN_TEST --download=on $MODEL_NAMES -} - -run_benchmarks() -{ - echo - echo "Running benchmarks:" - echo "======================" - - for (( i=0; i< ${#MODEL_NAMES[@]}; i++)); do - MODEL_NAME=${MODEL_NAMES[i]} - MODEL_PATH=$(find $NNFW_DIR/tests/framework/cache/$MODEL_NAME/ -name "*.tflite") - MODEL_PARAM=${MODEL_PARAMS[$i]} - - echo "$MODEL_NAME" - - local REPORT_MODEL_DIR=$REPORT_DIR/$MODEL_NAME - mkdir -p $REPORT_MODEL_DIR - - local OUT_FILE - - # TFLite Interpreter - OUT_FILE=$REPORT_MODEL_DIR/tflite_interpreter.out - echo - echo "{ // TFLite Interpreter" - LD_LIBRARY_PATH=$LD_LIBRARY_PATH $BENCHMARK_BIN --graph=$MODEL_PATH $MODEL_PARAM --use_nnapi=false 2> >(tee $OUT_FILE) - echo "} // TFLite Interpreter" - - # TFLite PureACL (CL) - OUT_FILE=$REPORT_MODEL_DIR/tflite_pureacl_cl.out - echo - echo "{ // TFLite PureACL(CL)" - LD_LIBRARY_PATH=$LD_LIBRARY_PATH $BENCHMARK_BIN --graph=$MODEL_PATH $MODEL_PARAM --use_nnapi=true 2> >(tee $OUT_FILE) - echo "} // TFLite_PureACL(CL)" - done -} - -# for debug -print_vars() -{ - echo SCRIPT_DIR=$SCRIPT_DIR - echo NNFW_DIR=$NNFW_DIR - echo RUN_TEST=$RUN_TEST - echo MODEL_IN=$MODEL_IN - echo BENCHMARK_BIN=$BENCHMARK_BIN - echo REPORT_DIR=$REPORT_DIR - echo MODEL_ROOT=$MODEL_ROOT -} - -if [ ! -e "$REPORT_DIR" ]; then - mkdir -p $REPORT_DIR -fi - -parse_args $@ -load_input -download_models -run_benchmarks diff --git a/tests/scripts/test_driver.sh b/tests/scripts/test_driver.sh index d856ed5..51820ce 100755 --- a/tests/scripts/test_driver.sh +++ b/tests/scripts/test_driver.sh @@ -34,11 +34,7 @@ function Usage() echo "--frameworktest_list_file - filepath of model list for test" echo "" echo "Following option is only needed when you want to test benchmark." - echo "--benchmark_acl - (default=off) run benchmark-acl" - echo "--benchmark - (default=off) run benchmark on neurun" - echo "--benchmark_pacl_op - (default=off) run benchmark per operation on pacl" echo "--benchmark_neurun_op - (default=off) run benchmark per operation on neurun" - echo "--benchmark_tflite_model - (default=off) run tflite_benchmark_model" echo "" echo "Following option is used for profiling." echo "--profile - (default=off) run operf" @@ -70,10 +66,7 @@ UNITTEST_ON="false" UNITTESTALL_ON="false" FRAMEWORKTEST_ON="false" VERIFICATION_ON="false" -BENCHMARK_ON="false" -BENCHMARK_PACL_OP_ON="false" BENCHMARK_NEURUN_OP_ON="false" -BENCHMARK_TFLITE_MODEL_ON="false" ACL_ENV_ON="false" PROFILE_ON="false" REPORT_DIR="" @@ -133,26 +126,10 @@ do ALLTEST_ON="false" VERIFICATION_ON="true" ;; - --benchmark) - ALLTEST_ON="false" - BENCHMARK_ON="true" - ;; - --benchmark_pacl_op) - ALLTEST_ON="false" - BENCHMARK_PACL_OP_ON="true" - ;; --benchmark_neurun_op) ALLTEST_ON="false" BENCHMARK_NEURUN_OP_ON="true" ;; - --benchmark_tflite_model) - ALLTEST_ON="false" - BENCHMARK_TFLITE_MODEL_ON="true" - ;; - # TODO Remove --benchmark_acl option after CI update - --benchmark_acl) - ALLTEST_ON="false" - ;; --acl_envon) ACL_ENV_ON="true" ;; @@ -248,36 +225,6 @@ if [ "$ALLTEST_ON" == "true" ] || [ "$VERIFICATION_ON" == "true" ]; then --frameworktest_list_file=${FRAMEWORKTEST_LIST_FILE:-} fi -# Run tflite_benchmark with tflite models -if [ "$BENCHMARK_ON" == "true" ]; then - if [ -z "$BENCHMARK_DRIVER_BIN" ]; then - DRIVER_BIN=$ARTIFACT_PATH/Product/out/bin/tflite_benchmark - else - DRIVER_BIN=$BENCHMARK_DRIVER_BIN - fi - - $TEST_DRIVER_DIR/run_benchmark_neurun.sh \ - --runtestsh=$RUN_TEST_SH \ - --driverbin=$DRIVER_BIN \ - --reportdir=$REPORT_DIR/benchmark -fi - -# Run tflite_benchmark from a list of tflite models. -# Each model has only one operator. -if [ "$BENCHMARK_PACL_OP_ON" == "true" ]; then - if [ -z "$BENCHMARK_DRIVER_BIN" ]; then - DRIVER_BIN=$ARTIFACT_PATH/Product/out/bin/tflite_benchmark - else - DRIVER_BIN=$BENCHMARK_DRIVER_BIN - fi - - $TEST_DRIVER_DIR/run_benchmark_pacl.sh \ - --runtestsh=$RUN_TEST_SH \ - --driverbin=$DRIVER_BIN \ - --reportdir=$REPORT_DIR/benchmark_op \ - --modelfilepath=$ARTIFACT_PATH/tests/framework -fi - if [ "$BENCHMARK_NEURUN_OP_ON" == "true" ]; then if [ -z "$BENCHMARK_DRIVER_BIN" ]; then DRIVER_BIN=$ARTIFACT_PATH/Product/out/bin/tflite_benchmark @@ -294,24 +241,16 @@ if [ "$BENCHMARK_NEURUN_OP_ON" == "true" ]; then fi # Make json file. Actually, this process is only needed on CI. That's why it is in test_driver.sh. -if [ "$BENCHMARK_ON" == "true" ] || [ "$BENCHMARK_PACL_OP_ON" == "true" ] || [ "$BENCHMARK_NEURUN_OP_ON" == "true" ]; then +if [ "$BENCHMARK_NEURUN_OP_ON" == "true" ]; then # functions to fill json with benchmark results source $ARTIFACT_PATH/tests/scripts/print_to_json.sh - if [ "$BENCHMARK_PACL_OP_ON" == "true" ] || [ "$BENCHMARK_NEURUN_OP_ON" == "true" ]; then + if [ "$BENCHMARK_NEURUN_OP_ON" == "true" ]; then print_to_json $REPORT_DIR/benchmark_op $REPORT_DIR "benchmark_op_result.json" else print_to_json $REPORT_DIR/benchmark $REPORT_DIR "benchmark_result.json" fi fi -# Run tflite_benchmark_model (= per-operation profiling tool). -# Each model can contain arbitrary number of operators. -if [ "$BENCHMARK_TFLITE_MODEL_ON" == "true" ]; then - $TEST_DRIVER_DIR/run_benchmark_tflite_model.sh \ - --reportdir=$REPORT_DIR/benchmark_tflite_model \ - --modelroot=$ARTIFACT_PATH/tests/framework/tests -fi - # Run profiling if [ "$PROFILE_ON" == "true" ]; then # FIXME: These driver and tflite test are set temporarily. Fix these to support flexibility -- 2.7.4