From 8be846c0d21e244192a2e784f6f451964ed34875 Mon Sep 17 00:00:00 2001 From: =?utf8?q?=EC=9D=B4=EC=83=81=EA=B7=9C/=EB=8F=99=EC=9E=91=EC=A0=9C?= =?utf8?q?=EC=96=B4Lab=28SR=29/Principal=20Engineer/=EC=82=BC=EC=84=B1?= =?utf8?q?=EC=A0=84=EC=9E=90?= Date: Tue, 11 Sep 2018 13:14:06 +0900 Subject: [PATCH] Add run_benchmark_tflite_model.sh (#2640) Related Issue: #2544 `run_benchmark_tflite_model.sh` will run `tflite_benchmark_model` tool with `run_benchmark_tflite_model.in` as input. `run_benchmark_tflite_model.in` can have several lines. Each line is a pair of (model name, its parameters for `tflite_benchmark_model`). Its delimeter is `\t`. For example, see `run_benchmark_tflite_mode.in`. Signed-off-by: Sanggyu Lee --- tools/test_driver/run_benchmark_tflite_model.in | 1 + tools/test_driver/run_benchmark_tflite_model.sh | 125 ++++++++++++++++++++++++ tools/test_driver/test_driver.sh | 10 +- 3 files changed, 127 insertions(+), 9 deletions(-) create mode 100644 tools/test_driver/run_benchmark_tflite_model.in create mode 100755 tools/test_driver/run_benchmark_tflite_model.sh diff --git a/tools/test_driver/run_benchmark_tflite_model.in b/tools/test_driver/run_benchmark_tflite_model.in new file mode 100644 index 0000000..1003ecc --- /dev/null +++ b/tools/test_driver/run_benchmark_tflite_model.in @@ -0,0 +1 @@ +MODELS/inception_nonslim --input_layer=Mul --input_layer_shape=1,299,299,3 --num_threads=1 --num_runs=1 diff --git a/tools/test_driver/run_benchmark_tflite_model.sh b/tools/test_driver/run_benchmark_tflite_model.sh new file mode 100755 index 0000000..50a2a5f --- /dev/null +++ b/tools/test_driver/run_benchmark_tflite_model.sh @@ -0,0 +1,125 @@ +#!/bin/bash +# +# Copyright (c) 2018 Samsung Electronics Co., Ltd. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +NNFW_DIR="$(dirname "$(dirname "${SCRIPT_DIR}")")" +REPORT_DIR="$NNFW_DIR/report/tflite_benchmark_model" +MODEL_ROOT="$NNFW_DIR/tests/framework/tests" +LD_LIBRARY_PATH="$NNFW_DIR/Product/out/lib" + +RUN_TEST=$NNFW_DIR/tests/framework/run_test.sh +MODEL_IN=${BASH_SOURCE[0]%.sh}.in +BENCHMARK_BIN=$NNFW_DIR/Product/out/bin/tflite_benchmark_model +MODEL_NAMES= +MODEL_PARAMS= + +source $SCRIPT_DIR/common.sh + +usage() +{ + echo + echo "Usage: LD_LIBRARY_PATH=Product/out/lib $(basename ${BASH_SOURCE[0]}) --reportdir=report --modelroot=modelroot" + echo +} + +parse_args() +{ + for i in "$@"; do + case $i in + -h|--help|help) + usage + exit 1 + ;; + --reportdir=*) + REPORT_DIR=${i#*=} + ;; + --modelroot=*) + MODEL_ROOT=${i#*=} + ;; + esac + shift + done +} + +load_input() +{ + mapfile -t MODEL_NAMES < <(cut -f1 "${MODEL_IN}") + mapfile -t MODEL_PARAMS < <(cut -f2- "${MODEL_IN}") + if [ "${#MODEL_NAMES[@]}" -eq 0 ]; then + echo "No model is found. Please check ${MODEL_IN} is correct." + exit 1 + fi +} + +download_models() +{ + $RUN_TEST --download=on $MODEL_NAMES +} + +run_benchmarks() +{ + echo + echo "Running benchmarks:" + echo "======================" + + for (( i=0; i< ${#MODEL_NAMES[@]}; i++)); do + MODEL_NAME=${MODEL_NAMES[i]} + MODEL_PATH=$(find $NNFW_DIR/tests/framework/cache/$MODEL_NAME/ -name "*.tflite") + MODEL_PARAM=${MODEL_PARAMS[$i]} + + echo "$MODEL_NAME" + + local REPORT_MODEL_DIR=$REPORT_DIR/$MODEL_NAME + mkdir -p $REPORT_MODEL_DIR + + local OUT_FILE + + # TFLite Interpreter + OUT_FILE=$REPORT_MODEL_DIR/tflite_interpreter.out + echo + echo "{ // TFLite Interpreter" + LD_LIBRARY_PATH=$LD_LIBRARY_PATH $BENCHMARK_BIN --graph=$MODEL_PATH $MODEL_PARAM --use_nnapi=false 2> >(tee $OUT_FILE) + echo "} // TFLite Interpreter" + + # TFLite PureACL (CL) + OUT_FILE=$REPORT_MODEL_DIR/tflite_pureacl_cl.out + echo + echo "{ // TFLite PureACL(CL)" + LD_LIBRARY_PATH=$LD_LIBRARY_PATH $BENCHMARK_BIN --graph=$MODEL_PATH $MODEL_PARAM --use_nnapi=true 2> >(tee $OUT_FILE) + echo "} // TFLite_PureACL(CL)" + done +} + +# for debug +print_vars() +{ + echo SCRIPT_DIR=$SCRIPT_DIR + echo NNFW_DIR=$NNFW_DIR + echo RUN_TEST=$RUN_TEST + echo MODEL_IN=$MODEL_IN + echo BENCHMARK_BIN=$BENCHMARK_BIN + echo REPORT_DIR=$REPORT_DIR + echo MODEL_ROOT=$MODEL_ROOT +} + +if [ ! -e "$REPORT_DIR" ]; then + mkdir -p $REPORT_DIR +fi + +parse_args $@ +load_input +download_models +run_benchmarks diff --git a/tools/test_driver/test_driver.sh b/tools/test_driver/test_driver.sh index 8d25cff..72be65d 100755 --- a/tools/test_driver/test_driver.sh +++ b/tools/test_driver/test_driver.sh @@ -307,17 +307,9 @@ fi # Run tflite_benchmark_model (= per-operation profiling tool). # Each model can contain arbitrary number of operators. if [ "$BENCHMARK_TFLITE_MODEL_ON" == "true" ]; then - if [ -z "$BENCHMARK_DRIVER_BIN" ]; then - DRIVER_BIN=$ARTIFACT_PATH/Product/out/bin/tflite_benchmark_model - else - DRIVER_BIN=$BENCHMARK_DRIVER_BIN - fi - $TEST_DRIVER_DIR/run_benchmark_tflite_model.sh \ - --runtestsh=$RUN_TEST_SH \ - --driverbin=$DRIVER_BIN \ --reportdir=$REPORT_DIR/benchmark_tflite_model \ - --modelfilepath=$ARTIFACT_PATH/tests/framework + --modelroot=$ARTIFACT_PATH/tests/framework/tests fi # Run profiling -- 2.7.4