Add run_benchmark_tflite_model.sh (#2640)

Related Issue: #2544

`run_benchmark_tflite_model.sh` will run `tflite_benchmark_model` tool
with `run_benchmark_tflite_model.in` as input.

`run_benchmark_tflite_model.in` can have several lines.
Each line is a pair of (model name, its parameters for `tflite_benchmark_model`).
Its delimeter is `\t`.

For example, see `run_benchmark_tflite_mode.in`.

Signed-off-by: Sanggyu Lee <sg5.lee@samsung.com>

diff --git a/tools/test_driver/run_benchmark_tflite_model.in b/tools/test_driver/run_benchmark_tflite_model.in
new file mode 100644 (file)
index 0000000..1003ecc
--- /dev/null
+++ b/tools/test_driver/run_benchmark_tflite_model.in
@@ -0,0 +1 @@
+MODELS/inception_nonslim       --input_layer=Mul --input_layer_shape=1,299,299,3 --num_threads=1 --num_runs=1

diff --git a/tools/test_driver/run_benchmark_tflite_model.sh b/tools/test_driver/run_benchmark_tflite_model.sh
new file mode 100755 (executable)
index 0000000..50a2a5f
--- /dev/null
+++ b/tools/test_driver/run_benchmark_tflite_model.sh
@@ -0,0 +1,125 @@
+#!/bin/bash
+#
+# Copyright (c) 2018 Samsung Electronics Co., Ltd. All Rights Reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+NNFW_DIR="$(dirname "$(dirname "${SCRIPT_DIR}")")"
+REPORT_DIR="$NNFW_DIR/report/tflite_benchmark_model"
+MODEL_ROOT="$NNFW_DIR/tests/framework/tests"
+LD_LIBRARY_PATH="$NNFW_DIR/Product/out/lib"
+
+RUN_TEST=$NNFW_DIR/tests/framework/run_test.sh
+MODEL_IN=${BASH_SOURCE[0]%.sh}.in
+BENCHMARK_BIN=$NNFW_DIR/Product/out/bin/tflite_benchmark_model
+MODEL_NAMES=
+MODEL_PARAMS=
+
+source $SCRIPT_DIR/common.sh
+
+usage()
+{
+  echo
+  echo "Usage: LD_LIBRARY_PATH=Product/out/lib $(basename ${BASH_SOURCE[0]}) --reportdir=report --modelroot=modelroot"
+  echo
+}
+
+parse_args()
+{
+  for i in "$@"; do
+    case $i in
+      -h|--help|help)
+        usage
+        exit 1
+        ;;
+      --reportdir=*)
+        REPORT_DIR=${i#*=}
+        ;;
+      --modelroot=*)
+        MODEL_ROOT=${i#*=}
+        ;;
+    esac
+    shift
+  done
+}
+
+load_input()
+{
+  mapfile -t MODEL_NAMES  < <(cut -f1  "${MODEL_IN}")
+  mapfile -t MODEL_PARAMS < <(cut -f2- "${MODEL_IN}")
+  if [ "${#MODEL_NAMES[@]}" -eq 0 ]; then
+    echo "No model is found. Please check ${MODEL_IN} is correct."
+    exit 1
+  fi
+}
+
+download_models()
+{
+  $RUN_TEST --download=on $MODEL_NAMES
+}
+
+run_benchmarks()
+{
+  echo
+  echo "Running benchmarks:"
+  echo "======================"
+
+  for (( i=0; i< ${#MODEL_NAMES[@]}; i++)); do
+    MODEL_NAME=${MODEL_NAMES[i]}
+    MODEL_PATH=$(find $NNFW_DIR/tests/framework/cache/$MODEL_NAME/ -name "*.tflite")
+    MODEL_PARAM=${MODEL_PARAMS[$i]}
+
+    echo "$MODEL_NAME"
+
+    local REPORT_MODEL_DIR=$REPORT_DIR/$MODEL_NAME
+    mkdir -p $REPORT_MODEL_DIR
+
+    local OUT_FILE
+
+    # TFLite Interpreter
+    OUT_FILE=$REPORT_MODEL_DIR/tflite_interpreter.out
+    echo
+    echo "{ // TFLite Interpreter"
+    LD_LIBRARY_PATH=$LD_LIBRARY_PATH $BENCHMARK_BIN --graph=$MODEL_PATH $MODEL_PARAM --use_nnapi=false 2> >(tee $OUT_FILE)
+    echo "} // TFLite Interpreter"
+
+    # TFLite PureACL (CL)
+    OUT_FILE=$REPORT_MODEL_DIR/tflite_pureacl_cl.out
+    echo
+    echo "{ // TFLite PureACL(CL)"
+    LD_LIBRARY_PATH=$LD_LIBRARY_PATH $BENCHMARK_BIN --graph=$MODEL_PATH $MODEL_PARAM --use_nnapi=true 2> >(tee $OUT_FILE)
+    echo "} // TFLite_PureACL(CL)"
+  done
+}
+
+# for debug
+print_vars()
+{
+  echo SCRIPT_DIR=$SCRIPT_DIR
+  echo NNFW_DIR=$NNFW_DIR
+  echo RUN_TEST=$RUN_TEST
+  echo MODEL_IN=$MODEL_IN
+  echo BENCHMARK_BIN=$BENCHMARK_BIN
+  echo REPORT_DIR=$REPORT_DIR
+  echo MODEL_ROOT=$MODEL_ROOT
+}
+
+if [ ! -e "$REPORT_DIR" ]; then
+    mkdir -p $REPORT_DIR
+fi
+
+parse_args $@
+load_input
+download_models
+run_benchmarks

diff --git a/tools/test_driver/test_driver.sh b/tools/test_driver/test_driver.sh
index 8d25cff..72be65d 100755 (executable)
--- a/tools/test_driver/test_driver.sh
+++ b/tools/test_driver/test_driver.sh
@@ -307,17 +307,9 @@ fi
 # Run tflite_benchmark_model (= per-operation profiling tool).
 # Each model can contain arbitrary number of operators.
 if [ "$BENCHMARK_TFLITE_MODEL_ON" == "true" ]; then
-    if [ -z "$BENCHMARK_DRIVER_BIN" ]; then
-        DRIVER_BIN=$ARTIFACT_PATH/Product/out/bin/tflite_benchmark_model
-    else
-        DRIVER_BIN=$BENCHMARK_DRIVER_BIN
-    fi
-
     $TEST_DRIVER_DIR/run_benchmark_tflite_model.sh \
-        --runtestsh=$RUN_TEST_SH \
-        --driverbin=$DRIVER_BIN \
         --reportdir=$REPORT_DIR/benchmark_tflite_model \
-        --modelfilepath=$ARTIFACT_PATH/tests/framework
+        --modelroot=$ARTIFACT_PATH/tests/framework/tests
 fi
 
 # Run profiling