+++ /dev/null
-#!/bin/bash
-#
-# Copyright (c) 2018 Samsung Electronics Co., Ltd. All Rights Reserved
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-MY_PATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-NNFW_HOME="$(dirname $(dirname ${MY_PATH}))"
-source $MY_PATH/common.sh
-
-BENCHMARK_RUN_TEST_SH=
-BENCHMARK_DRIVER_BIN=
-BENCHMARK_REPORT_DIR=
-BENCHMARK_MODELS_FILE=
-BENCHMARK_MODEL_LIST=
-MODEL_CACHE_ROOT_PATH=
-MODEL_TEST_ROOT_PATH=
-PURE_ACL_RT_ENV_FILE=$MY_PATH/benchmark_op_list.txt
-
-function Usage()
-{
- # TODO: Fill this
- echo "Usage: LD_LIBRARY_PATH=Product/out/lib/pureacl ./$0 --reportdir=report"
-}
-
-for i in "$@"
-do
- case $i in
- -h|--help|help)
- Usage
- exit 1
- ;;
- --runtestsh=*)
- BENCHMARK_RUN_TEST_SH=${i#*=}
- ;;
- --driverbin=*)
- BENCHMARK_DRIVER_BIN=${i#*=}
- ;;
- --reportdir=*)
- BENCHMARK_REPORT_DIR=${i#*=}
- BENCHMARK_MODELS_FILE=$BENCHMARK_REPORT_DIR/benchmark_op_models.txt
- ;;
- --modelfilepath=*)
- TEST_LIST_PATH=${i#*=}
- MODEL_CACHE_ROOT_PATH=$TEST_LIST_PATH/cache
- MODEL_TEST_ROOT_PATH=$TEST_LIST_PATH/tests
- ;;
- esac
- shift
-done
-
-function get_benchmark_op_list()
-{
- local TEST_DIRS="$@"
- local TESTS_TO_RUN=""
-
- if [[ $# -eq 0 ]]; then
- TEST_DIRS="."
- fi
-
- shift $#
-
- pushd $MODEL_TEST_ROOT_PATH > /dev/null
- for DIR in $TEST_DIRS; do
- if [ -d "$DIR" ]; then
- TESTS_FOUND=$(find "$DIR" -type f -name 'config.sh' -exec dirname {} \;| sed 's|^./||' | grep -v '^MODELS/' | sort)
- TESTS_TO_RUN="$TESTS_TO_RUN $TESTS_FOUND"
- fi
- done
- popd > /dev/null
-
- BENCHMARK_MODEL_LIST=$(echo "${TESTS_TO_RUN}")
-}
-
-function run_benchmark_test()
-{
- local DRIVER_BIN=$BENCHMARK_DRIVER_BIN
- local LOG_FILE=
- local RESULT_FILE=
- local RESULT=
- local REPORT_MODEL_DIR=
- local PAUSE_TIME_IN_SEC=15
-
- export COUNT=5
- echo "============================================"
- local i=0
- for MODEL in $BENCHMARK_MODEL_LIST; do
- STATUS="enabled"
- source $MODEL_TEST_ROOT_PATH/$MODEL/config.sh
-
- # Skip 'disabled' tests
- if [ $(tr '[:upper:]' '[:lower:]' <<< "$STATUS") == "disabled" ]; then
- continue
- fi
-
- echo "Benchmark test with `basename $DRIVER_BIN` & `echo $MODEL`"
- echo $MODEL >> $BENCHMARK_MODELS_FILE
-
- REPORT_MODEL_DIR=$BENCHMARK_REPORT_DIR/$MODEL
- mkdir -p $REPORT_MODEL_DIR
-
- # TFLite Interpreter
- unset USE_NNAPI
- run_benchmark_and_print "tflite_cpu_op" "TFLite Interpreter" $MODEL $REPORT_MODEL_DIR $PAUSE_TIME_IN_SEC $DRIVER_BIN $BENCHMARK_RUN_TEST_SH
-
- # PureACL-Neon
- export USE_NNAPI=1
- export NEON=1
- run_benchmark_and_print "tflite_nnrt_acl_neon_op" "PureACL-Neon" $MODEL $REPORT_MODEL_DIR $PAUSE_TIME_IN_SEC $DRIVER_BIN $BENCHMARK_RUN_TEST_SH
-
- # PureACL-CL
- export USE_NNAPI=1
- unset NEON
- run_benchmark_and_print "tflite_nnrt_acl_opencl_op" "PureACL-CL" $MODEL $REPORT_MODEL_DIR $PAUSE_TIME_IN_SEC $DRIVER_BIN $BENCHMARK_RUN_TEST_SH
-
- # Clean Up
- unset USE_NNAPI NEON
-
- if [[ $i -ne $(echo $BENCHMARK_MODEL_LIST | wc -w)-1 ]]; then
- echo ""
- fi
- i=$((i+1))
- done
- unset USE_NNAPI
- unset COUNT
- echo "============================================"
-}
-
-if [ ! -e "$BENCHMARK_REPORT_DIR" ]; then
- mkdir -p $BENCHMARK_REPORT_DIR
-fi
-
-get_benchmark_op_list
-
-rm -rf $BENCHMARK_MODELS_FILE
-
-echo ""
-run_benchmark_test
-echo ""
+++ /dev/null
-MODELS/inception_nonslim --input_layer=Mul --input_layer_shape=1,299,299,3 --num_threads=1 --num_runs=1
-MODELS/mobilenet --input_layer=input --input_layer_shape=1,128,128,3 --num_threads=1 --num_runs=1
+++ /dev/null
-#!/bin/bash
-#
-# Copyright (c) 2018 Samsung Electronics Co., Ltd. All Rights Reserved
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-NNFW_DIR="$(dirname "$(dirname "${SCRIPT_DIR}")")"
-REPORT_DIR="$NNFW_DIR/report/tflite_benchmark_model"
-MODEL_ROOT="$NNFW_DIR/tests/framework/tests"
-LD_LIBRARY_PATH="$NNFW_DIR/Product/out/lib/pureacl"
-
-RUN_TEST=$NNFW_DIR/tests/framework/run_test.sh
-MODEL_IN=${BASH_SOURCE[0]%.sh}.in
-BENCHMARK_BIN=$NNFW_DIR/Product/out/bin/tflite_benchmark_model
-MODEL_NAMES=
-MODEL_PARAMS=
-
-source $SCRIPT_DIR/common.sh
-
-usage()
-{
- echo
- echo "Usage: LD_LIBRARY_PATH=Product/out/lib/pureacl $(basename ${BASH_SOURCE[0]}) --reportdir=report --modelroot=modelroot"
- echo
-}
-
-parse_args()
-{
- for i in "$@"; do
- case $i in
- -h|--help|help)
- usage
- exit 1
- ;;
- --reportdir=*)
- REPORT_DIR=${i#*=}
- ;;
- --modelroot=*)
- MODEL_ROOT=${i#*=}
- ;;
- esac
- shift
- done
-}
-
-load_input()
-{
- mapfile -t MODEL_NAMES < <(cut -f1 "${MODEL_IN}")
- mapfile -t MODEL_PARAMS < <(cut -f2- "${MODEL_IN}")
- if [ "${#MODEL_NAMES[@]}" -eq 0 ]; then
- echo "No model is found. Please check ${MODEL_IN} is correct."
- exit 1
- fi
-}
-
-download_models()
-{
- $RUN_TEST --download=on $MODEL_NAMES
-}
-
-run_benchmarks()
-{
- echo
- echo "Running benchmarks:"
- echo "======================"
-
- for (( i=0; i< ${#MODEL_NAMES[@]}; i++)); do
- MODEL_NAME=${MODEL_NAMES[i]}
- MODEL_PATH=$(find $NNFW_DIR/tests/framework/cache/$MODEL_NAME/ -name "*.tflite")
- MODEL_PARAM=${MODEL_PARAMS[$i]}
-
- echo "$MODEL_NAME"
-
- local REPORT_MODEL_DIR=$REPORT_DIR/$MODEL_NAME
- mkdir -p $REPORT_MODEL_DIR
-
- local OUT_FILE
-
- # TFLite Interpreter
- OUT_FILE=$REPORT_MODEL_DIR/tflite_interpreter.out
- echo
- echo "{ // TFLite Interpreter"
- LD_LIBRARY_PATH=$LD_LIBRARY_PATH $BENCHMARK_BIN --graph=$MODEL_PATH $MODEL_PARAM --use_nnapi=false 2> >(tee $OUT_FILE)
- echo "} // TFLite Interpreter"
-
- # TFLite PureACL (CL)
- OUT_FILE=$REPORT_MODEL_DIR/tflite_pureacl_cl.out
- echo
- echo "{ // TFLite PureACL(CL)"
- LD_LIBRARY_PATH=$LD_LIBRARY_PATH $BENCHMARK_BIN --graph=$MODEL_PATH $MODEL_PARAM --use_nnapi=true 2> >(tee $OUT_FILE)
- echo "} // TFLite_PureACL(CL)"
- done
-}
-
-# for debug
-print_vars()
-{
- echo SCRIPT_DIR=$SCRIPT_DIR
- echo NNFW_DIR=$NNFW_DIR
- echo RUN_TEST=$RUN_TEST
- echo MODEL_IN=$MODEL_IN
- echo BENCHMARK_BIN=$BENCHMARK_BIN
- echo REPORT_DIR=$REPORT_DIR
- echo MODEL_ROOT=$MODEL_ROOT
-}
-
-if [ ! -e "$REPORT_DIR" ]; then
- mkdir -p $REPORT_DIR
-fi
-
-parse_args $@
-load_input
-download_models
-run_benchmarks
echo "--frameworktest_list_file - filepath of model list for test"
echo ""
echo "Following option is only needed when you want to test benchmark."
- echo "--benchmark_acl - (default=off) run benchmark-acl"
- echo "--benchmark - (default=off) run benchmark on neurun"
- echo "--benchmark_pacl_op - (default=off) run benchmark per operation on pacl"
echo "--benchmark_neurun_op - (default=off) run benchmark per operation on neurun"
- echo "--benchmark_tflite_model - (default=off) run tflite_benchmark_model"
echo ""
echo "Following option is used for profiling."
echo "--profile - (default=off) run operf"
UNITTESTALL_ON="false"
FRAMEWORKTEST_ON="false"
VERIFICATION_ON="false"
-BENCHMARK_ON="false"
-BENCHMARK_PACL_OP_ON="false"
BENCHMARK_NEURUN_OP_ON="false"
-BENCHMARK_TFLITE_MODEL_ON="false"
ACL_ENV_ON="false"
PROFILE_ON="false"
REPORT_DIR=""
ALLTEST_ON="false"
VERIFICATION_ON="true"
;;
- --benchmark)
- ALLTEST_ON="false"
- BENCHMARK_ON="true"
- ;;
- --benchmark_pacl_op)
- ALLTEST_ON="false"
- BENCHMARK_PACL_OP_ON="true"
- ;;
--benchmark_neurun_op)
ALLTEST_ON="false"
BENCHMARK_NEURUN_OP_ON="true"
;;
- --benchmark_tflite_model)
- ALLTEST_ON="false"
- BENCHMARK_TFLITE_MODEL_ON="true"
- ;;
- # TODO Remove --benchmark_acl option after CI update
- --benchmark_acl)
- ALLTEST_ON="false"
- ;;
--acl_envon)
ACL_ENV_ON="true"
;;
--frameworktest_list_file=${FRAMEWORKTEST_LIST_FILE:-}
fi
-# Run tflite_benchmark with tflite models
-if [ "$BENCHMARK_ON" == "true" ]; then
- if [ -z "$BENCHMARK_DRIVER_BIN" ]; then
- DRIVER_BIN=$ARTIFACT_PATH/Product/out/bin/tflite_benchmark
- else
- DRIVER_BIN=$BENCHMARK_DRIVER_BIN
- fi
-
- $TEST_DRIVER_DIR/run_benchmark_neurun.sh \
- --runtestsh=$RUN_TEST_SH \
- --driverbin=$DRIVER_BIN \
- --reportdir=$REPORT_DIR/benchmark
-fi
-
-# Run tflite_benchmark from a list of tflite models.
-# Each model has only one operator.
-if [ "$BENCHMARK_PACL_OP_ON" == "true" ]; then
- if [ -z "$BENCHMARK_DRIVER_BIN" ]; then
- DRIVER_BIN=$ARTIFACT_PATH/Product/out/bin/tflite_benchmark
- else
- DRIVER_BIN=$BENCHMARK_DRIVER_BIN
- fi
-
- $TEST_DRIVER_DIR/run_benchmark_pacl.sh \
- --runtestsh=$RUN_TEST_SH \
- --driverbin=$DRIVER_BIN \
- --reportdir=$REPORT_DIR/benchmark_op \
- --modelfilepath=$ARTIFACT_PATH/tests/framework
-fi
-
if [ "$BENCHMARK_NEURUN_OP_ON" == "true" ]; then
if [ -z "$BENCHMARK_DRIVER_BIN" ]; then
DRIVER_BIN=$ARTIFACT_PATH/Product/out/bin/tflite_benchmark
fi
# Make json file. Actually, this process is only needed on CI. That's why it is in test_driver.sh.
-if [ "$BENCHMARK_ON" == "true" ] || [ "$BENCHMARK_PACL_OP_ON" == "true" ] || [ "$BENCHMARK_NEURUN_OP_ON" == "true" ]; then
+if [ "$BENCHMARK_NEURUN_OP_ON" == "true" ]; then
# functions to fill json with benchmark results
source $ARTIFACT_PATH/tests/scripts/print_to_json.sh
- if [ "$BENCHMARK_PACL_OP_ON" == "true" ] || [ "$BENCHMARK_NEURUN_OP_ON" == "true" ]; then
+ if [ "$BENCHMARK_NEURUN_OP_ON" == "true" ]; then
print_to_json $REPORT_DIR/benchmark_op $REPORT_DIR "benchmark_op_result.json"
else
print_to_json $REPORT_DIR/benchmark $REPORT_DIR "benchmark_result.json"
fi
fi
-# Run tflite_benchmark_model (= per-operation profiling tool).
-# Each model can contain arbitrary number of operators.
-if [ "$BENCHMARK_TFLITE_MODEL_ON" == "true" ]; then
- $TEST_DRIVER_DIR/run_benchmark_tflite_model.sh \
- --reportdir=$REPORT_DIR/benchmark_tflite_model \
- --modelroot=$ARTIFACT_PATH/tests/framework/tests
-fi
-
# Run profiling
if [ "$PROFILE_ON" == "true" ]; then
# FIXME: These driver and tflite test are set temporarily. Fix these to support flexibility
projects / platform / core / ml / nnfw.git / commitdiff