3 # Copyright (c) 2020 Samsung Electronics Co., Ltd. All Rights Reserved
5 # Licensed under the Apache License, Version 2.0 (the "License");
6 # you may not use this file except in compliance with the License.
7 # You may obtain a copy of the License at
9 # http://www.apache.org/licenses/LICENSE-2.0
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 # See the License for the specific language governing permissions and
15 # limitations under the License.
18 # How to run benchmark testing
20 # This script is copy of test_scheduler_with_profiling.sh for Android.
21 # As Android does not provide bash, this with models/run_test_android.sh
22 # and common_android.sh, three scripts are modified for Android benchmark
23 # testing using Android shell.
24 # Test models are downloaded into models folder but as Android also doesn't
25 # provide downloading in shell script, user should push downloaded models
26 # to Android device also.
28 # 1. To download test models,
29 # run test_scheduler_with_profiling.sh from in Ubuntu/ARM device
30 # 2. You will have download models in tests/scripts/models/cache folder
31 # 3. Build for OneRT for Android
33 # adb shell mkdir -p /data/local/tmp/Product/report/benchmark
34 # adb push tests /data/local/tmp/.
35 # adb push Product/aarch64-android.release/out /data/local/tmp/Product/.
37 # 5. Run benchmark inside Android shell
38 # export LD_LIBRARY_PATH=/data/local/tmp/Product/out/lib
40 # sh /data/local/tmp/tests/scripts/test_scheduler_with_profiling_android.sh
43 MY_PATH="$( cd "$( dirname "$0" )" && pwd )"
45 SHELL_CMD=/system/bin/sh
47 source $MY_PATH/common_android.sh
50 # Run profiler BACKEND_CNT+1 times: on each run of the first BACKEND_CNT runs it will
51 # collect metrics for one unmeasured backend. On the last run metrics for data transfer
52 PROFILING_RUN_CNT=$((BACKEND_CNT+1))
53 TEST_DRIVER_DIR="$( cd "$( dirname "$0" )" && pwd )"
55 ARTIFACT_PATH="$TEST_DRIVER_DIR/../.."
56 BENCHMARK_DRIVER_BIN=$ARTIFACT_PATH/Product/out/bin/tflite_run
57 REPORT_DIR=$ARTIFACT_PATH/report
58 RUN_TEST_SH=$ARTIFACT_PATH/tests/scripts/models/run_test_android.sh
59 BENCHMARK_MODEL_LIST="MODELS/inception_nonslim MODELS/inception_slim MODELS/mobilenet"
61 if [ ! -e "$RUN_TEST_SH" ]; then
62 echo "Cannot find $RUN_TEST_SH"
66 BENCHMARK_REPORT_DIR=$REPORT_DIR/benchmark
67 BENCHMARK_MODELS_FILE=$BENCHMARK_REPORT_DIR/benchmark_models.txt
69 function run_without_sched()
71 local RESULT_SCH_INT=$1
72 local REPORT_MODEL_DIR=$2
77 #LOG_FILE=$REPORT_MODEL_DIR/tflite_${EXECUTOR,,}_$BACKEND.txt
78 LOG_FILE=$REPORT_MODEL_DIR/tflite_$EXECUTOR_$BACKEND.txt
79 export OP_BACKEND_ALLOPS=$BACKEND
80 export EXECUTOR=$EXECUTOR
82 print_with_dots "$EXECUTOR $BACKEND without scheduler"
84 RESULT=$(get_result_of_benchmark_test $BENCHMARK_DRIVER_BIN $MODEL $LOG_FILE)
86 # printf -v RESULT_INT '%d' $RESULT 2>/dev/null
87 RESULT_I=$(printf "%.0f" $RESULT)
88 RESULT_INT=$(expr $RESULT_I)
89 PERCENTAGE=$((100 - RESULT_SCH_INT * 100 / RESULT_INT))
90 echo "$RESULT ms. Parallel scheduler is $PERCENTAGE % faster"
93 function run_benchmark_test()
97 local REPORT_MODEL_DIR=
100 echo "============================================"
103 export BACKENDS="acl_cl;acl_neon;cpu"
104 # Remove metrics so that profiler can get metrics for operations
105 # with input&output sizes the same as the model
106 rm "exec_time.json" 2>/dev/null
107 for MODEL in $BENCHMARK_MODEL_LIST; do
109 echo "Benchmark test with `basename $BENCHMARK_DRIVER_BIN` & `echo $MODEL`"
110 echo $MODEL >> $BENCHMARK_MODELS_FILE
112 REPORT_MODEL_DIR=$BENCHMARK_REPORT_DIR/scheduler_benchmark/$MODEL
113 mkdir -p $REPORT_MODEL_DIR
115 ##################################################################################
116 # Get metrics by running profiler
117 ##################################################################################
118 export USE_SCHEDULER=1
119 export PROFILING_MODE=1
120 export EXECUTOR="Dataflow"
121 export ONERT_LOG_ENABLE=1
122 for j in 1 2 3 4; do # 1 to $PROFILING_RUN_CNT
123 # Save the verbose log of each run
124 LOG_FILE=$REPORT_MODEL_DIR/tflite_profiling_$j.txt
126 print_with_dots "Profiling run #$j out of $PROFILING_RUN_CNT"
128 $SHELL_CMD $RUN_TEST_SH --driverbin=$BENCHMARK_DRIVER_BIN $MODEL > $LOG_FILE 2>&1
130 if [[ $RET -ne 0 ]]; then
131 echo "Profiling $MODEL aborted in run#$j... exit code: $RET"
135 # Save the exec_time.json of each run
136 cp "exec_time.json" $REPORT_MODEL_DIR/"exec_time_$j.json"
138 unset ONERT_LOG_ENABLE
141 ##################################################################################
143 ##################################################################################
144 export PROFILING_MODE=0
146 ##################################################################################
147 # Run ParallelExecutor with scheduler
148 ##################################################################################
149 LOG_FILE=$REPORT_MODEL_DIR/tflite_parallel_with_scheduler.txt
150 export EXECUTOR="Parallel"
151 export GRAPH_DOT_DUMP=1
152 print_with_dots "Parallel with scheduler"
154 RESULT=$(get_result_of_benchmark_test $BENCHMARK_DRIVER_BIN $MODEL $LOG_FILE)
157 # printf -v RESULT_SCH_INT '%d' $RESULT 2>/dev/null
158 RESULT_I=$(printf "%.0f" $RESULT)
159 RESULT_SCH_INT=$(expr $RESULT_I)
161 mv "after_lower_subg-0.dot" $REPORT_MODEL_DIR/"after_lower_subg-0_parallel.dot"
163 ##################################################################################
164 # Run Linear executor with scheduler
165 ##################################################################################
166 LOG_FILE=$REPORT_MODEL_DIR/tflite_linear_with_scheduler.txt
167 export EXECUTOR="Linear"
168 export GRAPH_DOT_DUMP=1
169 print_with_dots "Linear with scheduler"
171 RESULT=$(get_result_of_benchmark_test $BENCHMARK_DRIVER_BIN $MODEL $LOG_FILE)
173 # printf -v RESULT_INT '%d' $RESULT 2>/dev/null
174 RESULT_I=$(printf "%.0f" $RESULT)
175 RESULT_INT=$(expr $RESULT_I)
177 PERCENTAGE=$((100 - $RESULT_SCH_INT * 100 / $RESULT_INT))
179 echo "$RESULT ms. Parallel scheduler is $PERCENTAGE % faster"
181 # Remove metrics so that for next model in profiler can get metrics
182 # for operations with input&output sizes the same as the model
183 mv "exec_time.json" $REPORT_MODEL_DIR
185 mv "after_lower_subg-0.dot" $REPORT_MODEL_DIR/"after_lower_subg-0_linear.dot"
188 ##################################################################################
190 ##################################################################################
191 export USE_SCHEDULER=0
193 # Run LinearExecutor on acl_cl without scheduler
194 run_without_sched $RESULT_SCH_INT $REPORT_MODEL_DIR $MODEL "Linear" "acl_cl"
196 # Run LinearExecutor on acl_neon without scheduler
197 run_without_sched $RESULT_SCH_INT $REPORT_MODEL_DIR $MODEL "Linear" "acl_neon"
199 # Run LinearExecutor on cpu without scheduler
200 # run_without_sched $RESULT_SCH_INT $REPORT_MODEL_DIR $MODEL "Linear" "cpu"
202 # Run ParallelExecutor on acl_cl without scheduler
203 run_without_sched $RESULT_SCH_INT $REPORT_MODEL_DIR $MODEL "Parallel" "acl_cl"
205 # Run ParallelExecutor on acl_neon without scheduler
206 run_without_sched $RESULT_SCH_INT $REPORT_MODEL_DIR $MODEL "Parallel" "acl_neon"
208 # Run ParallelExecutor on cpi without scheduler
209 # run_without_sched $RESULT_SCH_INT $REPORT_MODEL_DIR $MODEL "Parallel" "cpu"
211 if [[ $i -ne $(echo $BENCHMARK_MODEL_LIST | wc -w)-1 ]]; then
219 unset OP_BACKEND_ALLOPS
222 echo "============================================"