3 MY_PATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
4 source $MY_PATH/common.sh
7 # Run profiler BACKEND_CNT+1 times: on each run of the first BACKEND_CNT runs it will
8 # collect metrics for one unmeasured backend. On the last run metrics for data transfer
9 PROFILING_RUN_CNT=$((BACKEND_CNT+1))
10 TEST_DRIVER_DIR="$( cd "$( dirname "${BASH_SOURCE}" )" && pwd )"
11 ARTIFACT_PATH="$TEST_DRIVER_DIR/../.."
12 BENCHMARK_DRIVER_BIN=$ARTIFACT_PATH/Product/out/bin/tflite_run
13 REPORT_DIR=$ARTIFACT_PATH/report
14 RUN_TEST_SH=$ARTIFACT_PATH/tests/scripts/framework/run_test.sh
15 BENCHMARK_MODEL_LIST="MODELS/inception_nonslim MODELS/inception_slim MODELS/mobilenet"
17 if [ ! -e "$RUN_TEST_SH" ]; then
18 echo "Cannot find $RUN_TEST_SH"
23 BENCHMARK_REPORT_DIR=$REPORT_DIR/benchmark
24 BENCHMARK_MODELS_FILE=$BENCHMARK_REPORT_DIR/benchmark_models.txt
26 function run_without_sched()
28 local RESULT_SCH_INT=$1
29 local REPORT_MODEL_DIR=$2
34 LOG_FILE=$REPORT_MODEL_DIR/tflite_${EXECUTOR,,}_$BACKEND.txt
35 export OP_BACKEND_ALLOPS=$BACKEND
36 export EXECUTOR=$EXECUTOR
38 print_with_dots "$EXECUTOR $BACKEND without scheduler"
40 RESULT=$(get_result_of_benchmark_test $RUN_TEST_SH $BENCHMARK_DRIVER_BIN $MODEL $LOG_FILE)
42 printf -v RESULT_INT '%d' $RESULT 2>/dev/null
43 PERCENTAGE=$((100-RESULT_SCH_INT*100/RESULT_INT))
44 echo "$RESULT ms. Parallel scheduler is $PERCENTAGE% faster"
47 function run_benchmark_test()
51 local REPORT_MODEL_DIR=
54 echo "============================================"
57 export BACKENDS="acl_cl;acl_neon;cpu"
58 # Remove metrics so that profiler can get metrics for operations
59 # with input&output sizes the same as the model
60 rm "exec_time.json" 2>/dev/null
61 for MODEL in $BENCHMARK_MODEL_LIST; do
63 echo "Benchmark test with `basename $BENCHMARK_DRIVER_BIN` & `echo $MODEL`"
64 echo $MODEL >> $BENCHMARK_MODELS_FILE
66 REPORT_MODEL_DIR=$BENCHMARK_REPORT_DIR/scheduler_benchmark/$MODEL
67 mkdir -p $REPORT_MODEL_DIR
69 ##################################################################################
70 # Get metrics by running profiler
71 ##################################################################################
72 export USE_SCHEDULER=1
73 export PROFILING_MODE=1
74 export EXECUTOR="Dataflow"
75 export ONERT_LOG_ENABLE=1
76 for ((j = 1 ; j <= $PROFILING_RUN_CNT ; j++)); do
77 # Save the verbose log of each run
78 LOG_FILE=$REPORT_MODEL_DIR/tflite_profiling_$j.txt
80 print_with_dots "Profiling run #$j out of $PROFILING_RUN_CNT"
82 $RUN_TEST_SH --driverbin=$BENCHMARK_DRIVER_BIN $MODEL > $LOG_FILE 2>&1
84 if [[ $RET -ne 0 ]]; then
85 echo "Profiling $MODEL aborted in run#$j... exit code: $RET"xX
89 # Save the exec_time.json of each run
90 cp "exec_time.json" $REPORT_MODEL_DIR/"exec_time_$j.json"
92 unset ONERT_LOG_ENABLE
95 ##################################################################################
97 ##################################################################################
98 export PROFILING_MODE=0
100 ##################################################################################
101 # Run ParallelExecutor with scheduler
102 ##################################################################################
103 LOG_FILE=$REPORT_MODEL_DIR/tflite_parallel_with_scheduler.txt
104 export EXECUTOR="Parallel"
105 export GRAPH_DOT_DUMP=1
106 print_with_dots "Parallel with scheduler"
108 RESULT=$(get_result_of_benchmark_test $RUN_TEST_SH $BENCHMARK_DRIVER_BIN $MODEL $LOG_FILE)
111 printf -v RESULT_SCH_INT '%d' $RESULT 2>/dev/null
113 mv "after_lower.dot" $REPORT_MODEL_DIR/"after_lower_parallel.dot"
115 ##################################################################################
116 # Run Linear executor with scheduler
117 ##################################################################################
118 LOG_FILE=$REPORT_MODEL_DIR/tflite_linear_with_scheduler.txt
119 export EXECUTOR="Linear"
120 export GRAPH_DOT_DUMP=1
121 print_with_dots "Linear with scheduler"
123 RESULT=$(get_result_of_benchmark_test $RUN_TEST_SH $BENCHMARK_DRIVER_BIN $MODEL $LOG_FILE)
125 printf -v RESULT_INT '%d' $RESULT 2>/dev/null
126 PERCENTAGE=$((100-RESULT_SCH_INT*100/RESULT_INT))
127 echo "$RESULT ms. Parallel scheduler is $PERCENTAGE% faster"
129 # Remove metrics so that for next model in profiler can get metrics
130 # for operations with input&output sizes the same as the model
131 mv "exec_time.json" $REPORT_MODEL_DIR
133 mv "after_lower.dot" $REPORT_MODEL_DIR/"after_lower_linear.dot"
136 ##################################################################################
138 ##################################################################################
139 export USE_SCHEDULER=0
141 # Run LinearExecutor on acl_cl without scheduler
142 run_without_sched $RESULT_SCH_INT $REPORT_MODEL_DIR $MODEL "Linear" "acl_cl"
144 # Run LinearExecutor on acl_neon without scheduler
145 run_without_sched $RESULT_SCH_INT $REPORT_MODEL_DIR $MODEL "Linear" "acl_neon"
147 # Run ParallelExecutor on acl_cl without scheduler
148 run_without_sched $RESULT_SCH_INT $REPORT_MODEL_DIR $MODEL "Parallel" "acl_cl"
150 # Run ParallelExecutor on acl_neon without scheduler
151 run_without_sched $RESULT_SCH_INT $REPORT_MODEL_DIR $MODEL "Parallel" "acl_neon"
153 if [[ $i -ne $(echo $BENCHMARK_MODEL_LIST | wc -w)-1 ]]; then
161 unset OP_BACKEND_ALLOPS
164 echo "============================================"