From 20e3510240594ef4dff2246d7c13e5e92407ef9e Mon Sep 17 00:00:00 2001 From: =?utf8?q?=D0=94=D0=B8=D0=BB=D1=88=D0=BE=D0=B4=D0=B6=D0=BE=D0=BD=20?= =?utf8?q?=D0=A3=D0=BC=D1=80=D0=BE=D0=BD=D1=85=D0=BE=D0=BD=D0=BE=D0=B2?= =?utf8?q?=D0=B8=D1=87=20=D0=9F=D0=BE=D1=88=D1=88=D0=BE=D0=B5=D0=B2/AI=20T?= =?utf8?q?ools=20Lab=20/SRR/Engineer/=EC=82=BC=EC=84=B1=EC=A0=84=EC=9E=90?= Date: Fri, 19 Jul 2019 16:06:25 +0900 Subject: [PATCH] Add performance test for scheduler and profiler (#5675) * Add performance test for scheduler and profiler Add a script to test performance of scheduler and profiler. Since each model it compiles and executes 9 times, it takes much time to finish. Related issue: 5660 Signed-off-by: Dilshodzhon Poshshoev * Fix comment of PROFILING_RUN_CNT Signed-off-by: Dilshodzhon Poshshoev * Move exec_time.json to log dir, not remove Signed-off-by: Dilshodzhon Poshshoev * Add some more fixes Signed-off-by: Dilshodzhon Poshshoev --- tests/scripts/common.sh | 28 +++++++ tests/scripts/run_benchmark.sh | 28 ------- tests/scripts/test_performance.sh | 166 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 194 insertions(+), 28 deletions(-) create mode 100644 tests/scripts/test_performance.sh diff --git a/tests/scripts/common.sh b/tests/scripts/common.sh index 12a35fd..1e7ab39 100755 --- a/tests/scripts/common.sh +++ b/tests/scripts/common.sh @@ -32,3 +32,31 @@ function switch_nnfw_kernel_env() fi done } + +function get_result_of_benchmark_test() +{ + local RUN_TEST_SH=$1 + local DRIVER_BIN=$2 + local MODEL=$3 + local LOG_FILE=$4 + + local RET=0 + $RUN_TEST_SH --driverbin=$DRIVER_BIN $MODEL > $LOG_FILE 2>&1 + RET=$? + if [[ $RET -ne 0 ]]; then + echo "Testing $MODEL aborted... exit code: $RET" + exit $RET + fi + + local RESULT=`grep -E '^Mean:' $LOG_FILE | sed -e 's/ms//g' | awk '{print $2}'` + echo "$RESULT" +} + +function print_result_of_benchmark_test() +{ + local NAME=$1 + local RESULT=$2 + local RESULT_FILE=$3 + + echo "$NAME $RESULT" > $RESULT_FILE +} diff --git a/tests/scripts/run_benchmark.sh b/tests/scripts/run_benchmark.sh index 6980475..e22b400 100755 --- a/tests/scripts/run_benchmark.sh +++ b/tests/scripts/run_benchmark.sh @@ -51,34 +51,6 @@ do shift done -function get_result_of_benchmark_test() -{ - local RUN_TEST_SH=$1 - local DRIVER_BIN=$2 - local MODEL=$3 - local LOG_FILE=$4 - - local RET=0 - $RUN_TEST_SH --driverbin=$DRIVER_BIN $MODEL > $LOG_FILE 2>&1 - RET=$? - if [[ $RET -ne 0 ]]; then - echo "Testing $MODEL aborted... exit code: $RET" - exit $RET - fi - - local RESULT=`grep -E '^Mean:' $LOG_FILE | sed -e 's/ms//g' | awk '{print $2}'` - echo "$RESULT" -} - -function print_result_of_benchmark_test() -{ - local NAME=$1 - local RESULT=$2 - local RESULT_FILE=$3 - - echo "$NAME $RESULT" > $RESULT_FILE -} - function run_benchmark_test() { local DRIVER_BIN=$BENCHMARK_DRIVER_BIN diff --git a/tests/scripts/test_performance.sh b/tests/scripts/test_performance.sh new file mode 100644 index 0000000..3ed7b45 --- /dev/null +++ b/tests/scripts/test_performance.sh @@ -0,0 +1,166 @@ +#!/bin/bash + +MY_PATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +source $MY_PATH/common.sh + +PRINT_WIDTH=45 +BACKEND_CNT=3 +# Run profiler BACKEND_CNT+1 times: on each run of the first BACKEND_CNT runs it will +# collect metrics for one unmeasured backend. On the last run metrics for data transfer +PROFILING_RUN_CNT=$((BACKEND_CNT+1)) +TEST_DRIVER_DIR="$( cd "$( dirname "${BASH_SOURCE}" )" && pwd )" +ARTIFACT_PATH="$TEST_DRIVER_DIR/../.." +BENCHMARK_DRIVER_BIN=$ARTIFACT_PATH/Product/out/bin/tflite_benchmark +REPORT_DIR=$ARTIFACT_PATH/report +RUN_TEST_SH=$ARTIFACT_PATH/tests/framework/run_test.sh +BENCHMARK_MODEL_LIST="MODELS/inception_nonslim MODELS/inception_slim MODELS/mobilenet" + +if [ ! -e "$RUN_TEST_SH" ]; then + echo "Cannot find $RUN_TEST_SH" + exit 1 +fi + + +BENCHMARK_REPORT_DIR=$REPORT_DIR/benchmark +BENCHMARK_MODELS_FILE=$BENCHMARK_REPORT_DIR/benchmark_models.txt + +function print_with_dots() +{ + local MSG=$1 + pad=$(printf '%0.1s' "."{1..45}) + padlength=$((PRINT_WIDTH- ${#MSG})) + printf '%s' "$MSG" + printf '%*.*s ' 0 $padlength "$pad" +} + +function run_without_sched() +{ + local RESULT_SCH_INT=$1 + local REPORT_MODEL_DIR=$2 + local MODEL=$3 + local EXECUTOR=$4 + local BACKEND=$5 + + LOG_FILE=$REPORT_MODEL_DIR/tflite_${EXECUTOR,,}_$BACKEND.txt + export OP_BACKEND_ALLOPS=$BACKEND + export EXECUTOR=$EXECUTOR + + print_with_dots "$EXECUTOR $BACKEND without scheduler" + + RESULT=$(get_result_of_benchmark_test $RUN_TEST_SH $BENCHMARK_DRIVER_BIN $MODEL $LOG_FILE) + + printf -v RESULT_INT '%d' $RESULT 2>/dev/null + PERCENTAGE=$((100-RESULT_SCH_INT*100/RESULT_INT)) + echo "$RESULT ms. Scheduler is $PERCENTAGE% faster" +} + +function run_benchmark_test() +{ + local LOG_FILE= + local RESULT= + local REPORT_MODEL_DIR= + + export COUNT=5 + echo "============================================" + local i=0 + export USE_NNAPI=1 + export BACKENDS="acl_cl;acl_neon;cpu" + # Remove metrics so that profiler can get metrics for operations + # with input&output sizes the same as the model + rm "exec_time.json" 2>/dev/null + for MODEL in $BENCHMARK_MODEL_LIST; do + + echo "Benchmark test with `basename $BENCHMARK_DRIVER_BIN` & `echo $MODEL`" + echo $MODEL >> $BENCHMARK_MODELS_FILE + + REPORT_MODEL_DIR=$BENCHMARK_REPORT_DIR/scheduler_benchmark/$MODEL + mkdir -p $REPORT_MODEL_DIR + +################################################################################## + # Get metrics by running profiler +################################################################################## + export USE_SCHEDULER=1 + export PROFILING_MODE=1 + export EXECUTOR="Dataflow" + export NEURUN_LOG_ENABLE=1 + for ((j = 1 ; j <= $PROFILING_RUN_CNT ; j++)); do + # Save the verbose log of each run + LOG_FILE=$REPORT_MODEL_DIR/tflite_profiling_$j.txt + + print_with_dots "Profiling run #$j out of $PROFILING_RUN_CNT" + + $RUN_TEST_SH --driverbin=$BENCHMARK_DRIVER_BIN $MODEL > $LOG_FILE 2>&1 + RET=$? + if [[ $RET -ne 0 ]]; then + echo "Profiling $MODEL aborted in run#$j... exit code: $RET"xX + exit $RET + fi + echo "finished" + # Save the exec_time.json of each run + cp "exec_time.json" $REPORT_MODEL_DIR/"exec_time_$j.json" + done + unset NEURUN_LOG_ENABLE + + +################################################################################## + # Turn off profiling +################################################################################## + export PROFILING_MODE=0 + +################################################################################## + # Run ParallelExecutor with scheduler +################################################################################## + LOG_FILE=$REPORT_MODEL_DIR/tflite_parallel_with_scheduler.txt + export EXECUTOR="Parallel" + export GRAPH_DOT_DUMP=1 + print_with_dots "Parallel with scheduler" + + RESULT=$(get_result_of_benchmark_test $RUN_TEST_SH $BENCHMARK_DRIVER_BIN $MODEL $LOG_FILE) + echo "$RESULT ms" + + printf -v RESULT_SCH_INT '%d' $RESULT 2>/dev/null + + # Remove metrics so that for next model in profiler can get metrics + # for operations with input&output sizes the same as the model + mv "exec_time.json" $REPORT_MODEL_DIR + # Save the dot graph + mv "after_lower.dot" $REPORT_MODEL_DIR + unset GRAPH_DOT_DUMP + +################################################################################## + # Turn off scheduler +################################################################################## + export USE_SCHEDULER=0 + + # Run LinearExecutor on acl_cl without scheduler + run_without_sched $RESULT_SCH_INT $REPORT_MODEL_DIR $MODEL "Linear" "acl_cl" + + # Run LinearExecutor on acl_neon without scheduler + run_without_sched $RESULT_SCH_INT $REPORT_MODEL_DIR $MODEL "Linear" "acl_neon" + + # Run ParallelExecutor on acl_cl without scheduler + run_without_sched $RESULT_SCH_INT $REPORT_MODEL_DIR $MODEL "Parallel" "acl_cl" + + # Run ParallelExecutor on acl_neon without scheduler + run_without_sched $RESULT_SCH_INT $REPORT_MODEL_DIR $MODEL "Parallel" "acl_neon" + + if [[ $i -ne $(echo $BENCHMARK_MODEL_LIST | wc -w)-1 ]]; then + echo "" + fi + i=$((i+1)) + + unset USE_SCHEDULER + unset PROFILING_MODE + unset EXECUTOR + unset OP_BACKEND_ALLOPS + done + unset BACKENDS + echo "============================================" + unset COUNT + unset USE_NNAPI + +} + +echo "" +run_benchmark_test +echo "" -- 2.7.4