From: Дилшоджон Умронхонович Пошшоев/AI Tools Lab /SRR/Engineer/삼성전자 Date: Fri, 19 Jul 2019 07:06:25 +0000 (+0900) Subject: Add performance test for scheduler and profiler (#5675) X-Git-Tag: submit/tizen/20190809.050447~494 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=20e3510240594ef4dff2246d7c13e5e92407ef9e;p=platform%2Fcore%2Fml%2Fnnfw.git Add performance test for scheduler and profiler (#5675) * Add performance test for scheduler and profiler Add a script to test performance of scheduler and profiler. Since each model it compiles and executes 9 times, it takes much time to finish. Related issue: 5660 Signed-off-by: Dilshodzhon Poshshoev * Fix comment of PROFILING_RUN_CNT Signed-off-by: Dilshodzhon Poshshoev * Move exec_time.json to log dir, not remove Signed-off-by: Dilshodzhon Poshshoev * Add some more fixes Signed-off-by: Dilshodzhon Poshshoev --- diff --git a/tests/scripts/common.sh b/tests/scripts/common.sh index 12a35fd..1e7ab39 100755 --- a/tests/scripts/common.sh +++ b/tests/scripts/common.sh @@ -32,3 +32,31 @@ function switch_nnfw_kernel_env() fi done } + +function get_result_of_benchmark_test() +{ + local RUN_TEST_SH=$1 + local DRIVER_BIN=$2 + local MODEL=$3 + local LOG_FILE=$4 + + local RET=0 + $RUN_TEST_SH --driverbin=$DRIVER_BIN $MODEL > $LOG_FILE 2>&1 + RET=$? + if [[ $RET -ne 0 ]]; then + echo "Testing $MODEL aborted... exit code: $RET" + exit $RET + fi + + local RESULT=`grep -E '^Mean:' $LOG_FILE | sed -e 's/ms//g' | awk '{print $2}'` + echo "$RESULT" +} + +function print_result_of_benchmark_test() +{ + local NAME=$1 + local RESULT=$2 + local RESULT_FILE=$3 + + echo "$NAME $RESULT" > $RESULT_FILE +} diff --git a/tests/scripts/run_benchmark.sh b/tests/scripts/run_benchmark.sh index 6980475..e22b400 100755 --- a/tests/scripts/run_benchmark.sh +++ b/tests/scripts/run_benchmark.sh @@ -51,34 +51,6 @@ do shift done -function get_result_of_benchmark_test() -{ - local RUN_TEST_SH=$1 - local DRIVER_BIN=$2 - local MODEL=$3 - local LOG_FILE=$4 - - local RET=0 - $RUN_TEST_SH --driverbin=$DRIVER_BIN $MODEL > $LOG_FILE 2>&1 - RET=$? - if [[ $RET -ne 0 ]]; then - echo "Testing $MODEL aborted... exit code: $RET" - exit $RET - fi - - local RESULT=`grep -E '^Mean:' $LOG_FILE | sed -e 's/ms//g' | awk '{print $2}'` - echo "$RESULT" -} - -function print_result_of_benchmark_test() -{ - local NAME=$1 - local RESULT=$2 - local RESULT_FILE=$3 - - echo "$NAME $RESULT" > $RESULT_FILE -} - function run_benchmark_test() { local DRIVER_BIN=$BENCHMARK_DRIVER_BIN diff --git a/tests/scripts/test_performance.sh b/tests/scripts/test_performance.sh new file mode 100644 index 0000000..3ed7b45 --- /dev/null +++ b/tests/scripts/test_performance.sh @@ -0,0 +1,166 @@ +#!/bin/bash + +MY_PATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +source $MY_PATH/common.sh + +PRINT_WIDTH=45 +BACKEND_CNT=3 +# Run profiler BACKEND_CNT+1 times: on each run of the first BACKEND_CNT runs it will +# collect metrics for one unmeasured backend. On the last run metrics for data transfer +PROFILING_RUN_CNT=$((BACKEND_CNT+1)) +TEST_DRIVER_DIR="$( cd "$( dirname "${BASH_SOURCE}" )" && pwd )" +ARTIFACT_PATH="$TEST_DRIVER_DIR/../.." +BENCHMARK_DRIVER_BIN=$ARTIFACT_PATH/Product/out/bin/tflite_benchmark +REPORT_DIR=$ARTIFACT_PATH/report +RUN_TEST_SH=$ARTIFACT_PATH/tests/framework/run_test.sh +BENCHMARK_MODEL_LIST="MODELS/inception_nonslim MODELS/inception_slim MODELS/mobilenet" + +if [ ! -e "$RUN_TEST_SH" ]; then + echo "Cannot find $RUN_TEST_SH" + exit 1 +fi + + +BENCHMARK_REPORT_DIR=$REPORT_DIR/benchmark +BENCHMARK_MODELS_FILE=$BENCHMARK_REPORT_DIR/benchmark_models.txt + +function print_with_dots() +{ + local MSG=$1 + pad=$(printf '%0.1s' "."{1..45}) + padlength=$((PRINT_WIDTH- ${#MSG})) + printf '%s' "$MSG" + printf '%*.*s ' 0 $padlength "$pad" +} + +function run_without_sched() +{ + local RESULT_SCH_INT=$1 + local REPORT_MODEL_DIR=$2 + local MODEL=$3 + local EXECUTOR=$4 + local BACKEND=$5 + + LOG_FILE=$REPORT_MODEL_DIR/tflite_${EXECUTOR,,}_$BACKEND.txt + export OP_BACKEND_ALLOPS=$BACKEND + export EXECUTOR=$EXECUTOR + + print_with_dots "$EXECUTOR $BACKEND without scheduler" + + RESULT=$(get_result_of_benchmark_test $RUN_TEST_SH $BENCHMARK_DRIVER_BIN $MODEL $LOG_FILE) + + printf -v RESULT_INT '%d' $RESULT 2>/dev/null + PERCENTAGE=$((100-RESULT_SCH_INT*100/RESULT_INT)) + echo "$RESULT ms. Scheduler is $PERCENTAGE% faster" +} + +function run_benchmark_test() +{ + local LOG_FILE= + local RESULT= + local REPORT_MODEL_DIR= + + export COUNT=5 + echo "============================================" + local i=0 + export USE_NNAPI=1 + export BACKENDS="acl_cl;acl_neon;cpu" + # Remove metrics so that profiler can get metrics for operations + # with input&output sizes the same as the model + rm "exec_time.json" 2>/dev/null + for MODEL in $BENCHMARK_MODEL_LIST; do + + echo "Benchmark test with `basename $BENCHMARK_DRIVER_BIN` & `echo $MODEL`" + echo $MODEL >> $BENCHMARK_MODELS_FILE + + REPORT_MODEL_DIR=$BENCHMARK_REPORT_DIR/scheduler_benchmark/$MODEL + mkdir -p $REPORT_MODEL_DIR + +################################################################################## + # Get metrics by running profiler +################################################################################## + export USE_SCHEDULER=1 + export PROFILING_MODE=1 + export EXECUTOR="Dataflow" + export NEURUN_LOG_ENABLE=1 + for ((j = 1 ; j <= $PROFILING_RUN_CNT ; j++)); do + # Save the verbose log of each run + LOG_FILE=$REPORT_MODEL_DIR/tflite_profiling_$j.txt + + print_with_dots "Profiling run #$j out of $PROFILING_RUN_CNT" + + $RUN_TEST_SH --driverbin=$BENCHMARK_DRIVER_BIN $MODEL > $LOG_FILE 2>&1 + RET=$? + if [[ $RET -ne 0 ]]; then + echo "Profiling $MODEL aborted in run#$j... exit code: $RET"xX + exit $RET + fi + echo "finished" + # Save the exec_time.json of each run + cp "exec_time.json" $REPORT_MODEL_DIR/"exec_time_$j.json" + done + unset NEURUN_LOG_ENABLE + + +################################################################################## + # Turn off profiling +################################################################################## + export PROFILING_MODE=0 + +################################################################################## + # Run ParallelExecutor with scheduler +################################################################################## + LOG_FILE=$REPORT_MODEL_DIR/tflite_parallel_with_scheduler.txt + export EXECUTOR="Parallel" + export GRAPH_DOT_DUMP=1 + print_with_dots "Parallel with scheduler" + + RESULT=$(get_result_of_benchmark_test $RUN_TEST_SH $BENCHMARK_DRIVER_BIN $MODEL $LOG_FILE) + echo "$RESULT ms" + + printf -v RESULT_SCH_INT '%d' $RESULT 2>/dev/null + + # Remove metrics so that for next model in profiler can get metrics + # for operations with input&output sizes the same as the model + mv "exec_time.json" $REPORT_MODEL_DIR + # Save the dot graph + mv "after_lower.dot" $REPORT_MODEL_DIR + unset GRAPH_DOT_DUMP + +################################################################################## + # Turn off scheduler +################################################################################## + export USE_SCHEDULER=0 + + # Run LinearExecutor on acl_cl without scheduler + run_without_sched $RESULT_SCH_INT $REPORT_MODEL_DIR $MODEL "Linear" "acl_cl" + + # Run LinearExecutor on acl_neon without scheduler + run_without_sched $RESULT_SCH_INT $REPORT_MODEL_DIR $MODEL "Linear" "acl_neon" + + # Run ParallelExecutor on acl_cl without scheduler + run_without_sched $RESULT_SCH_INT $REPORT_MODEL_DIR $MODEL "Parallel" "acl_cl" + + # Run ParallelExecutor on acl_neon without scheduler + run_without_sched $RESULT_SCH_INT $REPORT_MODEL_DIR $MODEL "Parallel" "acl_neon" + + if [[ $i -ne $(echo $BENCHMARK_MODEL_LIST | wc -w)-1 ]]; then + echo "" + fi + i=$((i+1)) + + unset USE_SCHEDULER + unset PROFILING_MODE + unset EXECUTOR + unset OP_BACKEND_ALLOPS + done + unset BACKENDS + echo "============================================" + unset COUNT + unset USE_NNAPI + +} + +echo "" +run_benchmark_test +echo ""