tests/scripts/test-driver.sh

   1 #!/bin/bash
   2 #
   3 # Copyright (c) 2018 Samsung Electronics Co., Ltd. All Rights Reserved
   4 #
   5 # Licensed under the Apache License, Version 2.0 (the "License");
   6 # you may not use this file except in compliance with the License.
   7 # You may obtain a copy of the License at
   8 #
   9 #    http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 # Unless required by applicable law or agreed to in writing, software
  12 # distributed under the License is distributed on an "AS IS" BASIS,
  13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 # See the License for the specific language governing permissions and
  15 # limitations under the License.
  16
  17 set -e
  18 # NOTE: Supposed that this script would be executed with an artifact path.
  19 #       The artifact path has tests/(test suite) and Product/
  20 #       Reference this PR(https://github.sec.samsung.net/STAR/nnfw/pull/375).
  21
  22 function Usage()
  23 {
  24     echo "Usage: ./$0 --artifactpath=.    # run all tests"
  25     echo "Usage: ./$0 --artifactpath=/home/dragon/nnfw --frameworktest --verification --benchmark    # run fw test & verfication and benchmark"
  26     echo ""
  27     echo "--artifactpath            - (default={test-driver.sh's path}/../../) it should contain tests/ and Product/"
  28     echo ""
  29     echo "Following options are needed when you want to tests of specific types. If you don't pass any one, unittest and verification will be run"
  30     echo "--frameworktest           - (default=off) run framework test"
  31     echo "--verification            - (default=on) run verification"
  32     echo "--frameworktest_list_file - filepath of model list for test"
  33     echo ""
  34     echo "Following option is only needed when you want to test benchmark."
  35     echo "--benchmark_onert_op     - (default=off) run benchmark per operation on onert"
  36     echo ""
  37     echo "etc."
  38     echo "--framework_driverbin     - (default=../../Product/out/bin/tflite_run) runner for runnning framework tests"
  39     echo "--verification_driverbin  - (default=../../Product/out/bin/nnapi_test) runner for runnning verification tests"
  40     echo ""
  41     echo "--reportdir               - (default=\$ARTIFACT_PATH/report) directory to save report"
  42     echo ""
  43 }
  44
  45 TEST_DRIVER_DIR="$( cd "$( dirname "${BASH_SOURCE}" )" && pwd )"
  46 ARTIFACT_PATH="$TEST_DRIVER_DIR/../../"
  47 FRAMEWORK_DRIVER_BIN=""
  48 VERIFICATION_DRIVER_BIN=""
  49 ALLTEST_ON="true"
  50 FRAMEWORKTEST_ON="false"
  51 VERIFICATION_ON="false"
  52 BENCHMARK_ONERT_OP_ON="false"
  53 REPORT_DIR=""
  54
  55 for i in "$@"
  56 do
  57     case $i in
  58         -h|--help|help)
  59             Usage
  60             exit 1
  61             ;;
  62         --artifactpath=*)
  63             ARTIFACT_PATH=${i#*=}
  64             ;;
  65         --framework_driverbin=*)
  66             FRAMEWORK_DRIVER_BIN=${i#*=}
  67             ;;
  68         --verification_driverbin=*)
  69             VERIFICATION_DRIVER_BIN=${i#*=}
  70             ;;
  71         --frameworktest)
  72             ALLTEST_ON="false"
  73             FRAMEWORKTEST_ON="true"
  74             ;;
  75         --frameworktest_list_file=*)
  76             FRAMEWORKTEST_LIST_FILE=$PWD/${i#*=}
  77             if [ ! -e "$FRAMEWORKTEST_LIST_FILE" ]; then
  78                 echo "Pass on with proper frameworktest_list_file"
  79                 exit 1
  80             fi
  81             ;;
  82         --verification)
  83             ALLTEST_ON="false"
  84             VERIFICATION_ON="true"
  85             ;;
  86         --benchmark_onert_op)
  87             ALLTEST_ON="false"
  88             BENCHMARK_ONERT_OP_ON="true"
  89             ;;
  90         --reportdir=*)
  91             REPORT_DIR=${i#*=}
  92             ;;
  93         *)
  94             # Be careful that others params are handled as $ARTIFACT_PATH
  95             ARTIFACT_PATH="$i"
  96             ;;
  97     esac
  98     shift
  99 done
 100
 101 ARTIFACT_PATH="$(readlink -f $ARTIFACT_PATH)"
 102
 103 if [ -z "$UNIT_TEST_DIR" ]; then
 104     UNIT_TEST_DIR=$ARTIFACT_PATH/Product/out/unittest
 105 fi
 106
 107 if [ -z "$REPORT_DIR" ]; then
 108     REPORT_DIR=$ARTIFACT_PATH/report
 109 fi
 110
 111 source $TEST_DRIVER_DIR/common.sh
 112
 113 # Run tflite_run with various tflite models
 114 if [ "$FRAMEWORKTEST_ON" == "true" ]; then
 115     if [ -z "$FRAMEWORK_DRIVER_BIN" ]; then
 116         FRAMEWORK_DRIVER_BIN=$ARTIFACT_PATH/Product/out/bin/tflite_run
 117     fi
 118
 119     $TEST_DRIVER_DIR/test_framework.sh \
 120         --driverbin=$FRAMEWORK_DRIVER_BIN \
 121         --reportdir=$REPORT_DIR \
 122         --tapname=framework_test.tap \
 123         --logname=framework_test.log \
 124         --testname="Frameworktest" \
 125         --frameworktest_list_file=${FRAMEWORKTEST_LIST_FILE:-}
 126 fi
 127
 128 # Run nnapi_test with various tflite models
 129 if [ "$ALLTEST_ON" == "true" ] || [ "$VERIFICATION_ON" == "true" ]; then
 130     if [ -z "$VERIFICATION_DRIVER_BIN" ]; then
 131         VERIFICATION_DRIVER_BIN=$ARTIFACT_PATH/Product/out/bin/nnapi_test
 132     fi
 133
 134     # verification uses the same script as frameworktest does
 135     $TEST_DRIVER_DIR/test_framework.sh \
 136         --driverbin=$VERIFICATION_DRIVER_BIN \
 137         --reportdir=$REPORT_DIR \
 138         --tapname=verification_test.tap \
 139         --logname=verification_test.log \
 140         --testname="Verification" \
 141         --frameworktest_list_file=${FRAMEWORKTEST_LIST_FILE:-}
 142 fi
 143
 144 if [ "$BENCHMARK_ONERT_OP_ON" == "true" ]; then
 145     DRIVER_BIN=$ARTIFACT_PATH/Product/out/bin/tflite_run
 146
 147     $TEST_DRIVER_DIR/benchmark_nnapi.sh \
 148         --test_op \
 149         --driverbin=$DRIVER_BIN \
 150         --reportdir=$REPORT_DIR/benchmark_op \
 151         --modelfilepath=$ARTIFACT_PATH/tests/scripts/models
 152 fi
 153
 154 # Make json file. Actually, this process is only needed on CI. That's why it is in test-driver.sh.
 155 if [ "$BENCHMARK_ONERT_OP_ON" == "true" ]; then
 156     # functions to fill json with benchmark results
 157     source $ARTIFACT_PATH/tests/scripts/print_to_json.sh
 158     if [ "$BENCHMARK_ONERT_OP_ON" == "true" ]; then
 159         print_to_json $REPORT_DIR/benchmark_op $REPORT_DIR "benchmark_op_result.json"
 160     else
 161         print_to_json $REPORT_DIR/benchmark $REPORT_DIR "benchmark_result.json"
 162     fi
 163 fi